diff options
| author | Fabrice <fabrice@schaub-dev.xyz> | 2026-04-13 10:23:24 +0200 |
|---|---|---|
| committer | Fabrice <fabrice@schaub-dev.xyz> | 2026-04-13 11:31:27 +0200 |
| commit | 57da910f0f257f47b7d07739b231da1d502a4096 (patch) | |
| tree | da5237bec411be2ecda0867556c0b544e2d4bc89 /benches | |
| parent | 86da0af02dde3fa4600071593a73b1b9a267fb9a (diff) | |
benchmarks
adding unordered_map vs. cheesemap bench
adds tidwall's hashmap
adds abseil bench
plotting the results
Diffstat (limited to 'benches')
| -rw-r--r-- | benches/CMakeLists.txt | 55 | ||||
| -rw-r--r-- | benches/SPEC.txt | 234 | ||||
| m--------- | benches/abseil | 0 | ||||
| -rw-r--r-- | benches/abseil_bench.cpp | 87 | ||||
| -rw-r--r-- | benches/bench_common.hpp | 543 | ||||
| -rw-r--r-- | benches/cheesemap_bench.cpp | 80 | ||||
| m--------- | benches/gbench | 0 | ||||
| -rw-r--r-- | benches/results.py | 139 | ||||
| m--------- | benches/tidwall | 0 | ||||
| -rw-r--r-- | benches/tidwall_bench.cpp | 85 | ||||
| -rw-r--r-- | benches/unordered_map_bench.cpp | 87 |
11 files changed, 1310 insertions, 0 deletions
diff --git a/benches/CMakeLists.txt b/benches/CMakeLists.txt new file mode 100644 index 0000000..d096f4b --- /dev/null +++ b/benches/CMakeLists.txt @@ -0,0 +1,55 @@ +set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) +set(BENCHMARK_USE_BUNDLED_GTEST OFF CACHE BOOL "" FORCE) + +add_subdirectory("${cm_benchesdir}/gbench" EXCLUDE_FROM_ALL) + +set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE) +set(ABSL_BUILD_TESTING OFF CACHE BOOL "" FORCE) +add_subdirectory("${cm_benchesdir}/abseil" EXCLUDE_FROM_ALL) + +set(cm_bench_targets + cheesemap_bench + tidwall_bench + unordered_map_bench + abseil_bench +) + +foreach(target_name IN LISTS cm_bench_targets) + add_executable(${target_name} "${cm_benchesdir}/${target_name}.cpp") + + target_include_directories(${target_name} + PRIVATE + "${cm_topdir}" + "${cm_benchesdir}" + ) + + target_compile_options(${target_name} + PRIVATE + -Wall -Wextra -Werror + $<$<BOOL:${CM_ENABLE_NATIVE}>:-march=native> + ) + + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${target_name} PRIVATE -O3) + endif() + + target_link_libraries(${target_name} + PRIVATE + benchmark::benchmark_main + ) + + set_target_properties(${target_name} PROPERTIES + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + ) +endforeach() + +target_link_libraries(cheesemap_bench PRIVATE cheesemap) + +set(cm_tidwalldir "${cm_benchesdir}/tidwall") +target_include_directories(tidwall_bench PRIVATE "${cm_tidwalldir}") +target_sources(tidwall_bench PRIVATE "${cm_tidwalldir}/hashmap.c") + +target_link_libraries(abseil_bench PRIVATE absl::flat_hash_map) diff --git a/benches/SPEC.txt b/benches/SPEC.txt new file mode 100644 index 0000000..c0971f1 --- /dev/null +++ b/benches/SPEC.txt @@ -0,0 +1,234 @@ +Benchmark Specification + +Purpose + +This document defines the benchmark protocol for this repository. It is not tied to any specific data structure, language, runtime, or library. Its purpose is to ensure that all benchmark results are reproducible, comparable, and technically defensible. + +Scope + +This specification applies to all microbenchmarks and throughput benchmarks added to this repository, including native, managed, interpreted, and JIT-compiled implementations. + +Benchmark Goals + +The benchmark suite must measure: + +1. Steady-state operation cost for defined workloads. +2. Sensitivity to key shape, value shape, and hash quality. +3. Sensitivity to lookup hit rate, miss rate, insertion rate, and deletion rate. +4. Variance across repeated runs under a controlled environment. + +The benchmark suite must not: + +1. Mix unrelated workloads into a single reported result without exposing the individual phases. +2. Compare implementations that are not performing the same logical work. +3. Rely on implementation-specific shortcuts that bypass the workload under test. + +Common Rules + +1. Every benchmark case must define: + - dataset size + - key type + - value type + - hash function or hasher configuration + - equality semantics + - operation sequence + - hit and miss distribution + +2. Every benchmark case must use deterministic input generation from a fixed seed. + +3. Every benchmark case must prevent dead-code elimination by consuming benchmark outputs in a way visible to the optimizer barrier provided by the benchmark framework. + +4. Every implementation in a comparison set must execute the same logical dataset and the same operation sequence. + +5. Reported timings must be wall-clock timings measured by the benchmark framework unless otherwise stated. + +6. A benchmark must not include one-time setup cost inside the measured region unless the benchmark explicitly claims to measure setup cost. + +7. Data generation, fixture construction, and input shuffling must happen outside the timed region unless the benchmark case is explicitly a setup benchmark. + +Workload Categories + +Each benchmark must declare one of the following categories: + +1. Insert + Measures insertion of N unique keys into an empty container. + +2. LookupHit + Measures successful lookup of existing keys. + +3. LookupMiss + Measures unsuccessful lookup of absent keys. + +4. Erase + Measures deletion of existing keys. + +5. Mixed + Measures a fixed operation mix such as 80 percent lookup, 10 percent insert, 10 percent erase. + +6. Iterate + Measures full traversal of all live elements. + +Each category must be reported separately. A combined total may be reported as a derived value, but only if the component phases are also reported. + +Dataset Definitions + +Datasets must be named and versioned. Each dataset definition must specify: + +1. Key layout + Examples: u64, pair<u32,u32>, fixed-size struct, string handle. + +2. Value layout + Examples: u64, pointer-sized value, fixed-size payload, metadata struct. + +3. Entry count + +4. Key distribution + Examples: sequential then shuffled, pre-mixed random, clustered, adversarial. + +5. Lookup sets + - hit set + - miss set + +6. Hash quality profile + - good hash + - weak hash + - adversarial hash + +Recommended baseline datasets: + +1. Scalar + u64 key, u64 value + +2. HandlePayload + small fixed-size handle key, medium fixed-size payload value + +3. CompositeKey + composite fixed-size key, fixed-size metadata value + +Hash Policy + +Hash behavior must be explicit. + +1. If the benchmark compares containers that accept user-provided hashes, the benchmark must state the exact hash function used. + +2. If an implementation performs internal post-mixing or additional hash finalization, that is part of the implementation and must not be stripped out for benchmarking. + +3. Weak-hash and adversarial-hash cases must be reported separately from good-hash cases. + +4. Report hash quality as a dataset attribute, not as an informal note. + +Operation Semantics + +For all compared implementations: + +1. Insert must mean insertion or overwrite according to the implementation's normal semantics. + +2. LookupHit must verify that the returned value corresponds to the requested key. + +3. LookupMiss must verify absence. + +4. Erase must verify that the element existed and was removed. + +5. Mixed workloads must use a deterministic precomputed operation stream. + +6. If reserve or preallocation is used, that choice must be declared explicitly and applied consistently to all implementations that support an equivalent capability. + +Timing Rules + +1. Use at least 5 independent runs per benchmark case. + +2. Report: + - mean + - minimum + - maximum + - optionally standard deviation + +3. If a benchmark framework already handles warmup and statistical sampling, its configuration must be recorded. + +4. For JIT or managed runtimes, warmup policy must be explicit. + +5. If a benchmark includes separate phases, time each phase independently. + +Environment Control + +Each benchmark report must record: + +1. CPU model +2. core count +3. operating system +4. compiler and version +5. runtime and version +6. optimization flags +7. benchmark framework version +8. allocator configuration if relevant +9. SIMD configuration if relevant +10. thread count + +Recommended controls: + +1. Run on an otherwise idle machine. +2. Use a fixed CPU governor when possible. +3. Pin to a fixed core or NUMA node when possible. +4. Disable unrelated background load when possible. + +Output Format + +Every benchmark run must emit machine-readable output. + +Required raw fields: + +1. benchmark_name +2. implementation +3. language +4. dataset +5. workload_category +6. hash_quality +7. entry_count +8. run_index +9. time_unit +10. elapsed_value + +Recommended additional fields: + +1. simd_mode +2. allocator +3. compiler +4. compiler_flags +5. runtime +6. notes + +CSV or JSON output is acceptable. Raw per-run data must be preserved in the repository or in a referenced artifact location. + +Comparison Rules + +A comparison is valid only if: + +1. The compared implementations operate on equivalent datasets. +2. The compared implementations perform equivalent logical operations. +3. Reported numbers are taken from the same benchmark protocol revision. + +A comparison is not valid if: + +1. One implementation uses a specialized fast path that changes the benchmarked problem. +2. One implementation includes setup cost and another excludes it. +3. One implementation is measured with a different dataset shape or hit/miss mix. + +Revision Policy + +When the benchmark protocol changes, results produced under the old protocol must not be silently mixed with results from the new protocol. + +Each benchmark output set must record: + +1. protocol version +2. dataset version +3. implementation revision + +Minimum Acceptance Standard + +No benchmark result may be added to the README or other summary tables unless: + +1. the raw data exists +2. the benchmark case is reproducible from checked-in code or documented external commands +3. the environment and flags are recorded +4. at least 5 runs were collected + diff --git a/benches/abseil b/benches/abseil new file mode 160000 +Subproject 04b6110da90be12296037f9e91b8bd6d4b0d3ae diff --git a/benches/abseil_bench.cpp b/benches/abseil_bench.cpp new file mode 100644 index 0000000..813bf51 --- /dev/null +++ b/benches/abseil_bench.cpp @@ -0,0 +1,87 @@ +#include <benchmark/benchmark.h> + +#include "bench_common.hpp" + +namespace { + +using namespace cmbench; + +static void BM_Insert_Scalar(benchmark::State& state) { + bench_insert<AbseilAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"absl::flat_hash_map", "c++", "Scalar", "Insert"}); +} +static void BM_LookupHit_Scalar(benchmark::State& state) { + bench_lookup_hit<AbseilAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"absl::flat_hash_map", "c++", "Scalar", "LookupHit"}); +} +static void BM_LookupMiss_Scalar(benchmark::State& state) { + bench_lookup_miss<AbseilAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"absl::flat_hash_map", "c++", "Scalar", "LookupMiss"}); +} +static void BM_Erase_Scalar(benchmark::State& state) { + bench_erase<AbseilAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"absl::flat_hash_map", "c++", "Scalar", "Erase"}); +} + +static void BM_Insert_HandlePayload(benchmark::State& state) { + bench_insert<AbseilAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"absl::flat_hash_map", "c++", "HandlePayload", "Insert"}); +} +static void BM_LookupHit_HandlePayload(benchmark::State& state) { + bench_lookup_hit<AbseilAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"absl::flat_hash_map", "c++", "HandlePayload", "LookupHit"}); +} +static void BM_LookupMiss_HandlePayload(benchmark::State& state) { + bench_lookup_miss<AbseilAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"absl::flat_hash_map", "c++", "HandlePayload", "LookupMiss"}); +} +static void BM_Erase_HandlePayload(benchmark::State& state) { + bench_erase<AbseilAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"absl::flat_hash_map", "c++", "HandlePayload", "Erase"}); +} + +static void BM_Insert_CompositeKey(benchmark::State& state) { + bench_insert<AbseilAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"absl::flat_hash_map", "c++", "CompositeKey", "Insert"}); +} +static void BM_LookupHit_CompositeKey(benchmark::State& state) { + bench_lookup_hit<AbseilAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"absl::flat_hash_map", "c++", "CompositeKey", "LookupHit"}); +} +static void BM_LookupMiss_CompositeKey(benchmark::State& state) { + bench_lookup_miss<AbseilAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"absl::flat_hash_map", "c++", "CompositeKey", "LookupMiss"}); +} +static void BM_Erase_CompositeKey(benchmark::State& state) { + bench_erase<AbseilAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"absl::flat_hash_map", "c++", "CompositeKey", "Erase"}); +} + +} // namespace + +BENCHMARK(BM_Insert_Scalar); +BENCHMARK(BM_LookupHit_Scalar); +BENCHMARK(BM_LookupMiss_Scalar); +BENCHMARK(BM_Erase_Scalar); + +BENCHMARK(BM_Insert_HandlePayload); +BENCHMARK(BM_LookupHit_HandlePayload); +BENCHMARK(BM_LookupMiss_HandlePayload); +BENCHMARK(BM_Erase_HandlePayload); + +BENCHMARK(BM_Insert_CompositeKey); +BENCHMARK(BM_LookupHit_CompositeKey); +BENCHMARK(BM_LookupMiss_CompositeKey); +BENCHMARK(BM_Erase_CompositeKey); diff --git a/benches/bench_common.hpp b/benches/bench_common.hpp new file mode 100644 index 0000000..732ffb4 --- /dev/null +++ b/benches/bench_common.hpp @@ -0,0 +1,543 @@ +#ifndef CHEESEMAP_BENCH_COMMON_HPP +#define CHEESEMAP_BENCH_COMMON_HPP + +#include <benchmark/benchmark.h> + +#include <algorithm> +#include <array> +#include <cstdarg> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <random> +#include <string_view> +#include <type_traits> +#include <unordered_map> +#include <vector> + +#include "cheesemap.h" +#include "tidwall/hashmap.h" +#include "absl/container/flat_hash_map.h" + +namespace cmbench { + +inline constexpr std::int64_t kEntryCount = 1'000'000; +inline constexpr std::string_view kProtocolVersion = "1"; +inline constexpr std::string_view kDatasetVersion = "1"; +inline constexpr std::string_view kHashQuality = "good"; +inline constexpr std::string_view kHashName = "xxh_avalanche"; + +[[noreturn]] inline void panic_impl(const char* file, cm_u32 line, + const char* fmt, ...) { + std::fprintf(stderr, "panic at %s:%u: ", file, line); + va_list args; + va_start(args, fmt); + std::vfprintf(stderr, fmt, args); + va_end(args); + std::fputc('\n', stderr); + std::abort(); +} + +inline cm_u8* default_alloc(cm_usize size, cm_usize align, cm_u8* user) { + (void)user; + return static_cast<cm_u8*>(std::aligned_alloc(align, size)); +} + +inline void default_dealloc(cm_u8* ptr, cm_u8* user) { + (void)user; + std::free(ptr); +} + +inline std::uint64_t rotl64(std::uint64_t x, unsigned bits) { + return (x << bits) | (x >> (64 - bits)); +} + +inline std::uint64_t xxh64_avalanche(std::uint64_t x) { + x ^= x >> 33; + x *= 0xc2b2ae3d27d4eb4fULL; + x ^= x >> 29; + x *= 0x165667b19e3779f9ULL; + x ^= x >> 32; + return x; +} + +template <typename T> +inline std::array<std::uint64_t, (sizeof(T) + 7) / 8> as_words(const T& value) { + static_assert(std::is_trivially_copyable_v<T>); + std::array<std::uint64_t, (sizeof(T) + 7) / 8> words{}; + std::memcpy(words.data(), &value, sizeof(T)); + return words; +} + +template <typename T> +inline std::uint64_t hash_xxh_avalanche(const T& value) { + auto words = as_words(value); + std::uint64_t acc = 0x165667b19e3779f9ULL ^ sizeof(T); + for (std::uint64_t word : words) { + acc += word * 0x9e3779b185ebca87ULL; + acc = rotl64(acc, 31); + acc *= 0xc2b2ae3d27d4eb4fULL; + } + return xxh64_avalanche(acc); +} + +template <typename T> +inline bool byte_equal(const T& lhs, const T& rhs) { + static_assert(std::is_trivially_copyable_v<T>); + return std::memcmp(&lhs, &rhs, sizeof(T)) == 0; +} + +struct ScalarValue { + std::uint64_t value; +}; + +struct EntityId { + std::uint32_t index; + std::uint32_t generation; +}; + +struct ComponentPayload { + float position[3]; + float velocity[3]; + std::uint32_t archetype; + std::uint32_t chunk; +}; + +struct ComponentKey { + std::uint64_t archetype_mask; + std::uint32_t component_id; + std::uint32_t chunk; + std::uint32_t row; + std::uint32_t generation; +}; + +struct ComponentMeta { + std::uint64_t entity_mask; + std::uint32_t dense_index; + std::uint32_t sparse_index; + std::uint64_t version; +}; + +template <typename Key, typename Value> +struct Workload { + std::vector<Key> keys; + std::vector<Key> miss_keys; + std::vector<Value> values; +}; + +template <typename Key> +inline void shuffle_keys(std::vector<Key>& keys, std::mt19937_64& rng) { + std::shuffle(keys.begin(), keys.end(), rng); +} + +inline Workload<std::uint64_t, ScalarValue> make_scalar_workload( + std::size_t count) { + Workload<std::uint64_t, ScalarValue> workload{ + .keys = std::vector<std::uint64_t>(count), + .miss_keys = std::vector<std::uint64_t>(count), + .values = std::vector<ScalarValue>(count), + }; + + std::mt19937_64 rng(0xc0ffee5eedULL); + for (std::size_t i = 0; i < count; ++i) { + workload.keys[i] = xxh64_avalanche(i + 1); + workload.miss_keys[i] = xxh64_avalanche(i + 1 + count); + workload.values[i] = + ScalarValue{xxh64_avalanche(i ^ 0x123456789abcdef0ULL)}; + } + shuffle_keys(workload.keys, rng); + shuffle_keys(workload.miss_keys, rng); + return workload; +} + +inline Workload<EntityId, ComponentPayload> make_entity_workload( + std::size_t count) { + Workload<EntityId, ComponentPayload> workload{ + .keys = std::vector<EntityId>(count), + .miss_keys = std::vector<EntityId>(count), + .values = std::vector<ComponentPayload>(count), + }; + + std::mt19937_64 rng(0xEC500123ULL); + for (std::size_t i = 0; i < count; ++i) { + workload.keys[i] = EntityId{static_cast<std::uint32_t>(i), + static_cast<std::uint32_t>(i >> 12)}; + workload.miss_keys[i] = EntityId{static_cast<std::uint32_t>(i), + static_cast<std::uint32_t>((i >> 12) + 1)}; + + float base = static_cast<float>(i & 0xffff); + workload.values[i] = ComponentPayload{ + .position = {base + 0.1f, base + 0.2f, base + 0.3f}, + .velocity = {base + 1.1f, base + 1.2f, base + 1.3f}, + .archetype = static_cast<std::uint32_t>(xxh64_avalanche(i) & 0xffff), + .chunk = static_cast<std::uint32_t>(i / 64), + }; + } + shuffle_keys(workload.keys, rng); + shuffle_keys(workload.miss_keys, rng); + return workload; +} + +inline Workload<ComponentKey, ComponentMeta> make_component_workload( + std::size_t count) { + Workload<ComponentKey, ComponentMeta> workload{ + .keys = std::vector<ComponentKey>(count), + .miss_keys = std::vector<ComponentKey>(count), + .values = std::vector<ComponentMeta>(count), + }; + + std::mt19937_64 rng(0xA11CECC5ULL); + for (std::size_t i = 0; i < count; ++i) { + std::uint64_t archetype = xxh64_avalanche(i * 0x9e3779b97f4a7c15ULL); + workload.keys[i] = ComponentKey{ + .archetype_mask = archetype, + .component_id = static_cast<std::uint32_t>((i * 17) & 0xffff), + .chunk = static_cast<std::uint32_t>(i / 64), + .row = static_cast<std::uint32_t>(i % 64), + .generation = static_cast<std::uint32_t>(i >> 10), + }; + workload.miss_keys[i] = ComponentKey{ + .archetype_mask = archetype ^ 0xfeedfacecafebeefULL, + .component_id = static_cast<std::uint32_t>((i * 17) & 0xffff), + .chunk = static_cast<std::uint32_t>(i / 64), + .row = static_cast<std::uint32_t>(i % 64), + .generation = static_cast<std::uint32_t>(i >> 10), + }; + workload.values[i] = ComponentMeta{ + .entity_mask = xxh64_avalanche(i + 7), + .dense_index = static_cast<std::uint32_t>(i), + .sparse_index = + static_cast<std::uint32_t>(xxh64_avalanche(i) & 0xffffffffU), + .version = xxh64_avalanche(i ^ 0xabcdef01ULL), + }; + } + shuffle_keys(workload.keys, rng); + shuffle_keys(workload.miss_keys, rng); + return workload; +} + +inline const Workload<std::uint64_t, ScalarValue>& scalar_workload() { + static const auto workload = make_scalar_workload(kEntryCount); + return workload; +} + +inline const Workload<EntityId, ComponentPayload>& entity_workload() { + static const auto workload = make_entity_workload(kEntryCount); + return workload; +} + +inline const Workload<ComponentKey, ComponentMeta>& component_workload() { + static const auto workload = make_component_workload(kEntryCount); + return workload; +} + +template <typename T> +inline cm_hash_t cheesemap_hash_adapter(const cm_u8* key, cm_u8* user) { + (void)user; + return hash_xxh_avalanche(*reinterpret_cast<const T*>(key)); +} + +template <typename T> +inline bool cheesemap_equal_adapter(const cm_u8* lhs, const cm_u8* rhs, + cm_u8* user) { + (void)user; + return byte_equal(*reinterpret_cast<const T*>(lhs), + *reinterpret_cast<const T*>(rhs)); +} + +struct Meta { + std::string_view implementation; + std::string_view language; + std::string_view dataset; + std::string_view workload_category; +}; + +inline void set_common_metadata(benchmark::State& state, const Meta& meta) { + std::string label; + label.reserve(128); + label.append("implementation="); + label.append(meta.implementation); + label.append(",language="); + label.append(meta.language); + label.append(",dataset="); + label.append(meta.dataset); + label.append(",workload="); + label.append(meta.workload_category); + label.append(",hash_quality="); + label.append(kHashQuality); + label.append(",hash="); + label.append(kHashName); + label.append(",protocol="); + label.append(kProtocolVersion); + label.append(",dataset_version="); + label.append(kDatasetVersion); + state.SetLabel(label); + state.counters["entry_count"] = + benchmark::Counter(static_cast<double>(kEntryCount)); +} + +template <typename Key, typename Value> +struct CheesemapAdapter { + cheesemap map; + + CheesemapAdapter() { + cm_init(&map, sizeof(Key), alignof(Key), sizeof(Value), alignof(Value), + nullptr, cheesemap_hash_adapter<Key>, cheesemap_equal_adapter<Key>, + default_alloc, default_dealloc); + } + + ~CheesemapAdapter() { cm_drop(&map); } + + CheesemapAdapter(const CheesemapAdapter&) = delete; + CheesemapAdapter& operator=(const CheesemapAdapter&) = delete; + + void reserve(std::size_t count) { + if (!cm_reserve(&map, count)) + panic_impl(__FILE__, __LINE__, "reserve failed"); + } + + void insert(const Key& key, const Value& value) { + if (!cm_insert(&map, reinterpret_cast<const cm_u8*>(&key), + reinterpret_cast<const cm_u8*>(&value))) { + panic_impl(__FILE__, __LINE__, "insert failed"); + } + } + + Value* lookup_hit(const Key& key) { + cm_u8* value_ptr = nullptr; + if (!cm_lookup(&map, reinterpret_cast<const cm_u8*>(&key), &value_ptr)) + panic_impl(__FILE__, __LINE__, "lookup hit failed"); + return reinterpret_cast<Value*>(value_ptr); + } + + void lookup_miss(const Key& key) { + cm_u8* value_ptr = nullptr; + if (cm_lookup(&map, reinterpret_cast<const cm_u8*>(&key), &value_ptr)) + panic_impl(__FILE__, __LINE__, "lookup miss failed"); + benchmark::DoNotOptimize(value_ptr); + } + + void erase(const Key& key) { + if (!cm_remove(&map, reinterpret_cast<const cm_u8*>(&key), nullptr)) + panic_impl(__FILE__, __LINE__, "erase failed"); + } +}; + +template <typename Key> +struct UnorderedHashAdapter { + std::size_t operator()(const Key& key) const noexcept { + return static_cast<std::size_t>(hash_xxh_avalanche(key)); + } +}; + +template <typename Key> +struct UnorderedEqualAdapter { + bool operator()(const Key& lhs, const Key& rhs) const noexcept { + return byte_equal(lhs, rhs); + } +}; + +template <typename Key, typename Value> +struct UnorderedMapAdapter { + using Map = std::unordered_map<Key, Value, UnorderedHashAdapter<Key>, + UnorderedEqualAdapter<Key>>; + + Map map; + + void reserve(std::size_t count) { map.reserve(count); } + + void insert(const Key& key, const Value& value) { + map.insert_or_assign(key, value); + } + + Value* lookup_hit(const Key& key) { + auto it = map.find(key); + if (it == map.end()) panic_impl(__FILE__, __LINE__, "lookup hit failed"); + return &it->second; + } + + void lookup_miss(const Key& key) { + auto it = map.find(key); + if (it != map.end()) panic_impl(__FILE__, __LINE__, "lookup miss failed"); + benchmark::DoNotOptimize(it); + } + + void erase(const Key& key) { + if (map.erase(key) != 1) panic_impl(__FILE__, __LINE__, "erase failed"); + } +}; + +template <typename Key, typename Value> +struct TidwallEntry { + Key key; + Value value; +}; + +template <typename Key, typename Value> +struct TidwallAdapter { + using Entry = TidwallEntry<Key, Value>; + + hashmap* map = nullptr; + + static std::uint64_t hash_entry(const void* item, std::uint64_t seed0, + std::uint64_t seed1) { + (void)seed0; + (void)seed1; + const auto* entry = static_cast<const Entry*>(item); + return hash_xxh_avalanche(entry->key); + } + + static int compare_entry(const void* lhs, const void* rhs, void* udata) { + (void)udata; + const auto* left = static_cast<const Entry*>(lhs); + const auto* right = static_cast<const Entry*>(rhs); + return byte_equal(left->key, right->key) ? 0 : 1; + } + + TidwallAdapter() { + map = hashmap_new(sizeof(Entry), static_cast<std::size_t>(kEntryCount), 0, + 0, hash_entry, compare_entry, nullptr, nullptr); + if (map == nullptr) panic_impl(__FILE__, __LINE__, "hashmap_new failed"); + } + + ~TidwallAdapter() { + if (map != nullptr) hashmap_free(map); + } + + TidwallAdapter(const TidwallAdapter&) = delete; + TidwallAdapter& operator=(const TidwallAdapter&) = delete; + + void reserve(std::size_t count) { benchmark::DoNotOptimize(count); } + + void insert(const Key& key, const Value& value) { + const Entry entry{key, value}; + if (hashmap_set(map, &entry) == nullptr && hashmap_oom(map)) + panic_impl(__FILE__, __LINE__, "insert failed"); + } + + Value* lookup_hit(const Key& key) { + const Entry query{key, Value{}}; + auto* found = static_cast<const Entry*>(hashmap_get(map, &query)); + if (found == nullptr) panic_impl(__FILE__, __LINE__, "lookup hit failed"); + return const_cast<Value*>(&found->value); + } + + void lookup_miss(const Key& key) { + const Entry query{key, Value{}}; + auto* found = static_cast<const Entry*>(hashmap_get(map, &query)); + if (found != nullptr) panic_impl(__FILE__, __LINE__, "lookup miss failed"); + benchmark::DoNotOptimize(found); + } + + void erase(const Key& key) { + const Entry query{key, Value{}}; + if (hashmap_delete(map, &query) == nullptr) + panic_impl(__FILE__, __LINE__, "erase failed"); + } +}; + +template <typename Key, typename Value> +struct AbseilAdapter { + using Map = absl::flat_hash_map<Key, Value, UnorderedHashAdapter<Key>, + UnorderedEqualAdapter<Key>>; + + Map map; + + void reserve(std::size_t count) { map.reserve(count); } + + void insert(const Key& key, const Value& value) { + map.insert_or_assign(key, value); + } + + Value* lookup_hit(const Key& key) { + auto it = map.find(key); + if (it == map.end()) panic_impl(__FILE__, __LINE__, "lookup hit failed"); + return &it->second; + } + + void lookup_miss(const Key& key) { + auto it = map.find(key); + if (it != map.end()) panic_impl(__FILE__, __LINE__, "lookup miss failed"); + benchmark::DoNotOptimize(it); + } + + void erase(const Key& key) { + if (map.erase(key) != 1) panic_impl(__FILE__, __LINE__, "erase failed"); + } +}; + +template <typename Adapter, typename Key, typename Value> +inline void fill_container(Adapter& adapter, + const Workload<Key, Value>& workload) { + adapter.reserve(workload.keys.size()); + for (std::size_t i = 0; i < workload.keys.size(); ++i) + adapter.insert(workload.keys[i], workload.values[i]); +} + +template <typename Adapter, typename Key, typename Value> +inline void bench_insert(benchmark::State& state, + const Workload<Key, Value>& workload, + const Meta& meta) { + set_common_metadata(state, meta); + for (auto _ : state) { + Adapter adapter; + fill_container(adapter, workload); + benchmark::ClobberMemory(); + } +} + +template <typename Adapter, typename Key, typename Value> +inline void bench_lookup_hit(benchmark::State& state, + const Workload<Key, Value>& workload, + const Meta& meta) { + set_common_metadata(state, meta); + for (auto _ : state) { + state.PauseTiming(); + Adapter adapter; + fill_container(adapter, workload); + state.ResumeTiming(); + + for (const Key& key : workload.keys) { + Value* value_ptr = adapter.lookup_hit(key); + benchmark::DoNotOptimize(value_ptr); + } + benchmark::ClobberMemory(); + } +} + +template <typename Adapter, typename Key, typename Value> +inline void bench_lookup_miss(benchmark::State& state, + const Workload<Key, Value>& workload, + const Meta& meta) { + set_common_metadata(state, meta); + for (auto _ : state) { + state.PauseTiming(); + Adapter adapter; + fill_container(adapter, workload); + state.ResumeTiming(); + + for (const Key& key : workload.miss_keys) adapter.lookup_miss(key); + benchmark::ClobberMemory(); + } +} + +template <typename Adapter, typename Key, typename Value> +inline void bench_erase(benchmark::State& state, + const Workload<Key, Value>& workload, + const Meta& meta) { + set_common_metadata(state, meta); + for (auto _ : state) { + state.PauseTiming(); + Adapter adapter; + fill_container(adapter, workload); + state.ResumeTiming(); + + for (const Key& key : workload.keys) adapter.erase(key); + benchmark::ClobberMemory(); + } +} + +} // namespace cmbench + +#endif diff --git a/benches/cheesemap_bench.cpp b/benches/cheesemap_bench.cpp new file mode 100644 index 0000000..a85e54f --- /dev/null +++ b/benches/cheesemap_bench.cpp @@ -0,0 +1,80 @@ +#include <benchmark/benchmark.h> + +#include "bench_common.hpp" + +namespace { + +using namespace cmbench; + +static void BM_Insert_Scalar(benchmark::State& state) { + bench_insert<CheesemapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), {"cheesemap", "c", "Scalar", "Insert"}); +} +static void BM_LookupHit_Scalar(benchmark::State& state) { + bench_lookup_hit<CheesemapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), {"cheesemap", "c", "Scalar", "LookupHit"}); +} +static void BM_LookupMiss_Scalar(benchmark::State& state) { + bench_lookup_miss<CheesemapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), {"cheesemap", "c", "Scalar", "LookupMiss"}); +} +static void BM_Erase_Scalar(benchmark::State& state) { + bench_erase<CheesemapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), {"cheesemap", "c", "Scalar", "Erase"}); +} + +static void BM_Insert_HandlePayload(benchmark::State& state) { + bench_insert<CheesemapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), {"cheesemap", "c", "HandlePayload", "Insert"}); +} +static void BM_LookupHit_HandlePayload(benchmark::State& state) { + bench_lookup_hit<CheesemapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"cheesemap", "c", "HandlePayload", "LookupHit"}); +} +static void BM_LookupMiss_HandlePayload(benchmark::State& state) { + bench_lookup_miss<CheesemapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"cheesemap", "c", "HandlePayload", "LookupMiss"}); +} +static void BM_Erase_HandlePayload(benchmark::State& state) { + bench_erase<CheesemapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), {"cheesemap", "c", "HandlePayload", "Erase"}); +} + +static void BM_Insert_CompositeKey(benchmark::State& state) { + bench_insert<CheesemapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"cheesemap", "c", "CompositeKey", "Insert"}); +} +static void BM_LookupHit_CompositeKey(benchmark::State& state) { + bench_lookup_hit<CheesemapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"cheesemap", "c", "CompositeKey", "LookupHit"}); +} +static void BM_LookupMiss_CompositeKey(benchmark::State& state) { + bench_lookup_miss<CheesemapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"cheesemap", "c", "CompositeKey", "LookupMiss"}); +} +static void BM_Erase_CompositeKey(benchmark::State& state) { + bench_erase<CheesemapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), {"cheesemap", "c", "CompositeKey", "Erase"}); +} + +} // namespace + +BENCHMARK(BM_Insert_Scalar); +BENCHMARK(BM_LookupHit_Scalar); +BENCHMARK(BM_LookupMiss_Scalar); +BENCHMARK(BM_Erase_Scalar); + +BENCHMARK(BM_Insert_HandlePayload); +BENCHMARK(BM_LookupHit_HandlePayload); +BENCHMARK(BM_LookupMiss_HandlePayload); +BENCHMARK(BM_Erase_HandlePayload); + +BENCHMARK(BM_Insert_CompositeKey); +BENCHMARK(BM_LookupHit_CompositeKey); +BENCHMARK(BM_LookupMiss_CompositeKey); +BENCHMARK(BM_Erase_CompositeKey); diff --git a/benches/gbench b/benches/gbench new file mode 160000 +Subproject 7bf975066c82bac78c887786d0003f6e50b046f diff --git a/benches/results.py b/benches/results.py new file mode 100644 index 0000000..e6c6f4c --- /dev/null +++ b/benches/results.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Visualize benchmark results from Google Benchmark JSON output.""" + +import json +import sys +from pathlib import Path +from collections import defaultdict + +import matplotlib.pyplot as plt +import numpy as np + + +def load_benchmark_json(filepath): + """Load and parse a benchmark JSON file.""" + with open(filepath, "r") as f: + return json.load(f) + + +def extract_benchmark_data(json_data): + """Extract relevant benchmark data from JSON.""" + results = defaultdict(lambda: defaultdict(dict)) + + for bench in json_data["benchmarks"]: + if bench["aggregate_name"] != "mean": + continue + + label = bench.get("label", "") + + # Parse label + parts = {} + for item in label.split(","): + if "=" in item: + key, value = item.split("=", 1) + parts[key] = value + + implementation = parts.get("implementation", "unknown") + dataset = parts.get("dataset", "unknown") + workload = parts.get("workload", "unknown") + + # Convert nanoseconds to milliseconds + time_ms = bench["cpu_time"] / 1_000_000 + + results[implementation][dataset][workload] = time_ms + + return results + + +def create_comparison_chart(all_results, output_path): + """Create a grouped bar chart comparing all implementations.""" + + # Define workload order and colors + workloads = ["Insert", "LookupHit", "LookupMiss", "Erase"] + datasets = ["Scalar", "HandlePayload", "CompositeKey"] + + # Color scheme + colors = { + "Insert": "#3498db", + "LookupHit": "#2ecc71", + "LookupMiss": "#e74c3c", + "Erase": "#f39c12", + } + + implementations = list(all_results.keys()) + + # Create subplots for each dataset + fig, axes = plt.subplots(1, 3, figsize=(18, 6)) + title = "HashMap Benchmark Comparison (1M entries, lower is better)" + fig.suptitle(title, fontsize=16, fontweight="bold") + + for idx, dataset in enumerate(datasets): + ax = axes[idx] + + x = np.arange(len(implementations)) + width = 0.2 + + for i, workload in enumerate(workloads): + values = [] + for impl in implementations: + val = all_results[impl].get(dataset, {}).get(workload, 0) + values.append(val) + + offset = width * (i - 1.5) + ax.bar(x + offset, values, width, label=workload, color=colors[workload]) + + ax.set_ylabel("Time (ms)", fontsize=12) + ax.set_title(f"{dataset} Dataset", fontsize=14, fontweight="bold") + ax.set_xticks(x) + ax.set_xticklabels(implementations, rotation=15, ha="right") + ax.legend() + ax.grid(axis="y", alpha=0.3) + + plt.tight_layout() + plt.savefig(output_path, dpi=300, bbox_inches="tight") + print(f"Chart saved to: {output_path}") + + +def main(): + # Load all benchmark JSONs from parent directory + script_dir = Path(__file__).parent + root_dir = script_dir.parent + + json_files = { + "cheesemap": root_dir / "cheesemap.json", + "std::unordered_map": root_dir / "unordered.json", + "tidwall": root_dir / "tidwall.json", + "absl::flat_hash_map": root_dir / "abseil.json", + } + + all_results = {} + + for name, filepath in json_files.items(): + if not filepath.exists(): + print(f"Warning: {filepath} not found, skipping...") + continue + + print(f"Loading {filepath}...") + data = load_benchmark_json(filepath) + results = extract_benchmark_data(data) + + # Use the implementation name from the data if available + if results: + impl_name = list(results.keys())[0] + all_results[impl_name] = results[impl_name] + else: + all_results[name] = {} + + if not all_results: + print("Error: No valid benchmark data found!") + sys.exit(1) + + # Create visualizations in root directory + print("\nGenerating chart...") + create_comparison_chart(all_results, root_dir / "benchmarks.png") + + print("\nDone!") + + +if __name__ == "__main__": + main() diff --git a/benches/tidwall b/benches/tidwall new file mode 160000 +Subproject 5e475b4662622c51b8f375e7b2013e25f3c77b5 diff --git a/benches/tidwall_bench.cpp b/benches/tidwall_bench.cpp new file mode 100644 index 0000000..7780876 --- /dev/null +++ b/benches/tidwall_bench.cpp @@ -0,0 +1,85 @@ +#include <benchmark/benchmark.h> + +#include "bench_common.hpp" + +namespace { + +using namespace cmbench; + +static void BM_Insert_Scalar(benchmark::State& state) { + bench_insert<TidwallAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), {"tidwall_hashmap", "c", "Scalar", "Insert"}); +} +static void BM_LookupHit_Scalar(benchmark::State& state) { + bench_lookup_hit<TidwallAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"tidwall_hashmap", "c", "Scalar", "LookupHit"}); +} +static void BM_LookupMiss_Scalar(benchmark::State& state) { + bench_lookup_miss<TidwallAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"tidwall_hashmap", "c", "Scalar", "LookupMiss"}); +} +static void BM_Erase_Scalar(benchmark::State& state) { + bench_erase<TidwallAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), {"tidwall_hashmap", "c", "Scalar", "Erase"}); +} + +static void BM_Insert_HandlePayload(benchmark::State& state) { + bench_insert<TidwallAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"tidwall_hashmap", "c", "HandlePayload", "Insert"}); +} +static void BM_LookupHit_HandlePayload(benchmark::State& state) { + bench_lookup_hit<TidwallAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"tidwall_hashmap", "c", "HandlePayload", "LookupHit"}); +} +static void BM_LookupMiss_HandlePayload(benchmark::State& state) { + bench_lookup_miss<TidwallAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"tidwall_hashmap", "c", "HandlePayload", "LookupMiss"}); +} +static void BM_Erase_HandlePayload(benchmark::State& state) { + bench_erase<TidwallAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"tidwall_hashmap", "c", "HandlePayload", "Erase"}); +} + +static void BM_Insert_CompositeKey(benchmark::State& state) { + bench_insert<TidwallAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"tidwall_hashmap", "c", "CompositeKey", "Insert"}); +} +static void BM_LookupHit_CompositeKey(benchmark::State& state) { + bench_lookup_hit<TidwallAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"tidwall_hashmap", "c", "CompositeKey", "LookupHit"}); +} +static void BM_LookupMiss_CompositeKey(benchmark::State& state) { + bench_lookup_miss<TidwallAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"tidwall_hashmap", "c", "CompositeKey", "LookupMiss"}); +} +static void BM_Erase_CompositeKey(benchmark::State& state) { + bench_erase<TidwallAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"tidwall_hashmap", "c", "CompositeKey", "Erase"}); +} + +} // namespace + +BENCHMARK(BM_Insert_Scalar); +BENCHMARK(BM_LookupHit_Scalar); +BENCHMARK(BM_LookupMiss_Scalar); +BENCHMARK(BM_Erase_Scalar); + +BENCHMARK(BM_Insert_HandlePayload); +BENCHMARK(BM_LookupHit_HandlePayload); +BENCHMARK(BM_LookupMiss_HandlePayload); +BENCHMARK(BM_Erase_HandlePayload); + +BENCHMARK(BM_Insert_CompositeKey); +BENCHMARK(BM_LookupHit_CompositeKey); +BENCHMARK(BM_LookupMiss_CompositeKey); +BENCHMARK(BM_Erase_CompositeKey); diff --git a/benches/unordered_map_bench.cpp b/benches/unordered_map_bench.cpp new file mode 100644 index 0000000..a064ebc --- /dev/null +++ b/benches/unordered_map_bench.cpp @@ -0,0 +1,87 @@ +#include <benchmark/benchmark.h> + +#include "bench_common.hpp" + +namespace { + +using namespace cmbench; + +static void BM_Insert_Scalar(benchmark::State& state) { + bench_insert<UnorderedMapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"std::unordered_map", "c++", "Scalar", "Insert"}); +} +static void BM_LookupHit_Scalar(benchmark::State& state) { + bench_lookup_hit<UnorderedMapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"std::unordered_map", "c++", "Scalar", "LookupHit"}); +} +static void BM_LookupMiss_Scalar(benchmark::State& state) { + bench_lookup_miss<UnorderedMapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"std::unordered_map", "c++", "Scalar", "LookupMiss"}); +} +static void BM_Erase_Scalar(benchmark::State& state) { + bench_erase<UnorderedMapAdapter<std::uint64_t, ScalarValue>>( + state, scalar_workload(), + {"std::unordered_map", "c++", "Scalar", "Erase"}); +} + +static void BM_Insert_HandlePayload(benchmark::State& state) { + bench_insert<UnorderedMapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"std::unordered_map", "c++", "HandlePayload", "Insert"}); +} +static void BM_LookupHit_HandlePayload(benchmark::State& state) { + bench_lookup_hit<UnorderedMapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"std::unordered_map", "c++", "HandlePayload", "LookupHit"}); +} +static void BM_LookupMiss_HandlePayload(benchmark::State& state) { + bench_lookup_miss<UnorderedMapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"std::unordered_map", "c++", "HandlePayload", "LookupMiss"}); +} +static void BM_Erase_HandlePayload(benchmark::State& state) { + bench_erase<UnorderedMapAdapter<EntityId, ComponentPayload>>( + state, entity_workload(), + {"std::unordered_map", "c++", "HandlePayload", "Erase"}); +} + +static void BM_Insert_CompositeKey(benchmark::State& state) { + bench_insert<UnorderedMapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"std::unordered_map", "c++", "CompositeKey", "Insert"}); +} +static void BM_LookupHit_CompositeKey(benchmark::State& state) { + bench_lookup_hit<UnorderedMapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"std::unordered_map", "c++", "CompositeKey", "LookupHit"}); +} +static void BM_LookupMiss_CompositeKey(benchmark::State& state) { + bench_lookup_miss<UnorderedMapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"std::unordered_map", "c++", "CompositeKey", "LookupMiss"}); +} +static void BM_Erase_CompositeKey(benchmark::State& state) { + bench_erase<UnorderedMapAdapter<ComponentKey, ComponentMeta>>( + state, component_workload(), + {"std::unordered_map", "c++", "CompositeKey", "Erase"}); +} + +} // namespace + +BENCHMARK(BM_Insert_Scalar); +BENCHMARK(BM_LookupHit_Scalar); +BENCHMARK(BM_LookupMiss_Scalar); +BENCHMARK(BM_Erase_Scalar); + +BENCHMARK(BM_Insert_HandlePayload); +BENCHMARK(BM_LookupHit_HandlePayload); +BENCHMARK(BM_LookupMiss_HandlePayload); +BENCHMARK(BM_Erase_HandlePayload); + +BENCHMARK(BM_Insert_CompositeKey); +BENCHMARK(BM_LookupHit_CompositeKey); +BENCHMARK(BM_LookupMiss_CompositeKey); +BENCHMARK(BM_Erase_CompositeKey); |
