From fff9d7913aadc0b86f742227265147f6252534df Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 19 Nov 2026 10:57:47 -0800 Subject: [PATCH 1/2] REQ: keep the sketch valid when an allocation or an item copy throws Compactor: - The copy and converting constructors release their storage and the copies made when a copy throws, or when the converting constructor rejects the order of the items. - merge() works out the sections and the capacity first and allocates once, then copies the other compactor's items and destroys the copies if one throws. Only then does it change the state and the sections. It used to change them first, and leaked the copies made before a copy that threw. It compares with the sketch's comparator, not C(). - compact() allocates the space in the next level, and the growth that more sections need, before it flips or draws the coin and changes anything. A failed compaction used to leave the coin flipped (or a random bit drawn), and could change the sections and then fail to grow. Sketch: - update() compares the item with min and max and copies it first, and sets min and max only after the item is in level zero. - merge() copies the new min and max before it sets either. The same gaps are addressed in datasketches-java by apache/datasketches-java#808, except the item-loss window in compress(), which C++ does not have: compact() reserved the next level's space before moving any item. Results are byte-identical to before. Co-Authored-By: Claude Opus 5.5 --- req/include/req_compactor.hpp | 18 +- req/include/req_compactor_impl.hpp | 129 ++++++++++---- req/include/req_sketch.hpp | 7 + req/include/req_sketch_impl.hpp | 35 ++-- req/test/req_sketch_allocation_test.cpp | 226 ++++++++++++++++++++++++ req/test/req_sketch_test.cpp | 29 +++ 6 files changed, 392 insertions(+), 52 deletions(-) diff --git a/req/include/req_compactor.hpp b/req/include/req_compactor.hpp index 7c41ab52..60887144 100755 --- a/req/include/req_compactor.hpp +++ b/req/include/req_compactor.hpp @@ -119,7 +119,23 @@ class req_compactor { uint32_t capacity_; // 0 while items_ is nullptr T* items_; // nullptr until first use, see reserve() - bool ensure_enough_sections(); + // the section parameters, which change as the state advances + struct sections { + float size_raw; + uint32_t size; + uint8_t num; + uint32_t nom_capacity() const { return req_constants::MULTIPLIER * num * size; } + }; + sections get_sections() const; + void set_sections(const sections& s); + // one step of the section schedule for the given state: halves the section size (by sqrt(2)) and doubles + // the number of sections once the state reaches 2^(num - 1); returns false if the sections stay as they are + static bool next_sections(uint64_t state, sections& s); + // destroys the items and frees the storage + void destroy_items(); + // allocates storage and copies the items of other, for a constructor; releases both if a copy throws + template + void copy_items(const O& other); std::pair compute_compaction_range(uint32_t secs_to_compact) const; void grow(uint32_t new_capacity); void ensure_space(uint32_t num); diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp index bad8aa98..af6df0e7 100755 --- a/req/include/req_compactor_impl.hpp +++ b/req/include/req_compactor_impl.hpp @@ -50,9 +50,34 @@ items_(nullptr) // allocated on first use, see reserve() template req_compactor::~req_compactor() { + destroy_items(); +} + +template +void req_compactor::destroy_items() { if (items_ != nullptr) { for (auto it = begin(); it != end(); ++it) (*it).~T(); allocator_.deallocate(items_, capacity_); + items_ = nullptr; + } +} + +template +template +void req_compactor::copy_items(const O& other) { + if (other.items_ == nullptr) return; + items_ = allocator_.allocate(capacity_); + const uint32_t from = hra_ ? capacity_ - num_items_ : 0; + const uint32_t to = hra_ ? capacity_ : num_items_; + uint32_t i = from; + try { + for (; i < to; ++i) new (items_ + i) T(other.items_[i]); + } catch (...) { + // the destructor doesn't run if a constructor throws, so release the items here + for (uint32_t j = from; j < i; ++j) items_[j].~T(); + allocator_.deallocate(items_, capacity_); + items_ = nullptr; + throw; } } @@ -72,12 +97,7 @@ num_items_(other.num_items_), capacity_(other.capacity_), items_(nullptr) { - if (other.items_ != nullptr) { - items_ = allocator_.allocate(capacity_); - const uint32_t from = hra_ ? capacity_ - num_items_ : 0; - const uint32_t to = hra_ ? capacity_ : num_items_; - for (uint32_t i = from; i < to; ++i) new (items_ + i) T(other.items_[i]); - } + copy_items(other); } template @@ -155,14 +175,10 @@ num_items_(other.num_items_), capacity_(other.capacity_), items_(nullptr) { - if (other.items_ != nullptr) { - items_ = allocator_.allocate(capacity_); - const uint32_t from = hra_ ? capacity_ - num_items_ : 0; - const uint32_t to = hra_ ? capacity_ : num_items_; - for (uint32_t i = from; i < to; ++i) new (items_ + i) T(other.items_[i]); - if (sorted_ && !std::is_sorted(items_ + from, items_ + to, comparator_)) { - throw std::logic_error("items must be sorted"); - } + copy_items(other); + if (sorted_ && !std::is_sorted(begin(), end(), comparator_)) { + destroy_items(); // the destructor doesn't run if a constructor throws + throw std::logic_error("items must be sorted"); } } @@ -256,18 +272,42 @@ template void req_compactor::merge(FwdC&& other) { // TODO: swap if other is larger? if (lg_weight_ != other.lg_weight_) throw std::logic_error("weight mismatch"); - state_ |= other.state_; - while (ensure_enough_sections()) {} - ensure_space(other.get_num_items()); + const uint32_t num = other.get_num_items(); + // the sections once the states are combined + const uint64_t new_state = state_ | other.state_; + sections new_sections = get_sections(); + bool more_sections = false; + while (next_sections(new_state, new_sections)) more_sections = true; + + // allocate first, so that a failure changes nothing: + // the capacity the sections and the items need, grown the same way as before this was done up front + uint32_t new_capacity = capacity_; + if (more_sections && new_capacity < 2 * new_sections.nom_capacity()) new_capacity = 2 * new_sections.nom_capacity(); + if (num > 0) { + if (new_capacity == 0) new_capacity = 2 * new_sections.nom_capacity(); // as reserve() would + if (num_items_ + num > new_capacity) new_capacity = num_items_ + num + new_sections.nom_capacity(); + } + if (new_capacity != capacity_) grow(new_capacity); + + // copy (or move) the items of other into the free space; if a copy throws, destroy the copies + T* from = hra_ ? begin() - num : end(); + uint32_t i = 0; + try { + auto other_it = other.begin(); + for (; i < num; ++i, ++other_it) new (from + i) T(conditional_forward(*other_it)); + } catch (...) { + for (uint32_t j = 0; j < i; ++j) from[j].~T(); + throw; + } + + // nothing below allocates or copies + state_ = new_state; + if (more_sections) set_sections(new_sections); sort(); - auto offset = hra_ ? capacity_ - num_items_ : num_items_; - auto from = hra_ ? begin() - other.get_num_items() : end(); - auto to = from + other.get_num_items(); - auto other_it = other.begin(); - for (auto it = from; it != to; ++it, ++other_it) new (it) T(conditional_forward(*other_it)); - if (!other.sorted_) std::sort(from, to, comparator_); - if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : to, C()); - num_items_ += other.get_num_items(); + if (!other.sorted_) std::sort(from, from + num, comparator_); + const auto offset = hra_ ? capacity_ - num_items_ : num_items_; + if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : from + num, comparator_); + num_items_ += num; } template @@ -285,12 +325,18 @@ std::pair req_compactor::compact(req_compactor& nex const uint32_t secs_to_compact = std::min(count_trailing_zeros_in_u64(~state_) + 1, num_sections_); auto compaction_range = compute_compaction_range(secs_to_compact); if (compaction_range.second - compaction_range.first < 2) throw std::logic_error("compaction range error"); + const auto num = (compaction_range.second - compaction_range.first) / 2; + + // allocate first, so that a failure changes nothing: space in the next level, + // and the growth that more sections need once the state advances + next.ensure_space(num); + sections new_sections = get_sections(); + const bool more_sections = next_sections(state_ + 1, new_sections); + if (more_sections && capacity_ < 2 * new_sections.nom_capacity()) grow(2 * new_sections.nom_capacity()); if ((state_ & 1) == 1) { coin_ = !coin_; } // for odd flip coin; else { coin_ = random_utils::random_bit(); } // random coin flip - const auto num = (compaction_range.second - compaction_range.first) / 2; - next.ensure_space(num); auto next_middle = hra_ ? next.begin() : next.end(); auto next_empty = hra_ ? next.begin() - num : next.end(); promote_evens_or_odds(begin() + compaction_range.first, begin() + compaction_range.second, coin_, next_empty); @@ -300,7 +346,7 @@ std::pair req_compactor::compact(req_compactor& nex num_items_ -= compaction_range.second - compaction_range.first; ++state_; - ensure_enough_sections(); + if (more_sections) set_sections(new_sections); return std::pair( num, get_nom_capacity() - starting_nom_capacity @@ -308,16 +354,27 @@ std::pair req_compactor::compact(req_compactor& nex } template -bool req_compactor::ensure_enough_sections() { - const float ssr = section_size_raw_ / sqrtf(2); +auto req_compactor::get_sections() const -> sections { + return sections{section_size_raw_, section_size_, num_sections_}; +} + +template +void req_compactor::set_sections(const sections& s) { + section_size_raw_ = s.size_raw; + section_size_ = s.size; + num_sections_ = s.num; +} + +template +bool req_compactor::next_sections(uint64_t state, sections& s) { + const float ssr = s.size_raw / sqrtf(2); const uint32_t ne = nearest_even(ssr); - if (state_ >= static_cast(1ULL << (num_sections_ - 1)) - && section_size_ > req_constants::MIN_K + if (state >= static_cast(1ULL << (s.num - 1)) + && s.size > req_constants::MIN_K && ne >= req_constants::MIN_K) { - section_size_raw_ = ssr; - section_size_ = ne; - num_sections_ <<= 1; - if (capacity_ < 2 * get_nom_capacity()) grow(2 * get_nom_capacity()); + s.size_raw = ssr; + s.size = ne; + s.num <<= 1; return true; } return false; diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp index 8e3468f0..91af958e 100755 --- a/req/include/req_sketch.hpp +++ b/req/include/req_sketch.hpp @@ -197,6 +197,11 @@ class req_sketch { * If cross-language portability is required, callers should ensure that * the input string uses a compatible encoding (valid UTF-8). * The first update allocates storage for items, unless reserve() or a merge already did. + * If an allocation or a copy of the item throws before the item is added, the sketch is unchanged. + * If a compaction that follows fails to allocate, the item stays added, the sketch stays valid, + * and later updates compact it. + * If the comparator throws while items are sorted or merged, or does not impose a consistent + * total order, the state of the sketch is undefined. * @param item from a stream of items */ template @@ -208,6 +213,8 @@ class req_sketch { * both sketches were built using compatible string encodings. * The first merge of a non-empty sketch allocates storage for items, * unless reserve() or an update already did. + * If an allocation or a copy of an item throws, this sketch stays valid and usable, but may hold part of other. + * If the comparator throws while items are sorted or merged, the state of this sketch is undefined. * @param other sketch to merge into this one */ template diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index d6f4c8e4..474ace3b 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -174,17 +174,20 @@ template void req_sketch::update(FwdT&& item) { if (!check_update_item(item)) { return; } compactors_[0].reserve(); // first, so that a failure leaves the sketch unchanged - if (is_empty()) { - min_item_.emplace(item); - max_item_.emplace(item); - } else { - if (comparator_(item, *min_item_)) *min_item_ = item; - if (comparator_(*max_item_, item)) *max_item_ = item; - } + // compare and copy first, but change nothing until the item is in level zero, + // so that an exception leaves the sketch as it was + const bool is_new_min = is_empty() || comparator_(item, *min_item_); + const bool is_new_max = is_empty() || comparator_(*max_item_, item); + optional new_min; + optional new_max; + if (is_new_min) new_min = optional(static_cast(item)); + if (is_new_max) new_max = optional(static_cast(item)); + reset_sorted_view(); compactors_[0].append(std::forward(item)); + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); ++num_retained_; ++n_; - reset_sorted_view(); if (num_retained_ >= max_nom_size_) compress(); } @@ -198,13 +201,15 @@ void req_sketch::merge(FwdSk&& other) { try { // grow until this has at least as many compactors as other while (get_num_levels() < other.get_num_levels()) grow(); - if (is_empty()) { - min_item_.emplace(conditional_forward(*other.min_item_)); - max_item_.emplace(conditional_forward(*other.max_item_)); - } else { - if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward(*other.min_item_); - if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward(*other.max_item_); - } + // compare and copy first, so that a failed copy leaves min and max as they were + const bool is_new_min = is_empty() || comparator_(*other.min_item_, *min_item_); + const bool is_new_max = is_empty() || comparator_(*max_item_, *other.max_item_); + optional new_min; + optional new_max; + if (is_new_min) new_min = optional(conditional_forward(*other.min_item_)); + if (is_new_max) new_max = optional(conditional_forward(*other.max_item_)); + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); // merge the items in all height compactors for (size_t i = 0; i < other.get_num_levels(); ++i) { compactors_[i].merge(conditional_forward(other.compactors_[i])); diff --git a/req/test/req_sketch_allocation_test.cpp b/req/test/req_sketch_allocation_test.cpp index 1857cf0e..0602630f 100644 --- a/req/test/req_sketch_allocation_test.cpp +++ b/req/test/req_sketch_allocation_test.cpp @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #include @@ -113,6 +115,66 @@ void check_usable(sketch_type& sketch, allocation_control& control) { REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); } + +using compactor_type = req_compactor, failing_allocator>; + +std::vector bytes(const compactor_type& compactor) { + std::vector b(compactor.get_serialized_size_bytes(serde())); + compactor.serialize(b.data(), b.size(), serde()); + return b; +} + +long copies_allowed = -1; // copies left before a copy throws; negative means no limit + +// An item whose copy constructor throws once copies_allowed runs out. +// Each item owns memory from test_allocator, so an item that is never destroyed shows up as a leak. +struct fragile_item { + int* value; + explicit fragile_item(int v): value(test_allocator(0).allocate(1)) { *value = v; } + fragile_item(const fragile_item& other): value(nullptr) { + if (copies_allowed == 0) throw std::runtime_error("copy failed"); + if (copies_allowed > 0) --copies_allowed; + value = test_allocator(0).allocate(1); + *value = *other.value; + } + fragile_item(fragile_item&& other) noexcept: value(other.value) { other.value = nullptr; } + fragile_item& operator=(const fragile_item& other) { + fragile_item copy(other); + std::swap(value, copy.value); + return *this; + } + fragile_item& operator=(fragile_item&& other) noexcept { std::swap(value, other.value); return *this; } + ~fragile_item() { if (value != nullptr) test_allocator(0).deallocate(value, 1); } +}; + +struct fragile_less { + bool operator()(const fragile_item& a, const fragile_item& b) const { return *a.value < *b.value; } +}; + +using fragile_sketch = req_sketch>; + +fragile_sketch make_fragile_sketch(int first, int count, bool hra) { + fragile_sketch sketch(K, hra, fragile_less(), test_allocator(0)); + for (int i = first; i < first + count; ++i) sketch.update(fragile_item((i * 7919) % 10007)); + return sketch; +} + +// the retained items must account for n, and every item must be a live object +void check_consistent(const fragile_sketch& sketch) { + uint64_t total_weight = 0; + uint32_t num_items = 0; + for (const auto pair: sketch) { + REQUIRE(pair.first.value != nullptr); + total_weight += pair.second; + ++num_items; + } + REQUIRE(total_weight == sketch.get_n()); + REQUIRE(num_items == sketch.get_num_retained()); + if (!sketch.is_empty()) { + REQUIRE(*sketch.get_min_item().value <= *sketch.get_max_item().value); + } +} + } // namespace TEST_CASE("req sketch allocation: reserve allocates item storage up front", "[req_sketch]") { @@ -353,4 +415,168 @@ TEST_CASE("req sketch allocation: failure at each allocation of a merge", "[req_ REQUIRE(test_allocator_net_allocations == 0); } +// A compaction that fails to allocate must change neither compactor, and must draw no random bit, +// so that compacting again gives what the compaction would have given without the failure. +TEST_CASE("req compactor allocation: a failed compaction changes nothing", "[req_sketch]") { + allocation_control control; + long failures_tested = 0; + for (bool hra: {true, false}) { + CAPTURE(hra); + random_utils::random_bit.seed(3); + compactor_type level0(hra, 0, K, std::less(), failing_allocator(control)); + compactor_type level1(hra, 1, K, std::less(), failing_allocator(control)); + int value = 0; + // enough compactions to reach both coin cases and a change of the sections + for (int round = 0; round < 12; ++round) { + CAPTURE(round); + while (level0.get_num_items() < level0.get_nom_capacity()) level0.append(static_cast((value++ * 7919) % 10007)); + level0.sort(); + const compactor_type level0_before(level0); + const compactor_type level1_before(level1); + const auto bits_before = random_utils::random_bit; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + control.arm(fail_at); + try { + level0.compact(level1); + completed = true; + } catch (const std::bad_alloc&) { + ++failures_tested; + } + control.disarm(); + if (!completed) { + REQUIRE(bytes(level0) == bytes(level0_before)); + REQUIRE(bytes(level1) == bytes(level1_before)); + REQUIRE(random_utils::random_bit == bits_before); + } + } + // the same as the compaction without failures, from the same random state + const auto bits_after = random_utils::random_bit; + compactor_type expected0(level0_before); + compactor_type expected1(level1_before); + random_utils::random_bit = bits_before; + expected0.compact(expected1); + REQUIRE(random_utils::random_bit == bits_after); + REQUIRE(bytes(level0) == bytes(expected0)); + REQUIRE(bytes(level1) == bytes(expected1)); + } + } + REQUIRE(failures_tested > 0); +} + +// Fail each item copy made by the updates in turn: a failed update must leave the sketch as it was. +TEST_CASE("req sketch allocation: failure at each item copy of the updates", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + CAPTURE(hra); + fragile_sketch sketch(K, hra, fragile_less(), test_allocator(0)); + for (int i = 0; i < 300; ++i) { + const fragile_item item((i * 7919) % 10007); + const uint64_t n_before = sketch.get_n(); + const uint32_t num_retained_before = sketch.get_num_retained(); + const int min_before = sketch.is_empty() ? 0 : *sketch.get_min_item().value; + const int max_before = sketch.is_empty() ? 0 : *sketch.get_max_item().value; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(i, fail_at); + copies_allowed = fail_at; + try { + sketch.update(item); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + if (!completed) { + REQUIRE(sketch.get_n() == n_before); + REQUIRE(sketch.get_num_retained() == num_retained_before); + if (n_before > 0) { + REQUIRE(*sketch.get_min_item().value == min_before); + REQUIRE(*sketch.get_max_item().value == max_before); + } + } + check_consistent(sketch); + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each item copy made by merging from a const sketch in turn: the target must stay valid and usable. +TEST_CASE("req sketch allocation: failure at each item copy of a merge", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + for (int target_n: {0, 5, 1000}) { + for (int source_n: {5, 1000}) { + CAPTURE(hra, target_n, source_n); + const fragile_sketch source = make_fragile_sketch(100000, source_n, hra); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + fragile_sketch target = make_fragile_sketch(0, target_n, hra); + copies_allowed = fail_at; + try { + target.merge(source); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + REQUIRE(target.get_n() >= static_cast(target_n)); + REQUIRE(target.get_n() <= static_cast(target_n + source_n)); + if (completed) REQUIRE(target.get_n() == static_cast(target_n + source_n)); + check_consistent(target); + check_consistent(source); + REQUIRE(source.get_n() == static_cast(source_n)); + // the target stays usable + target.merge(source); + target.update(fragile_item(-1)); + check_consistent(target); + REQUIRE(*target.get_min_item().value == -1); + } + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req sketch allocation: failure at each item copy of the copy constructor", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + const fragile_sketch sketch = make_fragile_sketch(0, 1000, hra); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(hra, fail_at); + copies_allowed = fail_at; + try { + const fragile_sketch copy(sketch); + completed = true; + copies_allowed = -1; + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + } catch (const std::runtime_error&) {} + copies_allowed = -1; + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req sketch allocation: converting constructor that rejects the order releases the items", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + req_double_test_sketch sketch(K, true, std::less(), test_allocator(0)); + for (int i = 0; i < 1000; ++i) sketch.update(i); + REQUIRE(sketch.is_estimation_mode()); + using descending_sketch = req_sketch, test_allocator>; + REQUIRE_THROWS_AS(descending_sketch(sketch, std::greater(), test_allocator(0)), std::logic_error); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + + } /* namespace datasketches */ diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 3db56247..5167df84 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -763,4 +763,33 @@ TEST_CASE("req compactor: section growth stops at minimum section size", "[req_c } } +namespace { + +// a comparator that can't be default-constructed: the sketch must use its own comparator everywhere +struct ordered_by { + bool descending; + explicit ordered_by(bool d): descending(d) {} + bool operator()(int a, int b) const { return descending ? b < a : a < b; } +}; + +} // namespace + +TEST_CASE("req sketch: compaction and merge use the sketch's comparator", "[req_sketch]") { + req_sketch sketch(12, true, ordered_by(true)); + req_sketch other(12, true, ordered_by(true)); + for (int i = 0; i < 1000; ++i) { + sketch.update(i); + other.update(i + 1000); + } + sketch.merge(other); + REQUIRE(sketch.get_n() == 2000); + REQUIRE(sketch.is_estimation_mode()); + REQUIRE(sketch.get_min_item() == 1999); // first in descending order + REQUIRE(sketch.get_max_item() == 0); + // the converting constructor checks that the sorted levels are sorted by the comparator + const req_sketch copy(sketch, ordered_by(true)); + REQUIRE(copy.get_n() == 2000); + REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(0.05)); +} + } /* namespace datasketches */ From a68f18ec599c6ffb3081d93d3046f00bd0d73bc4 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 19 Nov 2026 16:53:38 -0800 Subject: [PATCH 2/2] REQ: don't pass an empty compactor's storage to the serde A compactor that has never held an item has no storage, so serialize() passed a null pointer with a count of 0, and the default serde passed it on to memcpy(). UBSan on Linux reports that as undefined behavior. Serialize the items only when there are some; the bytes are unchanged. Co-Authored-By: Claude Opus 5.5 --- req/include/req_compactor_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp index af6df0e7..a2eb0975 100755 --- a/req/include/req_compactor_impl.hpp +++ b/req/include/req_compactor_impl.hpp @@ -465,7 +465,7 @@ void req_compactor::serialize(std::ostream& os, const S& serde) const { const uint16_t padding = 0; write(os, padding); write(os, num_items_); - serde.serialize(os, begin(), num_items_); + if (num_items_ > 0) serde.serialize(os, begin(), num_items_); // an empty compactor may have no storage } template @@ -480,7 +480,7 @@ size_t req_compactor::serialize(void* dst, size_t capacity, const S& se const uint16_t padding = 0; ptr += copy_to_mem(padding, ptr); ptr += copy_to_mem(num_items_, ptr); - ptr += serde.serialize(ptr, end_ptr - ptr, begin(), num_items_); + if (num_items_ > 0) ptr += serde.serialize(ptr, end_ptr - ptr, begin(), num_items_); // an empty compactor may have no storage return ptr - static_cast(dst); }