diff --git a/req/include/req_compactor.hpp b/req/include/req_compactor.hpp index 7c41ab52..60887144 100755 --- a/req/include/req_compactor.hpp +++ b/req/include/req_compactor.hpp @@ -119,7 +119,23 @@ class req_compactor { uint32_t capacity_; // 0 while items_ is nullptr T* items_; // nullptr until first use, see reserve() - bool ensure_enough_sections(); + // the section parameters, which change as the state advances + struct sections { + float size_raw; + uint32_t size; + uint8_t num; + uint32_t nom_capacity() const { return req_constants::MULTIPLIER * num * size; } + }; + sections get_sections() const; + void set_sections(const sections& s); + // one step of the section schedule for the given state: halves the section size (by sqrt(2)) and doubles + // the number of sections once the state reaches 2^(num - 1); returns false if the sections stay as they are + static bool next_sections(uint64_t state, sections& s); + // destroys the items and frees the storage + void destroy_items(); + // allocates storage and copies the items of other, for a constructor; releases both if a copy throws + template + void copy_items(const O& other); std::pair compute_compaction_range(uint32_t secs_to_compact) const; void grow(uint32_t new_capacity); void ensure_space(uint32_t num); diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp index bad8aa98..a2eb0975 100755 --- a/req/include/req_compactor_impl.hpp +++ b/req/include/req_compactor_impl.hpp @@ -50,9 +50,34 @@ items_(nullptr) // allocated on first use, see reserve() template req_compactor::~req_compactor() { + destroy_items(); +} + +template +void req_compactor::destroy_items() { if (items_ != nullptr) { for (auto it = begin(); it != end(); ++it) (*it).~T(); allocator_.deallocate(items_, capacity_); + items_ = nullptr; + } +} + +template +template +void req_compactor::copy_items(const O& other) { + if (other.items_ == nullptr) return; + items_ = allocator_.allocate(capacity_); + const uint32_t from = hra_ ? capacity_ - num_items_ : 0; + const uint32_t to = hra_ ? capacity_ : num_items_; + uint32_t i = from; + try { + for (; i < to; ++i) new (items_ + i) T(other.items_[i]); + } catch (...) { + // the destructor doesn't run if a constructor throws, so release the items here + for (uint32_t j = from; j < i; ++j) items_[j].~T(); + allocator_.deallocate(items_, capacity_); + items_ = nullptr; + throw; } } @@ -72,12 +97,7 @@ num_items_(other.num_items_), capacity_(other.capacity_), items_(nullptr) { - if (other.items_ != nullptr) { - items_ = allocator_.allocate(capacity_); - const uint32_t from = hra_ ? capacity_ - num_items_ : 0; - const uint32_t to = hra_ ? capacity_ : num_items_; - for (uint32_t i = from; i < to; ++i) new (items_ + i) T(other.items_[i]); - } + copy_items(other); } template @@ -155,14 +175,10 @@ num_items_(other.num_items_), capacity_(other.capacity_), items_(nullptr) { - if (other.items_ != nullptr) { - items_ = allocator_.allocate(capacity_); - const uint32_t from = hra_ ? capacity_ - num_items_ : 0; - const uint32_t to = hra_ ? capacity_ : num_items_; - for (uint32_t i = from; i < to; ++i) new (items_ + i) T(other.items_[i]); - if (sorted_ && !std::is_sorted(items_ + from, items_ + to, comparator_)) { - throw std::logic_error("items must be sorted"); - } + copy_items(other); + if (sorted_ && !std::is_sorted(begin(), end(), comparator_)) { + destroy_items(); // the destructor doesn't run if a constructor throws + throw std::logic_error("items must be sorted"); } } @@ -256,18 +272,42 @@ template void req_compactor::merge(FwdC&& other) { // TODO: swap if other is larger? if (lg_weight_ != other.lg_weight_) throw std::logic_error("weight mismatch"); - state_ |= other.state_; - while (ensure_enough_sections()) {} - ensure_space(other.get_num_items()); + const uint32_t num = other.get_num_items(); + // the sections once the states are combined + const uint64_t new_state = state_ | other.state_; + sections new_sections = get_sections(); + bool more_sections = false; + while (next_sections(new_state, new_sections)) more_sections = true; + + // allocate first, so that a failure changes nothing: + // the capacity the sections and the items need, grown the same way as before this was done up front + uint32_t new_capacity = capacity_; + if (more_sections && new_capacity < 2 * new_sections.nom_capacity()) new_capacity = 2 * new_sections.nom_capacity(); + if (num > 0) { + if (new_capacity == 0) new_capacity = 2 * new_sections.nom_capacity(); // as reserve() would + if (num_items_ + num > new_capacity) new_capacity = num_items_ + num + new_sections.nom_capacity(); + } + if (new_capacity != capacity_) grow(new_capacity); + + // copy (or move) the items of other into the free space; if a copy throws, destroy the copies + T* from = hra_ ? begin() - num : end(); + uint32_t i = 0; + try { + auto other_it = other.begin(); + for (; i < num; ++i, ++other_it) new (from + i) T(conditional_forward(*other_it)); + } catch (...) { + for (uint32_t j = 0; j < i; ++j) from[j].~T(); + throw; + } + + // nothing below allocates or copies + state_ = new_state; + if (more_sections) set_sections(new_sections); sort(); - auto offset = hra_ ? capacity_ - num_items_ : num_items_; - auto from = hra_ ? begin() - other.get_num_items() : end(); - auto to = from + other.get_num_items(); - auto other_it = other.begin(); - for (auto it = from; it != to; ++it, ++other_it) new (it) T(conditional_forward(*other_it)); - if (!other.sorted_) std::sort(from, to, comparator_); - if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : to, C()); - num_items_ += other.get_num_items(); + if (!other.sorted_) std::sort(from, from + num, comparator_); + const auto offset = hra_ ? capacity_ - num_items_ : num_items_; + if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : from + num, comparator_); + num_items_ += num; } template @@ -285,12 +325,18 @@ std::pair req_compactor::compact(req_compactor& nex const uint32_t secs_to_compact = std::min(count_trailing_zeros_in_u64(~state_) + 1, num_sections_); auto compaction_range = compute_compaction_range(secs_to_compact); if (compaction_range.second - compaction_range.first < 2) throw std::logic_error("compaction range error"); + const auto num = (compaction_range.second - compaction_range.first) / 2; + + // allocate first, so that a failure changes nothing: space in the next level, + // and the growth that more sections need once the state advances + next.ensure_space(num); + sections new_sections = get_sections(); + const bool more_sections = next_sections(state_ + 1, new_sections); + if (more_sections && capacity_ < 2 * new_sections.nom_capacity()) grow(2 * new_sections.nom_capacity()); if ((state_ & 1) == 1) { coin_ = !coin_; } // for odd flip coin; else { coin_ = random_utils::random_bit(); } // random coin flip - const auto num = (compaction_range.second - compaction_range.first) / 2; - next.ensure_space(num); auto next_middle = hra_ ? next.begin() : next.end(); auto next_empty = hra_ ? next.begin() - num : next.end(); promote_evens_or_odds(begin() + compaction_range.first, begin() + compaction_range.second, coin_, next_empty); @@ -300,7 +346,7 @@ std::pair req_compactor::compact(req_compactor& nex num_items_ -= compaction_range.second - compaction_range.first; ++state_; - ensure_enough_sections(); + if (more_sections) set_sections(new_sections); return std::pair( num, get_nom_capacity() - starting_nom_capacity @@ -308,16 +354,27 @@ std::pair req_compactor::compact(req_compactor& nex } template -bool req_compactor::ensure_enough_sections() { - const float ssr = section_size_raw_ / sqrtf(2); +auto req_compactor::get_sections() const -> sections { + return sections{section_size_raw_, section_size_, num_sections_}; +} + +template +void req_compactor::set_sections(const sections& s) { + section_size_raw_ = s.size_raw; + section_size_ = s.size; + num_sections_ = s.num; +} + +template +bool req_compactor::next_sections(uint64_t state, sections& s) { + const float ssr = s.size_raw / sqrtf(2); const uint32_t ne = nearest_even(ssr); - if (state_ >= static_cast(1ULL << (num_sections_ - 1)) - && section_size_ > req_constants::MIN_K + if (state >= static_cast(1ULL << (s.num - 1)) + && s.size > req_constants::MIN_K && ne >= req_constants::MIN_K) { - section_size_raw_ = ssr; - section_size_ = ne; - num_sections_ <<= 1; - if (capacity_ < 2 * get_nom_capacity()) grow(2 * get_nom_capacity()); + s.size_raw = ssr; + s.size = ne; + s.num <<= 1; return true; } return false; @@ -408,7 +465,7 @@ void req_compactor::serialize(std::ostream& os, const S& serde) const { const uint16_t padding = 0; write(os, padding); write(os, num_items_); - serde.serialize(os, begin(), num_items_); + if (num_items_ > 0) serde.serialize(os, begin(), num_items_); // an empty compactor may have no storage } template @@ -423,7 +480,7 @@ size_t req_compactor::serialize(void* dst, size_t capacity, const S& se const uint16_t padding = 0; ptr += copy_to_mem(padding, ptr); ptr += copy_to_mem(num_items_, ptr); - ptr += serde.serialize(ptr, end_ptr - ptr, begin(), num_items_); + if (num_items_ > 0) ptr += serde.serialize(ptr, end_ptr - ptr, begin(), num_items_); // an empty compactor may have no storage return ptr - static_cast(dst); } diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp index 8e3468f0..91af958e 100755 --- a/req/include/req_sketch.hpp +++ b/req/include/req_sketch.hpp @@ -197,6 +197,11 @@ class req_sketch { * If cross-language portability is required, callers should ensure that * the input string uses a compatible encoding (valid UTF-8). * The first update allocates storage for items, unless reserve() or a merge already did. + * If an allocation or a copy of the item throws before the item is added, the sketch is unchanged. + * If a compaction that follows fails to allocate, the item stays added, the sketch stays valid, + * and later updates compact it. + * If the comparator throws while items are sorted or merged, or does not impose a consistent + * total order, the state of the sketch is undefined. * @param item from a stream of items */ template @@ -208,6 +213,8 @@ class req_sketch { * both sketches were built using compatible string encodings. * The first merge of a non-empty sketch allocates storage for items, * unless reserve() or an update already did. + * If an allocation or a copy of an item throws, this sketch stays valid and usable, but may hold part of other. + * If the comparator throws while items are sorted or merged, the state of this sketch is undefined. * @param other sketch to merge into this one */ template diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index d6f4c8e4..474ace3b 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -174,17 +174,20 @@ template void req_sketch::update(FwdT&& item) { if (!check_update_item(item)) { return; } compactors_[0].reserve(); // first, so that a failure leaves the sketch unchanged - if (is_empty()) { - min_item_.emplace(item); - max_item_.emplace(item); - } else { - if (comparator_(item, *min_item_)) *min_item_ = item; - if (comparator_(*max_item_, item)) *max_item_ = item; - } + // compare and copy first, but change nothing until the item is in level zero, + // so that an exception leaves the sketch as it was + const bool is_new_min = is_empty() || comparator_(item, *min_item_); + const bool is_new_max = is_empty() || comparator_(*max_item_, item); + optional new_min; + optional new_max; + if (is_new_min) new_min = optional(static_cast(item)); + if (is_new_max) new_max = optional(static_cast(item)); + reset_sorted_view(); compactors_[0].append(std::forward(item)); + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); ++num_retained_; ++n_; - reset_sorted_view(); if (num_retained_ >= max_nom_size_) compress(); } @@ -198,13 +201,15 @@ void req_sketch::merge(FwdSk&& other) { try { // grow until this has at least as many compactors as other while (get_num_levels() < other.get_num_levels()) grow(); - if (is_empty()) { - min_item_.emplace(conditional_forward(*other.min_item_)); - max_item_.emplace(conditional_forward(*other.max_item_)); - } else { - if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward(*other.min_item_); - if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward(*other.max_item_); - } + // compare and copy first, so that a failed copy leaves min and max as they were + const bool is_new_min = is_empty() || comparator_(*other.min_item_, *min_item_); + const bool is_new_max = is_empty() || comparator_(*max_item_, *other.max_item_); + optional new_min; + optional new_max; + if (is_new_min) new_min = optional(conditional_forward(*other.min_item_)); + if (is_new_max) new_max = optional(conditional_forward(*other.max_item_)); + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); // merge the items in all height compactors for (size_t i = 0; i < other.get_num_levels(); ++i) { compactors_[i].merge(conditional_forward(other.compactors_[i])); diff --git a/req/test/req_sketch_allocation_test.cpp b/req/test/req_sketch_allocation_test.cpp index 1857cf0e..0602630f 100644 --- a/req/test/req_sketch_allocation_test.cpp +++ b/req/test/req_sketch_allocation_test.cpp @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #include @@ -113,6 +115,66 @@ void check_usable(sketch_type& sketch, allocation_control& control) { REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); } + +using compactor_type = req_compactor, failing_allocator>; + +std::vector bytes(const compactor_type& compactor) { + std::vector b(compactor.get_serialized_size_bytes(serde())); + compactor.serialize(b.data(), b.size(), serde()); + return b; +} + +long copies_allowed = -1; // copies left before a copy throws; negative means no limit + +// An item whose copy constructor throws once copies_allowed runs out. +// Each item owns memory from test_allocator, so an item that is never destroyed shows up as a leak. +struct fragile_item { + int* value; + explicit fragile_item(int v): value(test_allocator(0).allocate(1)) { *value = v; } + fragile_item(const fragile_item& other): value(nullptr) { + if (copies_allowed == 0) throw std::runtime_error("copy failed"); + if (copies_allowed > 0) --copies_allowed; + value = test_allocator(0).allocate(1); + *value = *other.value; + } + fragile_item(fragile_item&& other) noexcept: value(other.value) { other.value = nullptr; } + fragile_item& operator=(const fragile_item& other) { + fragile_item copy(other); + std::swap(value, copy.value); + return *this; + } + fragile_item& operator=(fragile_item&& other) noexcept { std::swap(value, other.value); return *this; } + ~fragile_item() { if (value != nullptr) test_allocator(0).deallocate(value, 1); } +}; + +struct fragile_less { + bool operator()(const fragile_item& a, const fragile_item& b) const { return *a.value < *b.value; } +}; + +using fragile_sketch = req_sketch>; + +fragile_sketch make_fragile_sketch(int first, int count, bool hra) { + fragile_sketch sketch(K, hra, fragile_less(), test_allocator(0)); + for (int i = first; i < first + count; ++i) sketch.update(fragile_item((i * 7919) % 10007)); + return sketch; +} + +// the retained items must account for n, and every item must be a live object +void check_consistent(const fragile_sketch& sketch) { + uint64_t total_weight = 0; + uint32_t num_items = 0; + for (const auto pair: sketch) { + REQUIRE(pair.first.value != nullptr); + total_weight += pair.second; + ++num_items; + } + REQUIRE(total_weight == sketch.get_n()); + REQUIRE(num_items == sketch.get_num_retained()); + if (!sketch.is_empty()) { + REQUIRE(*sketch.get_min_item().value <= *sketch.get_max_item().value); + } +} + } // namespace TEST_CASE("req sketch allocation: reserve allocates item storage up front", "[req_sketch]") { @@ -353,4 +415,168 @@ TEST_CASE("req sketch allocation: failure at each allocation of a merge", "[req_ REQUIRE(test_allocator_net_allocations == 0); } +// A compaction that fails to allocate must change neither compactor, and must draw no random bit, +// so that compacting again gives what the compaction would have given without the failure. +TEST_CASE("req compactor allocation: a failed compaction changes nothing", "[req_sketch]") { + allocation_control control; + long failures_tested = 0; + for (bool hra: {true, false}) { + CAPTURE(hra); + random_utils::random_bit.seed(3); + compactor_type level0(hra, 0, K, std::less(), failing_allocator(control)); + compactor_type level1(hra, 1, K, std::less(), failing_allocator(control)); + int value = 0; + // enough compactions to reach both coin cases and a change of the sections + for (int round = 0; round < 12; ++round) { + CAPTURE(round); + while (level0.get_num_items() < level0.get_nom_capacity()) level0.append(static_cast((value++ * 7919) % 10007)); + level0.sort(); + const compactor_type level0_before(level0); + const compactor_type level1_before(level1); + const auto bits_before = random_utils::random_bit; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + control.arm(fail_at); + try { + level0.compact(level1); + completed = true; + } catch (const std::bad_alloc&) { + ++failures_tested; + } + control.disarm(); + if (!completed) { + REQUIRE(bytes(level0) == bytes(level0_before)); + REQUIRE(bytes(level1) == bytes(level1_before)); + REQUIRE(random_utils::random_bit == bits_before); + } + } + // the same as the compaction without failures, from the same random state + const auto bits_after = random_utils::random_bit; + compactor_type expected0(level0_before); + compactor_type expected1(level1_before); + random_utils::random_bit = bits_before; + expected0.compact(expected1); + REQUIRE(random_utils::random_bit == bits_after); + REQUIRE(bytes(level0) == bytes(expected0)); + REQUIRE(bytes(level1) == bytes(expected1)); + } + } + REQUIRE(failures_tested > 0); +} + +// Fail each item copy made by the updates in turn: a failed update must leave the sketch as it was. +TEST_CASE("req sketch allocation: failure at each item copy of the updates", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + CAPTURE(hra); + fragile_sketch sketch(K, hra, fragile_less(), test_allocator(0)); + for (int i = 0; i < 300; ++i) { + const fragile_item item((i * 7919) % 10007); + const uint64_t n_before = sketch.get_n(); + const uint32_t num_retained_before = sketch.get_num_retained(); + const int min_before = sketch.is_empty() ? 0 : *sketch.get_min_item().value; + const int max_before = sketch.is_empty() ? 0 : *sketch.get_max_item().value; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(i, fail_at); + copies_allowed = fail_at; + try { + sketch.update(item); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + if (!completed) { + REQUIRE(sketch.get_n() == n_before); + REQUIRE(sketch.get_num_retained() == num_retained_before); + if (n_before > 0) { + REQUIRE(*sketch.get_min_item().value == min_before); + REQUIRE(*sketch.get_max_item().value == max_before); + } + } + check_consistent(sketch); + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each item copy made by merging from a const sketch in turn: the target must stay valid and usable. +TEST_CASE("req sketch allocation: failure at each item copy of a merge", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + for (int target_n: {0, 5, 1000}) { + for (int source_n: {5, 1000}) { + CAPTURE(hra, target_n, source_n); + const fragile_sketch source = make_fragile_sketch(100000, source_n, hra); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + fragile_sketch target = make_fragile_sketch(0, target_n, hra); + copies_allowed = fail_at; + try { + target.merge(source); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + REQUIRE(target.get_n() >= static_cast(target_n)); + REQUIRE(target.get_n() <= static_cast(target_n + source_n)); + if (completed) REQUIRE(target.get_n() == static_cast(target_n + source_n)); + check_consistent(target); + check_consistent(source); + REQUIRE(source.get_n() == static_cast(source_n)); + // the target stays usable + target.merge(source); + target.update(fragile_item(-1)); + check_consistent(target); + REQUIRE(*target.get_min_item().value == -1); + } + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req sketch allocation: failure at each item copy of the copy constructor", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + const fragile_sketch sketch = make_fragile_sketch(0, 1000, hra); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(hra, fail_at); + copies_allowed = fail_at; + try { + const fragile_sketch copy(sketch); + completed = true; + copies_allowed = -1; + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + } catch (const std::runtime_error&) {} + copies_allowed = -1; + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req sketch allocation: converting constructor that rejects the order releases the items", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + req_double_test_sketch sketch(K, true, std::less(), test_allocator(0)); + for (int i = 0; i < 1000; ++i) sketch.update(i); + REQUIRE(sketch.is_estimation_mode()); + using descending_sketch = req_sketch, test_allocator>; + REQUIRE_THROWS_AS(descending_sketch(sketch, std::greater(), test_allocator(0)), std::logic_error); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + + } /* namespace datasketches */ diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 3db56247..5167df84 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -763,4 +763,33 @@ TEST_CASE("req compactor: section growth stops at minimum section size", "[req_c } } +namespace { + +// a comparator that can't be default-constructed: the sketch must use its own comparator everywhere +struct ordered_by { + bool descending; + explicit ordered_by(bool d): descending(d) {} + bool operator()(int a, int b) const { return descending ? b < a : a < b; } +}; + +} // namespace + +TEST_CASE("req sketch: compaction and merge use the sketch's comparator", "[req_sketch]") { + req_sketch sketch(12, true, ordered_by(true)); + req_sketch other(12, true, ordered_by(true)); + for (int i = 0; i < 1000; ++i) { + sketch.update(i); + other.update(i + 1000); + } + sketch.merge(other); + REQUIRE(sketch.get_n() == 2000); + REQUIRE(sketch.is_estimation_mode()); + REQUIRE(sketch.get_min_item() == 1999); // first in descending order + REQUIRE(sketch.get_max_item() == 0); + // the converting constructor checks that the sorted levels are sorted by the comparator + const req_sketch copy(sketch, ordered_by(true)); + REQUIRE(copy.get_n() == 2000); + REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(0.05)); +} + } /* namespace datasketches */