Repository navigation
KLL: weighted update, matching datasketches-java #582
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
ab49552
f3560c4
b81cdf5
2bdae6d
9e5b804
0ab4ab6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -193,6 +193,70 @@ void kll_sketch<T, C, A>::update(FwdT&& item) { | |
| reset_sorted_view(); | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| template<typename FwdT> | ||
| void kll_sketch<T, C, A>::update(FwdT&& item, uint64_t weight) { | ||
| if (!check_update_item(item)) { return; } | ||
| if (weight == 0) throw std::invalid_argument("weight must be at least 1"); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Throwing on
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Both follow datasketches-java. Its |
||
| if (weight > kll_constants::MAX_WEIGHT) { | ||
| throw std::invalid_argument("weight must be less than 2^61: " + std::to_string(weight)); | ||
| } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Checking
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed. #585 caps n at 2^63 - 1, Java's |
||
| if (weight == 1) { | ||
| update(std::forward<FwdT>(item)); | ||
| return; | ||
| } | ||
| allocate_items(); // first, so that a failure leaves the sketch unchanged | ||
| reset_sorted_view(); // before any change, so that a failed copy can't leave a stale view | ||
| if (weight < levels_[0]) { // fits into level zero without compaction | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is intentional. Java uses the same guard (
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Makes sense! One related detail just as an FYI (no action needed here): for |
||
| update_min_max(static_cast<const T&>(item)); | ||
| // construct each copy before counting it, so that a failed copy leaves a valid sketch | ||
| for (uint64_t i = 1; i < weight; ++i) { | ||
| new (&items_[levels_[0] - 1]) T(static_cast<const T&>(item)); | ||
| internal_update(); | ||
| } | ||
|
proost marked this conversation as resolved.
Resolved
|
||
| new (&items_[levels_[0] - 1]) T(std::forward<FwdT>(item)); | ||
| internal_update(); | ||
| } else { | ||
| merge(kll_sketch(k_, static_cast<const T&>(item), weight, comparator_, allocator_)); | ||
| } | ||
| } | ||
|
|
||
| // An item at level h carries weight 2^h, so one copy of the item at each level | ||
| // whose bit is set in the weight is an exact sketch of that many copies. | ||
| // The weight is less than 2^61, so this needs at most 61 levels. | ||
| template<typename T, typename C, typename A> | ||
| kll_sketch<T, C, A>::kll_sketch(uint16_t k, const T& item, uint64_t weight, const C& comparator, const A& allocator): | ||
| comparator_(comparator), | ||
| allocator_(allocator), | ||
| k_(k), | ||
| m_(kll_constants::DEFAULT_M), | ||
| min_k_(k), | ||
| num_levels_(static_cast<uint8_t>(64 - count_leading_zeros_in_u64(weight))), | ||
| is_level_zero_sorted_(true), | ||
| n_(weight), | ||
| levels_(num_levels_ + 1, 0, allocator), | ||
| items_(nullptr), | ||
| items_size_(0), | ||
| min_item_(item), | ||
| max_item_(item), | ||
| sorted_view_(nullptr) | ||
| { | ||
| for (uint8_t level = 0; level < num_levels_; ++level) { | ||
| levels_[level + 1] = levels_[level] + static_cast<uint32_t>((weight >> level) & 1); | ||
| } | ||
| items_size_ = levels_[num_levels_]; | ||
| items_ = allocator_.allocate(items_size_); | ||
| // the destructor doesn't run if a constructor throws, so release the items here | ||
| uint32_t num_constructed = 0; | ||
| try { | ||
| for (; num_constructed < items_size_; ++num_constructed) new (&items_[num_constructed]) T(item); | ||
| } catch (...) { | ||
| for (uint32_t i = 0; i < num_constructed; ++i) items_[i].~T(); | ||
| allocator_.deallocate(items_, items_size_); | ||
| throw; | ||
| } | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| void kll_sketch<T, C, A>::update_min_max(const T& item) { | ||
| if (is_empty()) { | ||
|
|
@@ -257,7 +321,7 @@ void kll_sketch<T, C, A>::merge(FwdSk&& other) { | |
| const uint32_t index = internal_update(); | ||
| new (&items_[index]) T(conditional_forward<FwdSk>(other.items_[i])); | ||
| } | ||
| if (other.num_levels_ >= 2) { merge_higher_levels(other, final_n); } | ||
| if (other.num_levels_ >= 2) { merge_higher_levels(std::forward<FwdSk>(other), final_n); } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Forwarding
The new Taking non-
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks, confirmed. Your fixture SEGVs under ASan. I was wrong in my earlier reply: the weighted update's merge copies no items only while this sketch has nothing at the levels it merges into. In estimation mode it still copied, after this sketch's items had already been moved out. The same was true of Fixed in #585. The merge now copies other's items into the work buffer first. If a copy throws, it destroys them, and this sketch is unchanged. After that, this sketch's items are merged in front of them using moves only, so nothing is copied from a |
||
| n_ = final_n; | ||
| if (other.is_estimation_mode()) { min_k_ = std::min(min_k_, other.min_k_); } | ||
| assert_correct_total_weight(); | ||
|
|
@@ -1097,7 +1161,13 @@ void kll_sketch<T, C, A>::reset_sorted_view() { | |
| template<typename T, typename C, typename A> | ||
| kll_sketch<T, C, A>::const_iterator::const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels): | ||
| items(items), levels(levels), num_levels(num_levels), index(items == nullptr ? levels[num_levels] : levels[0]), level(items == nullptr ? num_levels : 0), weight(1) | ||
| {} | ||
| { | ||
| // start at the first non-empty level: level zero can be empty after a merge | ||
| while (level < num_levels && levels[level] == levels[level + 1]) { | ||
| ++level; | ||
| weight *= 2; | ||
| } | ||
| } | ||
|
|
||
| template<typename T, typename C, typename A> | ||
| typename kll_sketch<T, C, A>::const_iterator& kll_sketch<T, C, A>::const_iterator::operator++() { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: when
weight < levels_[0](up to65,534forMAX_K),update(item, weight)runs anO(weight)copy loop into level zero rather thanO(log(weight)). Describing the two paths (direct level-zero insertion whenweightfits in the remaining free space, and merging a binary-level sketch otherwise) would avoid overstating the complexity bound for largek.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good point, fixed in #585. The doc now describes both paths.