From fff9d7913aadc0b86f742227265147f6252534df Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 19 Nov 2026 10:57:47 -0800 Subject: [PATCH 1/3] REQ: keep the sketch valid when an allocation or an item copy throws Compactor: - The copy and converting constructors release their storage and the copies made when a copy throws, or when the converting constructor rejects the order of the items. - merge() works out the sections and the capacity first and allocates once, then copies the other compactor's items and destroys the copies if one throws. Only then does it change the state and the sections. It used to change them first, and leaked the copies made before a copy that threw. It compares with the sketch's comparator, not C(). - compact() allocates the space in the next level, and the growth that more sections need, before it flips or draws the coin and changes anything. A failed compaction used to leave the coin flipped (or a random bit drawn), and could change the sections and then fail to grow. Sketch: - update() compares the item with min and max and copies it first, and sets min and max only after the item is in level zero. - merge() copies the new min and max before it sets either. The same gaps are addressed in datasketches-java by apache/datasketches-java#808, except the item-loss window in compress(), which C++ does not have: compact() reserved the next level's space before moving any item. Results are byte-identical to before. Co-Authored-By: Claude Opus 5.5 --- req/include/req_compactor.hpp | 18 +- req/include/req_compactor_impl.hpp | 129 ++++++++++---- req/include/req_sketch.hpp | 7 + req/include/req_sketch_impl.hpp | 35 ++-- req/test/req_sketch_allocation_test.cpp | 226 ++++++++++++++++++++++++ req/test/req_sketch_test.cpp | 29 +++ 6 files changed, 392 insertions(+), 52 deletions(-) diff --git a/req/include/req_compactor.hpp b/req/include/req_compactor.hpp index 7c41ab52..60887144 100755 --- a/req/include/req_compactor.hpp +++ b/req/include/req_compactor.hpp @@ -119,7 +119,23 @@ class req_compactor { uint32_t capacity_; // 0 while items_ is nullptr T* items_; // nullptr until first use, see reserve() - bool ensure_enough_sections(); + // the section parameters, which change as the state advances + struct sections { + float size_raw; + uint32_t size; + uint8_t num; + uint32_t nom_capacity() const { return req_constants::MULTIPLIER * num * size; } + }; + sections get_sections() const; + void set_sections(const sections& s); + // one step of the section schedule for the given state: halves the section size (by sqrt(2)) and doubles + // the number of sections once the state reaches 2^(num - 1); returns false if the sections stay as they are + static bool next_sections(uint64_t state, sections& s); + // destroys the items and frees the storage + void destroy_items(); + // allocates storage and copies the items of other, for a constructor; releases both if a copy throws + template + void copy_items(const O& other); std::pair compute_compaction_range(uint32_t secs_to_compact) const; void grow(uint32_t new_capacity); void ensure_space(uint32_t num); diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp index bad8aa98..af6df0e7 100755 --- a/req/include/req_compactor_impl.hpp +++ b/req/include/req_compactor_impl.hpp @@ -50,9 +50,34 @@ items_(nullptr) // allocated on first use, see reserve() template req_compactor::~req_compactor() { + destroy_items(); +} + +template +void req_compactor::destroy_items() { if (items_ != nullptr) { for (auto it = begin(); it != end(); ++it) (*it).~T(); allocator_.deallocate(items_, capacity_); + items_ = nullptr; + } +} + +template +template +void req_compactor::copy_items(const O& other) { + if (other.items_ == nullptr) return; + items_ = allocator_.allocate(capacity_); + const uint32_t from = hra_ ? capacity_ - num_items_ : 0; + const uint32_t to = hra_ ? capacity_ : num_items_; + uint32_t i = from; + try { + for (; i < to; ++i) new (items_ + i) T(other.items_[i]); + } catch (...) { + // the destructor doesn't run if a constructor throws, so release the items here + for (uint32_t j = from; j < i; ++j) items_[j].~T(); + allocator_.deallocate(items_, capacity_); + items_ = nullptr; + throw; } } @@ -72,12 +97,7 @@ num_items_(other.num_items_), capacity_(other.capacity_), items_(nullptr) { - if (other.items_ != nullptr) { - items_ = allocator_.allocate(capacity_); - const uint32_t from = hra_ ? capacity_ - num_items_ : 0; - const uint32_t to = hra_ ? capacity_ : num_items_; - for (uint32_t i = from; i < to; ++i) new (items_ + i) T(other.items_[i]); - } + copy_items(other); } template @@ -155,14 +175,10 @@ num_items_(other.num_items_), capacity_(other.capacity_), items_(nullptr) { - if (other.items_ != nullptr) { - items_ = allocator_.allocate(capacity_); - const uint32_t from = hra_ ? capacity_ - num_items_ : 0; - const uint32_t to = hra_ ? capacity_ : num_items_; - for (uint32_t i = from; i < to; ++i) new (items_ + i) T(other.items_[i]); - if (sorted_ && !std::is_sorted(items_ + from, items_ + to, comparator_)) { - throw std::logic_error("items must be sorted"); - } + copy_items(other); + if (sorted_ && !std::is_sorted(begin(), end(), comparator_)) { + destroy_items(); // the destructor doesn't run if a constructor throws + throw std::logic_error("items must be sorted"); } } @@ -256,18 +272,42 @@ template void req_compactor::merge(FwdC&& other) { // TODO: swap if other is larger? if (lg_weight_ != other.lg_weight_) throw std::logic_error("weight mismatch"); - state_ |= other.state_; - while (ensure_enough_sections()) {} - ensure_space(other.get_num_items()); + const uint32_t num = other.get_num_items(); + // the sections once the states are combined + const uint64_t new_state = state_ | other.state_; + sections new_sections = get_sections(); + bool more_sections = false; + while (next_sections(new_state, new_sections)) more_sections = true; + + // allocate first, so that a failure changes nothing: + // the capacity the sections and the items need, grown the same way as before this was done up front + uint32_t new_capacity = capacity_; + if (more_sections && new_capacity < 2 * new_sections.nom_capacity()) new_capacity = 2 * new_sections.nom_capacity(); + if (num > 0) { + if (new_capacity == 0) new_capacity = 2 * new_sections.nom_capacity(); // as reserve() would + if (num_items_ + num > new_capacity) new_capacity = num_items_ + num + new_sections.nom_capacity(); + } + if (new_capacity != capacity_) grow(new_capacity); + + // copy (or move) the items of other into the free space; if a copy throws, destroy the copies + T* from = hra_ ? begin() - num : end(); + uint32_t i = 0; + try { + auto other_it = other.begin(); + for (; i < num; ++i, ++other_it) new (from + i) T(conditional_forward(*other_it)); + } catch (...) { + for (uint32_t j = 0; j < i; ++j) from[j].~T(); + throw; + } + + // nothing below allocates or copies + state_ = new_state; + if (more_sections) set_sections(new_sections); sort(); - auto offset = hra_ ? capacity_ - num_items_ : num_items_; - auto from = hra_ ? begin() - other.get_num_items() : end(); - auto to = from + other.get_num_items(); - auto other_it = other.begin(); - for (auto it = from; it != to; ++it, ++other_it) new (it) T(conditional_forward(*other_it)); - if (!other.sorted_) std::sort(from, to, comparator_); - if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : to, C()); - num_items_ += other.get_num_items(); + if (!other.sorted_) std::sort(from, from + num, comparator_); + const auto offset = hra_ ? capacity_ - num_items_ : num_items_; + if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : from + num, comparator_); + num_items_ += num; } template @@ -285,12 +325,18 @@ std::pair req_compactor::compact(req_compactor& nex const uint32_t secs_to_compact = std::min(count_trailing_zeros_in_u64(~state_) + 1, num_sections_); auto compaction_range = compute_compaction_range(secs_to_compact); if (compaction_range.second - compaction_range.first < 2) throw std::logic_error("compaction range error"); + const auto num = (compaction_range.second - compaction_range.first) / 2; + + // allocate first, so that a failure changes nothing: space in the next level, + // and the growth that more sections need once the state advances + next.ensure_space(num); + sections new_sections = get_sections(); + const bool more_sections = next_sections(state_ + 1, new_sections); + if (more_sections && capacity_ < 2 * new_sections.nom_capacity()) grow(2 * new_sections.nom_capacity()); if ((state_ & 1) == 1) { coin_ = !coin_; } // for odd flip coin; else { coin_ = random_utils::random_bit(); } // random coin flip - const auto num = (compaction_range.second - compaction_range.first) / 2; - next.ensure_space(num); auto next_middle = hra_ ? next.begin() : next.end(); auto next_empty = hra_ ? next.begin() - num : next.end(); promote_evens_or_odds(begin() + compaction_range.first, begin() + compaction_range.second, coin_, next_empty); @@ -300,7 +346,7 @@ std::pair req_compactor::compact(req_compactor& nex num_items_ -= compaction_range.second - compaction_range.first; ++state_; - ensure_enough_sections(); + if (more_sections) set_sections(new_sections); return std::pair( num, get_nom_capacity() - starting_nom_capacity @@ -308,16 +354,27 @@ std::pair req_compactor::compact(req_compactor& nex } template -bool req_compactor::ensure_enough_sections() { - const float ssr = section_size_raw_ / sqrtf(2); +auto req_compactor::get_sections() const -> sections { + return sections{section_size_raw_, section_size_, num_sections_}; +} + +template +void req_compactor::set_sections(const sections& s) { + section_size_raw_ = s.size_raw; + section_size_ = s.size; + num_sections_ = s.num; +} + +template +bool req_compactor::next_sections(uint64_t state, sections& s) { + const float ssr = s.size_raw / sqrtf(2); const uint32_t ne = nearest_even(ssr); - if (state_ >= static_cast(1ULL << (num_sections_ - 1)) - && section_size_ > req_constants::MIN_K + if (state >= static_cast(1ULL << (s.num - 1)) + && s.size > req_constants::MIN_K && ne >= req_constants::MIN_K) { - section_size_raw_ = ssr; - section_size_ = ne; - num_sections_ <<= 1; - if (capacity_ < 2 * get_nom_capacity()) grow(2 * get_nom_capacity()); + s.size_raw = ssr; + s.size = ne; + s.num <<= 1; return true; } return false; diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp index 8e3468f0..91af958e 100755 --- a/req/include/req_sketch.hpp +++ b/req/include/req_sketch.hpp @@ -197,6 +197,11 @@ class req_sketch { * If cross-language portability is required, callers should ensure that * the input string uses a compatible encoding (valid UTF-8). * The first update allocates storage for items, unless reserve() or a merge already did. + * If an allocation or a copy of the item throws before the item is added, the sketch is unchanged. + * If a compaction that follows fails to allocate, the item stays added, the sketch stays valid, + * and later updates compact it. + * If the comparator throws while items are sorted or merged, or does not impose a consistent + * total order, the state of the sketch is undefined. * @param item from a stream of items */ template @@ -208,6 +213,8 @@ class req_sketch { * both sketches were built using compatible string encodings. * The first merge of a non-empty sketch allocates storage for items, * unless reserve() or an update already did. + * If an allocation or a copy of an item throws, this sketch stays valid and usable, but may hold part of other. + * If the comparator throws while items are sorted or merged, the state of this sketch is undefined. * @param other sketch to merge into this one */ template diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index d6f4c8e4..474ace3b 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -174,17 +174,20 @@ template void req_sketch::update(FwdT&& item) { if (!check_update_item(item)) { return; } compactors_[0].reserve(); // first, so that a failure leaves the sketch unchanged - if (is_empty()) { - min_item_.emplace(item); - max_item_.emplace(item); - } else { - if (comparator_(item, *min_item_)) *min_item_ = item; - if (comparator_(*max_item_, item)) *max_item_ = item; - } + // compare and copy first, but change nothing until the item is in level zero, + // so that an exception leaves the sketch as it was + const bool is_new_min = is_empty() || comparator_(item, *min_item_); + const bool is_new_max = is_empty() || comparator_(*max_item_, item); + optional new_min; + optional new_max; + if (is_new_min) new_min = optional(static_cast(item)); + if (is_new_max) new_max = optional(static_cast(item)); + reset_sorted_view(); compactors_[0].append(std::forward(item)); + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); ++num_retained_; ++n_; - reset_sorted_view(); if (num_retained_ >= max_nom_size_) compress(); } @@ -198,13 +201,15 @@ void req_sketch::merge(FwdSk&& other) { try { // grow until this has at least as many compactors as other while (get_num_levels() < other.get_num_levels()) grow(); - if (is_empty()) { - min_item_.emplace(conditional_forward(*other.min_item_)); - max_item_.emplace(conditional_forward(*other.max_item_)); - } else { - if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward(*other.min_item_); - if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward(*other.max_item_); - } + // compare and copy first, so that a failed copy leaves min and max as they were + const bool is_new_min = is_empty() || comparator_(*other.min_item_, *min_item_); + const bool is_new_max = is_empty() || comparator_(*max_item_, *other.max_item_); + optional new_min; + optional new_max; + if (is_new_min) new_min = optional(conditional_forward(*other.min_item_)); + if (is_new_max) new_max = optional(conditional_forward(*other.max_item_)); + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); // merge the items in all height compactors for (size_t i = 0; i < other.get_num_levels(); ++i) { compactors_[i].merge(conditional_forward(other.compactors_[i])); diff --git a/req/test/req_sketch_allocation_test.cpp b/req/test/req_sketch_allocation_test.cpp index 1857cf0e..0602630f 100644 --- a/req/test/req_sketch_allocation_test.cpp +++ b/req/test/req_sketch_allocation_test.cpp @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #include @@ -113,6 +115,66 @@ void check_usable(sketch_type& sketch, allocation_control& control) { REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); } + +using compactor_type = req_compactor, failing_allocator>; + +std::vector bytes(const compactor_type& compactor) { + std::vector b(compactor.get_serialized_size_bytes(serde())); + compactor.serialize(b.data(), b.size(), serde()); + return b; +} + +long copies_allowed = -1; // copies left before a copy throws; negative means no limit + +// An item whose copy constructor throws once copies_allowed runs out. +// Each item owns memory from test_allocator, so an item that is never destroyed shows up as a leak. +struct fragile_item { + int* value; + explicit fragile_item(int v): value(test_allocator(0).allocate(1)) { *value = v; } + fragile_item(const fragile_item& other): value(nullptr) { + if (copies_allowed == 0) throw std::runtime_error("copy failed"); + if (copies_allowed > 0) --copies_allowed; + value = test_allocator(0).allocate(1); + *value = *other.value; + } + fragile_item(fragile_item&& other) noexcept: value(other.value) { other.value = nullptr; } + fragile_item& operator=(const fragile_item& other) { + fragile_item copy(other); + std::swap(value, copy.value); + return *this; + } + fragile_item& operator=(fragile_item&& other) noexcept { std::swap(value, other.value); return *this; } + ~fragile_item() { if (value != nullptr) test_allocator(0).deallocate(value, 1); } +}; + +struct fragile_less { + bool operator()(const fragile_item& a, const fragile_item& b) const { return *a.value < *b.value; } +}; + +using fragile_sketch = req_sketch>; + +fragile_sketch make_fragile_sketch(int first, int count, bool hra) { + fragile_sketch sketch(K, hra, fragile_less(), test_allocator(0)); + for (int i = first; i < first + count; ++i) sketch.update(fragile_item((i * 7919) % 10007)); + return sketch; +} + +// the retained items must account for n, and every item must be a live object +void check_consistent(const fragile_sketch& sketch) { + uint64_t total_weight = 0; + uint32_t num_items = 0; + for (const auto pair: sketch) { + REQUIRE(pair.first.value != nullptr); + total_weight += pair.second; + ++num_items; + } + REQUIRE(total_weight == sketch.get_n()); + REQUIRE(num_items == sketch.get_num_retained()); + if (!sketch.is_empty()) { + REQUIRE(*sketch.get_min_item().value <= *sketch.get_max_item().value); + } +} + } // namespace TEST_CASE("req sketch allocation: reserve allocates item storage up front", "[req_sketch]") { @@ -353,4 +415,168 @@ TEST_CASE("req sketch allocation: failure at each allocation of a merge", "[req_ REQUIRE(test_allocator_net_allocations == 0); } +// A compaction that fails to allocate must change neither compactor, and must draw no random bit, +// so that compacting again gives what the compaction would have given without the failure. +TEST_CASE("req compactor allocation: a failed compaction changes nothing", "[req_sketch]") { + allocation_control control; + long failures_tested = 0; + for (bool hra: {true, false}) { + CAPTURE(hra); + random_utils::random_bit.seed(3); + compactor_type level0(hra, 0, K, std::less(), failing_allocator(control)); + compactor_type level1(hra, 1, K, std::less(), failing_allocator(control)); + int value = 0; + // enough compactions to reach both coin cases and a change of the sections + for (int round = 0; round < 12; ++round) { + CAPTURE(round); + while (level0.get_num_items() < level0.get_nom_capacity()) level0.append(static_cast((value++ * 7919) % 10007)); + level0.sort(); + const compactor_type level0_before(level0); + const compactor_type level1_before(level1); + const auto bits_before = random_utils::random_bit; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + control.arm(fail_at); + try { + level0.compact(level1); + completed = true; + } catch (const std::bad_alloc&) { + ++failures_tested; + } + control.disarm(); + if (!completed) { + REQUIRE(bytes(level0) == bytes(level0_before)); + REQUIRE(bytes(level1) == bytes(level1_before)); + REQUIRE(random_utils::random_bit == bits_before); + } + } + // the same as the compaction without failures, from the same random state + const auto bits_after = random_utils::random_bit; + compactor_type expected0(level0_before); + compactor_type expected1(level1_before); + random_utils::random_bit = bits_before; + expected0.compact(expected1); + REQUIRE(random_utils::random_bit == bits_after); + REQUIRE(bytes(level0) == bytes(expected0)); + REQUIRE(bytes(level1) == bytes(expected1)); + } + } + REQUIRE(failures_tested > 0); +} + +// Fail each item copy made by the updates in turn: a failed update must leave the sketch as it was. +TEST_CASE("req sketch allocation: failure at each item copy of the updates", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + CAPTURE(hra); + fragile_sketch sketch(K, hra, fragile_less(), test_allocator(0)); + for (int i = 0; i < 300; ++i) { + const fragile_item item((i * 7919) % 10007); + const uint64_t n_before = sketch.get_n(); + const uint32_t num_retained_before = sketch.get_num_retained(); + const int min_before = sketch.is_empty() ? 0 : *sketch.get_min_item().value; + const int max_before = sketch.is_empty() ? 0 : *sketch.get_max_item().value; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(i, fail_at); + copies_allowed = fail_at; + try { + sketch.update(item); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + if (!completed) { + REQUIRE(sketch.get_n() == n_before); + REQUIRE(sketch.get_num_retained() == num_retained_before); + if (n_before > 0) { + REQUIRE(*sketch.get_min_item().value == min_before); + REQUIRE(*sketch.get_max_item().value == max_before); + } + } + check_consistent(sketch); + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each item copy made by merging from a const sketch in turn: the target must stay valid and usable. +TEST_CASE("req sketch allocation: failure at each item copy of a merge", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + for (int target_n: {0, 5, 1000}) { + for (int source_n: {5, 1000}) { + CAPTURE(hra, target_n, source_n); + const fragile_sketch source = make_fragile_sketch(100000, source_n, hra); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + fragile_sketch target = make_fragile_sketch(0, target_n, hra); + copies_allowed = fail_at; + try { + target.merge(source); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + REQUIRE(target.get_n() >= static_cast(target_n)); + REQUIRE(target.get_n() <= static_cast(target_n + source_n)); + if (completed) REQUIRE(target.get_n() == static_cast(target_n + source_n)); + check_consistent(target); + check_consistent(source); + REQUIRE(source.get_n() == static_cast(source_n)); + // the target stays usable + target.merge(source); + target.update(fragile_item(-1)); + check_consistent(target); + REQUIRE(*target.get_min_item().value == -1); + } + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req sketch allocation: failure at each item copy of the copy constructor", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (bool hra: {true, false}) { + const fragile_sketch sketch = make_fragile_sketch(0, 1000, hra); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(hra, fail_at); + copies_allowed = fail_at; + try { + const fragile_sketch copy(sketch); + completed = true; + copies_allowed = -1; + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + } catch (const std::runtime_error&) {} + copies_allowed = -1; + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req sketch allocation: converting constructor that rejects the order releases the items", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + req_double_test_sketch sketch(K, true, std::less(), test_allocator(0)); + for (int i = 0; i < 1000; ++i) sketch.update(i); + REQUIRE(sketch.is_estimation_mode()); + using descending_sketch = req_sketch, test_allocator>; + REQUIRE_THROWS_AS(descending_sketch(sketch, std::greater(), test_allocator(0)), std::logic_error); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + + } /* namespace datasketches */ diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 3db56247..5167df84 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -763,4 +763,33 @@ TEST_CASE("req compactor: section growth stops at minimum section size", "[req_c } } +namespace { + +// a comparator that can't be default-constructed: the sketch must use its own comparator everywhere +struct ordered_by { + bool descending; + explicit ordered_by(bool d): descending(d) {} + bool operator()(int a, int b) const { return descending ? b < a : a < b; } +}; + +} // namespace + +TEST_CASE("req sketch: compaction and merge use the sketch's comparator", "[req_sketch]") { + req_sketch sketch(12, true, ordered_by(true)); + req_sketch other(12, true, ordered_by(true)); + for (int i = 0; i < 1000; ++i) { + sketch.update(i); + other.update(i + 1000); + } + sketch.merge(other); + REQUIRE(sketch.get_n() == 2000); + REQUIRE(sketch.is_estimation_mode()); + REQUIRE(sketch.get_min_item() == 1999); // first in descending order + REQUIRE(sketch.get_max_item() == 0); + // the converting constructor checks that the sorted levels are sorted by the comparator + const req_sketch copy(sketch, ordered_by(true)); + REQUIRE(copy.get_n() == 2000); + REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(0.05)); +} + } /* namespace datasketches */ From a68f18ec599c6ffb3081d93d3046f00bd0d73bc4 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 19 Nov 2026 16:53:38 -0800 Subject: [PATCH 2/3] REQ: don't pass an empty compactor's storage to the serde A compactor that has never held an item has no storage, so serialize() passed a null pointer with a count of 0, and the default serde passed it on to memcpy(). UBSan on Linux reports that as undefined behavior. Serialize the items only when there are some; the bytes are unchanged. Co-Authored-By: Claude Opus 5.5 --- req/include/req_compactor_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp index af6df0e7..a2eb0975 100755 --- a/req/include/req_compactor_impl.hpp +++ b/req/include/req_compactor_impl.hpp @@ -465,7 +465,7 @@ void req_compactor::serialize(std::ostream& os, const S& serde) const { const uint16_t padding = 0; write(os, padding); write(os, num_items_); - serde.serialize(os, begin(), num_items_); + if (num_items_ > 0) serde.serialize(os, begin(), num_items_); // an empty compactor may have no storage } template @@ -480,7 +480,7 @@ size_t req_compactor::serialize(void* dst, size_t capacity, const S& se const uint16_t padding = 0; ptr += copy_to_mem(padding, ptr); ptr += copy_to_mem(num_items_, ptr); - ptr += serde.serialize(ptr, end_ptr - ptr, begin(), num_items_); + if (num_items_ > 0) ptr += serde.serialize(ptr, end_ptr - ptr, begin(), num_items_); // an empty compactor may have no storage return ptr - static_cast(dst); } From a73df7cadf52dceadf03f5af4f3ed7b01d6749ba Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 10:02:20 -0800 Subject: [PATCH 3/3] REQ: fix the issues found in review - merge(): merging a sketch into itself merges a copy, as merging read other while changing this sketch. - merge(): merge the compactors from the highest level down, and set min and max with the first items of other that the sketch takes. A failed merge then leaves min and max unchanged unless items of other were added, and in that case a level above level zero holds items, so the sketch stays in estimation mode, where min and max are serialized. - update() and merge(): cap n at req_constants::MAX_N = 2^63 - 1, as KLL does. - Compactor append(): when level zero must grow, copy the item first, since it may be one of the items that growing moves and frees. - Compactor: stop doubling the sections at 48, which keeps 1 << (num_sections - 1) defined and stays within what deserialize accepts. - Compactor converting constructor: release the items if the comparator throws while the order is checked. - Compactor deserialize: an empty compactor gets no storage. - quantiles_sorted_view: check split points with the view's comparator instead of a default-constructed one, so get_CDF() and get_PMF() compile with comparators that can't be default-constructed and use the right order with ones that have state. Serialized results are byte-identical to master. Co-Authored-By: Claude Opus 5.5 --- common/include/quantiles_sorted_view.hpp | 8 +- req/include/req_common.hpp | 2 + req/include/req_compactor_impl.hpp | 42 ++++++--- req/include/req_sketch.hpp | 5 + req/include/req_sketch_impl.hpp | 31 ++++++- req/test/req_sketch_allocation_test.cpp | 89 ++++++++++++++++++ req/test/req_sketch_test.cpp | 113 ++++++++++++++++++++++- 7 files changed, 268 insertions(+), 22 deletions(-) diff --git a/common/include/quantiles_sorted_view.hpp b/common/include/quantiles_sorted_view.hpp index 0f51b7d2..ff0d732d 100755 --- a/common/include/quantiles_sorted_view.hpp +++ b/common/include/quantiles_sorted_view.hpp @@ -196,21 +196,21 @@ class quantiles_sorted_view { static inline Entry make_dummy_entry(uint64_t weight) { return Entry(nullptr, weight); } template::value, int>::type = 0> - static inline void check_split_points(const T* items, uint32_t size) { + inline void check_split_points(const T* items, uint32_t size) const { for (uint32_t i = 0; i < size ; i++) { if (std::isnan(items[i])) { throw std::invalid_argument("Values must not be NaN"); } - if ((i < (size - 1)) && !(Comparator()(items[i], items[i + 1]))) { + if ((i < (size - 1)) && !(comparator_(items[i], items[i + 1]))) { throw std::invalid_argument("Values must be unique and monotonically increasing"); } } } template::value, int>::type = 0> - static inline void check_split_points(const T* items, uint32_t size) { + inline void check_split_points(const T* items, uint32_t size) const { for (uint32_t i = 0; i < size ; i++) { - if ((i < (size - 1)) && !(Comparator()(items[i], items[i + 1]))) { + if ((i < (size - 1)) && !(comparator_(items[i], items[i + 1]))) { throw std::invalid_argument("Items must be unique and monotonically increasing"); } } diff --git a/req/include/req_common.hpp b/req/include/req_common.hpp index 4bd9f7f6..a10741c6 100755 --- a/req/include/req_common.hpp +++ b/req/include/req_common.hpp @@ -37,6 +37,8 @@ namespace req_constants { const uint8_t INIT_NUM_SECTIONS = 3; /// multiplier for nominal capacity const unsigned MULTIPLIER = 2; + /// maximum stream length n, the largest signed 64-bit integer, as in datasketches-java + const uint64_t MAX_N = (1ULL << 63) - 1; } } /* namespace datasketches */ diff --git a/req/include/req_compactor_impl.hpp b/req/include/req_compactor_impl.hpp index a2eb0975..cb75cdd5 100755 --- a/req/include/req_compactor_impl.hpp +++ b/req/include/req_compactor_impl.hpp @@ -176,8 +176,16 @@ capacity_(other.capacity_), items_(nullptr) { copy_items(other); - if (sorted_ && !std::is_sorted(begin(), end(), comparator_)) { - destroy_items(); // the destructor doesn't run if a constructor throws + // the destructor doesn't run if a constructor throws, and the comparator may throw too + bool items_sorted = true; + try { + items_sorted = !sorted_ || std::is_sorted(begin(), end(), comparator_); + } catch (...) { + destroy_items(); + throw; + } + if (!items_sorted) { + destroy_items(); throw std::logic_error("items must be sorted"); } } @@ -220,9 +228,17 @@ template template void req_compactor::append(FwdT&& item) { reserve(); - if (num_items_ == capacity_) grow(capacity_ + get_nom_capacity()); - const uint32_t i = hra_ ? capacity_ - num_items_ - 1 : num_items_; - new (items_ + i) T(std::forward(item)); + if (num_items_ == capacity_) { + // the item may be one of the items of this compactor, which growing destroys, + // so build it first; a failed copy then leaves the compactor as it was + T copy(std::forward(item)); + grow(capacity_ + get_nom_capacity()); + const uint32_t i = hra_ ? capacity_ - num_items_ - 1 : num_items_; + new (items_ + i) T(std::move(copy)); + } else { + const uint32_t i = hra_ ? capacity_ - num_items_ - 1 : num_items_; + new (items_ + i) T(std::forward(item)); + } ++num_items_; if (num_items_ > 1) sorted_ = false; } @@ -273,6 +289,7 @@ void req_compactor::merge(FwdC&& other) { // TODO: swap if other is larger? if (lg_weight_ != other.lg_weight_) throw std::logic_error("weight mismatch"); const uint32_t num = other.get_num_items(); + const bool other_sorted = other.sorted_; // the sections once the states are combined const uint64_t new_state = state_ | other.state_; sections new_sections = get_sections(); @@ -304,7 +321,7 @@ void req_compactor::merge(FwdC&& other) { state_ = new_state; if (more_sections) set_sections(new_sections); sort(); - if (!other.sorted_) std::sort(from, from + num, comparator_); + if (!other_sorted) std::sort(from, from + num, comparator_); const auto offset = hra_ ? capacity_ - num_items_ : num_items_; if (num_items_ > 0) std::inplace_merge(hra_ ? from : begin(), items_ + offset, hra_ ? end() : from + num, comparator_); num_items_ += num; @@ -369,7 +386,8 @@ template bool req_compactor::next_sections(uint64_t state, sections& s) { const float ssr = s.size_raw / sqrtf(2); const uint32_t ne = nearest_even(ssr); - if (state >= static_cast(1ULL << (s.num - 1)) + if (s.num <= 32 // at most 48 sections, so the shift below stays in range + && state >= static_cast(1ULL << (s.num - 1)) && s.size > req_constants::MIN_K && ne >= req_constants::MIN_K) { s.size_raw = ssr; @@ -515,8 +533,9 @@ template auto req_compactor::deserialize_items(std::istream& is, const S& serde, const A& allocator, uint32_t num) -> std::unique_ptr { A alloc(allocator); - std::unique_ptr items(alloc.allocate(num), items_deleter(allocator, false, num)); - serde.deserialize(is, items.get(), num); + // an empty compactor has no storage + std::unique_ptr items(num > 0 ? alloc.allocate(num) : nullptr, items_deleter(allocator, false, num)); + if (num > 0) serde.deserialize(is, items.get(), num); // serde did not throw, enable destructors items.get_deleter().set_destroy(true); if (!is.good()) throw std::runtime_error("error reading from std::istream"); @@ -572,8 +591,9 @@ auto req_compactor::deserialize_items(const void* bytes, size_t size, c const char* ptr = static_cast(bytes); const char* end_ptr = static_cast(bytes) + size; A alloc(allocator); - std::unique_ptr items(alloc.allocate(num), items_deleter(allocator, false, num)); - ptr += serde.deserialize(ptr, end_ptr - ptr, items.get(), num); + // an empty compactor has no storage + std::unique_ptr items(num > 0 ? alloc.allocate(num) : nullptr, items_deleter(allocator, false, num)); + if (num > 0) ptr += serde.deserialize(ptr, end_ptr - ptr, items.get(), num); // serde did not throw, enable destructors items.get_deleter().set_destroy(true); return std::pair, size_t>( diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp index 91af958e..409a24ca 100755 --- a/req/include/req_sketch.hpp +++ b/req/include/req_sketch.hpp @@ -203,6 +203,7 @@ class req_sketch { * If the comparator throws while items are sorted or merged, or does not impose a consistent * total order, the state of the sketch is undefined. * @param item from a stream of items + * @throw std::overflow_error if n would exceed req_constants::MAX_N, in which case the sketch is unchanged */ template void update(FwdT&& item); @@ -213,9 +214,11 @@ class req_sketch { * both sketches were built using compatible string encodings. * The first merge of a non-empty sketch allocates storage for items, * unless reserve() or an update already did. + * A sketch can be merged into itself. * If an allocation or a copy of an item throws, this sketch stays valid and usable, but may hold part of other. * If the comparator throws while items are sorted or merged, the state of this sketch is undefined. * @param other sketch to merge into this one + * @throw std::overflow_error if n would exceed req_constants::MAX_N, in which case the sketch is unchanged */ template void merge(FwdSk&& other); @@ -481,6 +484,8 @@ class req_sketch { void grow(); void update_max_nom_size(); void update_num_retained(); + // throws std::overflow_error if n_ + increase would exceed req_constants::MAX_N + void check_n_increase(uint64_t increase) const; // recomputes n, the number of retained items and the nominal size from the compactors, // after an allocation failed partway through an update or a merge void restore_counts(); diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index 474ace3b..620979c9 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -173,6 +173,7 @@ template template void req_sketch::update(FwdT&& item) { if (!check_update_item(item)) { return; } + check_n_increase(1); compactors_[0].reserve(); // first, so that a failure leaves the sketch unchanged // compare and copy first, but change nothing until the item is in level zero, // so that an exception leaves the sketch as it was @@ -196,6 +197,13 @@ template void req_sketch::merge(FwdSk&& other) { if (is_HRA() != other.is_HRA()) throw std::invalid_argument("merging HRA and LRA is not valid"); if (other.is_empty()) return; + if (static_cast(&other) == static_cast(this)) { + // merging reads other while it changes this, so merge a copy + req_sketch copy(other); + merge(std::move(copy)); + return; + } + check_n_increase(other.n_); reset_sorted_view(); const size_t num_levels = compactors_.size(); try { @@ -208,11 +216,19 @@ void req_sketch::merge(FwdSk&& other) { optional new_max; if (is_new_min) new_min = optional(conditional_forward(*other.min_item_)); if (is_new_max) new_max = optional(conditional_forward(*other.max_item_)); - if (is_new_min) min_item_ = std::move(new_min); - if (is_new_max) max_item_ = std::move(new_max); - // merge the items in all height compactors - for (size_t i = 0; i < other.get_num_levels(); ++i) { + // merge the items in all height compactors, the highest first, and set min and max + // with the first items of other that this sketch takes. Each compactor merge either completes or changes nothing, + // so if one fails, min and max are unchanged unless items of other were added, and then a level above + // level zero holds items, so the sketch stays in estimation mode, where min and max are serialized + bool min_max_set = false; + for (size_t i = other.get_num_levels(); i-- > 0;) { + const bool adds_items = other.compactors_[i].get_num_items() > 0; compactors_[i].merge(conditional_forward(other.compactors_[i])); + if (adds_items && !min_max_set) { + if (is_new_min) min_item_ = std::move(new_min); + if (is_new_max) max_item_ = std::move(new_max); + min_max_set = true; + } } } catch (...) { remove_empty_levels_above(num_levels); @@ -649,6 +665,13 @@ void req_sketch::update_num_retained() { for (const auto& compactor: compactors_) num_retained_ += compactor.get_num_items(); } +template +void req_sketch::check_n_increase(uint64_t increase) const { + if (n_ > req_constants::MAX_N || increase > req_constants::MAX_N - n_) { + throw std::overflow_error("n would exceed 2^63 - 1"); + } +} + template void req_sketch::restore_counts() { // the total weight of the retained items is always n diff --git a/req/test/req_sketch_allocation_test.cpp b/req/test/req_sketch_allocation_test.cpp index 0602630f..0ea85b35 100644 --- a/req/test/req_sketch_allocation_test.cpp +++ b/req/test/req_sketch_allocation_test.cpp @@ -96,6 +96,16 @@ void check_consistent(const sketch_type& sketch) { REQUIRE(sketch.is_estimation_mode() == (max_weight > 1)); } +// min and max must survive a round trip, which in exact mode derives them from the items +void check_min_max_round_trip(const sketch_type& sketch, allocation_control& control) { + if (sketch.is_empty()) return; + const auto b = sketch.serialize(); + const auto copy = sketch_type::deserialize(b.data(), b.size(), serde(), + std::less(), failing_allocator(control)); + REQUIRE(copy.get_min_item() == sketch.get_min_item()); + REQUIRE(copy.get_max_item() == sketch.get_max_item()); +} + // the sketch must still work: query it, update it, merge into it, and round-trip it void check_usable(sketch_type& sketch, allocation_control& control) { control.disarm(); @@ -387,6 +397,8 @@ TEST_CASE("req sketch allocation: failure at each allocation of a merge", "[req_ for (long fail_at = 0; ; ++fail_at) { sketch_type target = make_sketch(0, target_n, control, hra); const bool was_estimation_mode = target.is_estimation_mode(); + const std::string min_before = target.is_empty() ? "" : target.get_min_item(); + const std::string max_before = target.is_empty() ? "" : target.get_max_item(); control.arm(fail_at); try { target.merge(source); @@ -402,7 +414,14 @@ TEST_CASE("req sketch allocation: failure at each allocation of a merge", "[req_ // the target has the same levels as before if (target.get_n() == static_cast(target_n)) { REQUIRE(target.is_estimation_mode() == was_estimation_mode); + // and min and max are as they were + REQUIRE(target.is_empty() == (target_n == 0)); + if (target_n > 0) { + REQUIRE(target.get_min_item() == min_before); + REQUIRE(target.get_max_item() == max_before); + } } + check_min_max_round_trip(target, control); check_usable(target, control); } } @@ -516,12 +535,21 @@ TEST_CASE("req sketch allocation: failure at each item copy of a merge", "[req_s for (long fail_at = 0; !completed; ++fail_at) { CAPTURE(fail_at); fragile_sketch target = make_fragile_sketch(0, target_n, hra); + const int min_before = target.is_empty() ? 0 : *target.get_min_item().value; + const int max_before = target.is_empty() ? 0 : *target.get_max_item().value; copies_allowed = fail_at; try { target.merge(source); completed = true; } catch (const std::runtime_error&) {} copies_allowed = -1; + if (target.get_n() == static_cast(target_n)) { // nothing merged + REQUIRE(target.is_empty() == (target_n == 0)); + if (target_n > 0) { + REQUIRE(*target.get_min_item().value == min_before); + REQUIRE(*target.get_max_item().value == max_before); + } + } REQUIRE(target.get_n() >= static_cast(target_n)); REQUIRE(target.get_n() <= static_cast(target_n + source_n)); if (completed) REQUIRE(target.get_n() == static_cast(target_n + source_n)); @@ -578,5 +606,66 @@ TEST_CASE("req sketch allocation: converting constructor that rejects the order REQUIRE(test_allocator_net_allocations == 0); } +namespace { + +long comparisons_allowed = -1; // comparisons left before one throws; negative means no limit + +struct throwing_greater { + bool operator()(double a, double b) const { + if (comparisons_allowed == 0) throw std::runtime_error("comparison failed"); + if (comparisons_allowed > 0) --comparisons_allowed; + return a > b; + } +}; + +} // namespace + +TEST_CASE("req sketch allocation: converting constructor whose comparator throws releases the items", "[req_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + req_sketch, test_allocator> sketch(K, true, std::greater(), + test_allocator(0)); + for (int i = 0; i < 1000; ++i) sketch.update(i); + REQUIRE(sketch.is_estimation_mode()); + using target_sketch = req_sketch>; + long failures_tested = 0; + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + comparisons_allowed = fail_at; + try { + const target_sketch copy(sketch, throwing_greater(), test_allocator(0)); + completed = true; + comparisons_allowed = -1; + REQUIRE(copy.get_n() == sketch.get_n()); + } catch (const std::runtime_error&) { + ++failures_tested; + } + comparisons_allowed = -1; + } + REQUIRE(failures_tested > 0); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("req compactor allocation: deserializing an empty compactor allocates no item storage", "[req_sketch]") { + allocation_control control; + const compactor_type empty(true, 0, K, std::less(), failing_allocator(control)); + const auto b = bytes(empty); + control.arm(0); // any allocation throws + const auto result = compactor_type::deserialize(b.data(), b.size(), serde(), std::less(), + failing_allocator(control), true, true, K, 0); + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + s.write(reinterpret_cast(b.data()), b.size()); + const auto from_stream = compactor_type::deserialize(s, serde(), std::less(), + failing_allocator(control), true, true, K, 0); + control.disarm(); + REQUIRE(result.first.get_num_items() == 0); + REQUIRE(result.second == b.size()); + REQUIRE(from_stream.get_num_items() == 0); +} + } /* namespace datasketches */ diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 5167df84..d2e75fd7 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -26,6 +26,8 @@ #include #include #include +#include +#include namespace datasketches { @@ -735,11 +737,10 @@ TEST_CASE("req sketch: deserialize rejects inconsistent compactors", "[req_sketc using req_compactor_float = req_compactor, std::allocator>; -static req_compactor_float make_compactor_with_state(uint16_t k, uint64_t state) { +static req_compactor_float make_compactor_with_state(uint16_t k, uint64_t state, float section_size_raw, + uint8_t num_sections) { std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); - const float section_size_raw = k; // not used by merge const uint8_t lg_weight = 0; - const uint8_t num_sections = req_constants::INIT_NUM_SECTIONS; const uint16_t padding = 0; const uint32_t num_items = 0; s.write(reinterpret_cast(&state), sizeof(state)); @@ -752,6 +753,10 @@ static req_compactor_float make_compactor_with_state(uint16_t k, uint64_t state) k, 0); } +static req_compactor_float make_compactor_with_state(uint16_t k, uint64_t state) { + return make_compactor_with_state(k, state, k, req_constants::INIT_NUM_SECTIONS); // the section size is not used by merge +} + TEST_CASE("req compactor: section growth stops at minimum section size", "[req_compactor]") { const uint16_t ks[] = {6, 12, 14}; const uint32_t expected_nom_capacities[] = {48, 192, 192}; @@ -763,6 +768,98 @@ TEST_CASE("req compactor: section growth stops at minimum section size", "[req_c } } +TEST_CASE("req compactor: the number of sections stops at 48", "[req_compactor]") { + // 48 sections of 256 is where k = 1024 gets after four doublings; with a state of 2^47 or more, + // another doubling would give 96 sections, which deserialize rejects, and 1 << 95 after that + req_compactor_float c = make_compactor_with_state(1024, 1ULL << 47, 256, 48); + REQUIRE(c.get_nom_capacity() == 2 * 48 * 256); + c.merge(make_compactor_with_state(1024, 0)); + REQUIRE(c.get_nom_capacity() == 2 * 48 * 256); +} + +TEST_CASE("req sketch: merge into itself", "[req_sketch]") { + for (bool hra: {true, false}) { + for (int n: {2, 50, 1000}) { + CAPTURE(hra, n); + req_sketch sketch(12, hra); + for (int i = 0; i < n; ++i) sketch.update((i * 7919) % 10007); // level zero is not sorted + req_sketch expected(sketch); + const req_sketch copy(sketch); + random_utils::rand.seed(1); random_utils::random_bit.seed(1); + expected.merge(copy); + const auto expected_bytes = expected.serialize(); + + req_sketch lvalue(sketch); + random_utils::rand.seed(1); random_utils::random_bit.seed(1); + lvalue.merge(lvalue); + REQUIRE(lvalue.serialize() == expected_bytes); + + req_sketch rvalue(sketch); + random_utils::rand.seed(1); random_utils::random_bit.seed(1); + rvalue.merge(std::move(rvalue)); + REQUIRE(rvalue.serialize() == expected_bytes); + } + } + // level zero, unsorted before the merge, must be sorted after it + req_sketch sketch(12, true); + sketch.update(5); + sketch.update(1); + sketch.merge(sketch); + std::vector items; + for (const auto pair: sketch) items.push_back(pair.first); + REQUIRE(items == std::vector({1, 1, 5, 5})); +} + +TEST_CASE("req sketch: update with an item of the sketch after deserialize", "[req_sketch]") { + // after deserialize, level zero is full, so the update must not grow it before copying the item + for (bool hra: {true, false}) { + for (int n: {50, 1000}) { + CAPTURE(hra, n); + req_sketch sketch(12, hra); + for (int i = 0; i < n; ++i) sketch.update(std::string(64, 'a') + std::to_string(i)); + const auto bytes = sketch.serialize(); + auto copy = req_sketch::deserialize(bytes.data(), bytes.size()); + const std::string expected = (*copy.begin()).first; + auto count_expected = [©, &expected]() { + int count = 0; + for (const auto pair: copy) if (pair.first == expected) ++count; + return count; + }; + const int count_before = count_expected(); + copy.update((*copy.begin()).first); + REQUIRE(copy.get_n() == static_cast(n + 1)); + // the item added is a copy of the item, not of what growing left in its place + REQUIRE(count_expected() == count_before + 1); + } + } +} + +TEST_CASE("req sketch: n is capped at 2^63 - 1", "[req_sketch]") { + const size_t n_offset = 8; // after the preamble, in estimation mode + req_sketch sketch(12); + for (int i = 0; i < 1000; ++i) sketch.update(static_cast(i)); + auto bytes = sketch.serialize(); + put_value(bytes, n_offset, req_constants::MAX_N - 1); + auto big = req_sketch::deserialize(bytes.data(), bytes.size()); + REQUIRE(big.get_n() == req_constants::MAX_N - 1); + + SECTION("update") { + big.update(1.0f); + REQUIRE(big.get_n() == req_constants::MAX_N); + const auto before = big.serialize(); + REQUIRE_THROWS_AS(big.update(2.0f), std::overflow_error); + REQUIRE(big.serialize() == before); + } + + SECTION("merge") { + const auto before = big.serialize(); + REQUIRE_THROWS_AS(big.merge(sketch), std::overflow_error); + REQUIRE(big.serialize() == before); + REQUIRE_THROWS_AS(sketch.merge(big), std::overflow_error); + REQUIRE(sketch.get_n() == 1000); + } +} + namespace { // a comparator that can't be default-constructed: the sketch must use its own comparator everywhere @@ -790,6 +887,16 @@ TEST_CASE("req sketch: compaction and merge use the sketch's comparator", "[req_ const req_sketch copy(sketch, ordered_by(true)); REQUIRE(copy.get_n() == 2000); REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(0.05)); + // split points are checked with the sketch's comparator: descending here + const int split_points[] = {1500, 500}; + const auto cdf = sketch.get_CDF(split_points, 2); + REQUIRE(cdf.size() == 3); + REQUIRE(cdf[0] == Approx(0.25).margin(0.05)); + REQUIRE(cdf[1] == Approx(0.75).margin(0.05)); + const auto pmf = sketch.get_PMF(split_points, 2); + REQUIRE(pmf[1] == Approx(0.5).margin(0.05)); + const int ascending[] = {500, 1500}; + REQUIRE_THROWS_AS(sketch.get_CDF(ascending, 2), std::invalid_argument); } } /* namespace datasketches */