From 4fb57ca7fe23c857f28b3d6e6d8cf62862d63b64 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 10:50:49 -0800 Subject: [PATCH] REQ: limit N to Long.MAX_VALUE and keep N consistent after a failed merge; KLL items merge compares min and max before committing REQ - merge() throws SketchesArgumentException, with the sketch unchanged, if the combined N would exceed Long.MAX_VALUE. - update() throws SketchesStateException, with the sketch unchanged, if N is already Long.MAX_VALUE. - merge() sets N, min and max after the compactors are merged. If that merge is interrupted, N is advanced by the weight actually merged, and min and max include the other sketch's only if items were merged. - Javadoc for both methods names the exceptions. KLL - The items merge computes the new min and max before it commits N, the levels and the items, so a comparator that throws there leaves the sketch as it was after the level-0 updates. The doubles, floats and longs merges were checked and need no change. Co-Authored-By: Claude Opus 5.5 --- .../datasketches/kll/KllItemsHelper.java | 17 ++- .../apache/datasketches/req/ReqSketch.java | 62 +++++++- .../datasketches/kll/KllMergeFailureTest.java | 106 ++++++++++++++ .../datasketches/req/ReqMergeLimitsTest.java | 132 ++++++++++++++++++ 4 files changed, 302 insertions(+), 15 deletions(-) create mode 100644 src/test/java/org/apache/datasketches/kll/KllMergeFailureTest.java create mode 100644 src/test/java/org/apache/datasketches/req/ReqMergeLimitsTest.java diff --git a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java index 2ea651e13..15f70d33c 100644 --- a/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllItemsHelper.java @@ -236,6 +236,12 @@ static void mergeItemImpl(final KllItemsSketch mySketch, // } //end of updating levels above level 0 + //The new min and max items. The comparator may throw, so this is done before anything is committed. + final Object otherMin = otherItmSk.getMinItem(); + final Object otherMax = otherItmSk.getMaxItem(); + final Object newMin = myEmpty ? otherMin : Util.minT(myMin, otherMin, comp); + final Object newMax = myEmpty ? otherMax : Util.maxT(myMax, otherMax, comp); + //Update Preamble: mySketch.setN(finalN); if (otherItmSk.isEstimationMode()) { //otherwise the merge brings over exact items. @@ -248,15 +254,8 @@ static void mergeItemImpl(final KllItemsSketch mySketch, mySketch.setItemsArray(myNewItemsArr); //Update min, max items - final Object otherMin = otherItmSk.getMinItem(); - final Object otherMax = otherItmSk.getMaxItem(); - if (myEmpty) { - mySketch.setMinItem(otherMin); - mySketch.setMaxItem(otherMax); - } else { - mySketch.setMinItem(Util.minT(myMin, otherMin, comp)); - mySketch.setMaxItem(Util.maxT(myMax, otherMax, comp)); - } + mySketch.setMinItem(newMin); + mySketch.setMaxItem(newMax); assert KllHelper.sumTheSampleWeights(mySketch.getNumLevels(), mySketch.levelsArr) == mySketch.getN(); } diff --git a/src/main/java/org/apache/datasketches/req/ReqSketch.java b/src/main/java/org/apache/datasketches/req/ReqSketch.java index 0bf58d7f0..351fc9bee 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/ReqSketch.java @@ -28,6 +28,7 @@ import java.util.Random; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.quantilescommon.FloatsSketchSortedView; import org.apache.datasketches.quantilescommon.QuantileSearchCriteria; import org.apache.datasketches.quantilescommon.QuantilesAPI; @@ -403,6 +404,16 @@ public QuantilesFloatsSketchIterator iterator() { return new ReqSketchIterator(this); } + /** + * {@inheritDoc} + * + *

If an error interrupts the merge of the compactors, part of the other sketch may have been + * merged. N, the retained items and the nominal size are then recomputed from the compactors, and + * min and max include those of the other sketch if any of its items were merged.

+ * + * @throws SketchesArgumentException if the two sketches differ in their HighRankAccuracy setting, or if + * the combined N would exceed {@link Long#MAX_VALUE}. In both cases this sketch is unchanged. + */ @Override public ReqSketch merge(final ReqSketch other) { if ((other == null) || other.isEmpty()) { return this; } @@ -410,18 +421,33 @@ public ReqSketch merge(final ReqSketch other) { throw new SketchesArgumentException( "Both sketches must have the same HighRankAccuracy setting."); } + if (other.totalN > (Long.MAX_VALUE - totalN)) { + throw new SketchesArgumentException("The merged N would exceed Long.MAX_VALUE: " + + totalN + " + " + other.totalN); + } //allocate first: the first compactor's storage, then any new levels, before the counters change ensureCapacity(); //Grow until self has at least as many compactors as other while (getNumLevels() < other.getNumLevels()) { grow(); } - totalN += other.totalN; - //update min, max items, n - if (Float.isNaN(minItem) || (other.minItem < minItem)) { minItem = other.minItem; } - if (Float.isNaN(maxItem) || (other.maxItem > maxItem)) { maxItem = other.maxItem; } //Merge the items in all height compactors - for (int i = 0; i < other.getNumLevels(); i++) { - compactors.get(i).merge(other.compactors.get(i)); + final long startWeight = computeTotalWeight(); + try { + for (int i = 0; i < other.getNumLevels(); i++) { + compactors.get(i).merge(other.compactors.get(i)); + } + } catch (final RuntimeException | Error e) { + //Part of other may have been merged. Each level's items carry weight 2^level, so the weight + //added so far is the part of other's N that was merged. + final long addedWeight = computeTotalWeight() - startWeight; + if (addedWeight > 0) { updateMinMax(other); } + totalN += addedWeight; + maxNomSize = computeMaxNomSize(); + retItems = computeTotalRetainedItems(); + reqSV = null; + throw e; } + totalN += other.totalN; + updateMinMax(other); maxNomSize = computeMaxNomSize(); retItems = computeTotalRetainedItems(); if (retItems >= maxNomSize) { @@ -465,9 +491,17 @@ public String toString() { return sb.toString(); } + /** + * {@inheritDoc} + * + * @throws SketchesStateException if N is already {@link Long#MAX_VALUE}. The sketch is unchanged. + */ @Override public void update(final float item) { if (Float.isNaN(item)) { return; } + if (totalN == Long.MAX_VALUE) { + throw new SketchesStateException("N cannot exceed Long.MAX_VALUE."); + } ensureCapacity(); //first use allocates, before anything changes if (isEmpty()) { minItem = item; @@ -512,6 +546,17 @@ int computeMaxNomSize() { return cap; } + /** + * Computes the total weight of the retained items, where each item at level h has weight 2^h. + */ + long computeTotalWeight() { + long weight = 0; + for (final ReqCompactor c : compactors) { + weight += (long) c.getBuffer().getCount() << c.getLgWeight(); + } + return weight; + } + /** * Computes the retained Items for the sketch. */ @@ -577,6 +622,11 @@ private void compress() { if (reqDebug != null) { reqDebug.emitCompressDone(); } } + private void updateMinMax(final ReqSketch other) { + if (Float.isNaN(minItem) || (other.minItem < minItem)) { minItem = other.minItem; } + if (Float.isNaN(maxItem) || (other.maxItem > maxItem)) { maxItem = other.maxItem; } + } + private void grow() { addCompactor(new ReqCompactor((byte)getNumLevels(), hra, k, reqDebug)); } diff --git a/src/test/java/org/apache/datasketches/kll/KllMergeFailureTest.java b/src/test/java/org/apache/datasketches/kll/KllMergeFailureTest.java new file mode 100644 index 000000000..ca0d24076 --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllMergeFailureTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.kll; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Comparator; + +import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIteratorAPI; +import org.testng.annotations.Test; + +/** + * Checks that a KLL items sketch stays consistent when its comparator throws during a merge. + */ +public class KllMergeFailureTest { + private static final int K = 20; + private static final long SEED = 12345; + + /** Counts comparisons, and throws once a given number of further comparisons have been made. */ + private static final class CountingComparator implements Comparator { + int count = 0; + private int allowed = -1; //negative: never throws + + void failAfter(final int comparisons) { allowed = comparisons; } + + @Override + public int compare(final String a, final String b) { + if (allowed == 0) { + allowed = -1; + throw new IllegalStateException("comparator failure"); + } + if (allowed > 0) { allowed--; } + count++; + return a.compareTo(b); + } + } + + private static String item(final int i) { return String.format("%06d", i); } + + private static KllItemsSketch[] newPair(final CountingComparator comparator) { + final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe(); + KllSketch.random.setSeed(SEED); + final KllItemsSketch mine = KllItemsSketch.newHeapInstance(K, comparator, serDe); + for (int i = 0; i < 10 * K; i++) { mine.update(item(500_000 + i)); } + //ascending, so the other sketch's higher levels hold its smallest items + final KllItemsSketch other = KllItemsSketch.newHeapInstance(K, comparator, serDe); + for (int i = 0; i < 50 * K; i++) { other.update(item(i)); } + @SuppressWarnings("unchecked") + final KllItemsSketch[] pair = new KllItemsSketch[] {mine, other}; + return pair; + } + + //The last two comparisons of a merge find the new min and max. The comparator throws at the first + //of them. The min and max must still bound every retained item, and N must equal the total weight. + @Test + public void comparatorFailureAtMinMaxLeavesSketchConsistent() { + final CountingComparator counter = new CountingComparator(); + final KllItemsSketch[] dry = newPair(counter); + counter.count = 0; + dry[0].merge(dry[1]); + final int mergeComparisons = counter.count; + + final CountingComparator comparator = new CountingComparator(); + final KllItemsSketch[] pair = newPair(comparator); + final KllItemsSketch sk = pair[0]; + comparator.failAfter(mergeComparisons - 2); + try { + sk.merge(pair[1]); + fail("the comparator should have thrown"); + } catch (final IllegalStateException e) { + //expected + } + + final QuantilesGenericSketchIteratorAPI it = sk.iterator(); + long weight = 0; + while (it.next()) { + final String q = it.getQuantile(); + assertTrue(q.compareTo(sk.getMinItem()) >= 0, q + " < min " + sk.getMinItem()); + assertTrue(q.compareTo(sk.getMaxItem()) <= 0, q + " > max " + sk.getMaxItem()); + weight += it.getWeight(); + } + assertEquals(weight, sk.getN()); + sk.merge(pair[1]); //the sketch keeps working + } +} diff --git a/src/test/java/org/apache/datasketches/req/ReqMergeLimitsTest.java b/src/test/java/org/apache/datasketches/req/ReqMergeLimitsTest.java new file mode 100644 index 000000000..b3e02ea1c --- /dev/null +++ b/src/test/java/org/apache/datasketches/req/ReqMergeLimitsTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.req; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.List; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.SketchesStateException; +import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; +import org.testng.annotations.Test; + +/** + * Checks the limit on N, and that a merge interrupted partway leaves N consistent with the retained items. + */ +public class ReqMergeLimitsTest { + private static final int K = 12; + + private static ReqSketch newSketch(final boolean hra, final int n, final int offset) { + final ReqSketch sk = ReqSketch.builder().setK(K).setHighRankAccuracy(hra).build(); + for (int i = 0; i < n; i++) { sk.update(i + offset); } + return sk; + } + + //the same compactors and items as sk, but with the given N + private static ReqSketch withN(final ReqSketch sk, final long n) { + return new ReqSketch(sk.getK(), sk.getHighRankAccuracyMode(), n, sk.getMinItem(), sk.getMaxItem(), + sk.getCompactors()); + } + + @Test + public void mergeRejectsNAboveMax() { + for (final boolean hra : new boolean[] {true, false}) { + final ReqSketch other = newSketch(hra, 100, 0); + final ReqSketch sk = withN(newSketch(hra, 1000, 0), Long.MAX_VALUE - 99); + final byte[] before = sk.toByteArray(); + try { + sk.merge(other); + fail(); + } catch (final SketchesArgumentException e) { + //expected + } + assertEquals(sk.toByteArray(), before); + } + } + + @Test + public void updateRejectsNAboveMax() { + for (final boolean hra : new boolean[] {true, false}) { + final ReqSketch sk = withN(newSketch(hra, 1000, 0), Long.MAX_VALUE - 100); + sk.merge(newSketch(hra, 100, 0)); //reaches Long.MAX_VALUE exactly + assertEquals(sk.getN(), Long.MAX_VALUE); + final byte[] before = sk.toByteArray(); + try { + sk.update(1f); + fail(); + } catch (final SketchesStateException e) { + //expected + } + assertEquals(sk.toByteArray(), before); + sk.update(Float.NaN); //ignored, so allowed + } + } + + /** + * The other sketch's third compactor has the wrong lgWeight, so the merge fails after merging the + * first two levels. N must then count exactly the items that were merged. + */ + @Test + public void partialMergeKeepsNConsistent() { + for (final boolean hra : new boolean[] {true, false}) { + final ReqSketch sk = newSketch(hra, 500, 0); + final long startN = sk.getN(); + final List otherComps = new ArrayList<>(); + final int[] lgWeights = {0, 1, 5}; + long otherN = 0; + for (int h = 0; h < lgWeights.length; h++) { + final ReqCompactor c = new ReqCompactor((byte) lgWeights[h], hra, K, null); + for (int i = 0; i < 10; i++) { c.getBuffer().append(-1000 - (h * 10) - i); } + c.getBuffer().sort(); + otherComps.add(c); + otherN += 10L << lgWeights[h]; + } + final ReqSketch other = new ReqSketch(K, hra, otherN, -1029f, -1000f, otherComps); + try { + sk.merge(other); + fail(); + } catch (final AssertionError e) { + //expected: compactor lgWeights differ + } + assertEquals(sk.getN(), startN + 10 + 20); //levels 0 and 1 were merged + assertEquals(sk.getMinItem(), -1029f); //other's min, since some of its items were merged + assertEquals(sk.getMaxItem(), 499f); + checkConsistent(sk); + for (int i = 0; i < 10_000; i++) { sk.update(i); } + checkConsistent(sk); + final ReqSketch copy = ReqSketch.heapify(MemorySegment.ofArray(sk.toByteArray())); + assertEquals(copy.toByteArray(), sk.toByteArray()); + } + } + + private static void checkConsistent(final ReqSketch sk) { + assertEquals(sk.getNumRetained(), sk.computeTotalRetainedItems()); + assertEquals(sk.getMaxNomSize(), sk.computeMaxNomSize()); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while (it.next()) { weight += it.getWeight(); } + assertEquals(weight, sk.getN()); + } +}