Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,12 @@ static <T> void mergeItemImpl(final KllItemsSketch<T> mySketch,
//
} //end of updating levels above level 0

//The new min and max items. The comparator may throw, so this is done before anything is committed.
final Object otherMin = otherItmSk.getMinItem();
final Object otherMax = otherItmSk.getMaxItem();
final Object newMin = myEmpty ? otherMin : Util.minT(myMin, otherMin, comp);
final Object newMax = myEmpty ? otherMax : Util.maxT(myMax, otherMax, comp);

//Update Preamble:
mySketch.setN(finalN);
if (otherItmSk.isEstimationMode()) { //otherwise the merge brings over exact items.
Expand All @@ -248,15 +254,8 @@ static <T> void mergeItemImpl(final KllItemsSketch<T> mySketch,
mySketch.setItemsArray(myNewItemsArr);

//Update min, max items
final Object otherMin = otherItmSk.getMinItem();
final Object otherMax = otherItmSk.getMaxItem();
if (myEmpty) {
mySketch.setMinItem(otherMin);
mySketch.setMaxItem(otherMax);
} else {
mySketch.setMinItem(Util.minT(myMin, otherMin, comp));
mySketch.setMaxItem(Util.maxT(myMax, otherMax, comp));
}
mySketch.setMinItem(newMin);
mySketch.setMaxItem(newMax);
assert KllHelper.sumTheSampleWeights(mySketch.getNumLevels(), mySketch.levelsArr) == mySketch.getN();
}

Expand Down
62 changes: 56 additions & 6 deletions src/main/java/org/apache/datasketches/req/ReqSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.Random;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesStateException;
import org.apache.datasketches.quantilescommon.FloatsSketchSortedView;
import org.apache.datasketches.quantilescommon.QuantileSearchCriteria;
import org.apache.datasketches.quantilescommon.QuantilesAPI;
Expand Down Expand Up @@ -403,25 +404,50 @@ public QuantilesFloatsSketchIterator iterator() {
return new ReqSketchIterator(this);
}

/**
* {@inheritDoc}
*
* <p>If an error interrupts the merge of the compactors, part of the other sketch may have been
* merged. N, the retained items and the nominal size are then recomputed from the compactors, and
* min and max include those of the other sketch if any of its items were merged.</p>
*
* @throws SketchesArgumentException if the two sketches differ in their HighRankAccuracy setting, or if
* the combined N would exceed {@link Long#MAX_VALUE}. In both cases this sketch is unchanged.
*/
@Override
public ReqSketch merge(final ReqSketch other) {
if ((other == null) || other.isEmpty()) { return this; }
if (other.hra != hra) {
throw new SketchesArgumentException(
"Both sketches must have the same HighRankAccuracy setting.");
}
if (other.totalN > (Long.MAX_VALUE - totalN)) {
throw new SketchesArgumentException("The merged N would exceed Long.MAX_VALUE: "
+ totalN + " + " + other.totalN);
}
//allocate first: the first compactor's storage, then any new levels, before the counters change
ensureCapacity();
//Grow until self has at least as many compactors as other
while (getNumLevels() < other.getNumLevels()) { grow(); }
totalN += other.totalN;
//update min, max items, n
if (Float.isNaN(minItem) || (other.minItem < minItem)) { minItem = other.minItem; }
if (Float.isNaN(maxItem) || (other.maxItem > maxItem)) { maxItem = other.maxItem; }
//Merge the items in all height compactors
for (int i = 0; i < other.getNumLevels(); i++) {
compactors.get(i).merge(other.compactors.get(i));
final long startWeight = computeTotalWeight();
try {
for (int i = 0; i < other.getNumLevels(); i++) {
compactors.get(i).merge(other.compactors.get(i));
}
} catch (final RuntimeException | Error e) {
//Part of other may have been merged. Each level's items carry weight 2^level, so the weight
//added so far is the part of other's N that was merged.
final long addedWeight = computeTotalWeight() - startWeight;
if (addedWeight > 0) { updateMinMax(other); }
totalN += addedWeight;
maxNomSize = computeMaxNomSize();
retItems = computeTotalRetainedItems();
reqSV = null;
throw e;
}
totalN += other.totalN;
updateMinMax(other);
maxNomSize = computeMaxNomSize();
retItems = computeTotalRetainedItems();
if (retItems >= maxNomSize) {
Expand Down Expand Up @@ -465,9 +491,17 @@ public String toString() {
return sb.toString();
}

/**
* {@inheritDoc}
*
* @throws SketchesStateException if N is already {@link Long#MAX_VALUE}. The sketch is unchanged.
*/
@Override
public void update(final float item) {
if (Float.isNaN(item)) { return; }
if (totalN == Long.MAX_VALUE) {
throw new SketchesStateException("N cannot exceed Long.MAX_VALUE.");
}
ensureCapacity(); //first use allocates, before anything changes
if (isEmpty()) {
minItem = item;
Expand Down Expand Up @@ -512,6 +546,17 @@ int computeMaxNomSize() {
return cap;
}

/**
* Computes the total weight of the retained items, where each item at level h has weight 2^h.
*/
long computeTotalWeight() {
long weight = 0;
for (final ReqCompactor c : compactors) {
weight += (long) c.getBuffer().getCount() << c.getLgWeight();
}
return weight;
}

/**
* Computes the retained Items for the sketch.
*/
Expand Down Expand Up @@ -577,6 +622,11 @@ private void compress() {
if (reqDebug != null) { reqDebug.emitCompressDone(); }
}

private void updateMinMax(final ReqSketch other) {
if (Float.isNaN(minItem) || (other.minItem < minItem)) { minItem = other.minItem; }
if (Float.isNaN(maxItem) || (other.maxItem > maxItem)) { maxItem = other.maxItem; }
}

private void grow() {
addCompactor(new ReqCompactor((byte)getNumLevels(), hra, k, reqDebug));
}
Expand Down
106 changes: 106 additions & 0 deletions src/test/java/org/apache/datasketches/kll/KllMergeFailureTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/


package org.apache.datasketches.kll;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;

import java.util.Comparator;

import org.apache.datasketches.common.ArrayOfStringsSerDe;
import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIteratorAPI;
import org.testng.annotations.Test;

/**
* Checks that a KLL items sketch stays consistent when its comparator throws during a merge.
*/
public class KllMergeFailureTest {
private static final int K = 20;
private static final long SEED = 12345;

/** Counts comparisons, and throws once a given number of further comparisons have been made. */
private static final class CountingComparator implements Comparator<String> {
int count = 0;
private int allowed = -1; //negative: never throws

void failAfter(final int comparisons) { allowed = comparisons; }

@Override
public int compare(final String a, final String b) {
if (allowed == 0) {
allowed = -1;
throw new IllegalStateException("comparator failure");
}
if (allowed > 0) { allowed--; }
count++;
return a.compareTo(b);
}
}

private static String item(final int i) { return String.format("%06d", i); }

private static KllItemsSketch<String>[] newPair(final CountingComparator comparator) {
final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
KllSketch.random.setSeed(SEED);
final KllItemsSketch<String> mine = KllItemsSketch.newHeapInstance(K, comparator, serDe);
for (int i = 0; i < 10 * K; i++) { mine.update(item(500_000 + i)); }
//ascending, so the other sketch's higher levels hold its smallest items
final KllItemsSketch<String> other = KllItemsSketch.newHeapInstance(K, comparator, serDe);
for (int i = 0; i < 50 * K; i++) { other.update(item(i)); }
@SuppressWarnings("unchecked")
final KllItemsSketch<String>[] pair = new KllItemsSketch[] {mine, other};
return pair;
}

//The last two comparisons of a merge find the new min and max. The comparator throws at the first
//of them. The min and max must still bound every retained item, and N must equal the total weight.
@Test
public void comparatorFailureAtMinMaxLeavesSketchConsistent() {
final CountingComparator counter = new CountingComparator();
final KllItemsSketch<String>[] dry = newPair(counter);
counter.count = 0;
dry[0].merge(dry[1]);
final int mergeComparisons = counter.count;

final CountingComparator comparator = new CountingComparator();
final KllItemsSketch<String>[] pair = newPair(comparator);
final KllItemsSketch<String> sk = pair[0];
comparator.failAfter(mergeComparisons - 2);
try {
sk.merge(pair[1]);
fail("the comparator should have thrown");
} catch (final IllegalStateException e) {
//expected
}

final QuantilesGenericSketchIteratorAPI<String> it = sk.iterator();
long weight = 0;
while (it.next()) {
final String q = it.getQuantile();
assertTrue(q.compareTo(sk.getMinItem()) >= 0, q + " < min " + sk.getMinItem());
assertTrue(q.compareTo(sk.getMaxItem()) <= 0, q + " > max " + sk.getMaxItem());
weight += it.getWeight();
}
assertEquals(weight, sk.getN());
sk.merge(pair[1]); //the sketch keeps working
}
}
132 changes: 132 additions & 0 deletions src/test/java/org/apache/datasketches/req/ReqMergeLimitsTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/


package org.apache.datasketches.req;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;

import java.lang.foreign.MemorySegment;
import java.util.ArrayList;
import java.util.List;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesStateException;
import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator;
import org.testng.annotations.Test;

/**
* Checks the limit on N, and that a merge interrupted partway leaves N consistent with the retained items.
*/
public class ReqMergeLimitsTest {
private static final int K = 12;

private static ReqSketch newSketch(final boolean hra, final int n, final int offset) {
final ReqSketch sk = ReqSketch.builder().setK(K).setHighRankAccuracy(hra).build();
for (int i = 0; i < n; i++) { sk.update(i + offset); }
return sk;
}

//the same compactors and items as sk, but with the given N
private static ReqSketch withN(final ReqSketch sk, final long n) {
return new ReqSketch(sk.getK(), sk.getHighRankAccuracyMode(), n, sk.getMinItem(), sk.getMaxItem(),
sk.getCompactors());
}

@Test
public void mergeRejectsNAboveMax() {
for (final boolean hra : new boolean[] {true, false}) {
final ReqSketch other = newSketch(hra, 100, 0);
final ReqSketch sk = withN(newSketch(hra, 1000, 0), Long.MAX_VALUE - 99);
final byte[] before = sk.toByteArray();
try {
sk.merge(other);
fail();
} catch (final SketchesArgumentException e) {
//expected
}
assertEquals(sk.toByteArray(), before);
}
}

@Test
public void updateRejectsNAboveMax() {
for (final boolean hra : new boolean[] {true, false}) {
final ReqSketch sk = withN(newSketch(hra, 1000, 0), Long.MAX_VALUE - 100);
sk.merge(newSketch(hra, 100, 0)); //reaches Long.MAX_VALUE exactly
assertEquals(sk.getN(), Long.MAX_VALUE);
final byte[] before = sk.toByteArray();
try {
sk.update(1f);
fail();
} catch (final SketchesStateException e) {
//expected
}
assertEquals(sk.toByteArray(), before);
sk.update(Float.NaN); //ignored, so allowed
}
}

/**
* The other sketch's third compactor has the wrong lgWeight, so the merge fails after merging the
* first two levels. N must then count exactly the items that were merged.
*/
@Test
public void partialMergeKeepsNConsistent() {
for (final boolean hra : new boolean[] {true, false}) {
final ReqSketch sk = newSketch(hra, 500, 0);
final long startN = sk.getN();
final List<ReqCompactor> otherComps = new ArrayList<>();
final int[] lgWeights = {0, 1, 5};
long otherN = 0;
for (int h = 0; h < lgWeights.length; h++) {
final ReqCompactor c = new ReqCompactor((byte) lgWeights[h], hra, K, null);
for (int i = 0; i < 10; i++) { c.getBuffer().append(-1000 - (h * 10) - i); }
c.getBuffer().sort();
otherComps.add(c);
otherN += 10L << lgWeights[h];
}
final ReqSketch other = new ReqSketch(K, hra, otherN, -1029f, -1000f, otherComps);
try {
sk.merge(other);
fail();
} catch (final AssertionError e) {
//expected: compactor lgWeights differ
}
assertEquals(sk.getN(), startN + 10 + 20); //levels 0 and 1 were merged
assertEquals(sk.getMinItem(), -1029f); //other's min, since some of its items were merged
assertEquals(sk.getMaxItem(), 499f);
checkConsistent(sk);
for (int i = 0; i < 10_000; i++) { sk.update(i); }
checkConsistent(sk);
final ReqSketch copy = ReqSketch.heapify(MemorySegment.ofArray(sk.toByteArray()));
assertEquals(copy.toByteArray(), sk.toByteArray());
}
}

private static void checkConsistent(final ReqSketch sk) {
assertEquals(sk.getNumRetained(), sk.computeTotalRetainedItems());
assertEquals(sk.getMaxNomSize(), sk.computeMaxNomSize());
final QuantilesFloatsSketchIterator it = sk.iterator();
long weight = 0;
while (it.next()) { weight += it.getWeight(); }
assertEquals(weight, sk.getN());
}
}
Loading