Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ private void validate() {
final int capacityItems = KllHelper.computeTotalItemCapacity(k, m, numLevels);
levelsArr[numLevels] = capacityItems; //load the last one
sketchBytes = computeSketchBytes(srcSeg, sketchType, levelsArr, false, serDe);
checkN();
break;
}
case COMPACT_EMPTY: {
Expand Down Expand Up @@ -159,12 +160,22 @@ private void validate() {


sketchBytes = computeSketchBytes(srcSeg, sketchType, levelsArr, true, serDe);
checkN();
break;
}
default: break; //can not happen
}
}

//Each retained item has a weight of at least one, so N can be neither negative nor less than the
//number of retained items.
private void checkN() {
final int retainedItems = levelsArr[numLevels] - levelsArr[0];
if ((n < 0) || (n < retainedItems)) {
throw new SketchesArgumentException(N_LESS_THAN_RETAINED + "n: " + n + ", retained items: " + retainedItems);
}
}

static int computeSketchBytes( //for COMPACT_FULL or UPDATABLE only
final MemorySegment srcSeg,
final SketchType sketchType,
Expand Down Expand Up @@ -195,6 +206,7 @@ static int computeSketchBytes( //for COMPACT_FULL or UPDATABLE only
static final String EMPTY_FLAG_AND_COMPACT_FULL = "A compact full sketch should not have empty flag set. ";
static final String EMPTY_FLAG_AND_COMPACT_SINGLE = "A single item sketch should not have empty flag set. ";
//static final String N_AND_COMPACT_FULL = "A compact full sketch should have n > 1. ";
static final String N_LESS_THAN_RETAINED = "Possible corruption: n must be at least the number of retained items. ";
static final String SRC_NOT_KLL = "FamilyID Field must be: " + Family.KLL.getID() + ", NOT: ";
static final String MEMORY_TOO_SMALL = "A sketch MemorySegment image must be at least 8 bytes. ";

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/org/apache/datasketches/req/ReqCompactor.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static java.lang.Math.round;
import static org.apache.datasketches.common.Util.numberOfTrailingOnes;
import static org.apache.datasketches.req.BaseReqSketch.INIT_NUMBER_OF_SECTIONS;
import static org.apache.datasketches.req.ReqSketch.MAX_NUM_SECTIONS;
import static org.apache.datasketches.req.ReqSketch.MIN_K;
import static org.apache.datasketches.req.ReqSketch.NOM_CAP_MULT;

Expand Down Expand Up @@ -259,6 +260,7 @@ private boolean ensureEnoughSections() {
final float szf;
final int ne;
if ((state >= (1L << (numSections - 1)))
&& ((numSections << 1) <= MAX_NUM_SECTIONS)
&& (sectionSize > MIN_K)
&& ((ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K))
{
Expand Down
23 changes: 22 additions & 1 deletion src/main/java/org/apache/datasketches/req/ReqSerDe.java
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ static ReqSketch heapify(final MemorySegment seg) {
checkLgWeight(compactor, i);
compactors.add(compactor.reqCompactor);
}
checkTotalN(totalN, compactors);
final ReqSketch sk = new ReqSketch(k, hra, totalN, minItem, maxItem, compactors);
sk.setMaxNomSize(sk.computeMaxNomSize());
sk.setRetainedItems(sk.computeTotalRetainedItems());
Expand All @@ -200,6 +201,25 @@ static ReqSketch heapify(final MemorySegment seg) {
}
}

// Each item of the compactor at level h has weight 2^h, and compaction keeps the total weight, so N must
// equal the total weight of the retained items.
private static void checkTotalN(final long totalN, final List<ReqCompactor> compactors) {
long weight = 0;
for (int h = 0; h < compactors.size(); h++) {
final long count = compactors.get(h).getBuffer().getCount();
if ((count > 0) && ((h >= Long.SIZE - 1) || (count > ((Long.MAX_VALUE - weight) >>> h)))) {
weight = -1; //exceeds any valid N
break;
}
weight += count << h;
}
if (weight != totalN) {
throw new SketchesArgumentException("Possible corruption: N is " + totalN
+ ", but the retained items have a total weight of "
+ ((weight < 0) ? "more than Long.MAX_VALUE" : Long.toString(weight)));
}
}

// The sketch creates the compactor at level h with lgWeight h, so any other value is corrupt.
// An unchecked value would not fail. It would silently weight the items by 1L << lgWeight.
private static void checkLgWeight(final Compactor compactor, final int level) {
Expand All @@ -221,7 +241,8 @@ static final Compactor extractCompactor(final PositionalSegment posSeg, final in
final int count = posSeg.getInt();
// The section size starts at k and only shrinks, and as the number of sections doubles the section size
// shrinks by sqrt(2) to no less than MIN_K, so sectionSize * numSections stays below k * k.
if ((sectionSize < 1) || (sectionSize > k) || (numSections < 1)
// The number of sections never exceeds MAX_NUM_SECTIONS.
if ((sectionSize < 1) || (sectionSize > k) || (numSections < 1) || (numSections > ReqSketch.MAX_NUM_SECTIONS)
|| (((long) sectionSize * numSections) > ((long) k * k))) {
throw new SketchesArgumentException("Possible corruption: sectionSize: " + sectionSize
+ ", numSections: " + numSections + ", k: " + k);
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/apache/datasketches/req/ReqSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ static class CompactorReturn {
//static finals
static final byte MIN_K = 4;
static final byte NOM_CAP_MULT = 2;
//The number of sections starts at 3 and doubles: 48 takes 2^23 compactions of one compactor, and doubling
//again would take 2^47. It stops at 48, as in C++, so it always fits the byte that holds it.
static final byte MAX_NUM_SECTIONS = 48;

//finals
private final int k; //default is 12 (1% @ 95% Confidence)
Expand Down
89 changes: 89 additions & 0 deletions src/test/java/org/apache/datasketches/kll/KllCorruptNTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/


package org.apache.datasketches.kll;

import static org.apache.datasketches.kll.KllPreambleUtil.N_LONG_ADR;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.expectThrows;

import java.lang.foreign.MemorySegment;
import java.util.function.Consumer;

import org.apache.datasketches.common.ArrayOfStringsSerDe;
import org.apache.datasketches.common.ByteArrayUtil;
import org.apache.datasketches.common.SketchesArgumentException;
import org.testng.annotations.Test;

/**
* Checks that KLL images whose n is negative, or less than the number of retained items, are rejected.
*/
public class KllCorruptNTest {
private static final int K = 20;

private static byte[] withN(final byte[] image, final long n) {
final byte[] bytes = image.clone();
ByteArrayUtil.putLongLE(bytes, N_LONG_ADR, n);
return bytes;
}

private static void checkNValues(final byte[] image, final int retained,
final Consumer<MemorySegment> heapify) {
heapify.accept(MemorySegment.ofArray(withN(image, retained))); //the least valid n
for (final long n : new long[] {retained - 1L, -1L, Long.MIN_VALUE}) {
final MemorySegment seg = MemorySegment.ofArray(withN(image, n));
expectThrows(SketchesArgumentException.class, () -> heapify.accept(seg));
}
}

@Test
public void doublesRejectsBadN() {
final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K);
for (int i = 0; i < 1000; i++) { sk.update(i); }
final int retained = sk.getNumRetained();
checkNValues(sk.toByteArray(), retained, KllDoublesSketch::heapify);
checkNValues(sk.toByteArray(), retained, KllDoublesSketch::wrap);
checkNValues(KllHelper.toByteArray(sk, true), retained, KllDoublesSketch::heapify);
checkNValues(KllHelper.toByteArray(sk, true), retained, KllDoublesSketch::wrap);
assertEquals(KllDoublesSketch.heapify(MemorySegment.ofArray(sk.toByteArray())).getN(), 1000);
}

@Test
public void floatsAndLongsRejectBadN() {
final KllFloatsSketch fsk = KllFloatsSketch.newHeapInstance(K);
final KllLongsSketch lsk = KllLongsSketch.newHeapInstance(K);
for (int i = 0; i < 1000; i++) { fsk.update(i); lsk.update(i); }
checkNValues(fsk.toByteArray(), fsk.getNumRetained(), KllFloatsSketch::heapify);
checkNValues(KllHelper.toByteArray(fsk, true), fsk.getNumRetained(), KllFloatsSketch::wrap);
checkNValues(lsk.toByteArray(), lsk.getNumRetained(), KllLongsSketch::heapify);
checkNValues(KllHelper.toByteArray(lsk, true), lsk.getNumRetained(), KllLongsSketch::wrap);
}

@Test
public void itemsRejectsBadN() {
final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
final KllItemsSketch<String> sk = KllItemsSketch.newHeapInstance(K, String::compareTo, serDe);
for (int i = 0; i < 1000; i++) { sk.update(Integer.toString(i)); }
checkNValues(sk.toByteArray(), sk.getNumRetained(),
seg -> KllItemsSketch.heapify(seg, String::compareTo, serDe));
checkNValues(sk.toByteArray(), sk.getNumRetained(),
seg -> KllItemsSketch.wrap(seg, String::compareTo, serDe));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ public class ReqCorruptImageTest {
private static final int EST_SECTION_SIZE_FLOAT = 32; //first compactor, estimation mode
private static final int EST_NUM_SECTIONS_BYTE = 37;
private static final int EST_COUNT_INT = 40;
private static final int EXACT_COUNT_INT = 24; //the only compactor, exact mode
private static final int EXACT_COUNT_INT = 24;
private static final int EST_N_LONG = 8; //estimation mode //the only compactor, exact mode

private static byte[] image(final int n) {
final ReqSketch sk = ReqSketch.builder().setK(12).build();
Expand Down Expand Up @@ -87,6 +88,41 @@ public void checkSectionsTooLarge() {
checkRejected(negative);
}

@Test
public void checkTooManySections() {
final byte[] bytes = image(10_000);
bytes[EST_NUM_SECTIONS_BYTE] = 49; //49 * 2 is within k * k, but above the maximum of 48
ByteArrayUtil.putFloatLE(bytes, EST_SECTION_SIZE_FLOAT, 2f);
checkRejected(bytes);
bytes[EST_NUM_SECTIONS_BYTE] = 48; //the maximum is accepted
assertEquals(ReqSketch.heapify(MemorySegment.ofArray(bytes)).getN(), 10_000);
}

// A compactor whose state allows another doubling stops at the maximum number of sections,
// so it still fits the byte that holds it and the image can be read back.
@Test
public void checkSectionDoublingStopsAtMax() {
for (final boolean hra : new boolean[] {true, false}) {
final FloatBuffer buf = FloatBuffer.reconstruct(new float[0], 0, 0, 2 * 12 * 48, true, hra);
final ReqCompactor c = new ReqCompactor((byte) 0, hra, 1L << 50, 64f, ReqSketch.MAX_NUM_SECTIONS, buf);
c.merge(new ReqCompactor((byte) 0, hra, 64, null));
assertEquals(c.getNumSections(), ReqSketch.MAX_NUM_SECTIONS);
assertEquals(c.getSectionSize(), 64);
}
}

@Test
public void checkNNotEqualToWeight() {
final byte[] bytes = image(10_000);
assertEquals(ByteArrayUtil.getLongLE(bytes, EST_N_LONG), 10_000L);
ByteArrayUtil.putLongLE(bytes, EST_N_LONG, 10_001L);
checkRejected(bytes);
ByteArrayUtil.putLongLE(bytes, EST_N_LONG, -10_000L);
checkRejected(bytes);
ByteArrayUtil.putLongLE(bytes, EST_N_LONG, Long.MAX_VALUE);
checkRejected(bytes);
}

@Test
public void checkInvalidK() {
final byte[] bytes = image(10_000);
Expand Down
Loading