From c699629b8db281ab859c6fcdfbb99207f232567c Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 10:54:14 -0800 Subject: [PATCH] REQ and KLL: reject images whose N is inconsistent; cap REQ sections at 48 REQ - The number of sections of a compactor stops doubling at 48 (MAX_NUM_SECTIONS), as in C++. Doubling past it needs a state of 2^47, reachable only from a crafted image, and overflowed the byte that holds the number of sections, so the sketch could not be read back. - heapify rejects a compactor with more than 48 sections. - heapify rejects an estimation-mode image whose N is not the total weight of its retained items. KLL - Validation of full compact and updatable images rejects an n that is negative or less than the number of retained items. Co-Authored-By: Claude Opus 5.5 --- .../kll/KllMemorySegmentValidate.java | 12 +++ .../apache/datasketches/req/ReqCompactor.java | 2 + .../org/apache/datasketches/req/ReqSerDe.java | 23 ++++- .../apache/datasketches/req/ReqSketch.java | 3 + .../datasketches/kll/KllCorruptNTest.java | 89 +++++++++++++++++++ .../datasketches/req/ReqCorruptImageTest.java | 38 +++++++- 6 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/apache/datasketches/kll/KllCorruptNTest.java diff --git a/src/main/java/org/apache/datasketches/kll/KllMemorySegmentValidate.java b/src/main/java/org/apache/datasketches/kll/KllMemorySegmentValidate.java index 33f6904ec..e558b6251 100644 --- a/src/main/java/org/apache/datasketches/kll/KllMemorySegmentValidate.java +++ b/src/main/java/org/apache/datasketches/kll/KllMemorySegmentValidate.java @@ -126,6 +126,7 @@ private void validate() { final int capacityItems = KllHelper.computeTotalItemCapacity(k, m, numLevels); levelsArr[numLevels] = capacityItems; //load the last one sketchBytes = computeSketchBytes(srcSeg, sketchType, levelsArr, false, serDe); + checkN(); break; } case COMPACT_EMPTY: { @@ -159,12 +160,22 @@ private void validate() { sketchBytes = computeSketchBytes(srcSeg, sketchType, levelsArr, true, serDe); + checkN(); break; } default: break; //can not happen } } + //Each retained item has a weight of at least one, so N can be neither negative nor less than the + //number of retained items. + private void checkN() { + final int retainedItems = levelsArr[numLevels] - levelsArr[0]; + if ((n < 0) || (n < retainedItems)) { + throw new SketchesArgumentException(N_LESS_THAN_RETAINED + "n: " + n + ", retained items: " + retainedItems); + } + } + static int computeSketchBytes( //for COMPACT_FULL or UPDATABLE only final MemorySegment srcSeg, final SketchType sketchType, @@ -195,6 +206,7 @@ static int computeSketchBytes( //for COMPACT_FULL or UPDATABLE only static final String EMPTY_FLAG_AND_COMPACT_FULL = "A compact full sketch should not have empty flag set. "; static final String EMPTY_FLAG_AND_COMPACT_SINGLE = "A single item sketch should not have empty flag set. "; //static final String N_AND_COMPACT_FULL = "A compact full sketch should have n > 1. "; + static final String N_LESS_THAN_RETAINED = "Possible corruption: n must be at least the number of retained items. "; static final String SRC_NOT_KLL = "FamilyID Field must be: " + Family.KLL.getID() + ", NOT: "; static final String MEMORY_TOO_SMALL = "A sketch MemorySegment image must be at least 8 bytes. "; diff --git a/src/main/java/org/apache/datasketches/req/ReqCompactor.java b/src/main/java/org/apache/datasketches/req/ReqCompactor.java index 71aa4dfdc..d3b59f5fa 100644 --- a/src/main/java/org/apache/datasketches/req/ReqCompactor.java +++ b/src/main/java/org/apache/datasketches/req/ReqCompactor.java @@ -22,6 +22,7 @@ import static java.lang.Math.round; import static org.apache.datasketches.common.Util.numberOfTrailingOnes; import static org.apache.datasketches.req.BaseReqSketch.INIT_NUMBER_OF_SECTIONS; +import static org.apache.datasketches.req.ReqSketch.MAX_NUM_SECTIONS; import static org.apache.datasketches.req.ReqSketch.MIN_K; import static org.apache.datasketches.req.ReqSketch.NOM_CAP_MULT; @@ -259,6 +260,7 @@ private boolean ensureEnoughSections() { final float szf; final int ne; if ((state >= (1L << (numSections - 1))) + && ((numSections << 1) <= MAX_NUM_SECTIONS) && (sectionSize > MIN_K) && ((ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K)) { diff --git a/src/main/java/org/apache/datasketches/req/ReqSerDe.java b/src/main/java/org/apache/datasketches/req/ReqSerDe.java index 89a65e345..cf6e0e70d 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSerDe.java +++ b/src/main/java/org/apache/datasketches/req/ReqSerDe.java @@ -192,6 +192,7 @@ static ReqSketch heapify(final MemorySegment seg) { checkLgWeight(compactor, i); compactors.add(compactor.reqCompactor); } + checkTotalN(totalN, compactors); final ReqSketch sk = new ReqSketch(k, hra, totalN, minItem, maxItem, compactors); sk.setMaxNomSize(sk.computeMaxNomSize()); sk.setRetainedItems(sk.computeTotalRetainedItems()); @@ -200,6 +201,25 @@ static ReqSketch heapify(final MemorySegment seg) { } } + // Each item of the compactor at level h has weight 2^h, and compaction keeps the total weight, so N must + // equal the total weight of the retained items. + private static void checkTotalN(final long totalN, final List compactors) { + long weight = 0; + for (int h = 0; h < compactors.size(); h++) { + final long count = compactors.get(h).getBuffer().getCount(); + if ((count > 0) && ((h >= Long.SIZE - 1) || (count > ((Long.MAX_VALUE - weight) >>> h)))) { + weight = -1; //exceeds any valid N + break; + } + weight += count << h; + } + if (weight != totalN) { + throw new SketchesArgumentException("Possible corruption: N is " + totalN + + ", but the retained items have a total weight of " + + ((weight < 0) ? "more than Long.MAX_VALUE" : Long.toString(weight))); + } + } + // The sketch creates the compactor at level h with lgWeight h, so any other value is corrupt. // An unchecked value would not fail. It would silently weight the items by 1L << lgWeight. private static void checkLgWeight(final Compactor compactor, final int level) { @@ -221,7 +241,8 @@ static final Compactor extractCompactor(final PositionalSegment posSeg, final in final int count = posSeg.getInt(); // The section size starts at k and only shrinks, and as the number of sections doubles the section size // shrinks by sqrt(2) to no less than MIN_K, so sectionSize * numSections stays below k * k. - if ((sectionSize < 1) || (sectionSize > k) || (numSections < 1) + // The number of sections never exceeds MAX_NUM_SECTIONS. + if ((sectionSize < 1) || (sectionSize > k) || (numSections < 1) || (numSections > ReqSketch.MAX_NUM_SECTIONS) || (((long) sectionSize * numSections) > ((long) k * k))) { throw new SketchesArgumentException("Possible corruption: sectionSize: " + sectionSize + ", numSections: " + numSections + ", k: " + k); diff --git a/src/main/java/org/apache/datasketches/req/ReqSketch.java b/src/main/java/org/apache/datasketches/req/ReqSketch.java index 0bf58d7f0..30fb7eaa2 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/ReqSketch.java @@ -96,6 +96,9 @@ static class CompactorReturn { //static finals static final byte MIN_K = 4; static final byte NOM_CAP_MULT = 2; + //The number of sections starts at 3 and doubles: 48 takes 2^23 compactions of one compactor, and doubling + //again would take 2^47. It stops at 48, as in C++, so it always fits the byte that holds it. + static final byte MAX_NUM_SECTIONS = 48; //finals private final int k; //default is 12 (1% @ 95% Confidence) diff --git a/src/test/java/org/apache/datasketches/kll/KllCorruptNTest.java b/src/test/java/org/apache/datasketches/kll/KllCorruptNTest.java new file mode 100644 index 000000000..8b71408ad --- /dev/null +++ b/src/test/java/org/apache/datasketches/kll/KllCorruptNTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.kll; + +import static org.apache.datasketches.kll.KllPreambleUtil.N_LONG_ADR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.expectThrows; + +import java.lang.foreign.MemorySegment; +import java.util.function.Consumer; + +import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.ByteArrayUtil; +import org.apache.datasketches.common.SketchesArgumentException; +import org.testng.annotations.Test; + +/** + * Checks that KLL images whose n is negative, or less than the number of retained items, are rejected. + */ +public class KllCorruptNTest { + private static final int K = 20; + + private static byte[] withN(final byte[] image, final long n) { + final byte[] bytes = image.clone(); + ByteArrayUtil.putLongLE(bytes, N_LONG_ADR, n); + return bytes; + } + + private static void checkNValues(final byte[] image, final int retained, + final Consumer heapify) { + heapify.accept(MemorySegment.ofArray(withN(image, retained))); //the least valid n + for (final long n : new long[] {retained - 1L, -1L, Long.MIN_VALUE}) { + final MemorySegment seg = MemorySegment.ofArray(withN(image, n)); + expectThrows(SketchesArgumentException.class, () -> heapify.accept(seg)); + } + } + + @Test + public void doublesRejectsBadN() { + final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(K); + for (int i = 0; i < 1000; i++) { sk.update(i); } + final int retained = sk.getNumRetained(); + checkNValues(sk.toByteArray(), retained, KllDoublesSketch::heapify); + checkNValues(sk.toByteArray(), retained, KllDoublesSketch::wrap); + checkNValues(KllHelper.toByteArray(sk, true), retained, KllDoublesSketch::heapify); + checkNValues(KllHelper.toByteArray(sk, true), retained, KllDoublesSketch::wrap); + assertEquals(KllDoublesSketch.heapify(MemorySegment.ofArray(sk.toByteArray())).getN(), 1000); + } + + @Test + public void floatsAndLongsRejectBadN() { + final KllFloatsSketch fsk = KllFloatsSketch.newHeapInstance(K); + final KllLongsSketch lsk = KllLongsSketch.newHeapInstance(K); + for (int i = 0; i < 1000; i++) { fsk.update(i); lsk.update(i); } + checkNValues(fsk.toByteArray(), fsk.getNumRetained(), KllFloatsSketch::heapify); + checkNValues(KllHelper.toByteArray(fsk, true), fsk.getNumRetained(), KllFloatsSketch::wrap); + checkNValues(lsk.toByteArray(), lsk.getNumRetained(), KllLongsSketch::heapify); + checkNValues(KllHelper.toByteArray(lsk, true), lsk.getNumRetained(), KllLongsSketch::wrap); + } + + @Test + public void itemsRejectsBadN() { + final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe(); + final KllItemsSketch sk = KllItemsSketch.newHeapInstance(K, String::compareTo, serDe); + for (int i = 0; i < 1000; i++) { sk.update(Integer.toString(i)); } + checkNValues(sk.toByteArray(), sk.getNumRetained(), + seg -> KllItemsSketch.heapify(seg, String::compareTo, serDe)); + checkNValues(sk.toByteArray(), sk.getNumRetained(), + seg -> KllItemsSketch.wrap(seg, String::compareTo, serDe)); + } +} diff --git a/src/test/java/org/apache/datasketches/req/ReqCorruptImageTest.java b/src/test/java/org/apache/datasketches/req/ReqCorruptImageTest.java index c8f7ca939..bf1a19839 100644 --- a/src/test/java/org/apache/datasketches/req/ReqCorruptImageTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqCorruptImageTest.java @@ -37,7 +37,8 @@ public class ReqCorruptImageTest { private static final int EST_SECTION_SIZE_FLOAT = 32; //first compactor, estimation mode private static final int EST_NUM_SECTIONS_BYTE = 37; private static final int EST_COUNT_INT = 40; - private static final int EXACT_COUNT_INT = 24; //the only compactor, exact mode + private static final int EXACT_COUNT_INT = 24; + private static final int EST_N_LONG = 8; //estimation mode //the only compactor, exact mode private static byte[] image(final int n) { final ReqSketch sk = ReqSketch.builder().setK(12).build(); @@ -87,6 +88,41 @@ public void checkSectionsTooLarge() { checkRejected(negative); } + @Test + public void checkTooManySections() { + final byte[] bytes = image(10_000); + bytes[EST_NUM_SECTIONS_BYTE] = 49; //49 * 2 is within k * k, but above the maximum of 48 + ByteArrayUtil.putFloatLE(bytes, EST_SECTION_SIZE_FLOAT, 2f); + checkRejected(bytes); + bytes[EST_NUM_SECTIONS_BYTE] = 48; //the maximum is accepted + assertEquals(ReqSketch.heapify(MemorySegment.ofArray(bytes)).getN(), 10_000); + } + + // A compactor whose state allows another doubling stops at the maximum number of sections, + // so it still fits the byte that holds it and the image can be read back. + @Test + public void checkSectionDoublingStopsAtMax() { + for (final boolean hra : new boolean[] {true, false}) { + final FloatBuffer buf = FloatBuffer.reconstruct(new float[0], 0, 0, 2 * 12 * 48, true, hra); + final ReqCompactor c = new ReqCompactor((byte) 0, hra, 1L << 50, 64f, ReqSketch.MAX_NUM_SECTIONS, buf); + c.merge(new ReqCompactor((byte) 0, hra, 64, null)); + assertEquals(c.getNumSections(), ReqSketch.MAX_NUM_SECTIONS); + assertEquals(c.getSectionSize(), 64); + } + } + + @Test + public void checkNNotEqualToWeight() { + final byte[] bytes = image(10_000); + assertEquals(ByteArrayUtil.getLongLE(bytes, EST_N_LONG), 10_000L); + ByteArrayUtil.putLongLE(bytes, EST_N_LONG, 10_001L); + checkRejected(bytes); + ByteArrayUtil.putLongLE(bytes, EST_N_LONG, -10_000L); + checkRejected(bytes); + ByteArrayUtil.putLongLE(bytes, EST_N_LONG, Long.MAX_VALUE); + checkRejected(bytes); + } + @Test public void checkInvalidK() { final byte[] bytes = image(10_000);