From cdfd5450d441194b765bd7f43272f6c6b7fe8063 Mon Sep 17 00:00:00 2001 From: Chris Boumalhab Date: Tue, 10 Nov 2026 23:22:45 +0000 Subject: [PATCH 1/4] fix --- datasketches/PySerDe.py | 6 +++--- datasketches/TuplePolicy.py | 16 ++++++++++++++ include/py_serde.hpp | 13 +++++------ src/py_serde.cpp | 15 ++++++++----- tests/tuple_test.py | 43 +++++++++++++++++++++++++++++++++++-- 5 files changed, 77 insertions(+), 16 deletions(-) diff --git a/datasketches/PySerDe.py b/datasketches/PySerDe.py index fcaddfeb..16a353c5 100644 --- a/datasketches/PySerDe.py +++ b/datasketches/PySerDe.py @@ -47,11 +47,11 @@ def to_bytes(self, item: str): b.extend(map(ord,item)) return bytes(b) - def from_bytes(self, data: bytes, offset: int): - num_chars = int.from_bytes(data[offset:offset+3], 'little') + def from_bytes(self, data, offset: int): + num_chars = struct.unpack_from(' offset + len(data)): raise IndexError(f'num_chars read must be non-negative and not larger than the buffer. Found {num_chars}') - str = data[offset+4:offset+4+num_chars].decode() + str = bytes(data[offset+4:offset+4+num_chars]).decode() return (str, 4+num_chars) class PyIntsSerDe(PyObjectSerDe): diff --git a/datasketches/TuplePolicy.py b/datasketches/TuplePolicy.py index 760ebb7f..77eadae2 100644 --- a/datasketches/TuplePolicy.py +++ b/datasketches/TuplePolicy.py @@ -78,3 +78,19 @@ def update_summary(self, summary: int, update: int) -> int: def __call__(self, summary: int, update: int) -> int: return min(summary, update) + + +class KeepLastStringPolicy(TuplePolicy): + '''String-summary policy that keeps the most recent update.''' + + def __init__(self): + TuplePolicy.__init__(self) + + def create_summary(self) -> str: + return '' + + def update_summary(self, summary: str, update: str) -> str: + return update + + def __call__(self, summary: str, update: str) -> str: + return update diff --git a/include/py_serde.hpp b/include/py_serde.hpp index 1ff56988..3618945f 100644 --- a/include/py_serde.hpp +++ b/include/py_serde.hpp @@ -54,12 +54,13 @@ struct py_object_serde { /** * @brief Constructs an object from a serialized image, reading the * incoming buffer starting at the specified offset. - * - * @param bytes A buffer containing items from a serialized sketch - * @param offset The starting offset into the bytes buffer + * + * @param data A buffer-protocol object (memoryview or bytes) containing items + * from a serialized sketch + * @param offset The starting offset into the data buffer * @return A Python tuple of the reconstructed item and the total number of bytes read */ - virtual nb::tuple from_bytes(nb::bytes& bytes, size_t offset) const = 0; + virtual nb::tuple from_bytes(nb::object& data, size_t offset) const = 0; virtual ~py_object_serde() = default; @@ -94,10 +95,10 @@ struct PyObjectSerDe : public py_object_serde { ); } - nb::tuple from_bytes(nb::bytes& bytes, size_t offset) const override { + nb::tuple from_bytes(nb::object& data, size_t offset) const override { NB_OVERRIDE_PURE( from_bytes, // Name of function in C++ (must match Python name) - bytes, offset // Argument(s) + data, offset // Argument(s) ); } }; diff --git a/src/py_serde.cpp b/src/py_serde.cpp index a1896a63..e49ecfe6 100644 --- a/src/py_serde.cpp +++ b/src/py_serde.cpp @@ -42,9 +42,10 @@ void init_serde(nb::module_& m) { ":return: A :class:`bytes` object with the serialized object\n:rtype: bytes" ) .def("from_bytes", &py_object_serde::from_bytes, nb::arg("data"), nb::arg("offset"), - "Reads a bytes object starting from the given offest and returns a tuple of the reconstructed " + "Reads a buffer starting from the given offset and returns a tuple of the reconstructed " "object and the number of additional bytes read\n\n" - ":param data: A :class:`bytes` object from which to deserialize\n:type data: bytes\n" + ":param data: A buffer-protocol object (``memoryview`` or ``bytes``) from which to deserialize\n" + ":type data: memoryview or bytes\n" ":param offset: The offset, in bytes, at which to start reading\n:type offset: int\n" ":return: A :class:`tuple` with the reconstructed object and the number of bytes read\n" ":rtype: tuple(object, int)" @@ -78,12 +79,16 @@ namespace datasketches { bool error_from_python = false; nb::gil_scoped_acquire acquire; - // copy data into bytes only once - nb::bytes bytes(static_cast(ptr), capacity); + // zero-copy view over the incoming buffer; avoids a per-call buffer copy + // (this function is invoked per entry by the sketch deserialization loop) + nb::object view = nb::steal(PyMemoryView_FromMemory( + const_cast(static_cast(ptr)), + static_cast(capacity), + PyBUF_READ)); for (; i < num && !failure; ++i) { nb::tuple bytes_and_len; try { - bytes_and_len = from_bytes(bytes, bytes_read); + bytes_and_len = from_bytes(view, bytes_read); } catch (nb::python_error &e) { failure = true; error_from_python = true; diff --git a/tests/tuple_test.py b/tests/tuple_test.py index cf505fb1..1d00a598 100644 --- a/tests/tuple_test.py +++ b/tests/tuple_test.py @@ -21,8 +21,8 @@ from datasketches import compact_tuple_sketch, tuple_union from datasketches import tuple_intersection, tuple_a_not_b from datasketches import tuple_jaccard_similarity -from datasketches import tuple_jaccard_similarity, PyIntsSerDe -from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy +from datasketches import tuple_jaccard_similarity, PyIntsSerDe, PyStringsSerDe +from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy, KeepLastStringPolicy from datasketches import update_theta_sketch class TupleTest(unittest.TestCase): @@ -231,5 +231,44 @@ def generate_tuple_sketch(self, policy, n, lgk, value, offset=0): sk.update(i + offset, value) return sk + def test_tuple_strings_serde_round_trip(self): + # Short-string summary: exercises PyStringsSerDe over the memoryview path + # the C++ deserialize loop hands it one summary at a time. + lgk = 10 + n = 1 << 12 + value = 'alpha' + sk = update_tuple_sketch(KeepLastStringPolicy(), lgk) + for i in range(n): + sk.update(i, value) + compact = sk.compact() + + sk_bytes = compact.serialize(PyStringsSerDe()) + new_sk = compact_tuple_sketch.deserialize(sk_bytes, serde=PyStringsSerDe()) + + self.assertEqual(compact.num_retained, new_sk.num_retained) + self.assertEqual(compact.get_estimate(), new_sk.get_estimate()) + for pair in new_sk: + self.assertEqual(pair[1], value) + + def test_tuple_strings_serde_long_string(self): + # Long-string summary (len >= 256): the length prefix needs all 4 bytes, + # so this guards against a prior bug that read only 3 bytes of the length. + lgk = 8 + n = 1 << 10 + value = 'x' * 300 + sk = update_tuple_sketch(KeepLastStringPolicy(), lgk) + for i in range(n): + sk.update(i, value) + compact = sk.compact() + + sk_bytes = compact.serialize(PyStringsSerDe()) + new_sk = compact_tuple_sketch.deserialize(sk_bytes, serde=PyStringsSerDe()) + + self.assertEqual(compact.num_retained, new_sk.num_retained) + for pair in new_sk: + self.assertEqual(pair[1], value) + self.assertEqual(len(pair[1]), 300) + + if __name__ == '__main__': unittest.main() From 3d137586c7da0bf5c6be62582dbaf293279a8d33 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 19 Nov 2026 15:04:58 +0800 Subject: [PATCH 2/4] fix: keep deserialized summary views alive A Python SerDe can retain a slice of its input as the decoded value. A view created from a raw pointer does not retain the serialized sketch and can outlive its storage. Carry the original bytes through each sketch reader and build the view from that object. Translate the native cursor to an offset in the image so retained slices stay valid without copying the remaining input for every item. --- include/py_serde.hpp | 7 +++++++ include/quantile_conditional.hpp | 2 +- src/ebpps_wrapper.cpp | 4 +++- src/fi_wrapper.cpp | 2 +- src/py_serde.cpp | 14 +++++--------- src/tuple_wrapper.cpp | 2 +- src/vo_wrapper.cpp | 8 ++++++-- tests/tuple_test.py | 24 +++++++++++++++++++++++- 8 files changed, 47 insertions(+), 16 deletions(-) diff --git a/include/py_serde.hpp b/include/py_serde.hpp index 3618945f..dbb5e6d5 100644 --- a/include/py_serde.hpp +++ b/include/py_serde.hpp @@ -68,6 +68,13 @@ struct py_object_serde { // default implementations for C++ std::string and numeric types. size_t size_of_item(const nb::object& item) const; size_t serialize(void* ptr, size_t capacity, const nb::object* items, unsigned num) const; +}; + +// The sketch passes pointers into one serialized image, including for nested sketches. +struct py_object_deserializer { + const py_object_serde& serde; + const nb::bytes& bytes; + size_t deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const; }; diff --git a/include/quantile_conditional.hpp b/include/quantile_conditional.hpp index 2e3c55bf..db1b87e3 100644 --- a/include/quantile_conditional.hpp +++ b/include/quantile_conditional.hpp @@ -69,7 +69,7 @@ void add_serialization(nb::class_& clazz) { .def_static( "deserialize", [](const nb::bytes& bytes, datasketches::py_object_serde& serde) { - return SK::deserialize(bytes.c_str(), bytes.size(), serde); + return SK::deserialize(bytes.c_str(), bytes.size(), datasketches::py_object_deserializer{serde, bytes}); }, nb::arg("bytes"), nb::arg("serde"), "Deserializes the sketch from a bytes object using the provided serde." ); diff --git a/src/ebpps_wrapper.cpp b/src/ebpps_wrapper.cpp index d2f24f27..81085900 100644 --- a/src/ebpps_wrapper.cpp +++ b/src/ebpps_wrapper.cpp @@ -77,7 +77,9 @@ void bind_ebpps_sketch(nb::module_ &m, const char* name) { "Serializes the sketch into a bytes object") .def_static( "deserialize", - [](const nb::bytes& bytes, py_object_serde& serde) { return ebpps_sketch::deserialize(bytes.c_str(), bytes.size(), serde); }, + [](const nb::bytes& bytes, py_object_serde& serde) { + return ebpps_sketch::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); + }, nb::arg("bytes"), nb::arg("serde"), "Reads a bytes object and returns the corresponding ebpps_sketch") .def("__iter__", diff --git a/src/fi_wrapper.cpp b/src/fi_wrapper.cpp index 0fffcc16..dc24fb9b 100644 --- a/src/fi_wrapper.cpp +++ b/src/fi_wrapper.cpp @@ -160,7 +160,7 @@ void add_serialization(nb::class_::deserialize(bytes.c_str(), bytes.size(), serde); + return frequent_items_sketch::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); }, nb::arg("bytes"), nb::arg("serde"), "Reads a bytes object using the provided serde and returns the corresponding frequent_strings_sketch." ); diff --git a/src/py_serde.cpp b/src/py_serde.cpp index e49ecfe6..23ebc06a 100644 --- a/src/py_serde.cpp +++ b/src/py_serde.cpp @@ -72,23 +72,20 @@ namespace datasketches { return bytes_written; } - size_t py_object_serde::deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const { + size_t py_object_deserializer::deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const { size_t bytes_read = 0; unsigned i = 0; bool failure = false; bool error_from_python = false; nb::gil_scoped_acquire acquire; - // zero-copy view over the incoming buffer; avoids a per-call buffer copy - // (this function is invoked per entry by the sketch deserialization loop) - nb::object view = nb::steal(PyMemoryView_FromMemory( - const_cast(static_cast(ptr)), - static_cast(capacity), - PyBUF_READ)); + const size_t offset = static_cast(ptr) - bytes.c_str(); + nb::object view = nb::steal(PyMemoryView_FromObject(bytes.ptr())); + if (!view.is_valid()) throw nb::python_error(); for (; i < num && !failure; ++i) { nb::tuple bytes_and_len; try { - bytes_and_len = from_bytes(view, bytes_read); + bytes_and_len = serde.from_bytes(view, offset + bytes_read); } catch (nb::python_error &e) { failure = true; error_from_python = true; @@ -103,7 +100,6 @@ namespace datasketches { } new (&items[i]) nb::object(nb::cast(bytes_and_len[0])); - ptr = static_cast(ptr) + length; bytes_read += length; } diff --git a/src/tuple_wrapper.cpp b/src/tuple_wrapper.cpp index 6563e5bd..265561fd 100644 --- a/src/tuple_wrapper.cpp +++ b/src/tuple_wrapper.cpp @@ -145,7 +145,7 @@ void init_tuple(nb::module_ &m) { .def_static( "deserialize", [](const nb::bytes& bytes, py_object_serde& serde, uint64_t seed) { - return py_compact_tuple::deserialize(bytes.c_str(), bytes.size(), seed, serde); + return py_compact_tuple::deserialize(bytes.c_str(), bytes.size(), seed, py_object_deserializer{serde, bytes}); }, nb::arg("bytes"), nb::arg("serde"), nb::arg("seed")=DEFAULT_SEED, "Reads a bytes object and returns the corresponding compact_tuple_sketch" diff --git a/src/vo_wrapper.cpp b/src/vo_wrapper.cpp index ef0eb671..c129b2c5 100644 --- a/src/vo_wrapper.cpp +++ b/src/vo_wrapper.cpp @@ -85,7 +85,9 @@ void bind_vo_sketch(nb::module_ &m, const char* name) { "Serializes the sketch into a bytes object") .def_static( "deserialize", - [](const nb::bytes& bytes, py_object_serde& serde) { return var_opt_sketch::deserialize(bytes.c_str(), bytes.size(), serde); }, + [](const nb::bytes& bytes, py_object_serde& serde) { + return var_opt_sketch::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); + }, nb::arg("bytes"), nb::arg("serde"), "Reads a bytes object and returns the corresponding var opt sketch") .def("__iter__", @@ -127,7 +129,9 @@ void bind_vo_union(nb::module_ &m, const char* name) { "Serializes the union into a bytes object with the provided serde") .def_static( "deserialize", - [](const nb::bytes& bytes, py_object_serde& serde) { return var_opt_union::deserialize(bytes.c_str(), bytes.size(), serde); }, + [](const nb::bytes& bytes, py_object_serde& serde) { + return var_opt_union::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); + }, nb::arg("bytes"), nb::arg("serde"), "Constructs a var opt union from the given bytes using the provided serde") ; diff --git a/tests/tuple_test.py b/tests/tuple_test.py index 1d00a598..693304ae 100644 --- a/tests/tuple_test.py +++ b/tests/tuple_test.py @@ -21,7 +21,7 @@ from datasketches import compact_tuple_sketch, tuple_union from datasketches import tuple_intersection, tuple_a_not_b from datasketches import tuple_jaccard_similarity -from datasketches import tuple_jaccard_similarity, PyIntsSerDe, PyStringsSerDe +from datasketches import PyObjectSerDe, PyIntsSerDe, PyStringsSerDe from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy, KeepLastStringPolicy from datasketches import update_theta_sketch @@ -231,6 +231,28 @@ def generate_tuple_sketch(self, policy, n, lgk, value, offset=0): sk.update(i + offset, value) return sk + def test_retained_summary_views(self): + class DelimitedBytesSerDe(PyObjectSerDe): + def get_size(self, item): + return len(item) + 1 + + def to_bytes(self, item): + return item + b'\0' + + def from_bytes(self, data, offset): + end = offset + while data[end] != 0: + end += 1 + return memoryview(data)[offset:end], end + 1 - offset + + theta = update_theta_sketch() + theta.update(1) + theta.update(2) + sketch = compact_tuple_sketch(theta, b'alpha') + serde = DelimitedBytesSerDe() + restored = compact_tuple_sketch.deserialize(sketch.serialize(serde), serde) + self.assertEqual(dict(restored), dict(sketch)) + def test_tuple_strings_serde_round_trip(self): # Short-string summary: exercises PyStringsSerDe over the memoryview path # the C++ deserialize loop hands it one summary at a time. From ef71bf07a18f96a8794ca6461c0b3f18ac2310db Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 19 Nov 2026 15:05:52 +0800 Subject: [PATCH 3/4] fix: preserve bytes callbacks for custom SerDes Existing from_bytes implementations use bytes methods such as decode and index. Pass the original bytes object and the offset of each item directly to the callback so these readers work without copying the remaining input. --- datasketches/PySerDe.py | 4 ++-- include/py_serde.hpp | 7 +++---- src/py_serde.cpp | 10 ++++------ tests/tuple_test.py | 6 ++---- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/datasketches/PySerDe.py b/datasketches/PySerDe.py index 16a353c5..6146ac01 100644 --- a/datasketches/PySerDe.py +++ b/datasketches/PySerDe.py @@ -47,11 +47,11 @@ def to_bytes(self, item: str): b.extend(map(ord,item)) return bytes(b) - def from_bytes(self, data, offset: int): + def from_bytes(self, data: bytes, offset: int): num_chars = struct.unpack_from(' offset + len(data)): raise IndexError(f'num_chars read must be non-negative and not larger than the buffer. Found {num_chars}') - str = bytes(data[offset+4:offset+4+num_chars]).decode() + str = data[offset+4:offset+4+num_chars].decode() return (str, 4+num_chars) class PyIntsSerDe(PyObjectSerDe): diff --git a/include/py_serde.hpp b/include/py_serde.hpp index dbb5e6d5..7a463463 100644 --- a/include/py_serde.hpp +++ b/include/py_serde.hpp @@ -55,12 +55,11 @@ struct py_object_serde { * @brief Constructs an object from a serialized image, reading the * incoming buffer starting at the specified offset. * - * @param data A buffer-protocol object (memoryview or bytes) containing items - * from a serialized sketch + * @param data The bytes containing the serialized sketch * @param offset The starting offset into the data buffer * @return A Python tuple of the reconstructed item and the total number of bytes read */ - virtual nb::tuple from_bytes(nb::object& data, size_t offset) const = 0; + virtual nb::tuple from_bytes(const nb::bytes& data, size_t offset) const = 0; virtual ~py_object_serde() = default; @@ -102,7 +101,7 @@ struct PyObjectSerDe : public py_object_serde { ); } - nb::tuple from_bytes(nb::object& data, size_t offset) const override { + nb::tuple from_bytes(const nb::bytes& data, size_t offset) const override { NB_OVERRIDE_PURE( from_bytes, // Name of function in C++ (must match Python name) data, offset // Argument(s) diff --git a/src/py_serde.cpp b/src/py_serde.cpp index 23ebc06a..3549fa81 100644 --- a/src/py_serde.cpp +++ b/src/py_serde.cpp @@ -42,10 +42,10 @@ void init_serde(nb::module_& m) { ":return: A :class:`bytes` object with the serialized object\n:rtype: bytes" ) .def("from_bytes", &py_object_serde::from_bytes, nb::arg("data"), nb::arg("offset"), - "Reads a buffer starting from the given offset and returns a tuple of the reconstructed " + "Reads a bytes object starting from the given offset and returns a tuple of the reconstructed " "object and the number of additional bytes read\n\n" - ":param data: A buffer-protocol object (``memoryview`` or ``bytes``) from which to deserialize\n" - ":type data: memoryview or bytes\n" + ":param data: A :class:`bytes` object from which to deserialize\n" + ":type data: bytes\n" ":param offset: The offset, in bytes, at which to start reading\n:type offset: int\n" ":return: A :class:`tuple` with the reconstructed object and the number of bytes read\n" ":rtype: tuple(object, int)" @@ -80,12 +80,10 @@ namespace datasketches { nb::gil_scoped_acquire acquire; const size_t offset = static_cast(ptr) - bytes.c_str(); - nb::object view = nb::steal(PyMemoryView_FromObject(bytes.ptr())); - if (!view.is_valid()) throw nb::python_error(); for (; i < num && !failure; ++i) { nb::tuple bytes_and_len; try { - bytes_and_len = serde.from_bytes(view, offset + bytes_read); + bytes_and_len = serde.from_bytes(bytes, offset + bytes_read); } catch (nb::python_error &e) { failure = true; error_from_python = true; diff --git a/tests/tuple_test.py b/tests/tuple_test.py index 693304ae..11ac153d 100644 --- a/tests/tuple_test.py +++ b/tests/tuple_test.py @@ -240,10 +240,8 @@ def to_bytes(self, item): return item + b'\0' def from_bytes(self, data, offset): - end = offset - while data[end] != 0: - end += 1 - return memoryview(data)[offset:end], end + 1 - offset + end = data.index(b'\0', offset) + return memoryview(data)[offset:end], end - offset + 1 theta = update_theta_sketch() theta.update(1) From b71b4a08769fb9d7027c41305e42d8d55a56eb14 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 19 Nov 2026 15:07:12 +0800 Subject: [PATCH 4/4] fix: validate the full unsigned string length Read the four-byte little-endian length written by PyStringsSerDe and check that both the prefix and payload fit from the supplied offset. Python slices silently truncate, so a short payload could otherwise be returned as a complete string. Exercise a 16 MiB string and truncated records directly. These cases replace the repeated tuple fixtures and their string update policy. --- datasketches/PySerDe.py | 14 +++++++----- datasketches/TuplePolicy.py | 16 -------------- tests/serde_test.py | 42 ++++++++++++++++++++++++++++++++++++ tests/tuple_test.py | 43 ++----------------------------------- 4 files changed, 53 insertions(+), 62 deletions(-) create mode 100644 tests/serde_test.py diff --git a/datasketches/PySerDe.py b/datasketches/PySerDe.py index 6146ac01..97d42086 100644 --- a/datasketches/PySerDe.py +++ b/datasketches/PySerDe.py @@ -35,6 +35,7 @@ class PyStringsSerDe(PyObjectSerDe): '''Implements a simple string-encoding scheme where a string is written as ` `, with no null termination. + The byte count is an unsigned 32-bit little-endian integer. This format allows pre-allocating each string, at the cost of additional storage. Using this format, the serialized string consumes ``4 + len(item)`` bytes.''' @@ -48,11 +49,14 @@ def to_bytes(self, item: str): return bytes(b) def from_bytes(self, data: bytes, offset: int): - num_chars = struct.unpack_from(' offset + len(data)): - raise IndexError(f'num_chars read must be non-negative and not larger than the buffer. Found {num_chars}') - str = data[offset+4:offset+4+num_chars].decode() - return (str, 4+num_chars) + start = offset + 4 + if offset < 0 or start > len(data): + raise IndexError('String length prefix is outside the buffer') + num_bytes = struct.unpack_from(' len(data): + raise IndexError(f'String of {num_bytes} bytes exceeds the remaining buffer') + return (data[start:end].decode(), 4 + num_bytes) class PyIntsSerDe(PyObjectSerDe): '''Implements an integer encoding scheme where each integer is written diff --git a/datasketches/TuplePolicy.py b/datasketches/TuplePolicy.py index 77eadae2..760ebb7f 100644 --- a/datasketches/TuplePolicy.py +++ b/datasketches/TuplePolicy.py @@ -78,19 +78,3 @@ def update_summary(self, summary: int, update: int) -> int: def __call__(self, summary: int, update: int) -> int: return min(summary, update) - - -class KeepLastStringPolicy(TuplePolicy): - '''String-summary policy that keeps the most recent update.''' - - def __init__(self): - TuplePolicy.__init__(self) - - def create_summary(self) -> str: - return '' - - def update_summary(self, summary: str, update: str) -> str: - return update - - def __call__(self, summary: str, update: str) -> str: - return update diff --git a/tests/serde_test.py b/tests/serde_test.py new file mode 100644 index 00000000..c305169b --- /dev/null +++ b/tests/serde_test.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from datasketches import PyStringsSerDe + + +class SerDeTest(unittest.TestCase): + def test_long_string_round_trip(self): + serde = PyStringsSerDe() + value = 'x' * (1 << 24) + data = serde.to_bytes(value) + decoded, consumed = serde.from_bytes(data, 0) + self.assertEqual(consumed, len(data)) + self.assertEqual(decoded, value) + + def test_truncated_string(self): + serde = PyStringsSerDe() + first = serde.to_bytes('first') + second = serde.to_bytes('second') + for tail in (second[:3], second[:-1]): + with self.subTest(tail=tail), self.assertRaises(IndexError): + serde.from_bytes(first + tail, len(first)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tuple_test.py b/tests/tuple_test.py index 11ac153d..c2c58f48 100644 --- a/tests/tuple_test.py +++ b/tests/tuple_test.py @@ -21,8 +21,8 @@ from datasketches import compact_tuple_sketch, tuple_union from datasketches import tuple_intersection, tuple_a_not_b from datasketches import tuple_jaccard_similarity -from datasketches import PyObjectSerDe, PyIntsSerDe, PyStringsSerDe -from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy, KeepLastStringPolicy +from datasketches import PyObjectSerDe, PyIntsSerDe +from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy from datasketches import update_theta_sketch class TupleTest(unittest.TestCase): @@ -251,44 +251,5 @@ def from_bytes(self, data, offset): restored = compact_tuple_sketch.deserialize(sketch.serialize(serde), serde) self.assertEqual(dict(restored), dict(sketch)) - def test_tuple_strings_serde_round_trip(self): - # Short-string summary: exercises PyStringsSerDe over the memoryview path - # the C++ deserialize loop hands it one summary at a time. - lgk = 10 - n = 1 << 12 - value = 'alpha' - sk = update_tuple_sketch(KeepLastStringPolicy(), lgk) - for i in range(n): - sk.update(i, value) - compact = sk.compact() - - sk_bytes = compact.serialize(PyStringsSerDe()) - new_sk = compact_tuple_sketch.deserialize(sk_bytes, serde=PyStringsSerDe()) - - self.assertEqual(compact.num_retained, new_sk.num_retained) - self.assertEqual(compact.get_estimate(), new_sk.get_estimate()) - for pair in new_sk: - self.assertEqual(pair[1], value) - - def test_tuple_strings_serde_long_string(self): - # Long-string summary (len >= 256): the length prefix needs all 4 bytes, - # so this guards against a prior bug that read only 3 bytes of the length. - lgk = 8 - n = 1 << 10 - value = 'x' * 300 - sk = update_tuple_sketch(KeepLastStringPolicy(), lgk) - for i in range(n): - sk.update(i, value) - compact = sk.compact() - - sk_bytes = compact.serialize(PyStringsSerDe()) - new_sk = compact_tuple_sketch.deserialize(sk_bytes, serde=PyStringsSerDe()) - - self.assertEqual(compact.num_retained, new_sk.num_retained) - for pair in new_sk: - self.assertEqual(pair[1], value) - self.assertEqual(len(pair[1]), 300) - - if __name__ == '__main__': unittest.main()