diff --git a/datasketches/PySerDe.py b/datasketches/PySerDe.py index fcaddfeb..97d42086 100644 --- a/datasketches/PySerDe.py +++ b/datasketches/PySerDe.py @@ -35,6 +35,7 @@ class PyStringsSerDe(PyObjectSerDe): '''Implements a simple string-encoding scheme where a string is written as ` `, with no null termination. + The byte count is an unsigned 32-bit little-endian integer. This format allows pre-allocating each string, at the cost of additional storage. Using this format, the serialized string consumes ``4 + len(item)`` bytes.''' @@ -48,11 +49,14 @@ def to_bytes(self, item: str): return bytes(b) def from_bytes(self, data: bytes, offset: int): - num_chars = int.from_bytes(data[offset:offset+3], 'little') - if (num_chars < 0 or num_chars > offset + len(data)): - raise IndexError(f'num_chars read must be non-negative and not larger than the buffer. Found {num_chars}') - str = data[offset+4:offset+4+num_chars].decode() - return (str, 4+num_chars) + start = offset + 4 + if offset < 0 or start > len(data): + raise IndexError('String length prefix is outside the buffer') + num_bytes = struct.unpack_from(' len(data): + raise IndexError(f'String of {num_bytes} bytes exceeds the remaining buffer') + return (data[start:end].decode(), 4 + num_bytes) class PyIntsSerDe(PyObjectSerDe): '''Implements an integer encoding scheme where each integer is written diff --git a/include/py_serde.hpp b/include/py_serde.hpp index 1ff56988..7a463463 100644 --- a/include/py_serde.hpp +++ b/include/py_serde.hpp @@ -54,12 +54,12 @@ struct py_object_serde { /** * @brief Constructs an object from a serialized image, reading the * incoming buffer starting at the specified offset. - * - * @param bytes A buffer containing items from a serialized sketch - * @param offset The starting offset into the bytes buffer + * + * @param data The bytes containing the serialized sketch + * @param offset The starting offset into the data buffer * @return A Python tuple of the reconstructed item and the total number of bytes read */ - virtual nb::tuple from_bytes(nb::bytes& bytes, size_t offset) const = 0; + virtual nb::tuple from_bytes(const nb::bytes& data, size_t offset) const = 0; virtual ~py_object_serde() = default; @@ -67,6 +67,13 @@ struct py_object_serde { // default implementations for C++ std::string and numeric types. size_t size_of_item(const nb::object& item) const; size_t serialize(void* ptr, size_t capacity, const nb::object* items, unsigned num) const; +}; + +// The sketch passes pointers into one serialized image, including for nested sketches. +struct py_object_deserializer { + const py_object_serde& serde; + const nb::bytes& bytes; + size_t deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const; }; @@ -94,10 +101,10 @@ struct PyObjectSerDe : public py_object_serde { ); } - nb::tuple from_bytes(nb::bytes& bytes, size_t offset) const override { + nb::tuple from_bytes(const nb::bytes& data, size_t offset) const override { NB_OVERRIDE_PURE( from_bytes, // Name of function in C++ (must match Python name) - bytes, offset // Argument(s) + data, offset // Argument(s) ); } }; diff --git a/include/quantile_conditional.hpp b/include/quantile_conditional.hpp index 2e3c55bf..db1b87e3 100644 --- a/include/quantile_conditional.hpp +++ b/include/quantile_conditional.hpp @@ -69,7 +69,7 @@ void add_serialization(nb::class_& clazz) { .def_static( "deserialize", [](const nb::bytes& bytes, datasketches::py_object_serde& serde) { - return SK::deserialize(bytes.c_str(), bytes.size(), serde); + return SK::deserialize(bytes.c_str(), bytes.size(), datasketches::py_object_deserializer{serde, bytes}); }, nb::arg("bytes"), nb::arg("serde"), "Deserializes the sketch from a bytes object using the provided serde." ); diff --git a/src/ebpps_wrapper.cpp b/src/ebpps_wrapper.cpp index d2f24f27..81085900 100644 --- a/src/ebpps_wrapper.cpp +++ b/src/ebpps_wrapper.cpp @@ -77,7 +77,9 @@ void bind_ebpps_sketch(nb::module_ &m, const char* name) { "Serializes the sketch into a bytes object") .def_static( "deserialize", - [](const nb::bytes& bytes, py_object_serde& serde) { return ebpps_sketch::deserialize(bytes.c_str(), bytes.size(), serde); }, + [](const nb::bytes& bytes, py_object_serde& serde) { + return ebpps_sketch::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); + }, nb::arg("bytes"), nb::arg("serde"), "Reads a bytes object and returns the corresponding ebpps_sketch") .def("__iter__", diff --git a/src/fi_wrapper.cpp b/src/fi_wrapper.cpp index 0fffcc16..dc24fb9b 100644 --- a/src/fi_wrapper.cpp +++ b/src/fi_wrapper.cpp @@ -160,7 +160,7 @@ void add_serialization(nb::class_::deserialize(bytes.c_str(), bytes.size(), serde); + return frequent_items_sketch::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); }, nb::arg("bytes"), nb::arg("serde"), "Reads a bytes object using the provided serde and returns the corresponding frequent_strings_sketch." ); diff --git a/src/py_serde.cpp b/src/py_serde.cpp index a1896a63..3549fa81 100644 --- a/src/py_serde.cpp +++ b/src/py_serde.cpp @@ -42,9 +42,10 @@ void init_serde(nb::module_& m) { ":return: A :class:`bytes` object with the serialized object\n:rtype: bytes" ) .def("from_bytes", &py_object_serde::from_bytes, nb::arg("data"), nb::arg("offset"), - "Reads a bytes object starting from the given offest and returns a tuple of the reconstructed " + "Reads a bytes object starting from the given offset and returns a tuple of the reconstructed " "object and the number of additional bytes read\n\n" - ":param data: A :class:`bytes` object from which to deserialize\n:type data: bytes\n" + ":param data: A :class:`bytes` object from which to deserialize\n" + ":type data: bytes\n" ":param offset: The offset, in bytes, at which to start reading\n:type offset: int\n" ":return: A :class:`tuple` with the reconstructed object and the number of bytes read\n" ":rtype: tuple(object, int)" @@ -71,19 +72,18 @@ namespace datasketches { return bytes_written; } - size_t py_object_serde::deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const { + size_t py_object_deserializer::deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const { size_t bytes_read = 0; unsigned i = 0; bool failure = false; bool error_from_python = false; nb::gil_scoped_acquire acquire; - // copy data into bytes only once - nb::bytes bytes(static_cast(ptr), capacity); + const size_t offset = static_cast(ptr) - bytes.c_str(); for (; i < num && !failure; ++i) { nb::tuple bytes_and_len; try { - bytes_and_len = from_bytes(bytes, bytes_read); + bytes_and_len = serde.from_bytes(bytes, offset + bytes_read); } catch (nb::python_error &e) { failure = true; error_from_python = true; @@ -98,7 +98,6 @@ namespace datasketches { } new (&items[i]) nb::object(nb::cast(bytes_and_len[0])); - ptr = static_cast(ptr) + length; bytes_read += length; } diff --git a/src/tuple_wrapper.cpp b/src/tuple_wrapper.cpp index 6563e5bd..265561fd 100644 --- a/src/tuple_wrapper.cpp +++ b/src/tuple_wrapper.cpp @@ -145,7 +145,7 @@ void init_tuple(nb::module_ &m) { .def_static( "deserialize", [](const nb::bytes& bytes, py_object_serde& serde, uint64_t seed) { - return py_compact_tuple::deserialize(bytes.c_str(), bytes.size(), seed, serde); + return py_compact_tuple::deserialize(bytes.c_str(), bytes.size(), seed, py_object_deserializer{serde, bytes}); }, nb::arg("bytes"), nb::arg("serde"), nb::arg("seed")=DEFAULT_SEED, "Reads a bytes object and returns the corresponding compact_tuple_sketch" diff --git a/src/vo_wrapper.cpp b/src/vo_wrapper.cpp index ef0eb671..c129b2c5 100644 --- a/src/vo_wrapper.cpp +++ b/src/vo_wrapper.cpp @@ -85,7 +85,9 @@ void bind_vo_sketch(nb::module_ &m, const char* name) { "Serializes the sketch into a bytes object") .def_static( "deserialize", - [](const nb::bytes& bytes, py_object_serde& serde) { return var_opt_sketch::deserialize(bytes.c_str(), bytes.size(), serde); }, + [](const nb::bytes& bytes, py_object_serde& serde) { + return var_opt_sketch::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); + }, nb::arg("bytes"), nb::arg("serde"), "Reads a bytes object and returns the corresponding var opt sketch") .def("__iter__", @@ -127,7 +129,9 @@ void bind_vo_union(nb::module_ &m, const char* name) { "Serializes the union into a bytes object with the provided serde") .def_static( "deserialize", - [](const nb::bytes& bytes, py_object_serde& serde) { return var_opt_union::deserialize(bytes.c_str(), bytes.size(), serde); }, + [](const nb::bytes& bytes, py_object_serde& serde) { + return var_opt_union::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes}); + }, nb::arg("bytes"), nb::arg("serde"), "Constructs a var opt union from the given bytes using the provided serde") ; diff --git a/tests/serde_test.py b/tests/serde_test.py new file mode 100644 index 00000000..c305169b --- /dev/null +++ b/tests/serde_test.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from datasketches import PyStringsSerDe + + +class SerDeTest(unittest.TestCase): + def test_long_string_round_trip(self): + serde = PyStringsSerDe() + value = 'x' * (1 << 24) + data = serde.to_bytes(value) + decoded, consumed = serde.from_bytes(data, 0) + self.assertEqual(consumed, len(data)) + self.assertEqual(decoded, value) + + def test_truncated_string(self): + serde = PyStringsSerDe() + first = serde.to_bytes('first') + second = serde.to_bytes('second') + for tail in (second[:3], second[:-1]): + with self.subTest(tail=tail), self.assertRaises(IndexError): + serde.from_bytes(first + tail, len(first)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tuple_test.py b/tests/tuple_test.py index cf505fb1..c2c58f48 100644 --- a/tests/tuple_test.py +++ b/tests/tuple_test.py @@ -21,7 +21,7 @@ from datasketches import compact_tuple_sketch, tuple_union from datasketches import tuple_intersection, tuple_a_not_b from datasketches import tuple_jaccard_similarity -from datasketches import tuple_jaccard_similarity, PyIntsSerDe +from datasketches import PyObjectSerDe, PyIntsSerDe from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy from datasketches import update_theta_sketch @@ -231,5 +231,25 @@ def generate_tuple_sketch(self, policy, n, lgk, value, offset=0): sk.update(i + offset, value) return sk + def test_retained_summary_views(self): + class DelimitedBytesSerDe(PyObjectSerDe): + def get_size(self, item): + return len(item) + 1 + + def to_bytes(self, item): + return item + b'\0' + + def from_bytes(self, data, offset): + end = data.index(b'\0', offset) + return memoryview(data)[offset:end], end - offset + 1 + + theta = update_theta_sketch() + theta.update(1) + theta.update(2) + sketch = compact_tuple_sketch(theta, b'alpha') + serde = DelimitedBytesSerDe() + restored = compact_tuple_sketch.deserialize(sketch.serialize(serde), serde) + self.assertEqual(dict(restored), dict(sketch)) + if __name__ == '__main__': unittest.main()