Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions datasketches/PySerDe.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
class PyStringsSerDe(PyObjectSerDe):
'''Implements a simple string-encoding scheme where a string is
written as `<num_bytes> <string_contents>`, with no null termination.
The byte count is an unsigned 32-bit little-endian integer.
This format allows pre-allocating each string, at the cost of
additional storage. Using this format, the serialized string consumes
``4 + len(item)`` bytes.'''
Expand All @@ -48,11 +49,14 @@ def to_bytes(self, item: str):
return bytes(b)

def from_bytes(self, data: bytes, offset: int):
num_chars = int.from_bytes(data[offset:offset+3], 'little')
if (num_chars < 0 or num_chars > offset + len(data)):
raise IndexError(f'num_chars read must be non-negative and not larger than the buffer. Found {num_chars}')
str = data[offset+4:offset+4+num_chars].decode()
return (str, 4+num_chars)
start = offset + 4
if offset < 0 or start > len(data):
raise IndexError('String length prefix is outside the buffer')
num_bytes = struct.unpack_from('<I', data, offset)[0]
end = start + num_bytes
if end > len(data):
raise IndexError(f'String of {num_bytes} bytes exceeds the remaining buffer')
return (data[start:end].decode(), 4 + num_bytes)

class PyIntsSerDe(PyObjectSerDe):
'''Implements an integer encoding scheme where each integer is written
Expand Down
19 changes: 13 additions & 6 deletions include/py_serde.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,19 +54,26 @@ struct py_object_serde {
/**
* @brief Constructs an object from a serialized image, reading the
* incoming buffer starting at the specified offset.
*
* @param bytes A buffer containing items from a serialized sketch
* @param offset The starting offset into the bytes buffer
*
* @param data The bytes containing the serialized sketch
* @param offset The starting offset into the data buffer
* @return A Python tuple of the reconstructed item and the total number of bytes read
*/
virtual nb::tuple from_bytes(nb::bytes& bytes, size_t offset) const = 0;
virtual nb::tuple from_bytes(const nb::bytes& data, size_t offset) const = 0;

virtual ~py_object_serde() = default;

// these methods are required by the serde interface; see common/include/serde.hpp for
// default implementations for C++ std::string and numeric types.
size_t size_of_item(const nb::object& item) const;
size_t serialize(void* ptr, size_t capacity, const nb::object* items, unsigned num) const;
};

// The sketch passes pointers into one serialized image, including for nested sketches.
struct py_object_deserializer {
const py_object_serde& serde;
const nb::bytes& bytes;

size_t deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const;
};

Expand Down Expand Up @@ -94,10 +101,10 @@ struct PyObjectSerDe : public py_object_serde {
);
}

nb::tuple from_bytes(nb::bytes& bytes, size_t offset) const override {
nb::tuple from_bytes(const nb::bytes& data, size_t offset) const override {
NB_OVERRIDE_PURE(
from_bytes, // Name of function in C++ (must match Python name)
bytes, offset // Argument(s)
data, offset // Argument(s)
);
}
};
Expand Down
2 changes: 1 addition & 1 deletion include/quantile_conditional.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ void add_serialization(nb::class_<SK>& clazz) {
.def_static(
"deserialize",
[](const nb::bytes& bytes, datasketches::py_object_serde& serde) {
return SK::deserialize(bytes.c_str(), bytes.size(), serde);
return SK::deserialize(bytes.c_str(), bytes.size(), datasketches::py_object_deserializer{serde, bytes});
}, nb::arg("bytes"), nb::arg("serde"),
"Deserializes the sketch from a bytes object using the provided serde."
);
Expand Down
4 changes: 3 additions & 1 deletion src/ebpps_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ void bind_ebpps_sketch(nb::module_ &m, const char* name) {
"Serializes the sketch into a bytes object")
.def_static(
"deserialize",
[](const nb::bytes& bytes, py_object_serde& serde) { return ebpps_sketch<T>::deserialize(bytes.c_str(), bytes.size(), serde); },
[](const nb::bytes& bytes, py_object_serde& serde) {
return ebpps_sketch<T>::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes});
},
nb::arg("bytes"), nb::arg("serde"),
"Reads a bytes object and returns the corresponding ebpps_sketch")
.def("__iter__",
Expand Down
2 changes: 1 addition & 1 deletion src/fi_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ void add_serialization(nb::class_<datasketches::frequent_items_sketch<T, W, H, E
.def_static(
"deserialize",
[](const nb::bytes& bytes, py_object_serde& serde) {
return frequent_items_sketch<T, W, H, E>::deserialize(bytes.c_str(), bytes.size(), serde);
return frequent_items_sketch<T, W, H, E>::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes});
}, nb::arg("bytes"), nb::arg("serde"),
"Reads a bytes object using the provided serde and returns the corresponding frequent_strings_sketch."
);
Expand Down
13 changes: 6 additions & 7 deletions src/py_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ void init_serde(nb::module_& m) {
":return: A :class:`bytes` object with the serialized object\n:rtype: bytes"
)
.def("from_bytes", &py_object_serde::from_bytes, nb::arg("data"), nb::arg("offset"),
"Reads a bytes object starting from the given offest and returns a tuple of the reconstructed "
"Reads a bytes object starting from the given offset and returns a tuple of the reconstructed "
"object and the number of additional bytes read\n\n"
":param data: A :class:`bytes` object from which to deserialize\n:type data: bytes\n"
":param data: A :class:`bytes` object from which to deserialize\n"
":type data: bytes\n"
":param offset: The offset, in bytes, at which to start reading\n:type offset: int\n"
":return: A :class:`tuple` with the reconstructed object and the number of bytes read\n"
":rtype: tuple(object, int)"
Expand All @@ -71,19 +72,18 @@ namespace datasketches {
return bytes_written;
}

size_t py_object_serde::deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const {
size_t py_object_deserializer::deserialize(const void* ptr, size_t capacity, nb::object* items, unsigned num) const {
size_t bytes_read = 0;
unsigned i = 0;
bool failure = false;
bool error_from_python = false;
nb::gil_scoped_acquire acquire;

// copy data into bytes only once
nb::bytes bytes(static_cast<const char*>(ptr), capacity);
const size_t offset = static_cast<const char*>(ptr) - bytes.c_str();
for (; i < num && !failure; ++i) {
nb::tuple bytes_and_len;
try {
bytes_and_len = from_bytes(bytes, bytes_read);
bytes_and_len = serde.from_bytes(bytes, offset + bytes_read);
} catch (nb::python_error &e) {
failure = true;
error_from_python = true;
Expand All @@ -98,7 +98,6 @@ namespace datasketches {
}

new (&items[i]) nb::object(nb::cast<nb::object>(bytes_and_len[0]));
ptr = static_cast<const char*>(ptr) + length;
bytes_read += length;
}

Expand Down
2 changes: 1 addition & 1 deletion src/tuple_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ void init_tuple(nb::module_ &m) {
.def_static(
"deserialize",
[](const nb::bytes& bytes, py_object_serde& serde, uint64_t seed) {
return py_compact_tuple::deserialize(bytes.c_str(), bytes.size(), seed, serde);
return py_compact_tuple::deserialize(bytes.c_str(), bytes.size(), seed, py_object_deserializer{serde, bytes});
},
nb::arg("bytes"), nb::arg("serde"), nb::arg("seed")=DEFAULT_SEED,
"Reads a bytes object and returns the corresponding compact_tuple_sketch"
Expand Down
8 changes: 6 additions & 2 deletions src/vo_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ void bind_vo_sketch(nb::module_ &m, const char* name) {
"Serializes the sketch into a bytes object")
.def_static(
"deserialize",
[](const nb::bytes& bytes, py_object_serde& serde) { return var_opt_sketch<T>::deserialize(bytes.c_str(), bytes.size(), serde); },
[](const nb::bytes& bytes, py_object_serde& serde) {
return var_opt_sketch<T>::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes});
},
nb::arg("bytes"), nb::arg("serde"),
"Reads a bytes object and returns the corresponding var opt sketch")
.def("__iter__",
Expand Down Expand Up @@ -127,7 +129,9 @@ void bind_vo_union(nb::module_ &m, const char* name) {
"Serializes the union into a bytes object with the provided serde")
.def_static(
"deserialize",
[](const nb::bytes& bytes, py_object_serde& serde) { return var_opt_union<T>::deserialize(bytes.c_str(), bytes.size(), serde); },
[](const nb::bytes& bytes, py_object_serde& serde) {
return var_opt_union<T>::deserialize(bytes.c_str(), bytes.size(), py_object_deserializer{serde, bytes});
},
nb::arg("bytes"), nb::arg("serde"),
"Constructs a var opt union from the given bytes using the provided serde")
;
Expand Down
42 changes: 42 additions & 0 deletions tests/serde_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import unittest

from datasketches import PyStringsSerDe


class SerDeTest(unittest.TestCase):
def test_long_string_round_trip(self):
serde = PyStringsSerDe()
value = 'x' * (1 << 24)
data = serde.to_bytes(value)
decoded, consumed = serde.from_bytes(data, 0)
self.assertEqual(consumed, len(data))
self.assertEqual(decoded, value)

def test_truncated_string(self):
serde = PyStringsSerDe()
first = serde.to_bytes('first')
second = serde.to_bytes('second')
for tail in (second[:3], second[:-1]):
with self.subTest(tail=tail), self.assertRaises(IndexError):
serde.from_bytes(first + tail, len(first))


if __name__ == '__main__':
unittest.main()
22 changes: 21 additions & 1 deletion tests/tuple_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from datasketches import compact_tuple_sketch, tuple_union
from datasketches import tuple_intersection, tuple_a_not_b
from datasketches import tuple_jaccard_similarity
from datasketches import tuple_jaccard_similarity, PyIntsSerDe
from datasketches import PyObjectSerDe, PyIntsSerDe
from datasketches import AccumulatorPolicy, MaxIntPolicy, MinIntPolicy
from datasketches import update_theta_sketch

Expand Down Expand Up @@ -231,5 +231,25 @@ def generate_tuple_sketch(self, policy, n, lgk, value, offset=0):
sk.update(i + offset, value)
return sk

def test_retained_summary_views(self):
class DelimitedBytesSerDe(PyObjectSerDe):
def get_size(self, item):
return len(item) + 1

def to_bytes(self, item):
return item + b'\0'

def from_bytes(self, data, offset):
end = data.index(b'\0', offset)
return memoryview(data)[offset:end], end - offset + 1

theta = update_theta_sketch()
theta.update(1)
theta.update(2)
sketch = compact_tuple_sketch(theta, b'alpha')
serde = DelimitedBytesSerDe()
restored = compact_tuple_sketch.deserialize(sketch.serialize(serde), serde)
self.assertEqual(dict(restored), dict(sketch))

if __name__ == '__main__':
unittest.main()
Loading