diff --git a/density/include/density_sketch.hpp b/density/include/density_sketch.hpp index 57f29eaf..8a064b1a 100755 --- a/density/include/density_sketch.hpp +++ b/density/include/density_sketch.hpp @@ -201,7 +201,8 @@ class density_sketch { enum flags { RESERVED0, RESERVED1, IS_EMPTY }; static const uint8_t PREAMBLE_INTS_SHORT = 3; static const uint8_t PREAMBLE_INTS_LONG = 6; - static const uint8_t FAMILY_ID = 19; + static const uint8_t FAMILY_ID = 23; + static const uint8_t LEGACY_FAMILY_ID = 19; static const uint8_t SERIAL_VERSION = 1; static const size_t LEVELS_ARRAY_START = 5; @@ -217,7 +218,7 @@ class density_sketch { static void check_k(uint16_t k); static void check_serial_version(uint8_t serial_version); - static void check_family_id(uint8_t family_id); + static void check_family_id(uint8_t family_id, uint8_t serial_version); static void check_header_validity(uint8_t preamble_ints, uint8_t flags_byte, uint8_t serial_version); density_sketch(uint16_t k, uint32_t dim, uint32_t num_retained, uint64_t n, Levels&& levels, diff --git a/density/include/density_sketch_impl.hpp b/density/include/density_sketch_impl.hpp index a5a5bc0f..019b0bf6 100755 --- a/density/include/density_sketch_impl.hpp +++ b/density/include/density_sketch_impl.hpp @@ -272,12 +272,12 @@ density_sketch density_sketch::deserialize(std::istream& is, c read(is); // unused const auto dim = read(is); - check_k(k); // do we have constraints? - check_serial_version(serial_version); // a little redundant with the header check - check_family_id(family_id); + if (!is.good()) throw std::runtime_error("error reading from std::istream"); + check_serial_version(serial_version); + check_family_id(family_id, serial_version); check_header_validity(preamble_ints, flags_byte, serial_version); + check_k(k); - if (!is.good()) throw std::runtime_error("error reading from std::istream"); const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0; if (is_empty) { return density_sketch(k, dim, kernel, allocator); @@ -330,10 +330,10 @@ density_sketch density_sketch::deserialize(const void* bytes, uint32_t dim; ptr += copy_from_mem(ptr, dim); - check_k(k); - check_serial_version(serial_version); // a little redundant with the header check - check_family_id(family_id); + check_serial_version(serial_version); + check_family_id(family_id, serial_version); check_header_validity(preamble_ints, flags_byte, serial_version); + check_k(k); const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0; if (is_empty) { @@ -392,8 +392,10 @@ void density_sketch::check_serial_version(uint8_t serial_version) { } template -void density_sketch::check_family_id(uint8_t family_id) { - if (family_id == FAMILY_ID) +void density_sketch::check_family_id(uint8_t family_id, uint8_t serial_version) { + // Earlier releases wrote family ID 19, which collided with EBPPS. Accept it for v1 images; + // check_header_validity distinguishes Density (preamble_ints 3 or 6) from EBPPS (1 or 5). + if (family_id == FAMILY_ID || (family_id == LEGACY_FAMILY_ID && serial_version == 1)) return; else throw std::invalid_argument("Possible corruption. Family id does not indicate density sketch: " + std::to_string(family_id)); @@ -401,17 +403,21 @@ void density_sketch::check_family_id(uint8_t family_id) { template void density_sketch::check_header_validity(uint8_t preamble_ints, uint8_t flags_byte, uint8_t serial_version) { - const bool empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0; + const uint8_t empty_mask = 1 << flags::IS_EMPTY; + const bool valid_flags = (flags_byte & ~empty_mask) == 0; + const bool empty = (flags_byte & empty_mask) != 0; - if ((empty && preamble_ints == PREAMBLE_INTS_SHORT) - || (!empty && preamble_ints == PREAMBLE_INTS_LONG)) - return; + if (valid_flags + && ((empty && preamble_ints == PREAMBLE_INTS_SHORT) + || (!empty && preamble_ints == PREAMBLE_INTS_LONG))) + return; else { std::ostringstream os; os << "Possible sketch corruption. Inconsistent state: " - << "preamble_ints = " << preamble_ints + << "preamble_ints = " << static_cast(preamble_ints) + << ", flags = " << static_cast(flags_byte) << ", empty = " << (empty ? "true" : "false") - << ", serialization_version = " << serial_version; + << ", serialization_version = " << static_cast(serial_version); throw std::invalid_argument(os.str()); } } diff --git a/density/test/density_sketch_test.cpp b/density/test/density_sketch_test.cpp index f8c5d528..6fd88c9e 100755 --- a/density/test/density_sketch_test.cpp +++ b/density/test/density_sketch_test.cpp @@ -111,6 +111,8 @@ TEST_CASE("custom kernel", "[density_sketch]") { TEST_CASE("serialize empty", "[density_sketch]") { density_sketch sk(10, 2); auto bytes = sk.serialize(); + REQUIRE(bytes.size() >= 3); + REQUIRE(bytes[2] == 23); auto sk2 = density_sketch::deserialize(bytes.data(), bytes.size()); REQUIRE(sk2.is_empty()); REQUIRE(!sk2.is_estimation_mode()); @@ -121,6 +123,9 @@ TEST_CASE("serialize empty", "[density_sketch]") { std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); sk.serialize(s); + const std::string stream_bytes = s.str(); + REQUIRE(stream_bytes.size() >= 3); + REQUIRE(static_cast(stream_bytes[2]) == 23); auto sk3 = density_sketch::deserialize(s); REQUIRE(sk3.is_empty()); REQUIRE(!sk3.is_estimation_mode()); @@ -143,6 +148,8 @@ TEST_CASE("serialize bytes", "[density_sketch]") { // exact mode auto bytes = sk.serialize(); + REQUIRE(bytes.size() >= 3); + REQUIRE(bytes[2] == 23); auto sk2 = density_sketch::deserialize(bytes.data(), bytes.size()); REQUIRE(!sk2.is_empty()); REQUIRE(!sk2.is_estimation_mode()); @@ -168,6 +175,8 @@ TEST_CASE("serialize bytes", "[density_sketch]") { REQUIRE(sk.is_estimation_mode()); bytes = sk.serialize(); + REQUIRE(bytes.size() >= 3); + REQUIRE(bytes[2] == 23); sk2 = density_sketch::deserialize(bytes.data(), bytes.size()); REQUIRE(!sk2.is_empty()); REQUIRE(sk2.is_estimation_mode()); @@ -199,6 +208,8 @@ TEST_CASE("serialize stream", "[density_sketch]") { // exact mode std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); sk.serialize(s); + REQUIRE(s.str().size() >= 3); + REQUIRE(static_cast(s.str()[2]) == 23); auto sk2 = density_sketch::deserialize(s); REQUIRE(!sk2.is_empty()); REQUIRE(!sk2.is_estimation_mode()); @@ -225,6 +236,8 @@ TEST_CASE("serialize stream", "[density_sketch]") { std::stringstream s2(std::ios::in | std::ios::out | std::ios::binary); sk.serialize(s2); + REQUIRE(s2.str().size() >= 3); + REQUIRE(static_cast(s2.str()[2]) == 23); sk2 = density_sketch::deserialize(s2); REQUIRE(!sk2.is_empty()); REQUIRE(sk2.is_estimation_mode()); @@ -271,4 +284,200 @@ TEST_CASE("density sketch: custom allocator", "[density_sketch]") { REQUIRE(test_allocator_net_allocations == 0); } +TEST_CASE("density sketch: legacy family id 19 compatibility", "[density_sketch]") { + // Legacy empty Density sketch (k = 10, dim = 2, family_id = 19) + const std::vector legacy_empty = { + 3, 1, 19, 4, + 10, 0, 0, 0, + 2, 0, 0, 0 + }; + { + auto sk = density_sketch::deserialize(legacy_empty.data(), legacy_empty.size()); + REQUIRE(sk.is_empty()); + REQUIRE(sk.get_k() == 10); + REQUIRE(sk.get_dim() == 2); + REQUIRE(sk.get_n() == 0); + REQUIRE(sk.get_num_retained() == 0); + + auto upgraded = sk.serialize(); + REQUIRE(upgraded.size() == legacy_empty.size()); + REQUIRE(upgraded[2] == 23); + + std::stringstream is( + std::string(reinterpret_cast(legacy_empty.data()), legacy_empty.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + auto sk_stream = density_sketch::deserialize(is); + REQUIRE(sk_stream.is_empty()); + REQUIRE(sk_stream.get_k() == 10); + REQUIRE(sk_stream.get_dim() == 2); + + std::stringstream os(std::ios::in | std::ios::out | std::ios::binary); + sk_stream.serialize(os); + REQUIRE(static_cast(os.str()[2]) == 23); + } + + // Legacy single-point Density sketch (k = 10, dim = 2, point = [1.0, 2.0], family_id = 19) + const std::vector legacy_single = { + 6, 1, 19, 0, + 10, 0, 0, 0, + 2, 0, 0, 0, + 1, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40 + }; + { + auto sk = density_sketch::deserialize(legacy_single.data(), legacy_single.size()); + REQUIRE(!sk.is_empty()); + REQUIRE(sk.get_k() == 10); + REQUIRE(sk.get_dim() == 2); + REQUIRE(sk.get_n() == 1); + REQUIRE(sk.get_num_retained() == 1); + REQUIRE(sk.get_estimate({1.0, 2.0}) == 1.0); + + auto upgraded = sk.serialize(); + REQUIRE(upgraded.size() == legacy_single.size()); + REQUIRE(upgraded[2] == 23); + upgraded[2] = 19; + REQUIRE(upgraded == legacy_single); + + std::stringstream is( + std::string(reinterpret_cast(legacy_single.data()), legacy_single.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + auto sk_stream = density_sketch::deserialize(is); + REQUIRE(sk_stream.get_n() == 1); + REQUIRE(sk_stream.get_estimate({1.0, 2.0}) == 1.0); + } + + // Legacy estimation-mode Density sketch round-trip and re-serialization to family ID 23 + density_sketch original(10, 3); + for (uint32_t i = 0; i < 1031; ++i) { + const double val = static_cast(i); + original.update(std::vector({val, std::sqrt(val), -val})); + } + REQUIRE(original.is_estimation_mode()); + + auto legacy_bytes = original.serialize(); + REQUIRE(legacy_bytes[2] == 23); + legacy_bytes[2] = 19; + + auto from_bytes = density_sketch::deserialize(legacy_bytes.data(), legacy_bytes.size()); + REQUIRE(from_bytes.is_estimation_mode()); + REQUIRE(from_bytes.get_k() == original.get_k()); + REQUIRE(from_bytes.get_dim() == original.get_dim()); + REQUIRE(from_bytes.get_n() == original.get_n()); + REQUIRE(from_bytes.get_num_retained() == original.get_num_retained()); + REQUIRE(from_bytes.get_estimate({5.0, std::sqrt(5.0), -5.0}) == original.get_estimate({5.0, std::sqrt(5.0), -5.0})); + REQUIRE(from_bytes.serialize()[2] == 23); + + std::stringstream legacy_stream( + std::string(reinterpret_cast(legacy_bytes.data()), legacy_bytes.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + auto from_stream = density_sketch::deserialize(legacy_stream); + REQUIRE(from_stream.get_n() == original.get_n()); + REQUIRE(from_stream.get_num_retained() == original.get_num_retained()); + REQUIRE(from_stream.get_estimate({5.0, std::sqrt(5.0), -5.0}) == original.get_estimate({5.0, std::sqrt(5.0), -5.0})); + std::stringstream out_stream(std::ios::in | std::ios::out | std::ios::binary); + from_stream.serialize(out_stream); + REQUIRE(static_cast(out_stream.str()[2]) == 23); +} + +TEST_CASE("density sketch: reject invalid family and ebpps headers", "[density_sketch]") { + // Unrelated family ID (20 = T-Digest) + const std::vector wrong_family = { + 3, 1, 20, 4, + 10, 0, 0, 0, + 2, 0, 0, 0 + }; + REQUIRE_THROWS_AS( + density_sketch::deserialize(wrong_family.data(), wrong_family.size()), + std::invalid_argument + ); + { + std::stringstream is( + std::string(reinterpret_cast(wrong_family.data()), wrong_family.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + REQUIRE_THROWS_AS(density_sketch::deserialize(is), std::invalid_argument); + } + + // Empty EBPPS header (preamble_longs = 1, ser_ver = 1, family_id = 19, flags = 4, 8 bytes) + const std::vector ebpps_empty = { + 1, 1, 19, 4, + 10, 0, 0, 0 + }; + REQUIRE_THROWS_AS( + density_sketch::deserialize(ebpps_empty.data(), ebpps_empty.size()), + std::out_of_range + ); + { + std::stringstream is( + std::string(reinterpret_cast(ebpps_empty.data()), ebpps_empty.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + REQUIRE_THROWS_AS(density_sketch::deserialize(is), std::runtime_error); + } + + // Padded empty EBPPS header (preamble_longs = 1, family_id = 19, flags = 4, 12 bytes) + const std::vector ebpps_empty_padded = { + 1, 1, 19, 4, + 10, 0, 0, 0, + 2, 0, 0, 0 + }; + REQUIRE_THROWS_AS( + density_sketch::deserialize(ebpps_empty_padded.data(), ebpps_empty_padded.size()), + std::invalid_argument + ); + { + std::stringstream is( + std::string(reinterpret_cast(ebpps_empty_padded.data()), ebpps_empty_padded.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + REQUIRE_THROWS_AS(density_sketch::deserialize(is), std::invalid_argument); + } + + // Non-empty EBPPS header (preamble_longs = 5, ser_ver = 1, family_id = 19, flags = 0 / 8) + for (uint8_t flags : {uint8_t{0}, uint8_t{8}}) { + std::vector ebpps_non_empty(40, 0); + ebpps_non_empty[0] = 5; + ebpps_non_empty[1] = 1; + ebpps_non_empty[2] = 19; + ebpps_non_empty[3] = flags; + ebpps_non_empty[4] = 10; + REQUIRE_THROWS_AS( + density_sketch::deserialize(ebpps_non_empty.data(), ebpps_non_empty.size()), + std::invalid_argument + ); + std::stringstream is( + std::string(reinterpret_cast(ebpps_non_empty.data()), ebpps_non_empty.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + REQUIRE_THROWS_AS(density_sketch::deserialize(is), std::invalid_argument); + } + + // Unexpected flag bits (bit 3 = 8) must be rejected for both legacy (19) and current (23) IDs + for (uint8_t fam : {uint8_t{19}, uint8_t{23}}) { + std::vector bad_flags(24, 0); + bad_flags[0] = 6; + bad_flags[1] = 1; + bad_flags[2] = fam; + bad_flags[3] = 8; + bad_flags[4] = 10; + bad_flags[8] = 2; + REQUIRE_THROWS_AS( + density_sketch::deserialize(bad_flags.data(), bad_flags.size()), + std::invalid_argument + ); + std::stringstream is( + std::string(reinterpret_cast(bad_flags.data()), bad_flags.size()), + std::ios::in | std::ios::out | std::ios::binary + ); + REQUIRE_THROWS_AS(density_sketch::deserialize(is), std::invalid_argument); + } +} + } /* namespace datasketches */