From f656f535e13741a08031db096da70218f8bff22f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 31 Aug 2021 15:04:06 +0200 Subject: [PATCH 01/32] :construction: begin BON8 implementation --- .../nlohmann/detail/input/input_adapters.hpp | 2 +- .../nlohmann/detail/output/binary_writer.hpp | 200 ++++++++++- include/nlohmann/json.hpp | 17 + single_include/nlohmann/json.hpp | 219 +++++++++++- test/src/unit-bon8.cpp | 315 ++++++++++++++++++ 5 files changed, 739 insertions(+), 14 deletions(-) create mode 100644 test/src/unit-bon8.cpp diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 6df58a1cd..1aed3664e 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -23,7 +23,7 @@ namespace nlohmann namespace detail { /// the supported input formats -enum class input_format_t { json, cbor, msgpack, ubjson, bson }; +enum class input_format_t { json, cbor, msgpack, ubjson, bson, bon8 }; //////////////////// // input adapters // diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 42fd50e32..8a999a613 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -923,6 +923,158 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + */ + void write_bon8(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xFA)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF9) + : to_char_type(0xF8)); + break; + } + + case value_t::number_float: + { + // special values + if (j.m_value.number_float == -1.0) + { + oa->write_character(to_char_type(0xFB)); + break; + } + if (j.m_value.number_float == 0.0) + { + oa->write_character(to_char_type(0xFC)); + break; + } + if (j.m_value.number_float == 1.0) + { + oa->write_character(to_char_type(0xFD)); + break; + } + + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + break; + } + + case value_t::string: + { + // empty string: use end-of-text symbol + if (j.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + break; + } + + // write strings as is + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + const auto N = j.m_value.array->size(); + if (N <= 4) + { + // start array with count (80..84) + oa->write_character(to_char_type(0x80 + N)); + } + else + { + // start array + oa->write_character(to_char_type(0x85)); + } + + // write each element + for (std::size_t i = 0; i < N; ++i) + { + const auto& el = j.m_value.array->operator[](i); + + // check if 0xFF after nonempty string and string is required + if (i > 0) + { + const auto& prev = j.m_value.array->operator[](i - 1); + if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + write_bon8(el); + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + } + break; + } + + case value_t::object: + { + const auto N = j.m_value.object->size(); + if (N <= 4) + { + // start object with count (86..8A) + oa->write_character(to_char_type(0x86 + N)); + } + else + { + // start object + oa->write_character(to_char_type(0x8B)); + } + + // write each element + for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) + { + const auto& key = it->first; + const auto& value = it->second; + + write_bon8(key); + + // check if we need a 0xFF separator between key and value + if (!key.empty() && value.is_string()) + { + oa->write_character(to_char_type(0xFF)); + } + + write_bon8(value); + + // check if we need a 0xFF separator between the value and the next key + if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + private: ////////// // BSON // @@ -1524,6 +1676,20 @@ class binary_writer return 'D'; // float 64 } + ////////// + // BON8 // + ////////// + + static constexpr CharType get_bon8_float_prefix(float /*unused*/) + { + return to_char_type(0x8E); + } + + static constexpr CharType get_bon8_float_prefix(double /*unused*/) + { + return to_char_type(0x8F); + } + /////////////////////// // Utility functions // /////////////////////// @@ -1566,16 +1732,38 @@ class binary_writer static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + switch (format) + { + case input_format_t::cbor: + get_cbor_float_prefix(static_cast(n)); + break; + case input_format_t::msgpack: + get_msgpack_float_prefix(static_cast(n)); + break; + case input_format_t::bon8: + get_bon8_float_prefix(static_cast(n)); + break; + default: + break; + } write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + switch (format) + { + case input_format_t::cbor: + get_cbor_float_prefix(n); + break; + case input_format_t::msgpack: + get_msgpack_float_prefix(n); + break; + case input_format_t::bon8: + get_bon8_float_prefix(n); + break; + default: + break; + } write_number(n); } #ifdef __GNUC__ diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index fc3e60828..650730509 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -7611,6 +7611,23 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + static std::vector to_bon8(const basic_json& j) + { + std::vector result; + to_bon8(j, result); + return result; + } + + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + /*! @brief create a JSON value from an input in CBOR format diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 8959265da..8c7b3d1b6 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5354,7 +5354,7 @@ namespace nlohmann namespace detail { /// the supported input formats -enum class input_format_t { json, cbor, msgpack, ubjson, bson }; +enum class input_format_t { json, cbor, msgpack, ubjson, bson, bon8 }; //////////////////// // input adapters // @@ -14495,6 +14495,158 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + */ + void write_bon8(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xFA)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF9) + : to_char_type(0xF8)); + break; + } + + case value_t::number_float: + { + // special values + if (j.m_value.number_float == -1.0) + { + oa->write_character(to_char_type(0xFB)); + break; + } + if (j.m_value.number_float == 0.0) + { + oa->write_character(to_char_type(0xFC)); + break; + } + if (j.m_value.number_float == 1.0) + { + oa->write_character(to_char_type(0xFD)); + break; + } + + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + break; + } + + case value_t::string: + { + // empty string: use end-of-text symbol + if (j.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + break; + } + + // write strings as is + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + const auto N = j.m_value.array->size(); + if (N <= 4) + { + // start array with count (80..84) + oa->write_character(to_char_type(0x80 + N)); + } + else + { + // start array + oa->write_character(to_char_type(0x85)); + } + + // write each element + for (std::size_t i = 0; i < N; ++i) + { + const auto& el = j.m_value.array->operator[](i); + + // check if 0xFF after nonempty string and string is required + if (i > 0) + { + const auto& prev = j.m_value.array->operator[](i - 1); + if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + write_bon8(el); + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + } + break; + } + + case value_t::object: + { + const auto N = j.m_value.object->size(); + if (N <= 4) + { + // start object with count (86..8A) + oa->write_character(to_char_type(0x86 + N)); + } + else + { + // start object + oa->write_character(to_char_type(0x8B)); + } + + // write each element + for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) + { + const auto& key = it->first; + const auto& value = it->second; + + write_bon8(key); + + // check if we need a 0xFF separator between key and value + if (!key.empty() && value.is_string()) + { + oa->write_character(to_char_type(0xFF)); + } + + write_bon8(value); + + // check if we need a 0xFF separator between the value and the next key + if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + private: ////////// // BSON // @@ -15096,6 +15248,20 @@ class binary_writer return 'D'; // float 64 } + ////////// + // BON8 // + ////////// + + static constexpr CharType get_bon8_float_prefix(float /*unused*/) + { + return to_char_type(0x8E); + } + + static constexpr CharType get_bon8_float_prefix(double /*unused*/) + { + return to_char_type(0x8F); + } + /////////////////////// // Utility functions // /////////////////////// @@ -15138,16 +15304,38 @@ class binary_writer static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + switch (format) + { + case input_format_t::cbor: + get_cbor_float_prefix(static_cast(n)); + break; + case input_format_t::msgpack: + get_msgpack_float_prefix(static_cast(n)); + break; + case input_format_t::bon8: + get_bon8_float_prefix(static_cast(n)); + break; + default: + break; + } write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + switch (format) + { + case input_format_t::cbor: + get_cbor_float_prefix(n); + break; + case input_format_t::msgpack: + get_msgpack_float_prefix(n); + break; + case input_format_t::bon8: + get_bon8_float_prefix(n); + break; + default: + break; + } write_number(n); } #ifdef __GNUC__ @@ -25016,6 +25204,23 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + static std::vector to_bon8(const basic_json& j) + { + std::vector result; + to_bon8(j, result); + return result; + } + + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + /*! @brief create a JSON value from an input in CBOR format diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp new file mode 100644 index 000000000..3c17ba46b --- /dev/null +++ b/test/src/unit-bon8.cpp @@ -0,0 +1,315 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ (test suite) +| | |__ | | | | | | version 3.10.2 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2019 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include "test_utils.hpp" + +namespace +{ +class SaxCountdown +{ + public: + explicit SaxCountdown(const int count) : events_left(count) + {} + + bool null() + { + return events_left-- > 0; + } + + bool boolean(bool /*unused*/) + { + return events_left-- > 0; + } + + bool number_integer(json::number_integer_t /*unused*/) + { + return events_left-- > 0; + } + + bool number_unsigned(json::number_unsigned_t /*unused*/) + { + return events_left-- > 0; + } + + bool number_float(json::number_float_t /*unused*/, const std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool string(std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool binary(std::vector& /*unused*/) + { + return events_left-- > 0; + } + + bool start_object(std::size_t /*unused*/) + { + return events_left-- > 0; + } + + bool key(std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool end_object() + { + return events_left-- > 0; + } + + bool start_array(std::size_t /*unused*/) + { + return events_left-- > 0; + } + + bool end_array() + { + return events_left-- > 0; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& /*unused*/) // NOLINT(readability-convert-member-functions-to-static) + { + return false; + } + + private: + int events_left = 0; +}; +} // namespace + +TEST_CASE("BON8") +{ + SECTION("individual values") + { + SECTION("discarded") + { + // discarded values are not serialized + json j = json::value_t::discarded; + const auto result = json::to_bon8(j); + CHECK(result.empty()); + } + + SECTION("null") + { + json j = nullptr; + std::vector expected = {0xFA}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("boolean") + { + SECTION("true") + { + json j = true; + std::vector expected = {0xF9}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("false") + { + json j = false; + std::vector expected = {0xF8}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("floating-point numbers") + { + SECTION("-1.0") + { + json j = -1.0; + std::vector expected = {0xFB}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("0.0") + { + json j = 0.0; + std::vector expected = {0xFC}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("1.0") + { + json j = 1.0; + std::vector expected = {0xFD}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("string") + { + SECTION("empty string") + { + json j = ""; + std::vector expected = {0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("other strings") + { + json j = "This is a string."; + std::vector expected = {'T', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 's', 't', 'r', 'i', 'n', 'g', '.'}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("array") + { + SECTION("array with count") + { + SECTION("empty array") + { + json j = json::array(); + std::vector expected = {0x80}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[false]") + { + json j = {false}; + std::vector expected = {0x81, 0xF8}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[false, null]") + { + json j = {false, nullptr}; + std::vector expected = {0x82, 0xF8, 0xFA}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[false, null, true]") + { + json j = {false, nullptr, true}; + std::vector expected = {0x83, 0xF8, 0xFA, 0xF9}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[false, null, true, 1.0]") + { + json j = {false, nullptr, true, 1.0}; + std::vector expected = {0x84, 0xF8, 0xFA, 0xF9, 0xFD}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[\"s\", \"s\"]") + { + json j = {"s", "s"}; + std::vector expected = {0x82, 's', 0xFF, 's'}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[\"\", \"s\"]") + { + json j = {"", "s"}; + std::vector expected = {0x82, 0xFF, 's'}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("array without count") + { + SECTION("[false, null, true, 1.0, [], 0.0]") + { + json j = {false, nullptr, true, 1.0, json::array(), 0.0}; + std::vector expected = {0x85, 0xF8, 0xFA, 0xF9, 0xFD, 0x80, 0xFC, 0xFE}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + } + + SECTION("object") + { + SECTION("object with count") + { + SECTION("empty object") + { + json j = json::object(); + std::vector expected = {0x86}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("{\"foo\": null}") + { + json j = {{"foo", nullptr}}; + std::vector expected = {0x87, 'f', 'o', 'o', 0xFA}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("{\"\": true, \"foo\": null}") + { + json j = {{"", true}, {"foo", nullptr}}; + std::vector expected = {0x88, 0xFF, 0xF9, 'f', 'o', 'o', 0xFA}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("{\"a\": \"\", \"c\": \"d\"}") + { + json j = {{"a", ""}, {"c", "d"}}; + std::vector expected = {0x88, 'a', 0xFF, 0xFF, 'c', 0xFF, 'd'}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + } + } +} From e6518ff2d61393d6c536053671523901c9e714a3 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 4 Sep 2021 14:54:54 +0200 Subject: [PATCH 02/32] :construction: add integer support --- .../nlohmann/detail/output/binary_writer.hpp | 116 +++++- single_include/nlohmann/json.hpp | 116 +++++- test/src/unit-bon8.cpp | 362 ++++++++++++++++++ 3 files changed, 566 insertions(+), 28 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 8a999a613..20b553ac5 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -944,27 +944,42 @@ class binary_writer break; } + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned > std::numeric_limits::max()) + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); + } + write_bon8_integer(static_cast(j.m_value.number_unsigned)); + break; + } + + case value_t::number_integer: + { + write_bon8_integer(j.m_value.number_integer); + break; + } + case value_t::number_float: { // special values if (j.m_value.number_float == -1.0) { oa->write_character(to_char_type(0xFB)); - break; } - if (j.m_value.number_float == 0.0) + else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) { oa->write_character(to_char_type(0xFC)); - break; } - if (j.m_value.number_float == 1.0) + else if (j.m_value.number_float == 1.0) { oa->write_character(to_char_type(0xFD)); - break; } - - // write float with prefix - write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + else + { + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + } break; } @@ -1680,6 +1695,79 @@ class binary_writer // BON8 // ////////// + void write_bon8_integer(typename BasicJsonType::number_integer_t value) + { + if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) + { + // 64 bit integers + oa->write_character(0x8D); + write_number(static_cast(value)); + } + else if (value < -33554432 || value > 67108863) + { + // 32 bit integers + oa->write_character(0x8C); + write_number(static_cast(value)); + } + else if (value < -262144) + { + JSON_ASSERT(value >= -33554432); + value = -value - 1; + oa->write_character(0xF0 + (value >> 22 & 0x07)); + oa->write_character(0xC0 + (value >> 16 & 0x3F)); + oa->write_character(value >> 8); + oa->write_character(value); + } + else if (value < -1920) + { + JSON_ASSERT(value >= -262144); + value = -value - 1; + oa->write_character(0xE0 + (value >> 14 & 0x0F)); + oa->write_character(0xC0 + (value >> 8 & 0x3F)); + oa->write_character(value); + } + else if (value < -10) + { + JSON_ASSERT(value >= -1920); + value = -value - 1; + oa->write_character(0xC2 + (value >> 6 & 0x1F)); + oa->write_character(0xC0 + (value & 0x3F)); + } + else if (value < 0) + { + JSON_ASSERT(value >= -10); + value = -value - 1; + oa->write_character(0xB8 + value); + } + else if (value <= 39) + { + JSON_ASSERT(value >= 0); + oa->write_character(0x90 + value); + } + else if (value <= 3839) + { + JSON_ASSERT(value >= 0); + oa->write_character(0xC2 + (value >> 7 & 0x1F)); + oa->write_character(value & 0x7F); + } + else if (value <= 524287) + { + JSON_ASSERT(value >= 0); + oa->write_character(0xE0 + (value >> 15 & 0x0F)); + oa->write_character(value >> 8 & 0x7F); + oa->write_character(value); + } + else + { + JSON_ASSERT(value >= 0); + JSON_ASSERT(value <= 67108863); + oa->write_character(0xF0 + (value >> 23 & 0x17)); + oa->write_character(value >> 16 & 0x7F); + oa->write_character(value >> 8); + oa->write_character(value); + } + } + static constexpr CharType get_bon8_float_prefix(float /*unused*/) { return to_char_type(0x8E); @@ -1735,13 +1823,13 @@ class binary_writer switch (format) { case input_format_t::cbor: - get_cbor_float_prefix(static_cast(n)); + oa->write_character(get_cbor_float_prefix(static_cast(n))); break; case input_format_t::msgpack: - get_msgpack_float_prefix(static_cast(n)); + oa->write_character(get_msgpack_float_prefix(static_cast(n))); break; case input_format_t::bon8: - get_bon8_float_prefix(static_cast(n)); + oa->write_character(get_bon8_float_prefix(static_cast(n))); break; default: break; @@ -1753,13 +1841,13 @@ class binary_writer switch (format) { case input_format_t::cbor: - get_cbor_float_prefix(n); + oa->write_character(get_cbor_float_prefix(n)); break; case input_format_t::msgpack: - get_msgpack_float_prefix(n); + oa->write_character(get_msgpack_float_prefix(n)); break; case input_format_t::bon8: - get_bon8_float_prefix(n); + oa->write_character(get_bon8_float_prefix(n)); break; default: break; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 8c7b3d1b6..1eb2347a9 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -14516,27 +14516,42 @@ class binary_writer break; } + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned > std::numeric_limits::max()) + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); + } + write_bon8_integer(static_cast(j.m_value.number_unsigned)); + break; + } + + case value_t::number_integer: + { + write_bon8_integer(j.m_value.number_integer); + break; + } + case value_t::number_float: { // special values if (j.m_value.number_float == -1.0) { oa->write_character(to_char_type(0xFB)); - break; } - if (j.m_value.number_float == 0.0) + else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) { oa->write_character(to_char_type(0xFC)); - break; } - if (j.m_value.number_float == 1.0) + else if (j.m_value.number_float == 1.0) { oa->write_character(to_char_type(0xFD)); - break; } - - // write float with prefix - write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + else + { + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + } break; } @@ -15252,6 +15267,79 @@ class binary_writer // BON8 // ////////// + void write_bon8_integer(typename BasicJsonType::number_integer_t value) + { + if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) + { + // 64 bit integers + oa->write_character(0x8D); + write_number(static_cast(value)); + } + else if (value < -33554432 || value > 67108863) + { + // 32 bit integers + oa->write_character(0x8C); + write_number(static_cast(value)); + } + else if (value < -262144) + { + JSON_ASSERT(value >= -33554432); + value = -value - 1; + oa->write_character(0xF0 + (value >> 22 & 0x07)); + oa->write_character(0xC0 + (value >> 16 & 0x3F)); + oa->write_character(value >> 8); + oa->write_character(value); + } + else if (value < -1920) + { + JSON_ASSERT(value >= -262144); + value = -value - 1; + oa->write_character(0xE0 + (value >> 14 & 0x0F)); + oa->write_character(0xC0 + (value >> 8 & 0x3F)); + oa->write_character(value); + } + else if (value < -10) + { + JSON_ASSERT(value >= -1920); + value = -value - 1; + oa->write_character(0xC2 + (value >> 6 & 0x1F)); + oa->write_character(0xC0 + (value & 0x3F)); + } + else if (value < 0) + { + JSON_ASSERT(value >= -10); + value = -value - 1; + oa->write_character(0xB8 + value); + } + else if (value <= 39) + { + JSON_ASSERT(value >= 0); + oa->write_character(0x90 + value); + } + else if (value <= 3839) + { + JSON_ASSERT(value >= 0); + oa->write_character(0xC2 + (value >> 7 & 0x1F)); + oa->write_character(value & 0x7F); + } + else if (value <= 524287) + { + JSON_ASSERT(value >= 0); + oa->write_character(0xE0 + (value >> 15 & 0x0F)); + oa->write_character(value >> 8 & 0x7F); + oa->write_character(value); + } + else + { + JSON_ASSERT(value >= 0); + JSON_ASSERT(value <= 67108863); + oa->write_character(0xF0 + (value >> 23 & 0x17)); + oa->write_character(value >> 16 & 0x7F); + oa->write_character(value >> 8); + oa->write_character(value); + } + } + static constexpr CharType get_bon8_float_prefix(float /*unused*/) { return to_char_type(0x8E); @@ -15307,13 +15395,13 @@ class binary_writer switch (format) { case input_format_t::cbor: - get_cbor_float_prefix(static_cast(n)); + oa->write_character(get_cbor_float_prefix(static_cast(n))); break; case input_format_t::msgpack: - get_msgpack_float_prefix(static_cast(n)); + oa->write_character(get_msgpack_float_prefix(static_cast(n))); break; case input_format_t::bon8: - get_bon8_float_prefix(static_cast(n)); + oa->write_character(get_bon8_float_prefix(static_cast(n))); break; default: break; @@ -15325,13 +15413,13 @@ class binary_writer switch (format) { case input_format_t::cbor: - get_cbor_float_prefix(n); + oa->write_character(get_cbor_float_prefix(n)); break; case input_format_t::msgpack: - get_msgpack_float_prefix(n); + oa->write_character(get_msgpack_float_prefix(n)); break; case input_format_t::bon8: - get_bon8_float_prefix(n); + oa->write_character(get_bon8_float_prefix(n)); break; default: break; diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 3c17ba46b..e74b0eabd 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -155,6 +155,360 @@ TEST_CASE("BON8") } } + SECTION("unsigned integers") + { + SECTION("0..39") + { + SECTION("0") + { + json j = 0U; + std::vector expected = {0x90}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("39") + { + json j = 39U; + std::vector expected = {0xB7}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("40..3839") + { + SECTION("40") + { + json j = 40U; + std::vector expected = {0xC2, 0x28}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("3839") + { + json j = 3839U; + std::vector expected = {0xDF, 0x7F}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("3840..524287") + { + SECTION("3840") + { + json j = 3840U; + std::vector expected = {0xE0, 0x0F, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("524287") + { + json j = 524287U; + std::vector expected = {0xEF, 0x7F, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("524288..67108863") + { + SECTION("524288") + { + json j = 524288U; + std::vector expected = {0xF0, 0x08, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("67108863") + { + json j = 67108863U; + std::vector expected = {0xF7, 0x7F, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("67108864..2147483647 (int32max)") + { + SECTION("67108864") + { + json j = 67108864U; + std::vector expected = {0x8C, 0x04, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("2147483647 (int32max)") + { + json j = 2147483647U; + std::vector expected = {0x8C, 0x7F, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("2147483648..9223372036854775807 (int64max)") + { + SECTION("2147483648") + { + json j = 2147483648U; + std::vector expected = {0x8D, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("9223372036854775807 (int64max)") + { + json j = 9223372036854775807U; + std::vector expected = {0x8D, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("greater than int64max") + { + json j = 9223372036854775808U; + CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.out_of_range.407] integer number 9223372036854775808 cannot be represented by BON8 as it does not fit int64", json::out_of_range); + } + } + + SECTION("signed integers") + { + SECTION("-9223372036854775808 (int64min)..-2147483649") + { + SECTION("-9223372036854775808") + { + json j = INT64_MIN; + std::vector expected = {0x8D, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-2147483649") + { + json j = -2147483649; + std::vector expected = {0x8D, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("-2147483648 (int32min)..-33554433") + { + SECTION("-2147483648") + { + json j = -2147483648; + std::vector expected = {0x8C, 0x80, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-33554433") + { + json j = -33554433; + std::vector expected = {0x8C, 0xFD, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("-33554432..-262145") + { + SECTION("-33554432") + { + json j = -33554432; + std::vector expected = {0xF7, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-262145") + { + json j = -262145; + std::vector expected = {0xF0, 0xC4, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("-262144..-1921") + { + SECTION("-262144") + { + json j = -262144; + std::vector expected = {0xEF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-1921") + { + json j = -1921; + std::vector expected = {0xE0, 0xC7, 0x80}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("-1920..-11") + { + SECTION("-1920") + { + json j = -1920; + std::vector expected = {0xDF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-11") + { + json j = -11; + std::vector expected = {0xC2, 0xCA}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("-10..-1") + { + SECTION("-10") + { + json j = -10; + std::vector expected = {0xC1}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-1") + { + json j = -1; + std::vector expected = {0xB8}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("0..39") + { + SECTION("0") + { + json j = 0; + std::vector expected = {0x90}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("39") + { + json j = 39; + std::vector expected = {0xB7}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("40..3839") + { + SECTION("40") + { + json j = 40; + std::vector expected = {0xC2, 0x28}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("3839") + { + json j = 3839; + std::vector expected = {0xDF, 0x7F}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("3840..524287") + { + SECTION("3840") + { + json j = 3840; + std::vector expected = {0xE0, 0x0F, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("524287") + { + json j = 524287; + std::vector expected = {0xEF, 0x7F, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("524288..67108863") + { + SECTION("524288") + { + json j = 524288; + std::vector expected = {0xF0, 0x08, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("67108863") + { + json j = 67108863; + std::vector expected = {0xF7, 0x7F, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("67108864..2147483647 (int32max)") + { + SECTION("67108864") + { + json j = 67108864; + std::vector expected = {0x8C, 0x04, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("2147483647 (int32max)") + { + json j = 2147483647; + std::vector expected = {0x8C, 0x7F, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + + SECTION("2147483648..9223372036854775807 (int64max)") + { + SECTION("2147483648") + { + json j = 2147483648; + std::vector expected = {0x8D, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("9223372036854775807 (int64max)") + { + json j = 9223372036854775807; + std::vector expected = {0x8D, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } + } + SECTION("floating-point numbers") { SECTION("-1.0") @@ -180,6 +534,14 @@ TEST_CASE("BON8") const auto result = json::to_bon8(j); CHECK(result == expected); } + + SECTION("-0.0") + { + json j = -0.0; + std::vector expected = {0x8E, 0x80, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } } SECTION("string") From 6cf42d1b5598048effcfe57809e20d37be26f28a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 4 Sep 2021 15:54:17 +0200 Subject: [PATCH 03/32] :rotating_light: fix warnings --- .../nlohmann/detail/input/binary_reader.hpp | 2 + .../nlohmann/detail/output/binary_writer.hpp | 54 ++++++++++-------- single_include/nlohmann/json.hpp | 56 +++++++++++-------- 3 files changed, 64 insertions(+), 48 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index f1b65ba54..2cb7bc769 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -120,6 +120,7 @@ class binary_reader result = parse_ubjson_internal(); break; + case input_format_t::bon8: // LCOV_EXCL_LINE case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -2496,6 +2497,7 @@ class binary_reader error_msg += "BSON"; break; + case input_format_t::bon8: // LCOV_EXCL_LINE case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 20b553ac5..da57f5766 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -946,7 +946,7 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned > std::numeric_limits::max()) + if (j.m_value.number_unsigned > (std::numeric_limits::max)()) { JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); } @@ -1697,74 +1697,74 @@ class binary_writer void write_bon8_integer(typename BasicJsonType::number_integer_t value) { - if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) { // 64 bit integers - oa->write_character(0x8D); + oa->write_character(to_char_type(0x8D)); write_number(static_cast(value)); } else if (value < -33554432 || value > 67108863) { // 32 bit integers - oa->write_character(0x8C); + oa->write_character(to_char_type(0x8C)); write_number(static_cast(value)); } else if (value < -262144) { JSON_ASSERT(value >= -33554432); value = -value - 1; - oa->write_character(0xF0 + (value >> 22 & 0x07)); - oa->write_character(0xC0 + (value >> 16 & 0x3F)); - oa->write_character(value >> 8); - oa->write_character(value); + oa->write_character(to_char_type(0xF0 + (value >> 22 & 0x07))); + oa->write_character(to_char_type(0xC0 + (value >> 16 & 0x3F))); + oa->write_character(to_char_type(value >> 8)); + oa->write_character(to_char_type(value)); } else if (value < -1920) { JSON_ASSERT(value >= -262144); value = -value - 1; - oa->write_character(0xE0 + (value >> 14 & 0x0F)); - oa->write_character(0xC0 + (value >> 8 & 0x3F)); - oa->write_character(value); + oa->write_character(to_char_type(0xE0 + (value >> 14 & 0x0F))); + oa->write_character(to_char_type(0xC0 + (value >> 8 & 0x3F))); + oa->write_character(to_char_type(value)); } else if (value < -10) { JSON_ASSERT(value >= -1920); value = -value - 1; - oa->write_character(0xC2 + (value >> 6 & 0x1F)); - oa->write_character(0xC0 + (value & 0x3F)); + oa->write_character(to_char_type(0xC2 + (value >> 6 & 0x1F))); + oa->write_character(to_char_type(0xC0 + (value & 0x3F))); } else if (value < 0) { JSON_ASSERT(value >= -10); value = -value - 1; - oa->write_character(0xB8 + value); + oa->write_character(to_char_type(0xB8 + value)); } else if (value <= 39) { JSON_ASSERT(value >= 0); - oa->write_character(0x90 + value); + oa->write_character(to_char_type(0x90 + value)); } else if (value <= 3839) { JSON_ASSERT(value >= 0); - oa->write_character(0xC2 + (value >> 7 & 0x1F)); - oa->write_character(value & 0x7F); + oa->write_character(to_char_type(0xC2 + (value >> 7 & 0x1F))); + oa->write_character(to_char_type(value & 0x7F)); } else if (value <= 524287) { JSON_ASSERT(value >= 0); - oa->write_character(0xE0 + (value >> 15 & 0x0F)); - oa->write_character(value >> 8 & 0x7F); - oa->write_character(value); + oa->write_character(to_char_type(0xE0 + (value >> 15 & 0x0F))); + oa->write_character(to_char_type(value >> 8 & 0x7F)); + oa->write_character(to_char_type(value)); } else { JSON_ASSERT(value >= 0); JSON_ASSERT(value <= 67108863); - oa->write_character(0xF0 + (value >> 23 & 0x17)); - oa->write_character(value >> 16 & 0x7F); - oa->write_character(value >> 8); - oa->write_character(value); + oa->write_character(to_char_type(0xF0 + (value >> 23 & 0x17))); + oa->write_character(to_char_type(value >> 16 & 0x7F)); + oa->write_character(to_char_type(value >> 8)); + oa->write_character(to_char_type(value)); } } @@ -1831,6 +1831,9 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(static_cast(n))); break; + case input_format_t::bson: + case input_format_t::json: + case input_format_t::ubjson: default: break; } @@ -1849,6 +1852,9 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(n)); break; + case input_format_t::bson: + case input_format_t::json: + case input_format_t::ubjson: default: break; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1eb2347a9..bfd0c16df 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8414,6 +8414,7 @@ class binary_reader result = parse_ubjson_internal(); break; + case input_format_t::bon8: // LCOV_EXCL_LINE case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -10790,6 +10791,7 @@ class binary_reader error_msg += "BSON"; break; + case input_format_t::bon8: // LCOV_EXCL_LINE case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -14518,7 +14520,7 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned > std::numeric_limits::max()) + if (j.m_value.number_unsigned > (std::numeric_limits::max)()) { JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); } @@ -15269,74 +15271,74 @@ class binary_writer void write_bon8_integer(typename BasicJsonType::number_integer_t value) { - if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) { // 64 bit integers - oa->write_character(0x8D); + oa->write_character(to_char_type(0x8D)); write_number(static_cast(value)); } else if (value < -33554432 || value > 67108863) { // 32 bit integers - oa->write_character(0x8C); + oa->write_character(to_char_type(0x8C)); write_number(static_cast(value)); } else if (value < -262144) { JSON_ASSERT(value >= -33554432); value = -value - 1; - oa->write_character(0xF0 + (value >> 22 & 0x07)); - oa->write_character(0xC0 + (value >> 16 & 0x3F)); - oa->write_character(value >> 8); - oa->write_character(value); + oa->write_character(to_char_type(0xF0 + (value >> 22 & 0x07))); + oa->write_character(to_char_type(0xC0 + (value >> 16 & 0x3F))); + oa->write_character(to_char_type(value >> 8)); + oa->write_character(to_char_type(value)); } else if (value < -1920) { JSON_ASSERT(value >= -262144); value = -value - 1; - oa->write_character(0xE0 + (value >> 14 & 0x0F)); - oa->write_character(0xC0 + (value >> 8 & 0x3F)); - oa->write_character(value); + oa->write_character(to_char_type(0xE0 + (value >> 14 & 0x0F))); + oa->write_character(to_char_type(0xC0 + (value >> 8 & 0x3F))); + oa->write_character(to_char_type(value)); } else if (value < -10) { JSON_ASSERT(value >= -1920); value = -value - 1; - oa->write_character(0xC2 + (value >> 6 & 0x1F)); - oa->write_character(0xC0 + (value & 0x3F)); + oa->write_character(to_char_type(0xC2 + (value >> 6 & 0x1F))); + oa->write_character(to_char_type(0xC0 + (value & 0x3F))); } else if (value < 0) { JSON_ASSERT(value >= -10); value = -value - 1; - oa->write_character(0xB8 + value); + oa->write_character(to_char_type(0xB8 + value)); } else if (value <= 39) { JSON_ASSERT(value >= 0); - oa->write_character(0x90 + value); + oa->write_character(to_char_type(0x90 + value)); } else if (value <= 3839) { JSON_ASSERT(value >= 0); - oa->write_character(0xC2 + (value >> 7 & 0x1F)); - oa->write_character(value & 0x7F); + oa->write_character(to_char_type(0xC2 + (value >> 7 & 0x1F))); + oa->write_character(to_char_type(value & 0x7F)); } else if (value <= 524287) { JSON_ASSERT(value >= 0); - oa->write_character(0xE0 + (value >> 15 & 0x0F)); - oa->write_character(value >> 8 & 0x7F); - oa->write_character(value); + oa->write_character(to_char_type(0xE0 + (value >> 15 & 0x0F))); + oa->write_character(to_char_type(value >> 8 & 0x7F)); + oa->write_character(to_char_type(value)); } else { JSON_ASSERT(value >= 0); JSON_ASSERT(value <= 67108863); - oa->write_character(0xF0 + (value >> 23 & 0x17)); - oa->write_character(value >> 16 & 0x7F); - oa->write_character(value >> 8); - oa->write_character(value); + oa->write_character(to_char_type(0xF0 + (value >> 23 & 0x17))); + oa->write_character(to_char_type(value >> 16 & 0x7F)); + oa->write_character(to_char_type(value >> 8)); + oa->write_character(to_char_type(value)); } } @@ -15403,6 +15405,9 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(static_cast(n))); break; + case input_format_t::bson: + case input_format_t::json: + case input_format_t::ubjson: default: break; } @@ -15421,6 +15426,9 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(n)); break; + case input_format_t::bson: + case input_format_t::json: + case input_format_t::ubjson: default: break; } From 526bee2bfbc517ab58e04fd4674daddf5257d324 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 4 Sep 2021 22:36:43 +0200 Subject: [PATCH 04/32] :rotating_light: fix warnings --- .../nlohmann/detail/output/binary_writer.hpp | 47 ++++++++++--------- single_include/nlohmann/json.hpp | 47 ++++++++++--------- test/src/unit-bon8.cpp | 4 +- 3 files changed, 50 insertions(+), 48 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index da57f5766..437a86cd8 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -946,7 +946,7 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned > (std::numeric_limits::max)()) + if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) { JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); } @@ -1005,7 +1005,7 @@ class binary_writer if (N <= 4) { // start array with count (80..84) - oa->write_character(to_char_type(0x80 + N)); + oa->write_character(static_cast(0x80 + N)); } else { @@ -1045,7 +1045,7 @@ class binary_writer if (N <= 4) { // start object with count (86..8A) - oa->write_character(to_char_type(0x86 + N)); + oa->write_character(static_cast(0x86 + N)); } else { @@ -1084,6 +1084,7 @@ class binary_writer break; } + case value_t::binary: case value_t::discarded: default: break; @@ -1713,58 +1714,58 @@ class binary_writer { JSON_ASSERT(value >= -33554432); value = -value - 1; - oa->write_character(to_char_type(0xF0 + (value >> 22 & 0x07))); - oa->write_character(to_char_type(0xC0 + (value >> 16 & 0x3F))); - oa->write_character(to_char_type(value >> 8)); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xF0 + (value >> 22 & 0x07))); + oa->write_character(static_cast(0xC0 + (value >> 16 & 0x3F))); + oa->write_character(static_cast(value >> 8)); + oa->write_character(static_cast(value)); } else if (value < -1920) { JSON_ASSERT(value >= -262144); value = -value - 1; - oa->write_character(to_char_type(0xE0 + (value >> 14 & 0x0F))); - oa->write_character(to_char_type(0xC0 + (value >> 8 & 0x3F))); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xE0 + (value >> 14 & 0x0F))); + oa->write_character(static_cast(0xC0 + (value >> 8 & 0x3F))); + oa->write_character(static_cast(value)); } else if (value < -10) { JSON_ASSERT(value >= -1920); value = -value - 1; - oa->write_character(to_char_type(0xC2 + (value >> 6 & 0x1F))); - oa->write_character(to_char_type(0xC0 + (value & 0x3F))); + oa->write_character(static_cast(0xC2 + (value >> 6 & 0x1F))); + oa->write_character(static_cast(0xC0 + (value & 0x3F))); } else if (value < 0) { JSON_ASSERT(value >= -10); value = -value - 1; - oa->write_character(to_char_type(0xB8 + value)); + oa->write_character(static_cast(0xB8 + value)); } else if (value <= 39) { JSON_ASSERT(value >= 0); - oa->write_character(to_char_type(0x90 + value)); + oa->write_character(static_cast(0x90 + value)); } else if (value <= 3839) { JSON_ASSERT(value >= 0); - oa->write_character(to_char_type(0xC2 + (value >> 7 & 0x1F))); - oa->write_character(to_char_type(value & 0x7F)); + oa->write_character(static_cast(0xC2 + (value >> 7 & 0x1F))); + oa->write_character(static_cast(value & 0x7F)); } else if (value <= 524287) { JSON_ASSERT(value >= 0); - oa->write_character(to_char_type(0xE0 + (value >> 15 & 0x0F))); - oa->write_character(to_char_type(value >> 8 & 0x7F)); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xE0 + (value >> 15 & 0x0F))); + oa->write_character(static_cast(value >> 8 & 0x7F)); + oa->write_character(static_cast(value)); } else { JSON_ASSERT(value >= 0); JSON_ASSERT(value <= 67108863); - oa->write_character(to_char_type(0xF0 + (value >> 23 & 0x17))); - oa->write_character(to_char_type(value >> 16 & 0x7F)); - oa->write_character(to_char_type(value >> 8)); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xF0 + (value >> 23 & 0x17))); + oa->write_character(static_cast(value >> 16 & 0x7F)); + oa->write_character(static_cast(value >> 8)); + oa->write_character(static_cast(value)); } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index bfd0c16df..3876db03e 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -14520,7 +14520,7 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned > (std::numeric_limits::max)()) + if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) { JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); } @@ -14579,7 +14579,7 @@ class binary_writer if (N <= 4) { // start array with count (80..84) - oa->write_character(to_char_type(0x80 + N)); + oa->write_character(static_cast(0x80 + N)); } else { @@ -14619,7 +14619,7 @@ class binary_writer if (N <= 4) { // start object with count (86..8A) - oa->write_character(to_char_type(0x86 + N)); + oa->write_character(static_cast(0x86 + N)); } else { @@ -14658,6 +14658,7 @@ class binary_writer break; } + case value_t::binary: case value_t::discarded: default: break; @@ -15287,58 +15288,58 @@ class binary_writer { JSON_ASSERT(value >= -33554432); value = -value - 1; - oa->write_character(to_char_type(0xF0 + (value >> 22 & 0x07))); - oa->write_character(to_char_type(0xC0 + (value >> 16 & 0x3F))); - oa->write_character(to_char_type(value >> 8)); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xF0 + (value >> 22 & 0x07))); + oa->write_character(static_cast(0xC0 + (value >> 16 & 0x3F))); + oa->write_character(static_cast(value >> 8)); + oa->write_character(static_cast(value)); } else if (value < -1920) { JSON_ASSERT(value >= -262144); value = -value - 1; - oa->write_character(to_char_type(0xE0 + (value >> 14 & 0x0F))); - oa->write_character(to_char_type(0xC0 + (value >> 8 & 0x3F))); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xE0 + (value >> 14 & 0x0F))); + oa->write_character(static_cast(0xC0 + (value >> 8 & 0x3F))); + oa->write_character(static_cast(value)); } else if (value < -10) { JSON_ASSERT(value >= -1920); value = -value - 1; - oa->write_character(to_char_type(0xC2 + (value >> 6 & 0x1F))); - oa->write_character(to_char_type(0xC0 + (value & 0x3F))); + oa->write_character(static_cast(0xC2 + (value >> 6 & 0x1F))); + oa->write_character(static_cast(0xC0 + (value & 0x3F))); } else if (value < 0) { JSON_ASSERT(value >= -10); value = -value - 1; - oa->write_character(to_char_type(0xB8 + value)); + oa->write_character(static_cast(0xB8 + value)); } else if (value <= 39) { JSON_ASSERT(value >= 0); - oa->write_character(to_char_type(0x90 + value)); + oa->write_character(static_cast(0x90 + value)); } else if (value <= 3839) { JSON_ASSERT(value >= 0); - oa->write_character(to_char_type(0xC2 + (value >> 7 & 0x1F))); - oa->write_character(to_char_type(value & 0x7F)); + oa->write_character(static_cast(0xC2 + (value >> 7 & 0x1F))); + oa->write_character(static_cast(value & 0x7F)); } else if (value <= 524287) { JSON_ASSERT(value >= 0); - oa->write_character(to_char_type(0xE0 + (value >> 15 & 0x0F))); - oa->write_character(to_char_type(value >> 8 & 0x7F)); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xE0 + (value >> 15 & 0x0F))); + oa->write_character(static_cast(value >> 8 & 0x7F)); + oa->write_character(static_cast(value)); } else { JSON_ASSERT(value >= 0); JSON_ASSERT(value <= 67108863); - oa->write_character(to_char_type(0xF0 + (value >> 23 & 0x17))); - oa->write_character(to_char_type(value >> 16 & 0x7F)); - oa->write_character(to_char_type(value >> 8)); - oa->write_character(to_char_type(value)); + oa->write_character(static_cast(0xF0 + (value >> 23 & 0x17))); + oa->write_character(static_cast(value >> 16 & 0x7F)); + oa->write_character(static_cast(value >> 8)); + oa->write_character(static_cast(value)); } } diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index e74b0eabd..253f6d51f 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -292,7 +292,7 @@ TEST_CASE("BON8") SECTION("-2147483649") { - json j = -2147483649; + json j = -2147483649L; std::vector expected = {0x8D, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); @@ -303,7 +303,7 @@ TEST_CASE("BON8") { SECTION("-2147483648") { - json j = -2147483648; + json j = -2147483648L; std::vector expected = {0x8C, 0x80, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); From f16465c614bfbae85358634aa5de90464171509c Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 4 Sep 2021 22:48:08 +0200 Subject: [PATCH 05/32] :rotating_light: fix warnings --- test/src/unit-bon8.cpp | 78 ------------------------------------------ 1 file changed, 78 deletions(-) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 253f6d51f..ca6c90401 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -38,84 +38,6 @@ using nlohmann::json; #include #include "test_utils.hpp" -namespace -{ -class SaxCountdown -{ - public: - explicit SaxCountdown(const int count) : events_left(count) - {} - - bool null() - { - return events_left-- > 0; - } - - bool boolean(bool /*unused*/) - { - return events_left-- > 0; - } - - bool number_integer(json::number_integer_t /*unused*/) - { - return events_left-- > 0; - } - - bool number_unsigned(json::number_unsigned_t /*unused*/) - { - return events_left-- > 0; - } - - bool number_float(json::number_float_t /*unused*/, const std::string& /*unused*/) - { - return events_left-- > 0; - } - - bool string(std::string& /*unused*/) - { - return events_left-- > 0; - } - - bool binary(std::vector& /*unused*/) - { - return events_left-- > 0; - } - - bool start_object(std::size_t /*unused*/) - { - return events_left-- > 0; - } - - bool key(std::string& /*unused*/) - { - return events_left-- > 0; - } - - bool end_object() - { - return events_left-- > 0; - } - - bool start_array(std::size_t /*unused*/) - { - return events_left-- > 0; - } - - bool end_array() - { - return events_left-- > 0; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& /*unused*/) // NOLINT(readability-convert-member-functions-to-static) - { - return false; - } - - private: - int events_left = 0; -}; -} // namespace - TEST_CASE("BON8") { SECTION("individual values") From 78e59f502bfb4f27397620c7f796c84e1d750521 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 4 Sep 2021 22:59:43 +0200 Subject: [PATCH 06/32] :rotating_light: fix warnings --- test/src/unit-bon8.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index ca6c90401..ba9f9173f 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -214,7 +214,8 @@ TEST_CASE("BON8") SECTION("-2147483649") { - json j = -2147483649L; + // cannot use -2147483649 directly, see https://developercommunity.visualstudio.com/t/-2147483648-c4146-error/141813#T-N229960 + json j = -2147483648 - 1; std::vector expected = {0x8D, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); @@ -225,7 +226,8 @@ TEST_CASE("BON8") { SECTION("-2147483648") { - json j = -2147483648L; + // cannot use -2147483648 directly, see https://developercommunity.visualstudio.com/t/-2147483648-c4146-error/141813#T-N229960 + json j = -2147483647 - 1; std::vector expected = {0x8C, 0x80, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); From b4f2cf769a1fc2393943bdcbe631b8bfc113154b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 4 Sep 2021 23:26:37 +0200 Subject: [PATCH 07/32] :rotating_light: fix warnings --- test/src/unit-bon8.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index ba9f9173f..6dd1f610f 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -215,7 +215,7 @@ TEST_CASE("BON8") SECTION("-2147483649") { // cannot use -2147483649 directly, see https://developercommunity.visualstudio.com/t/-2147483648-c4146-error/141813#T-N229960 - json j = -2147483648 - 1; + json j = std::int64_t(-2147483647) - 2; std::vector expected = {0x8D, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); From f444d9c14f761faf86a0189991ddcb598a206ef7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 5 Sep 2021 09:25:58 +0200 Subject: [PATCH 08/32] :white_check_mark: improve coverage --- test/src/unit-bon8.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 6dd1f610f..4d8d2546a 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -596,6 +596,17 @@ TEST_CASE("BON8") CHECK(result == expected); } } + + SECTION("object without count") + { + SECTION("{\"one\": 1, \"two\": 2, \"three\": 3, \"four\": 4, \"five\": 5}") + { + json j = R"({"one": 1, "two": 2, "three": 3, "four": 4, "five": 5})"_json; + std::vector expected = {0x8b, 'f', 'i', 'v', 'e', 0x95, 'f', 'o', 'u', 'r', 0x94, 'o', 'n', 'e', 0x91, 't', 'h', 'r', 'e', 'e', 0x93, 't', 'w', 'o', 0x92, 0xFE}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + } } } } From 62487fcf68738ba3cc32225d7a99d00dda56ebe4 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 5 Sep 2021 11:26:12 +0200 Subject: [PATCH 09/32] :white_check_mark: improve coverage --- .../nlohmann/detail/output/binary_writer.hpp | 337 +++++++++--------- single_include/nlohmann/json.hpp | 337 +++++++++--------- test/src/unit-bon8.cpp | 40 ++- 3 files changed, 392 insertions(+), 322 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 437a86cd8..5f0fb55d8 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -928,166 +928,10 @@ class binary_writer */ void write_bon8(const BasicJsonType& j) { - switch (j.type()) + const bool last_written_value_is_string = write_bon8_internal(j); + if (last_written_value_is_string) { - case value_t::null: - { - oa->write_character(to_char_type(0xFA)); - break; - } - - case value_t::boolean: - { - oa->write_character(j.m_value.boolean - ? to_char_type(0xF9) - : to_char_type(0xF8)); - break; - } - - case value_t::number_unsigned: - { - if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) - { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); - } - write_bon8_integer(static_cast(j.m_value.number_unsigned)); - break; - } - - case value_t::number_integer: - { - write_bon8_integer(j.m_value.number_integer); - break; - } - - case value_t::number_float: - { - // special values - if (j.m_value.number_float == -1.0) - { - oa->write_character(to_char_type(0xFB)); - } - else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) - { - oa->write_character(to_char_type(0xFC)); - } - else if (j.m_value.number_float == 1.0) - { - oa->write_character(to_char_type(0xFD)); - } - else - { - // write float with prefix - write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); - } - break; - } - - case value_t::string: - { - // empty string: use end-of-text symbol - if (j.m_value.string->empty()) - { - oa->write_character(to_char_type(0xFF)); - break; - } - - // write strings as is - oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); - break; - } - - case value_t::array: - { - const auto N = j.m_value.array->size(); - if (N <= 4) - { - // start array with count (80..84) - oa->write_character(static_cast(0x80 + N)); - } - else - { - // start array - oa->write_character(to_char_type(0x85)); - } - - // write each element - for (std::size_t i = 0; i < N; ++i) - { - const auto& el = j.m_value.array->operator[](i); - - // check if 0xFF after nonempty string and string is required - if (i > 0) - { - const auto& prev = j.m_value.array->operator[](i - 1); - if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) - { - oa->write_character(to_char_type(0xFF)); - } - } - - write_bon8(el); - } - - if (N > 4) - { - // end of container - oa->write_character(to_char_type(0xFE)); - } - break; - } - - case value_t::object: - { - const auto N = j.m_value.object->size(); - if (N <= 4) - { - // start object with count (86..8A) - oa->write_character(static_cast(0x86 + N)); - } - else - { - // start object - oa->write_character(to_char_type(0x8B)); - } - - // write each element - for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) - { - const auto& key = it->first; - const auto& value = it->second; - - write_bon8(key); - - // check if we need a 0xFF separator between key and value - if (!key.empty() && value.is_string()) - { - oa->write_character(to_char_type(0xFF)); - } - - write_bon8(value); - - // check if we need a 0xFF separator between the value and the next key - if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) - { - oa->write_character(to_char_type(0xFF)); - } - } - - if (N > 4) - { - // end of container - oa->write_character(to_char_type(0xFE)); - } - break; - } - - case value_t::binary: - case value_t::discarded: - default: - break; + oa->write_character(to_char_type(0xFF)); } } @@ -1696,6 +1540,181 @@ class binary_writer // BON8 // ////////// + /*! + * @param j + * @return whether the last written value was a string + */ + bool write_bon8_internal(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xFA)); + return false; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF9) + : to_char_type(0xF8)); + return false; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); + } + write_bon8_integer(static_cast(j.m_value.number_unsigned)); + return false; + } + + case value_t::number_integer: + { + write_bon8_integer(j.m_value.number_integer); + return false; + } + + case value_t::number_float: + { + // special values + if (j.m_value.number_float == -1.0) + { + oa->write_character(to_char_type(0xFB)); + } + else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) + { + oa->write_character(to_char_type(0xFC)); + } + else if (j.m_value.number_float == 1.0) + { + oa->write_character(to_char_type(0xFD)); + } + else + { + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + } + return false; + } + + case value_t::string: + { + // empty string: use end-of-text symbol + if (j.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + return false; // already wrote 0xFF byte + } + + // write strings as is + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + return true; + } + + case value_t::array: + { + bool last_written_value_is_string = false; + const auto N = j.m_value.array->size(); + if (N <= 4) + { + // start array with count (80..84) + oa->write_character(static_cast(0x80 + N)); + } + else + { + // start array + oa->write_character(to_char_type(0x85)); + } + + // write each element + for (std::size_t i = 0; i < N; ++i) + { + const auto& el = j.m_value.array->operator[](i); + + // check if 0xFF after nonempty string and string is required + if (i > 0) + { + const auto& prev = j.m_value.array->operator[](i - 1); + if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + last_written_value_is_string = write_bon8_internal(el); + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + last_written_value_is_string = false; // 0xFE is not a string byte + } + + return last_written_value_is_string; + } + + case value_t::object: + { + bool last_written_value_is_string = false; + const auto N = j.m_value.object->size(); + if (N <= 4) + { + // start object with count (86..8A) + oa->write_character(static_cast(0x86 + N)); + } + else + { + // start object + oa->write_character(to_char_type(0x8B)); + } + + // write each element + for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) + { + const auto& key = it->first; + const auto& value = it->second; + + write_bon8_internal(key); + + // check if we need a 0xFF separator between key and value + if (!key.empty() && value.is_string()) + { + oa->write_character(to_char_type(0xFF)); + } + + last_written_value_is_string = write_bon8_internal(value); + + // check if we need a 0xFF separator between the value and the next key + if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + last_written_value_is_string = false; // 0xFE is not a string byte + } + + return last_written_value_is_string; + } + + case value_t::binary: + case value_t::discarded: + default: + return false; + } + } + void write_bon8_integer(typename BasicJsonType::number_integer_t value) { if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3876db03e..6d1a489a4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -14502,166 +14502,10 @@ class binary_writer */ void write_bon8(const BasicJsonType& j) { - switch (j.type()) + const bool last_written_value_is_string = write_bon8_internal(j); + if (last_written_value_is_string) { - case value_t::null: - { - oa->write_character(to_char_type(0xFA)); - break; - } - - case value_t::boolean: - { - oa->write_character(j.m_value.boolean - ? to_char_type(0xF9) - : to_char_type(0xF8)); - break; - } - - case value_t::number_unsigned: - { - if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) - { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); - } - write_bon8_integer(static_cast(j.m_value.number_unsigned)); - break; - } - - case value_t::number_integer: - { - write_bon8_integer(j.m_value.number_integer); - break; - } - - case value_t::number_float: - { - // special values - if (j.m_value.number_float == -1.0) - { - oa->write_character(to_char_type(0xFB)); - } - else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) - { - oa->write_character(to_char_type(0xFC)); - } - else if (j.m_value.number_float == 1.0) - { - oa->write_character(to_char_type(0xFD)); - } - else - { - // write float with prefix - write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); - } - break; - } - - case value_t::string: - { - // empty string: use end-of-text symbol - if (j.m_value.string->empty()) - { - oa->write_character(to_char_type(0xFF)); - break; - } - - // write strings as is - oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); - break; - } - - case value_t::array: - { - const auto N = j.m_value.array->size(); - if (N <= 4) - { - // start array with count (80..84) - oa->write_character(static_cast(0x80 + N)); - } - else - { - // start array - oa->write_character(to_char_type(0x85)); - } - - // write each element - for (std::size_t i = 0; i < N; ++i) - { - const auto& el = j.m_value.array->operator[](i); - - // check if 0xFF after nonempty string and string is required - if (i > 0) - { - const auto& prev = j.m_value.array->operator[](i - 1); - if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) - { - oa->write_character(to_char_type(0xFF)); - } - } - - write_bon8(el); - } - - if (N > 4) - { - // end of container - oa->write_character(to_char_type(0xFE)); - } - break; - } - - case value_t::object: - { - const auto N = j.m_value.object->size(); - if (N <= 4) - { - // start object with count (86..8A) - oa->write_character(static_cast(0x86 + N)); - } - else - { - // start object - oa->write_character(to_char_type(0x8B)); - } - - // write each element - for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) - { - const auto& key = it->first; - const auto& value = it->second; - - write_bon8(key); - - // check if we need a 0xFF separator between key and value - if (!key.empty() && value.is_string()) - { - oa->write_character(to_char_type(0xFF)); - } - - write_bon8(value); - - // check if we need a 0xFF separator between the value and the next key - if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) - { - oa->write_character(to_char_type(0xFF)); - } - } - - if (N > 4) - { - // end of container - oa->write_character(to_char_type(0xFE)); - } - break; - } - - case value_t::binary: - case value_t::discarded: - default: - break; + oa->write_character(to_char_type(0xFF)); } } @@ -15270,6 +15114,181 @@ class binary_writer // BON8 // ////////// + /*! + * @param j + * @return whether the last written value was a string + */ + bool write_bon8_internal(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xFA)); + return false; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF9) + : to_char_type(0xF8)); + return false; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); + } + write_bon8_integer(static_cast(j.m_value.number_unsigned)); + return false; + } + + case value_t::number_integer: + { + write_bon8_integer(j.m_value.number_integer); + return false; + } + + case value_t::number_float: + { + // special values + if (j.m_value.number_float == -1.0) + { + oa->write_character(to_char_type(0xFB)); + } + else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) + { + oa->write_character(to_char_type(0xFC)); + } + else if (j.m_value.number_float == 1.0) + { + oa->write_character(to_char_type(0xFD)); + } + else + { + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + } + return false; + } + + case value_t::string: + { + // empty string: use end-of-text symbol + if (j.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + return false; // already wrote 0xFF byte + } + + // write strings as is + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + return true; + } + + case value_t::array: + { + bool last_written_value_is_string = false; + const auto N = j.m_value.array->size(); + if (N <= 4) + { + // start array with count (80..84) + oa->write_character(static_cast(0x80 + N)); + } + else + { + // start array + oa->write_character(to_char_type(0x85)); + } + + // write each element + for (std::size_t i = 0; i < N; ++i) + { + const auto& el = j.m_value.array->operator[](i); + + // check if 0xFF after nonempty string and string is required + if (i > 0) + { + const auto& prev = j.m_value.array->operator[](i - 1); + if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + last_written_value_is_string = write_bon8_internal(el); + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + last_written_value_is_string = false; // 0xFE is not a string byte + } + + return last_written_value_is_string; + } + + case value_t::object: + { + bool last_written_value_is_string = false; + const auto N = j.m_value.object->size(); + if (N <= 4) + { + // start object with count (86..8A) + oa->write_character(static_cast(0x86 + N)); + } + else + { + // start object + oa->write_character(to_char_type(0x8B)); + } + + // write each element + for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) + { + const auto& key = it->first; + const auto& value = it->second; + + write_bon8_internal(key); + + // check if we need a 0xFF separator between key and value + if (!key.empty() && value.is_string()) + { + oa->write_character(to_char_type(0xFF)); + } + + last_written_value_is_string = write_bon8_internal(value); + + // check if we need a 0xFF separator between the value and the next key + if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + last_written_value_is_string = false; // 0xFE is not a string byte + } + + return last_written_value_is_string; + } + + case value_t::binary: + case value_t::discarded: + default: + return false; + } + } + void write_bon8_integer(typename BasicJsonType::number_integer_t value) { if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 4d8d2546a..7be727343 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -481,7 +481,7 @@ TEST_CASE("BON8") SECTION("other strings") { json j = "This is a string."; - std::vector expected = {'T', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 's', 't', 'r', 'i', 'n', 'g', '.'}; + std::vector expected = {'T', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 's', 't', 'r', 'i', 'n', 'g', '.', 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } @@ -534,7 +534,7 @@ TEST_CASE("BON8") SECTION("[\"s\", \"s\"]") { json j = {"s", "s"}; - std::vector expected = {0x82, 's', 0xFF, 's'}; + std::vector expected = {0x82, 's', 0xFF, 's', 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } @@ -542,7 +542,31 @@ TEST_CASE("BON8") SECTION("[\"\", \"s\"]") { json j = {"", "s"}; - std::vector expected = {0x82, 0xFF, 's'}; + std::vector expected = {0x82, 0xFF, 's', 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[[[\"foo\"]]]") + { + json j = R"([[["foo"]]])"_json; + std::vector expected = {0x81, 0x81, 0x81, 'f', 'o', 'o', 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[[[1]]]") + { + json j = R"([[[1]]])"_json; + std::vector expected = {0x81, 0x81, 0x81, 0x91}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("[[[\"\"]]]") + { + json j = R"([[[""]]])"_json; + std::vector expected = {0x81, 0x81, 0x81, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } @@ -591,7 +615,15 @@ TEST_CASE("BON8") SECTION("{\"a\": \"\", \"c\": \"d\"}") { json j = {{"a", ""}, {"c", "d"}}; - std::vector expected = {0x88, 'a', 0xFF, 0xFF, 'c', 0xFF, 'd'}; + std::vector expected = {0x88, 'a', 0xFF, 0xFF, 'c', 0xFF, 'd', 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("{\"a\": \"b\", \"c\": \"d\"}") + { + json j = {{"a", "b"}, {"c", "d"}}; + std::vector expected = {0x88, 'a', 0xFF, 'b', 0xFF, 'c', 0xFF, 'd', 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } From 623f0e96e2341aaad4e0f976396f0e1cb1b53ac2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 5 Sep 2021 12:44:39 +0200 Subject: [PATCH 10/32] :white_check_mark: improve coverage --- .../nlohmann/detail/output/binary_writer.hpp | 4 ++ single_include/nlohmann/json.hpp | 4 ++ test/src/unit-bon8.cpp | 71 +++++++++++++------ 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 5f0fb55d8..f995d31ce 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1851,11 +1851,13 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(static_cast(n))); break; + // LCOV_EXCL_START case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: default: break; + // LCOV_EXCL_STOP } write_number(static_cast(n)); } @@ -1872,11 +1874,13 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(n)); break; + // LCOV_EXCL_START case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: default: break; + // LCOV_EXCL_STOP } write_number(n); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6d1a489a4..feda56644 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15425,11 +15425,13 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(static_cast(n))); break; + // LCOV_EXCL_START case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: default: break; + // LCOV_EXCL_STOP } write_number(static_cast(n)); } @@ -15446,11 +15448,13 @@ class binary_writer case input_format_t::bon8: oa->write_character(get_bon8_float_prefix(n)); break; + // LCOV_EXCL_START case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: default: break; + // LCOV_EXCL_STOP } write_number(n); } diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 7be727343..4a56d7775 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -435,36 +435,61 @@ TEST_CASE("BON8") SECTION("floating-point numbers") { - SECTION("-1.0") + SECTION("special values") { - json j = -1.0; - std::vector expected = {0xFB}; - const auto result = json::to_bon8(j); - CHECK(result == expected); + SECTION("-1.0") + { + json j = -1.0; + std::vector expected = {0xFB}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("0.0") + { + json j = 0.0; + std::vector expected = {0xFC}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("1.0") + { + json j = 1.0; + std::vector expected = {0xFD}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-0.0") + { + json j = -0.0; + std::vector expected = {0x8E, 0x80, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } } - SECTION("0.0") + SECTION("floats") { - json j = 0.0; - std::vector expected = {0xFC}; - const auto result = json::to_bon8(j); - CHECK(result == expected); + SECTION("2.0") + { + json j = 2.0; + std::vector expected = {0x8E, 0x40, 0x00, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } } - SECTION("1.0") + SECTION("doubles") { - json j = 1.0; - std::vector expected = {0xFD}; - const auto result = json::to_bon8(j); - CHECK(result == expected); - } - - SECTION("-0.0") - { - json j = -0.0; - std::vector expected = {0x8E, 0x80, 0x00, 0x00, 0x00}; - const auto result = json::to_bon8(j); - CHECK(result == expected); + SECTION("100000000.1") + { + json j = 100000000.1; + std::vector expected = {0x8F, 0x41, 0x97, 0xD7, 0x84, 0x00, 0x66, 0x66, 0x66}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } } } From d798ca24aa075e9b78eb0b814cbae3cd45a2e598 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 13:22:23 +0200 Subject: [PATCH 11/32] :recycle: adjust positive integer representation --- .../nlohmann/detail/output/binary_writer.hpp | 17 ++-- single_include/nlohmann/json.hpp | 17 ++-- test/src/unit-bon8.cpp | 80 +++++++++---------- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index f995d31ce..3f0b786b6 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1723,7 +1723,7 @@ class binary_writer oa->write_character(to_char_type(0x8D)); write_number(static_cast(value)); } - else if (value < -33554432 || value > 67108863) + else if (value < -33554432 || value > 67637031) { // 32 bit integers oa->write_character(to_char_type(0x8C)); @@ -1764,23 +1764,26 @@ class binary_writer JSON_ASSERT(value >= 0); oa->write_character(static_cast(0x90 + value)); } - else if (value <= 3839) + else if (value <= 3879) { - JSON_ASSERT(value >= 0); + JSON_ASSERT(value >= 40); + value -= 40; oa->write_character(static_cast(0xC2 + (value >> 7 & 0x1F))); oa->write_character(static_cast(value & 0x7F)); } - else if (value <= 524287) + else if (value <= 528167) { - JSON_ASSERT(value >= 0); + JSON_ASSERT(value >= 3880); + value -= 3880; oa->write_character(static_cast(0xE0 + (value >> 15 & 0x0F))); oa->write_character(static_cast(value >> 8 & 0x7F)); oa->write_character(static_cast(value)); } else { - JSON_ASSERT(value >= 0); - JSON_ASSERT(value <= 67108863); + JSON_ASSERT(value >= 528168); + JSON_ASSERT(value <= 67637031); + value -= 528168; oa->write_character(static_cast(0xF0 + (value >> 23 & 0x17))); oa->write_character(static_cast(value >> 16 & 0x7F)); oa->write_character(static_cast(value >> 8)); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index feda56644..e50c9a751 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15297,7 +15297,7 @@ class binary_writer oa->write_character(to_char_type(0x8D)); write_number(static_cast(value)); } - else if (value < -33554432 || value > 67108863) + else if (value < -33554432 || value > 67637031) { // 32 bit integers oa->write_character(to_char_type(0x8C)); @@ -15338,23 +15338,26 @@ class binary_writer JSON_ASSERT(value >= 0); oa->write_character(static_cast(0x90 + value)); } - else if (value <= 3839) + else if (value <= 3879) { - JSON_ASSERT(value >= 0); + JSON_ASSERT(value >= 40); + value -= 40; oa->write_character(static_cast(0xC2 + (value >> 7 & 0x1F))); oa->write_character(static_cast(value & 0x7F)); } - else if (value <= 524287) + else if (value <= 528167) { - JSON_ASSERT(value >= 0); + JSON_ASSERT(value >= 3880); + value -= 3880; oa->write_character(static_cast(0xE0 + (value >> 15 & 0x0F))); oa->write_character(static_cast(value >> 8 & 0x7F)); oa->write_character(static_cast(value)); } else { - JSON_ASSERT(value >= 0); - JSON_ASSERT(value <= 67108863); + JSON_ASSERT(value >= 528168); + JSON_ASSERT(value <= 67637031); + value -= 528168; oa->write_character(static_cast(0xF0 + (value >> 23 & 0x17))); oa->write_character(static_cast(value >> 16 & 0x7F)); oa->write_character(static_cast(value >> 8)); diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 4a56d7775..b5135b033 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -98,69 +98,69 @@ TEST_CASE("BON8") } } - SECTION("40..3839") + SECTION("40..3879") { SECTION("40") { json j = 40U; - std::vector expected = {0xC2, 0x28}; + std::vector expected = {0xC2, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("3839") + SECTION("3879") { - json j = 3839U; + json j = 3879U; std::vector expected = {0xDF, 0x7F}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("3840..524287") + SECTION("3880..524287") { - SECTION("3840") + SECTION("3880") { - json j = 3840U; - std::vector expected = {0xE0, 0x0F, 0x00}; + json j = 3880U; + std::vector expected = {0xE0, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("524287") + SECTION("528167") { - json j = 524287U; + json j = 528167U; std::vector expected = {0xEF, 0x7F, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("524288..67108863") + SECTION("528168..67637031") { - SECTION("524288") + SECTION("528168") { - json j = 524288U; - std::vector expected = {0xF0, 0x08, 0x00, 0x00}; + json j = 528168U; + std::vector expected = {0xF0, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("67108863") + SECTION("67637031") { - json j = 67108863U; + json j = 67637031U; std::vector expected = {0xF7, 0x7F, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("67108864..2147483647 (int32max)") + SECTION("67637032..2147483647 (int32max)") { - SECTION("67108864") + SECTION("67637032") { - json j = 67108864U; - std::vector expected = {0x8C, 0x04, 0x00, 0x00, 0x00}; + json j = 67637032U; + std::vector expected = {0x8C, 0x04, 0x08, 0x0F, 0x28}; const auto result = json::to_bon8(j); CHECK(result == expected); } @@ -337,69 +337,69 @@ TEST_CASE("BON8") } } - SECTION("40..3839") + SECTION("40..3879") { SECTION("40") { json j = 40; - std::vector expected = {0xC2, 0x28}; + std::vector expected = {0xC2, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("3839") + SECTION("3879") { - json j = 3839; + json j = 3879; std::vector expected = {0xDF, 0x7F}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("3840..524287") + SECTION("3880..524287") { - SECTION("3840") + SECTION("3880") { - json j = 3840; - std::vector expected = {0xE0, 0x0F, 0x00}; + json j = 3880; + std::vector expected = {0xE0, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("524287") + SECTION("528167") { - json j = 524287; + json j = 528167; std::vector expected = {0xEF, 0x7F, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("524288..67108863") + SECTION("528168..67637031") { - SECTION("524288") + SECTION("528168") { - json j = 524288; - std::vector expected = {0xF0, 0x08, 0x00, 0x00}; + json j = 528168; + std::vector expected = {0xF0, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("67108863") + SECTION("67637031") { - json j = 67108863; + json j = 67637031; std::vector expected = {0xF7, 0x7F, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("67108864..2147483647 (int32max)") + SECTION("67637032..2147483647 (int32max)") { - SECTION("67108864") + SECTION("67637032") { - json j = 67108864; - std::vector expected = {0x8C, 0x04, 0x00, 0x00, 0x00}; + json j = 67637032; + std::vector expected = {0x8C, 0x04, 0x08, 0x0F, 0x28}; const auto result = json::to_bon8(j); CHECK(result == expected); } From 6bfd21e3215035b31a8bcc7b05e4214d0b964e24 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 13:50:54 +0200 Subject: [PATCH 12/32] :recycle: adjust negative integer representation --- .../nlohmann/detail/output/binary_writer.hpp | 24 +++++------ single_include/nlohmann/json.hpp | 24 +++++------ test/src/unit-bon8.cpp | 40 +++++++++---------- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 3f0b786b6..ee59af149 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1723,40 +1723,40 @@ class binary_writer oa->write_character(to_char_type(0x8D)); write_number(static_cast(value)); } - else if (value < -33554432 || value > 67637031) + else if (value < -33818506 || value > 67637031) { // 32 bit integers oa->write_character(to_char_type(0x8C)); write_number(static_cast(value)); } - else if (value < -262144) + else if (value <= -264075) { - JSON_ASSERT(value >= -33554432); - value = -value - 1; + JSON_ASSERT(value >= -33818506); + value = -(value + 264075); oa->write_character(static_cast(0xF0 + (value >> 22 & 0x07))); oa->write_character(static_cast(0xC0 + (value >> 16 & 0x3F))); oa->write_character(static_cast(value >> 8)); oa->write_character(static_cast(value)); } - else if (value < -1920) + else if (value <= -1931) { - JSON_ASSERT(value >= -262144); - value = -value - 1; + JSON_ASSERT(value >= -264074); + value = -(value + 1931); oa->write_character(static_cast(0xE0 + (value >> 14 & 0x0F))); oa->write_character(static_cast(0xC0 + (value >> 8 & 0x3F))); oa->write_character(static_cast(value)); } - else if (value < -10) + else if (value <= -11) { - JSON_ASSERT(value >= -1920); - value = -value - 1; + JSON_ASSERT(value >= -1930); + value = -(value + 11); oa->write_character(static_cast(0xC2 + (value >> 6 & 0x1F))); oa->write_character(static_cast(0xC0 + (value & 0x3F))); } - else if (value < 0) + else if (value <= -1) { JSON_ASSERT(value >= -10); - value = -value - 1; + value = -(value + 1); oa->write_character(static_cast(0xB8 + value)); } else if (value <= 39) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index e50c9a751..eae9dedb4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15297,40 +15297,40 @@ class binary_writer oa->write_character(to_char_type(0x8D)); write_number(static_cast(value)); } - else if (value < -33554432 || value > 67637031) + else if (value < -33818506 || value > 67637031) { // 32 bit integers oa->write_character(to_char_type(0x8C)); write_number(static_cast(value)); } - else if (value < -262144) + else if (value <= -264075) { - JSON_ASSERT(value >= -33554432); - value = -value - 1; + JSON_ASSERT(value >= -33818506); + value = -(value + 264075); oa->write_character(static_cast(0xF0 + (value >> 22 & 0x07))); oa->write_character(static_cast(0xC0 + (value >> 16 & 0x3F))); oa->write_character(static_cast(value >> 8)); oa->write_character(static_cast(value)); } - else if (value < -1920) + else if (value <= -1931) { - JSON_ASSERT(value >= -262144); - value = -value - 1; + JSON_ASSERT(value >= -264074); + value = -(value + 1931); oa->write_character(static_cast(0xE0 + (value >> 14 & 0x0F))); oa->write_character(static_cast(0xC0 + (value >> 8 & 0x3F))); oa->write_character(static_cast(value)); } - else if (value < -10) + else if (value <= -11) { - JSON_ASSERT(value >= -1920); - value = -value - 1; + JSON_ASSERT(value >= -1930); + value = -(value + 11); oa->write_character(static_cast(0xC2 + (value >> 6 & 0x1F))); oa->write_character(static_cast(0xC0 + (value & 0x3F))); } - else if (value < 0) + else if (value <= -1) { JSON_ASSERT(value >= -10); - value = -value - 1; + value = -(value + 1); oa->write_character(static_cast(0xB8 + value)); } else if (value <= 39) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index b5135b033..9f6848cf6 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -222,7 +222,7 @@ TEST_CASE("BON8") } } - SECTION("-2147483648 (int32min)..-33554433") + SECTION("-2147483648 (int32min)..-33818507") { SECTION("-2147483648") { @@ -233,58 +233,58 @@ TEST_CASE("BON8") CHECK(result == expected); } - SECTION("-33554433") + SECTION("-33818507") { - json j = -33554433; - std::vector expected = {0x8C, 0xFD, 0xFF, 0xFF, 0xFF}; + json j = -33818507; + std::vector expected = {0x8C, 0xFD, 0xFB, 0xF8, 0x75}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("-33554432..-262145") + SECTION("-33818506..-264075") { - SECTION("-33554432") + SECTION("-33818506") { - json j = -33554432; + json j = -33818506; std::vector expected = {0xF7, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("-262145") + SECTION("-264075") { - json j = -262145; - std::vector expected = {0xF0, 0xC4, 0x00, 0x00}; + json j = -264075; + std::vector expected = {0xF0, 0xC0, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("-262144..-1921") + SECTION("-264074..-1931") { - SECTION("-262144") + SECTION("-264074") { - json j = -262144; + json j = -264074; std::vector expected = {0xEF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); } - SECTION("-1921") + SECTION("-1931") { - json j = -1921; - std::vector expected = {0xE0, 0xC7, 0x80}; + json j = -1931; + std::vector expected = {0xE0, 0xC0, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); } } - SECTION("-1920..-11") + SECTION("-1930..-11") { - SECTION("-1920") + SECTION("-1930") { - json j = -1920; + json j = -1930; std::vector expected = {0xDF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); @@ -293,7 +293,7 @@ TEST_CASE("BON8") SECTION("-11") { json j = -11; - std::vector expected = {0xC2, 0xCA}; + std::vector expected = {0xC2, 0xC0}; const auto result = json::to_bon8(j); CHECK(result == expected); } From 5221115ff11c6d3c3ca781771242b0c13712a88c Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 14:27:38 +0200 Subject: [PATCH 13/32] :recycle: implement floating-point special values --- .../nlohmann/detail/output/binary_writer.hpp | 14 ++++++++--- single_include/nlohmann/json.hpp | 14 ++++++++--- test/src/unit-bon8.cpp | 24 +++++++++++++++++++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index ee59af149..451bb4f15 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1593,6 +1593,14 @@ class binary_writer { oa->write_character(to_char_type(0xFD)); } + else if (std::isnan(j.m_value.number_float)) + { + oa->write_character(to_char_type(0x8E)); + oa->write_character(to_char_type(0x7F)); + oa->write_character(to_char_type(0x80)); + oa->write_character(to_char_type(0x00)); + oa->write_character(to_char_type(0x01)); + } else { // write float with prefix @@ -1839,9 +1847,9 @@ class binary_writer #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wfloat-equal" #endif - if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && - static_cast(n) <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(n)) == static_cast(n)) + if (std::isnan(n) || std::isinf(n) || (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n))) { switch (format) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index eae9dedb4..1c13057e1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15167,6 +15167,14 @@ class binary_writer { oa->write_character(to_char_type(0xFD)); } + else if (std::isnan(j.m_value.number_float)) + { + oa->write_character(to_char_type(0x8E)); + oa->write_character(to_char_type(0x7F)); + oa->write_character(to_char_type(0x80)); + oa->write_character(to_char_type(0x00)); + oa->write_character(to_char_type(0x01)); + } else { // write float with prefix @@ -15413,9 +15421,9 @@ class binary_writer #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wfloat-equal" #endif - if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && - static_cast(n) <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(n)) == static_cast(n)) + if (std::isnan(n) || std::isinf(n) || (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n))) { switch (format) { diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 9f6848cf6..06cf57073 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -468,6 +468,30 @@ TEST_CASE("BON8") const auto result = json::to_bon8(j); CHECK(result == expected); } + + SECTION("NAN") + { + json j = NAN; + std::vector expected = {0x8E, 0x7F, 0x80, 0x00, 0x01}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("infinity") + { + json j = INFINITY; + std::vector expected = {0x8E, 0x7F, 0x80, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } + + SECTION("-infinity") + { + json j = -INFINITY; + std::vector expected = {0x8E, 0xFF, 0x80, 0x00, 0x00}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + } } SECTION("floats") From 7c55a91004250ca9353296952b0b27df62c37f2d Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 14:59:21 +0200 Subject: [PATCH 14/32] :memo: add documentation --- doc/examples/to_bon8.cpp | 21 +++++++ doc/examples/to_bon8.output | 1 + doc/mkdocs/docs/api/basic_json/index.md | 1 + .../docs/api/basic_json/input_format_t.md | 2 +- doc/mkdocs/docs/api/basic_json/to_bon8.md | 57 +++++++++++++++++++ .../docs/features/binary_formats/bson.md | 2 +- .../docs/features/binary_formats/index.md | 3 + doc/mkdocs/mkdocs.yml | 1 + include/nlohmann/json.hpp | 53 +++++++++++++++++ single_include/nlohmann/json.hpp | 53 +++++++++++++++++ 10 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 doc/examples/to_bon8.cpp create mode 100644 doc/examples/to_bon8.output create mode 100644 doc/mkdocs/docs/api/basic_json/to_bon8.md diff --git a/doc/examples/to_bon8.cpp b/doc/examples/to_bon8.cpp new file mode 100644 index 000000000..2b4e311b4 --- /dev/null +++ b/doc/examples/to_bon8.cpp @@ -0,0 +1,21 @@ +#include +#include +#include + +using json = nlohmann::json; + +int main() +{ + // create a JSON value + json j = R"({"compact": true, "schema": 0})"_json; + + // serialize it to BON8 + std::vector v = json::to_bon8(j); + + // print the vector content + for (auto& byte : v) + { + std::cout << "0x" << std::hex << std::setw(2) << std::setfill('0') << (int)byte << " "; + } + std::cout << std::endl; +} diff --git a/doc/examples/to_bon8.output b/doc/examples/to_bon8.output new file mode 100644 index 000000000..8cce605dd --- /dev/null +++ b/doc/examples/to_bon8.output @@ -0,0 +1 @@ +0x88 0x63 0x6f 0x6d 0x70 0x61 0x63 0x74 0xf9 0x73 0x63 0x68 0x65 0x6d 0x61 0x90 diff --git a/doc/mkdocs/docs/api/basic_json/index.md b/doc/mkdocs/docs/api/basic_json/index.md index e8841e850..517a207ca 100644 --- a/doc/mkdocs/docs/api/basic_json/index.md +++ b/doc/mkdocs/docs/api/basic_json/index.md @@ -230,6 +230,7 @@ Access to the JSON value - [**from_cbor**](from_cbor.md) (static) - create a JSON value from an input in CBOR format - [**from_msgpack**](from_msgpack.md) (static) - create a JSON value from an input in MessagePack format - [**from_ubjson**](from_ubjson.md) (static) - create a JSON value from an input in UBJSON format +- [**to_bon8**](to_bon8.md) (static) - create a BON8 serialization of a given JSON value - [**to_bson**](to_bson.md) (static) - create a BSON serialization of a given JSON value - [**to_cbor**](to_cbor.md) (static) - create a CBOR serialization of a given JSON value - [**to_msgpack**](to_msgpack.md) (static) - create a MessagePack serialization of a given JSON value diff --git a/doc/mkdocs/docs/api/basic_json/input_format_t.md b/doc/mkdocs/docs/api/basic_json/input_format_t.md index 783085d8e..b12ef4641 100644 --- a/doc/mkdocs/docs/api/basic_json/input_format_t.md +++ b/doc/mkdocs/docs/api/basic_json/input_format_t.md @@ -25,7 +25,7 @@ ubjson : UBJSON (Universal Binary JSON) bson -: BSON (Bin­ary JSON) +: BSON (Binary JSON) ## Version history diff --git a/doc/mkdocs/docs/api/basic_json/to_bon8.md b/doc/mkdocs/docs/api/basic_json/to_bon8.md new file mode 100644 index 000000000..277fd3151 --- /dev/null +++ b/doc/mkdocs/docs/api/basic_json/to_bon8.md @@ -0,0 +1,57 @@ +# basic_json::to_bon8 + +```cpp +// (1) +static std::vector to_bon8(const basic_json& j); + +// (2) +static void to_bon8(const basic_json& j, detail::output_adapter o); +static void to_bon8(const basic_json& j, detail::output_adapter o); +``` + +Serializes a given JSON value `j` to a byte vector using the BON8 serialization format. BON8 is a binary serialization +format which aims to be more compact than JSON itself, yet more efficient to parse. + +1. Returns a byte vector containing the BON8 serialization. +2. Writes the BON8 serialization to an output adapter. + +## Parameters + +`j` (in) +: JSON value to serialize + +`o` (in) +: output adapter to write serialization to + +## Return value + +1. BON8 serialization as byte vector +2. / + +## Exception safety + +Strong guarantee: if an exception is thrown, there are no changes in the JSON value. + +## Complexity + +Linear in the size of the JSON value `j`. + +## Example + +??? example + + The example shows the serialization of a JSON value to a byte vector in BON8 format. + + ```cpp + --8<-- "examples/to_bon8.cpp" + ``` + + Output: + + ```json + --8<-- "examples/to_bon8.output" + ``` + +## Version history + +- Added in version 3.11.0. diff --git a/doc/mkdocs/docs/features/binary_formats/bson.md b/doc/mkdocs/docs/features/binary_formats/bson.md index 0ed2a786e..e8b06f84b 100644 --- a/doc/mkdocs/docs/features/binary_formats/bson.md +++ b/doc/mkdocs/docs/features/binary_formats/bson.md @@ -1,6 +1,6 @@ # BSON -BSON, short for Bin­ary JSON, is a bin­ary-en­coded seri­al­iz­a­tion of JSON-like doc­u­ments. Like JSON, BSON sup­ports the em­bed­ding of doc­u­ments and ar­rays with­in oth­er doc­u­ments and ar­rays. BSON also con­tains ex­ten­sions that al­low rep­res­ent­a­tion of data types that are not part of the JSON spec. For ex­ample, BSON has a Date type and a BinData type. +BSON, short for Binary JSON, is a binary-encoded serialization of JSON-like documents. Like JSON, BSON supports the embedding of documents and arrays within other documents and arrays. BSON also contains extensions that allow representation of data types that are not part of the JSON spec. For example, BSON has a Date type and a BinData type. !!! abstract "References" diff --git a/doc/mkdocs/docs/features/binary_formats/index.md b/doc/mkdocs/docs/features/binary_formats/index.md index 279009d11..bfa18c3c6 100644 --- a/doc/mkdocs/docs/features/binary_formats/index.md +++ b/doc/mkdocs/docs/features/binary_formats/index.md @@ -6,6 +6,7 @@ Though JSON is a ubiquitous data format, it is not a very compact format suitabl - [CBOR](cbor.md) (Concise Binary Object Representation), - [MessagePack](messagepack.md), and - [UBJSON](ubjson.md) (Universal Binary JSON) +- BON8 to efficiently encode JSON values to byte vectors and to decode such vectors. @@ -19,6 +20,7 @@ to efficiently encode JSON values to byte vectors and to decode such vectors. | CBOR | complete | incomplete, but all JSON types are supported | | MessagePack | complete | complete | | UBJSON | complete | complete | +| BON8 | complete | not yet implemented | ### Binary values @@ -28,6 +30,7 @@ to efficiently encode JSON values to byte vectors and to decode such vectors. | CBOR | supported | supported | | MessagePack | supported | supported | | UBJSON | not supported | not supported | +| BON8 | not supported | not supported | See [binary values](../binary_values.md) for more information. diff --git a/doc/mkdocs/mkdocs.yml b/doc/mkdocs/mkdocs.yml index 7aa6e2c5d..cd12656c3 100644 --- a/doc/mkdocs/mkdocs.yml +++ b/doc/mkdocs/mkdocs.yml @@ -168,6 +168,7 @@ nav: - api/basic_json/sax_parse.md - api/basic_json/size.md - api/basic_json/string_t.md + - api/basic_json/to_bon8.md - api/basic_json/to_bson.md - api/basic_json/to_cbor.md - api/basic_json/to_msgpack.md diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 650730509..ff57fbeaf 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -7303,6 +7303,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format @sa see @ref to_ubjson(const basic_json&, const bool, const bool) for the related UBJSON format + @sa see @ref to_bson(const basic_json&) for the related BSON format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format @since version 2.0.9; compact representation of floating-point numbers since version 3.8.0 @@ -7399,6 +7401,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec @sa see @ref to_cbor(const basic_json& for the related CBOR format @sa see @ref to_ubjson(const basic_json&, const bool, const bool) for the related UBJSON format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format @since version 2.0.9 */ @@ -7502,6 +7505,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec analogous deserialization @sa see @ref to_cbor(const basic_json& for the related CBOR format @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format @since version 3.1.0 */ @@ -7582,6 +7586,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec related UBJSON format @sa see @ref to_cbor(const basic_json&) for the related CBOR format @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format */ static std::vector to_bson(const basic_json& j) { @@ -7611,6 +7616,44 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + /*! + @brief Serializes the given JSON object `j` to BON8 and returns a vector + containing the corresponding BON8-representation. + + BON8 is a binary format that encodes JSON values using those byte sequences + that would form invalid UTF-8 strings. As a result, strings do not need to + be re-encoded. + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a BON8 value. + + @note The following values can **not** be converted to a BON8 value: + - integers larger than 9223372036854775807 + + @throw out_of_range.407 if `j.is_number_unsigned() && j.get() > 9223372036854775807` + + @note Any BON8 output created via @ref to_bon8 can be successfully parsed + by @ref from_bon8. + + @param[in] j JSON value to serialize + @return BON8 serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in BON8 format.,to_bon8} + + @sa https://github.com/ttauri-project/ttauri/blob/main/docs/BON8.md + @sa see @ref from_bon8(detail::input_adapter&&, const bool strict) for the + analogous deserialization + @sa see @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + @sa see @ref to_cbor(const basic_json&) for the related CBOR format + @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa see @ref to_bson(const basic_json&) for the related BSON format + + @since version 3.11.0 + */ static std::vector to_bon8(const basic_json& j) { std::vector result; @@ -7618,11 +7661,21 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return result; } + /*! + @brief Serializes the given JSON object `j` to BON8 and forwards the + corresponding BON8-representation to the given output_adapter `o`. + @param j The JSON object to convert to BON8. + @param o The output adapter that receives the binary BON8 representation. + @sa see @ref to_bon8(const basic_json&) + */ static void to_bon8(const basic_json& j, detail::output_adapter o) { binary_writer(o).write_bon8(j); } + /*! + @copydoc to_bon8(const basic_json&, detail::output_adapter) + */ static void to_bon8(const basic_json& j, detail::output_adapter o) { binary_writer(o).write_bon8(j); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1c13057e1..546bc5e95 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -25027,6 +25027,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format @sa see @ref to_ubjson(const basic_json&, const bool, const bool) for the related UBJSON format + @sa see @ref to_bson(const basic_json&) for the related BSON format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format @since version 2.0.9; compact representation of floating-point numbers since version 3.8.0 @@ -25123,6 +25125,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec @sa see @ref to_cbor(const basic_json& for the related CBOR format @sa see @ref to_ubjson(const basic_json&, const bool, const bool) for the related UBJSON format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format @since version 2.0.9 */ @@ -25226,6 +25229,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec analogous deserialization @sa see @ref to_cbor(const basic_json& for the related CBOR format @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format @since version 3.1.0 */ @@ -25306,6 +25310,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec related UBJSON format @sa see @ref to_cbor(const basic_json&) for the related CBOR format @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa see @ref to_bon8(const basic_json&) for the related BON8 format */ static std::vector to_bson(const basic_json& j) { @@ -25335,6 +25340,44 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + /*! + @brief Serializes the given JSON object `j` to BON8 and returns a vector + containing the corresponding BON8-representation. + + BON8 is a binary format that encodes JSON values using those byte sequences + that would form invalid UTF-8 strings. As a result, strings do not need to + be re-encoded. + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a BON8 value. + + @note The following values can **not** be converted to a BON8 value: + - integers larger than 9223372036854775807 + + @throw out_of_range.407 if `j.is_number_unsigned() && j.get() > 9223372036854775807` + + @note Any BON8 output created via @ref to_bon8 can be successfully parsed + by @ref from_bon8. + + @param[in] j JSON value to serialize + @return BON8 serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in BON8 format.,to_bon8} + + @sa https://github.com/ttauri-project/ttauri/blob/main/docs/BON8.md + @sa see @ref from_bon8(detail::input_adapter&&, const bool strict) for the + analogous deserialization + @sa see @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + @sa see @ref to_cbor(const basic_json&) for the related CBOR format + @sa see @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa see @ref to_bson(const basic_json&) for the related BSON format + + @since version 3.11.0 + */ static std::vector to_bon8(const basic_json& j) { std::vector result; @@ -25342,11 +25385,21 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return result; } + /*! + @brief Serializes the given JSON object `j` to BON8 and forwards the + corresponding BON8-representation to the given output_adapter `o`. + @param j The JSON object to convert to BON8. + @param o The output adapter that receives the binary BON8 representation. + @sa see @ref to_bon8(const basic_json&) + */ static void to_bon8(const basic_json& j, detail::output_adapter o) { binary_writer(o).write_bon8(j); } + /*! + @copydoc to_bon8(const basic_json&, detail::output_adapter) + */ static void to_bon8(const basic_json& j, detail::output_adapter o) { binary_writer(o).write_bon8(j); From e9b1ab4e0a191a178be27696858678a0fd04f978 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 15:40:47 +0200 Subject: [PATCH 15/32] :memo: update table for binary formats --- .../docs/features/binary_formats/index.md | 24 ++- test/src/unit-binary_formats.cpp | 202 ++++++++++++++++++ 2 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 test/src/unit-binary_formats.cpp diff --git a/doc/mkdocs/docs/features/binary_formats/index.md b/doc/mkdocs/docs/features/binary_formats/index.md index bfa18c3c6..9ad0ce335 100644 --- a/doc/mkdocs/docs/features/binary_formats/index.md +++ b/doc/mkdocs/docs/features/binary_formats/index.md @@ -36,13 +36,21 @@ See [binary values](../binary_values.md) for more information. ### Sizes -| Format | canada.json | twitter.json | citm_catalog.json | jeopardy.json | -| ------------------ | ----------- | ------------ | ----------------- | ------------- | -| BSON | 85,8 % | 95,2 % | 95,8 % | 106,7 % | -| CBOR | 50,5 % | 86,3 % | 68,4 % | 88,0 % | -| MessagePack | 50,6 % | 86,0 % | 68,5 % | 87,9 % | -| UBJSON | 53,2 % | 91,3 % | 78,2 % | 96,6 % | -| UBJSON (size) | 58,6 % | 92,3 % | 86,8 % | 97,4 % | -| UBJSON (size+type) | 55,9 % | 92,3 % | 85,0 % | 95,0 % | +| Format | [canada.json](https://github.com/nlohmann/json_test_data/blob/master/nativejson-benchmark/canada.json) | [twitter.json](https://github.com/nlohmann/json_test_data/blob/master/nativejson-benchmark/twitter.json) | [citm_catalog.json](https://github.com/nlohmann/json_test_data/blob/master/nativejson-benchmark/citm_catalog.json) | [jeopardy.json](https://github.com/nlohmann/json_test_data/blob/master/jeopardy/jeopardy.json) | [sample.json](https://github.com/nlohmann/json_test_data/blob/master/json_testsuite/sample.json) | +| ------------------ | ----------- | ------------ | ----------------- | ------------- | ------------- | +| BSON | 85,8 % | 95,2 % | 95,8 % | 106,7 % (1) | N/A (2) | +| CBOR | 50,5 % | 86,3 % | 68,4 % | 88,0 % | 87,2 % | +| MessagePack | 50,6 % | 86,0 % | 68,5 % | 87,9 % | 87,2 % | +| UBJSON | 53,2 % | 91,3 % | 78,2 % | 96,6 % | 88,2 % | +| UBJSON (size) | 58,6 % | 92,3 % | 86,8 % | 97,4 % | 89,3 % | +| UBJSON (size+type) | 55,9 % | 92,3 % | 85,0 % | 95,0 % | 89,5 % | +| BON8 | 50,5 % | 83,8 % | 63,5 % | 87,5 % | 85,6 % | Sizes compared to minified JSON value. + +Notes: + +- (1) The JSON value is an array that needed to be wrapped in an object to be processed by BSON. We used an empty object key for minimal overhead. +- (2) The JSON value contained a string with code point `U+0000` which cannot be represented by BSON. + +The JSON files are part of the [nlohmann/json_test_data](https://github.com/nlohmann/json_test_data) repository. diff --git a/test/src/unit-binary_formats.cpp b/test/src/unit-binary_formats.cpp new file mode 100644 index 000000000..d9f69ea2f --- /dev/null +++ b/test/src/unit-binary_formats.cpp @@ -0,0 +1,202 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ (test suite) +| | |__ | | | | | | version 3.10.2 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2019 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include + +TEST_CASE("Binary Formats" * doctest::skip()) +{ + SECTION("canada.json") + { + const auto* filename = TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"; + json j = json::parse(std::ifstream(filename)); + + const auto json_size = j.dump().size(); + const auto bson_size = json::to_bson(j).size(); + const auto cbor_size = json::to_cbor(j).size(); + const auto msgpack_size = json::to_msgpack(j).size(); + const auto ubjson_1_size = json::to_ubjson(j).size(); + const auto ubjson_2_size = json::to_ubjson(j, true).size(); + const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bon8_size = json::to_bon8(j).size(); + + CHECK(json_size == 2090303); + CHECK(bson_size == 1794522); + CHECK(cbor_size == 1055552); + CHECK(msgpack_size == 1056145); + CHECK(ubjson_1_size == 1112030); + CHECK(ubjson_2_size == 1224148); + CHECK(ubjson_3_size == 1169069); + CHECK(bon8_size == 1055789); + + CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(85.849)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(50.497)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(50.526)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(53.199)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(58.563)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(55.928)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(50.509)); + } + + SECTION("twitter.json") + { + const auto* filename = TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"; + json j = json::parse(std::ifstream(filename)); + + const auto json_size = j.dump().size(); + const auto bson_size = json::to_bson(j).size(); + const auto cbor_size = json::to_cbor(j).size(); + const auto msgpack_size = json::to_msgpack(j).size(); + const auto ubjson_1_size = json::to_ubjson(j).size(); + const auto ubjson_2_size = json::to_ubjson(j, true).size(); + const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bon8_size = json::to_bon8(j).size(); + + CHECK(json_size == 466906); + CHECK(bson_size == 444568); + CHECK(cbor_size == 402814); + CHECK(msgpack_size == 401510); + CHECK(ubjson_1_size == 426160); + CHECK(ubjson_2_size == 430788); + CHECK(ubjson_3_size == 430798); + CHECK(bon8_size == 391172); + + CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(95.215)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(86.273)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(85.993)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(91.273)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(92.264)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(92.266)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(83.779)); + } + + SECTION("citm_catalog.json") + { + const auto* filename = TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"; + json j = json::parse(std::ifstream(filename)); + + const auto json_size = j.dump().size(); + const auto bson_size = json::to_bson(j).size(); + const auto cbor_size = json::to_cbor(j).size(); + const auto msgpack_size = json::to_msgpack(j).size(); + const auto ubjson_1_size = json::to_ubjson(j).size(); + const auto ubjson_2_size = json::to_ubjson(j, true).size(); + const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bon8_size = json::to_bon8(j).size(); + + CHECK(json_size == 500299); + CHECK(bson_size == 479430); + CHECK(cbor_size == 342373); + CHECK(msgpack_size == 342473); + CHECK(ubjson_1_size == 391463); + CHECK(ubjson_2_size == 434239); + CHECK(ubjson_3_size == 425073); + CHECK(bon8_size == 317877); + + CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(95.828)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(68.433)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(68.453)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(78.245)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(86.795)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(84.963)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(63.537)); + } + + SECTION("jeopardy.json") + { + const auto* filename = TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json"; + json j = json::parse(std::ifstream(filename)); + + const auto json_size = j.dump().size(); + const auto bson_size = json::to_bson({{"", j}}).size(); // wrap array in object for BSON + const auto cbor_size = json::to_cbor(j).size(); + const auto msgpack_size = json::to_msgpack(j).size(); + const auto ubjson_1_size = json::to_ubjson(j).size(); + const auto ubjson_2_size = json::to_ubjson(j, true).size(); + const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bon8_size = json::to_bon8(j).size(); + + CHECK(json_size == 52508728); + CHECK(bson_size == 56008520); + CHECK(cbor_size == 46187320); + CHECK(msgpack_size == 46158575); + CHECK(ubjson_1_size == 50710965); + CHECK(ubjson_2_size == 51144830); + CHECK(ubjson_3_size == 49861422); + CHECK(bon8_size == 45942080); + + CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(106.665)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(87.961)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(87.906)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(96.576)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(97.402)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(94.958)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(87.494)); + } + + SECTION("sample.json") + { + const auto* filename = TEST_DATA_DIRECTORY "/json_testsuite/sample.json"; + json j = json::parse(std::ifstream(filename)); + + const auto json_size = j.dump().size(); + // BSON cannot process the file as it contains code point U+0000 + const auto cbor_size = json::to_cbor(j).size(); + const auto msgpack_size = json::to_msgpack(j).size(); + const auto ubjson_1_size = json::to_ubjson(j).size(); + const auto ubjson_2_size = json::to_ubjson(j, true).size(); + const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bon8_size = json::to_bon8(j).size(); + + CHECK(json_size == 168677); + CHECK(cbor_size == 147095); + CHECK(msgpack_size == 147017); + CHECK(ubjson_1_size == 148695); + CHECK(ubjson_2_size == 150569); + CHECK(ubjson_3_size == 150883); + CHECK(bon8_size == 144463); + + CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(87.205)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(87.158)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(88.153)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(89.264)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(89.450)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(85.644)); + } +} From 627eefa5ad65b86aa8692d5f273c84b75595a255 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 16:01:10 +0200 Subject: [PATCH 16/32] :checkered_flag: set flag for MSVC for test-binary_formats --- test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 447192cb0..a00e38aa0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -102,6 +102,7 @@ endif() # avoid stack overflow, see https://github.com/nlohmann/json/issues/2955 if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set_property(TARGET test-binary_formats APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000") set_property(TARGET test-cbor APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000") set_property(TARGET test-msgpack APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000") set_property(TARGET test-ubjson APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000") From e27c12706829672d97c2760827728efc187359e1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 22:46:49 +0200 Subject: [PATCH 17/32] :construction: start BON8 parser --- .../nlohmann/detail/input/binary_reader.hpp | 185 ++++++++++++++- include/nlohmann/json.hpp | 30 +++ single_include/nlohmann/json.hpp | 215 +++++++++++++++++- test/src/unit-bon8.cpp | 34 +++ 4 files changed, 460 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 2cb7bc769..234017f7d 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -120,7 +120,10 @@ class binary_reader result = parse_ubjson_internal(); break; - case input_format_t::bon8: // LCOV_EXCL_LINE + case input_format_t::bon8: + result = parse_bon8_internal(true); + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -2300,6 +2303,181 @@ class binary_reader } } + ////////// + // BON8 // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character should + be considered instead (false) + + @return whether a valid BON8 value was passed to the SAX parser + */ + bool parse_bon8_internal(const bool get_char) + { + switch (get_char ? get() : current) + { + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + return get_bon8_array(static_cast(current - 0x80)); + + case 0x85: + return get_bon8_array(static_cast(-1)); + + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + return get_bon8_object(static_cast(current - 0x86)); + + case 0x8B: + return get_bon8_object(static_cast(-1)); + + case 0x8C: + { + std::int32_t number{}; + return get_number(input_format_t::bon8, number) && sax->number_integer(number); + } + + case 0x8D: + { + std::int64_t number{}; + return get_number(input_format_t::bon8, number) && sax->number_integer(number); + } + + case 0x8E: + { + float number{}; + return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + } + + case 0x8F: + { + double number{}; + return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + } + + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return sax->number_unsigned(current - 0x90); + + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xC0: + case 0xC1: + return sax->number_integer(0xB7 - current); + + case 0xF8: + return sax->boolean(false); + + case 0xF9: + return sax->boolean(true); + + case 0xFA: + return sax->null(); + + case 0xFB: + return sax->number_float(-1.0, ""); + + case 0xFC: + return sax->number_float(0.0, ""); + + case 0xFD: + return sax->number_float(1.0, ""); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + bool get_bon8_array(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(true))) + { + return false; + } + } + } + else + { + while (get() != 0xFE) + { + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + bool get_bon8_object(const std::size_t len) + { + return false; + } + /////////////////////// // Utility functions // /////////////////////// @@ -2497,7 +2675,10 @@ class binary_reader error_msg += "BSON"; break; - case input_format_t::bon8: // LCOV_EXCL_LINE + case input_format_t::bon8: + error_msg += "BON8"; + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index ff57fbeaf..86510b4a8 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -8211,6 +8211,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bon8(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bon8, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_bon8(InputType&&, const bool, const bool) + */ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bon8(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bon8, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + /// @} ////////////////////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 546bc5e95..2cea5f013 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8414,7 +8414,10 @@ class binary_reader result = parse_ubjson_internal(); break; - case input_format_t::bon8: // LCOV_EXCL_LINE + case input_format_t::bon8: + result = parse_bon8_internal(true); + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -10594,6 +10597,181 @@ class binary_reader } } + ////////// + // BON8 // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character should + be considered instead (false) + + @return whether a valid BON8 value was passed to the SAX parser + */ + bool parse_bon8_internal(const bool get_char) + { + switch (get_char ? get() : current) + { + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + return get_bon8_array(static_cast(current - 0x80)); + + case 0x85: + return get_bon8_array(static_cast(-1)); + + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + return get_bon8_object(static_cast(current - 0x86)); + + case 0x8B: + return get_bon8_object(static_cast(-1)); + + case 0x8C: + { + std::int32_t number{}; + return get_number(input_format_t::bon8, number) && sax->number_integer(number); + } + + case 0x8D: + { + std::int64_t number{}; + return get_number(input_format_t::bon8, number) && sax->number_integer(number); + } + + case 0x8E: + { + float number{}; + return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + } + + case 0x8F: + { + double number{}; + return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + } + + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return sax->number_unsigned(current - 0x90); + + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xC0: + case 0xC1: + return sax->number_integer(0xB7 - current); + + case 0xF8: + return sax->boolean(false); + + case 0xF9: + return sax->boolean(true); + + case 0xFA: + return sax->null(); + + case 0xFB: + return sax->number_float(-1.0, ""); + + case 0xFC: + return sax->number_float(0.0, ""); + + case 0xFD: + return sax->number_float(1.0, ""); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + bool get_bon8_array(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(true))) + { + return false; + } + } + } + else + { + while (get() != 0xFE) + { + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + bool get_bon8_object(const std::size_t len) + { + return false; + } + /////////////////////// // Utility functions // /////////////////////// @@ -10791,7 +10969,10 @@ class binary_reader error_msg += "BSON"; break; - case input_format_t::bon8: // LCOV_EXCL_LINE + case input_format_t::bon8: + error_msg += "BON8"; + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -25935,6 +26116,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bon8(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bon8, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_bon8(InputType&&, const bool, const bool) + */ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bon8(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bon8, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + /// @} ////////////////////////// diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 06cf57073..1858e05b9 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -56,6 +56,7 @@ TEST_CASE("BON8") std::vector expected = {0xFA}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("boolean") @@ -66,6 +67,7 @@ TEST_CASE("BON8") std::vector expected = {0xF9}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("false") @@ -74,6 +76,7 @@ TEST_CASE("BON8") std::vector expected = {0xF8}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -87,6 +90,7 @@ TEST_CASE("BON8") std::vector expected = {0x90}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("39") @@ -95,6 +99,7 @@ TEST_CASE("BON8") std::vector expected = {0xB7}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -163,6 +168,7 @@ TEST_CASE("BON8") std::vector expected = {0x8C, 0x04, 0x08, 0x0F, 0x28}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("2147483647 (int32max)") @@ -171,6 +177,7 @@ TEST_CASE("BON8") std::vector expected = {0x8C, 0x7F, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -182,6 +189,7 @@ TEST_CASE("BON8") std::vector expected = {0x8D, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("9223372036854775807 (int64max)") @@ -190,6 +198,7 @@ TEST_CASE("BON8") std::vector expected = {0x8D, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -210,6 +219,7 @@ TEST_CASE("BON8") std::vector expected = {0x8D, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("-2147483649") @@ -219,6 +229,7 @@ TEST_CASE("BON8") std::vector expected = {0x8D, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -231,6 +242,7 @@ TEST_CASE("BON8") std::vector expected = {0x8C, 0x80, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("-33818507") @@ -239,6 +251,7 @@ TEST_CASE("BON8") std::vector expected = {0x8C, 0xFD, 0xFB, 0xF8, 0x75}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -307,6 +320,7 @@ TEST_CASE("BON8") std::vector expected = {0xC1}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("-1") @@ -315,6 +329,7 @@ TEST_CASE("BON8") std::vector expected = {0xB8}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -326,6 +341,7 @@ TEST_CASE("BON8") std::vector expected = {0x90}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("39") @@ -334,6 +350,7 @@ TEST_CASE("BON8") std::vector expected = {0xB7}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -443,6 +460,7 @@ TEST_CASE("BON8") std::vector expected = {0xFB}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("0.0") @@ -451,6 +469,7 @@ TEST_CASE("BON8") std::vector expected = {0xFC}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("1.0") @@ -459,6 +478,7 @@ TEST_CASE("BON8") std::vector expected = {0xFD}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("-0.0") @@ -467,6 +487,7 @@ TEST_CASE("BON8") std::vector expected = {0x8E, 0x80, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("NAN") @@ -475,6 +496,8 @@ TEST_CASE("BON8") std::vector expected = {0x8E, 0x7F, 0x80, 0x00, 0x01}; const auto result = json::to_bon8(j); CHECK(result == expected); + json::number_float_t d{json::from_bon8(result)}; + CHECK(std::isnan(d)); } SECTION("infinity") @@ -483,6 +506,7 @@ TEST_CASE("BON8") std::vector expected = {0x8E, 0x7F, 0x80, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("-infinity") @@ -491,6 +515,7 @@ TEST_CASE("BON8") std::vector expected = {0x8E, 0xFF, 0x80, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -502,6 +527,7 @@ TEST_CASE("BON8") std::vector expected = {0x8E, 0x40, 0x00, 0x00, 0x00}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -513,6 +539,7 @@ TEST_CASE("BON8") std::vector expected = {0x8F, 0x41, 0x97, 0xD7, 0x84, 0x00, 0x66, 0x66, 0x66}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } } @@ -546,6 +573,7 @@ TEST_CASE("BON8") std::vector expected = {0x80}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[false]") @@ -554,6 +582,7 @@ TEST_CASE("BON8") std::vector expected = {0x81, 0xF8}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[false, null]") @@ -562,6 +591,7 @@ TEST_CASE("BON8") std::vector expected = {0x82, 0xF8, 0xFA}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[false, null, true]") @@ -570,6 +600,7 @@ TEST_CASE("BON8") std::vector expected = {0x83, 0xF8, 0xFA, 0xF9}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[false, null, true, 1.0]") @@ -578,6 +609,7 @@ TEST_CASE("BON8") std::vector expected = {0x84, 0xF8, 0xFA, 0xF9, 0xFD}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[\"s\", \"s\"]") @@ -610,6 +642,7 @@ TEST_CASE("BON8") std::vector expected = {0x81, 0x81, 0x81, 0x91}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[[[\"\"]]]") @@ -629,6 +662,7 @@ TEST_CASE("BON8") std::vector expected = {0x85, 0xF8, 0xFA, 0xF9, 0xFD, 0x80, 0xFC, 0xFE}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } } From b932217c15bac104605d676055de1facfd0a6594 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Sep 2021 23:28:16 +0200 Subject: [PATCH 18/32] :construction: add object and string parser for BON8 --- .../nlohmann/detail/input/binary_reader.hpp | 100 +++++++++++++++++- single_include/nlohmann/json.hpp | 100 +++++++++++++++++- test/src/unit-bon8.cpp | 8 ++ 3 files changed, 204 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 234017f7d..655c30f21 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2434,11 +2434,17 @@ class binary_reader case 0xFD: return sax->number_float(1.0, ""); - default: // anything else + case 0xFE: { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); } + + default: + { + string_t s; + return get_bon8_string(s) && sax->string(s); + } } } @@ -2475,7 +2481,97 @@ class binary_reader bool get_bon8_object(const std::size_t len) { - return false; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + if (len != 0) + { + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_bon8_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFE) + { + if (JSON_HEDLEY_UNLIKELY(!get_bon8_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + key.clear(); + } + } + } + + return sax->end_object(); + } + + bool get_bon8_string(string_t& result) + { + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + + if ((current & 0x80) == 0x00) + { + result.push_back(static_cast(current)); + get(); + } + else if ((current & 0xE0) == 0xC0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + get(); + } + else if ((current & 0xF0) == 0xE0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + result.push_back(static_cast(get())); + get(); + } + else if ((current & 0xF8) == 0xF0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + result.push_back(static_cast(get())); + result.push_back(static_cast(get())); + get(); + } + else if (current == 0xFF) + { + get(); + return true; + } + else + { + return true; + } + } } /////////////////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 2cea5f013..af4c3caa0 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10728,11 +10728,17 @@ class binary_reader case 0xFD: return sax->number_float(1.0, ""); - default: // anything else + case 0xFE: { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); } + + default: + { + string_t s; + return get_bon8_string(s) && sax->string(s); + } } } @@ -10769,7 +10775,97 @@ class binary_reader bool get_bon8_object(const std::size_t len) { - return false; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + if (len != 0) + { + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_bon8_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFE) + { + if (JSON_HEDLEY_UNLIKELY(!get_bon8_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + key.clear(); + } + } + } + + return sax->end_object(); + } + + bool get_bon8_string(string_t& result) + { + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + + if ((current & 0x80) == 0x00) + { + result.push_back(static_cast(current)); + get(); + } + else if ((current & 0xE0) == 0xC0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + get(); + } + else if ((current & 0xF0) == 0xE0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + result.push_back(static_cast(get())); + get(); + } + else if ((current & 0xF8) == 0xF0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + result.push_back(static_cast(get())); + result.push_back(static_cast(get())); + get(); + } + else if (current == 0xFF) + { + get(); + return true; + } + else + { + return true; + } + } } /////////////////////// diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 1858e05b9..471deeb2d 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -552,6 +552,7 @@ TEST_CASE("BON8") std::vector expected = {0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("other strings") @@ -560,6 +561,7 @@ TEST_CASE("BON8") std::vector expected = {'T', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 's', 't', 'r', 'i', 'n', 'g', '.', 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -634,6 +636,7 @@ TEST_CASE("BON8") std::vector expected = {0x81, 0x81, 0x81, 'f', 'o', 'o', 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("[[[1]]]") @@ -651,6 +654,7 @@ TEST_CASE("BON8") std::vector expected = {0x81, 0x81, 0x81, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } @@ -677,6 +681,7 @@ TEST_CASE("BON8") std::vector expected = {0x86}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("{\"foo\": null}") @@ -685,6 +690,7 @@ TEST_CASE("BON8") std::vector expected = {0x87, 'f', 'o', 'o', 0xFA}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("{\"\": true, \"foo\": null}") @@ -693,6 +699,7 @@ TEST_CASE("BON8") std::vector expected = {0x88, 0xFF, 0xF9, 'f', 'o', 'o', 0xFA}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } SECTION("{\"a\": \"\", \"c\": \"d\"}") @@ -720,6 +727,7 @@ TEST_CASE("BON8") std::vector expected = {0x8b, 'f', 'i', 'v', 'e', 0x95, 'f', 'o', 'u', 'r', 0x94, 'o', 'n', 'e', 0x91, 't', 'h', 'r', 'e', 'e', 0x93, 't', 'w', 'o', 0x92, 0xFE}; const auto result = json::to_bon8(j); CHECK(result == expected); + CHECK(json::from_bon8(result) == j); } } } From 95d75c2346508db2f388b872a35b7f587e6d0109 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 12 Sep 2021 13:33:45 +0200 Subject: [PATCH 19/32] :rotating_light: fix warnings --- include/nlohmann/detail/input/binary_reader.hpp | 4 ++-- single_include/nlohmann/json.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 655c30f21..1e3f565ee 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2402,7 +2402,7 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return sax->number_unsigned(current - 0x90); + return sax->number_unsigned(static_cast(current) - static_cast(0x90)); case 0xB8: case 0xB9: @@ -2414,7 +2414,7 @@ class binary_reader case 0xBF: case 0xC0: case 0xC1: - return sax->number_integer(0xB7 - current); + return sax->number_integer(static_cast(0xB7) - static_cast(current)); case 0xF8: return sax->boolean(false); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index af4c3caa0..610206761 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10696,7 +10696,7 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return sax->number_unsigned(current - 0x90); + return sax->number_unsigned(static_cast(current) - static_cast(0x90)); case 0xB8: case 0xB9: @@ -10708,7 +10708,7 @@ class binary_reader case 0xBF: case 0xC0: case 0xC1: - return sax->number_integer(0xB7 - current); + return sax->number_integer(static_cast(0xB7) - static_cast(current)); case 0xF8: return sax->boolean(false); From e6013608ccbeef56cc5d057c61e50c4c90d556a3 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 12 Sep 2021 19:04:24 +0200 Subject: [PATCH 20/32] :white_check_mark: improve coverage --- test/src/unit-bon8.cpp | 116 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 471deeb2d..91c8af5bf 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -38,6 +38,84 @@ using nlohmann::json; #include #include "test_utils.hpp" +namespace +{ +class SaxCountdown +{ + public: + explicit SaxCountdown(const int count) : events_left(count) + {} + + bool null() + { + return events_left-- > 0; + } + + bool boolean(bool /*unused*/) + { + return events_left-- > 0; + } + + bool number_integer(json::number_integer_t /*unused*/) + { + return events_left-- > 0; + } + + bool number_unsigned(json::number_unsigned_t /*unused*/) + { + return events_left-- > 0; + } + + bool number_float(json::number_float_t /*unused*/, const std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool string(std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool binary(std::vector& /*unused*/) + { + return events_left-- > 0; + } + + bool start_object(std::size_t /*unused*/) + { + return events_left-- > 0; + } + + bool key(std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool end_object() + { + return events_left-- > 0; + } + + bool start_array(std::size_t /*unused*/) + { + return events_left-- > 0; + } + + bool end_array() + { + return events_left-- > 0; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& /*unused*/) // NOLINT(readability-convert-member-functions-to-static) + { + return false; + } + + private: + int events_left = 0; +}; +} // namespace + TEST_CASE("BON8") { SECTION("individual values") @@ -732,4 +810,42 @@ TEST_CASE("BON8") } } } + + SECTION("SAX aborts") + { + SECTION("start_array(len)") + { + std::vector v = {0x80}; + SaxCountdown scp(0); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } + + SECTION("error in array") + { + std::vector v = {0x81}; + SaxCountdown scp(1000); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } + + SECTION("start_object(len)") + { + std::vector v = {0x86}; + SaxCountdown scp(0); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } + + SECTION("key()") + { + std::vector v = {0x87, 'f', 'o', 'o', 0xFF, 0xFA}; + SaxCountdown scp(1); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } + + SECTION("error in object") + { + std::vector v = {0x87, 'f', 'o', 'o', 0xFF}; + SaxCountdown scp(1000); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } + } } From cf81564797b26b145a72d3f47150e034c5dff147 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 12 Sep 2021 22:21:08 +0200 Subject: [PATCH 21/32] :white_check_mark: improve coverage --- .../nlohmann/detail/input/binary_reader.hpp | 24 +++++++++ single_include/nlohmann/json.hpp | 24 +++++++++ test/src/unit-bon8.cpp | 51 ++++++++++++++++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 1e3f565ee..66500069f 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2545,21 +2545,45 @@ class binary_reader { result.push_back(static_cast(current)); result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } get(); } else if ((current & 0xF0) == 0xE0) { result.push_back(static_cast(current)); result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } get(); } else if ((current & 0xF8) == 0xF0) { result.push_back(static_cast(current)); result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } get(); } else if (current == 0xFF) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 862330ea0..467c22171 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10839,21 +10839,45 @@ class binary_reader { result.push_back(static_cast(current)); result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } get(); } else if ((current & 0xF0) == 0xE0) { result.push_back(static_cast(current)); result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } get(); } else if ((current & 0xF8) == 0xF0) { result.push_back(static_cast(current)); result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } get(); } else if (current == 0xFF) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index 91c8af5bf..db243e6fd 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -641,6 +641,39 @@ TEST_CASE("BON8") CHECK(result == expected); CHECK(json::from_bon8(result) == j); } + + SECTION("multi-byte, 2 bytes") + { + json j = "\xC2\xA3"; + std::vector expected = {0xC2, 0xA3, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + CHECK(json::from_bon8(result) == j); + } + + SECTION("multi-byte, 3 bytes") + { + json j = "\xEF\xB8\xBB"; + std::vector expected = {0xEF, 0xB8, 0xBB, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + CHECK(json::from_bon8(result) == j); + } + + SECTION("multi-byte, 4 bytes") + { + json j = "\xF0\x9F\x80\x84"; + std::vector expected = {0xF0, 0x9F, 0x80, 0x84, 0xFF}; + const auto result = json::to_bon8(j); + CHECK(result == expected); + CHECK(json::from_bon8(result) == j); + } + + SECTION("invalid string") + { + std::vector v = {0xF0, 0x9F, 0x80, 0x84}; + CHECK_THROWS_WITH_AS(json::from_bon8(v), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 string: unexpected end of input", json::parse_error); + } } SECTION("array") @@ -820,13 +853,20 @@ TEST_CASE("BON8") CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); } - SECTION("error in array") + SECTION("error in array with size") { std::vector v = {0x81}; SaxCountdown scp(1000); CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); } + SECTION("error in array without size") + { + std::vector v = {0x85}; + SaxCountdown scp(1000); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } + SECTION("start_object(len)") { std::vector v = {0x86}; @@ -841,11 +881,18 @@ TEST_CASE("BON8") CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); } - SECTION("error in object") + SECTION("error in object with size") { std::vector v = {0x87, 'f', 'o', 'o', 0xFF}; SaxCountdown scp(1000); CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); } + + SECTION("error in object without size") + { + std::vector v = {0x8B, 'f', 'o', 'o', 0xFF}; + SaxCountdown scp(1000); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bon8)); + } } } From cec6b216950ee91fd93cf831941123f9f39019cf Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 12 Sep 2021 22:48:34 +0200 Subject: [PATCH 22/32] :rotating_light: fix warning --- test/src/unit-bon8.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/src/unit-bon8.cpp b/test/src/unit-bon8.cpp index db243e6fd..047e4e5af 100644 --- a/test/src/unit-bon8.cpp +++ b/test/src/unit-bon8.cpp @@ -672,7 +672,8 @@ TEST_CASE("BON8") SECTION("invalid string") { std::vector v = {0xF0, 0x9F, 0x80, 0x84}; - CHECK_THROWS_WITH_AS(json::from_bon8(v), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 string: unexpected end of input", json::parse_error); + json j; + CHECK_THROWS_WITH_AS(j = json::from_bon8(v), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 string: unexpected end of input", json::parse_error); } } From 9105f5514e2094de8a583df83368c0b16febf658 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 31 Dec 2021 14:57:35 +0100 Subject: [PATCH 23/32] :twisted_rightwards_arrows: merge develop branch --- include/nlohmann/json.hpp | 23 +++++++++++++++++++++++ single_include/nlohmann/json.hpp | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 29476fced..9121afaa4 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3939,6 +3939,29 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static std::vector to_bon8(const basic_json& j) + { + std::vector result; + to_bon8(j, result); + return result; + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + /// @brief create a JSON value from an input in CBOR format /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ template diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f8bc4107d..2bb398764 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21721,6 +21721,29 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static std::vector to_bon8(const basic_json& j) + { + std::vector result; + to_bon8(j, result); + return result; + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + /// @brief create a JSON value from an input in CBOR format /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ template From 4ada06d860bb2baa8cdd6bbfcc1189f7c842c613 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 May 2022 20:01:10 +0200 Subject: [PATCH 24/32] :twisted_rightwards_arrows: merge develop --- .../nlohmann/detail/input/binary_reader.hpp | 2 +- .../nlohmann/detail/output/binary_writer.hpp | 2 +- single_include/nlohmann/json.hpp | 693 +++++++++++++++++- 3 files changed, 685 insertions(+), 12 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 9323c5f05..971781ca2 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2776,7 +2776,7 @@ class binary_reader case 0xFE: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), nullptr)); } default: diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 07a115c67..d14feb6be 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1758,7 +1758,7 @@ class binary_writer { if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", j)); + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", &j)); } write_bon8_integer(static_cast(j.m_value.number_unsigned)); return false; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index ad777c92c..082c3c057 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5525,7 +5525,7 @@ namespace nlohmann namespace detail { /// the supported input formats -enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata }; +enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata, bon8 }; //////////////////// // input adapters // @@ -8590,6 +8590,10 @@ class binary_reader result = parse_ubjson_internal(); break; + case input_format_t::bon8: + result = parse_bon8_internal(true); + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -11105,6 +11109,301 @@ class binary_reader } } + ////////// + // BON8 // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character should + be considered instead (false) + + @return whether a valid BON8 value was passed to the SAX parser + */ + bool parse_bon8_internal(const bool get_char) + { + switch (get_char ? get() : current) + { + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + return get_bon8_array(static_cast(current - 0x80)); + + case 0x85: + return get_bon8_array(static_cast(-1)); + + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + return get_bon8_object(static_cast(current - 0x86)); + + case 0x8B: + return get_bon8_object(static_cast(-1)); + + case 0x8C: + { + std::int32_t number{}; + return get_number(input_format_t::bon8, number) && sax->number_integer(number); + } + + case 0x8D: + { + std::int64_t number{}; + return get_number(input_format_t::bon8, number) && sax->number_integer(number); + } + + case 0x8E: + { + float number{}; + return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + } + + case 0x8F: + { + double number{}; + return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + } + + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return sax->number_unsigned(static_cast(current) - static_cast(0x90)); + + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xC0: + case 0xC1: + return sax->number_integer(static_cast(0xB7) - static_cast(current)); + + case 0xF8: + return sax->boolean(false); + + case 0xF9: + return sax->boolean(true); + + case 0xFA: + return sax->null(); + + case 0xFB: + return sax->number_float(-1.0, ""); + + case 0xFC: + return sax->number_float(0.0, ""); + + case 0xFD: + return sax->number_float(1.0, ""); + + case 0xFE: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bon8, "invalid byte: 0x" + last_token, "value"), nullptr)); + } + + default: + { + string_t s; + return get_bon8_string(s) && sax->string(s); + } + } + } + + bool get_bon8_array(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(true))) + { + return false; + } + } + } + else + { + while (get() != 0xFE) + { + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + bool get_bon8_object(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + if (len != 0) + { + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_bon8_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFE) + { + if (JSON_HEDLEY_UNLIKELY(!get_bon8_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bon8_internal(false))) + { + return false; + } + key.clear(); + } + } + } + + return sax->end_object(); + } + + bool get_bon8_string(string_t& result) + { + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + + if ((current & 0x80) == 0x00) + { + result.push_back(static_cast(current)); + get(); + } + else if ((current & 0xE0) == 0xC0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + get(); + } + else if ((current & 0xF0) == 0xE0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + get(); + } + else if ((current & 0xF8) == 0xF0) + { + result.push_back(static_cast(current)); + result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + result.push_back(static_cast(get())); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bon8, "string"))) + { + return false; + } + get(); + } + else if (current == 0xFF) + { + get(); + return true; + } + else + { + return true; + } + } + } + /////////////////////// // Utility functions // /////////////////////// @@ -11304,6 +11603,10 @@ class binary_reader error_msg += "BSON"; break; + case input_format_t::bon8: + error_msg += "BON8"; + break; + case input_format_t::bjdata: error_msg += "BJData"; break; @@ -14929,6 +15232,18 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + */ + void write_bon8(const BasicJsonType& j) + { + const bool last_written_value_is_string = write_bon8_internal(j); + if (last_written_value_is_string) + { + oa->write_character(to_char_type(0xFF)); + } + } + private: ////////// // BSON // @@ -15705,6 +16020,279 @@ class binary_writer return false; } + ////////// + // BON8 // + ////////// + + /*! + * @param j + * @return whether the last written value was a string + */ + bool write_bon8_internal(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xFA)); + return false; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF9) + : to_char_type(0xF8)); + return false; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BON8 as it does not fit int64", &j)); + } + write_bon8_integer(static_cast(j.m_value.number_unsigned)); + return false; + } + + case value_t::number_integer: + { + write_bon8_integer(j.m_value.number_integer); + return false; + } + + case value_t::number_float: + { + // special values + if (j.m_value.number_float == -1.0) + { + oa->write_character(to_char_type(0xFB)); + } + else if (j.m_value.number_float == 0.0 && !std::signbit(j.m_value.number_float)) + { + oa->write_character(to_char_type(0xFC)); + } + else if (j.m_value.number_float == 1.0) + { + oa->write_character(to_char_type(0xFD)); + } + else if (std::isnan(j.m_value.number_float)) + { + oa->write_character(to_char_type(0x8E)); + oa->write_character(to_char_type(0x7F)); + oa->write_character(to_char_type(0x80)); + oa->write_character(to_char_type(0x00)); + oa->write_character(to_char_type(0x01)); + } + else + { + // write float with prefix + write_compact_float(j.m_value.number_float, detail::input_format_t::bon8); + } + return false; + } + + case value_t::string: + { + // empty string: use end-of-text symbol + if (j.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + return false; // already wrote 0xFF byte + } + + // write strings as is + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + return true; + } + + case value_t::array: + { + bool last_written_value_is_string = false; + const auto N = j.m_value.array->size(); + if (N <= 4) + { + // start array with count (80..84) + oa->write_character(static_cast(0x80 + N)); + } + else + { + // start array + oa->write_character(to_char_type(0x85)); + } + + // write each element + for (std::size_t i = 0; i < N; ++i) + { + const auto& el = j.m_value.array->operator[](i); + + // check if 0xFF after nonempty string and string is required + if (i > 0) + { + const auto& prev = j.m_value.array->operator[](i - 1); + if (el.is_string() && prev.is_string() && !prev.m_value.string->empty()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + last_written_value_is_string = write_bon8_internal(el); + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + last_written_value_is_string = false; // 0xFE is not a string byte + } + + return last_written_value_is_string; + } + + case value_t::object: + { + bool last_written_value_is_string = false; + const auto N = j.m_value.object->size(); + if (N <= 4) + { + // start object with count (86..8A) + oa->write_character(static_cast(0x86 + N)); + } + else + { + // start object + oa->write_character(to_char_type(0x8B)); + } + + // write each element + for (auto it = j.m_value.object->begin(); it != j.m_value.object->end(); ++it) + { + const auto& key = it->first; + const auto& value = it->second; + + write_bon8_internal(key); + + // check if we need a 0xFF separator between key and value + if (!key.empty() && value.is_string()) + { + oa->write_character(to_char_type(0xFF)); + } + + last_written_value_is_string = write_bon8_internal(value); + + // check if we need a 0xFF separator between the value and the next key + if (value.is_string() && !value.m_value.string->empty() && std::next(it) != j.m_value.object->end()) + { + oa->write_character(to_char_type(0xFF)); + } + } + + if (N > 4) + { + // end of container + oa->write_character(to_char_type(0xFE)); + last_written_value_is_string = false; // 0xFE is not a string byte + } + + return last_written_value_is_string; + } + + case value_t::binary: + case value_t::discarded: + default: + return false; + } + } + + void write_bon8_integer(typename BasicJsonType::number_integer_t value) + { + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) + { + // 64 bit integers + oa->write_character(to_char_type(0x8D)); + write_number(static_cast(value)); + } + else if (value < -33818506 || value > 67637031) + { + // 32 bit integers + oa->write_character(to_char_type(0x8C)); + write_number(static_cast(value)); + } + else if (value <= -264075) + { + JSON_ASSERT(value >= -33818506); + value = -(value + 264075); + oa->write_character(static_cast(0xF0 + (value >> 22 & 0x07))); + oa->write_character(static_cast(0xC0 + (value >> 16 & 0x3F))); + oa->write_character(static_cast(value >> 8)); + oa->write_character(static_cast(value)); + } + else if (value <= -1931) + { + JSON_ASSERT(value >= -264074); + value = -(value + 1931); + oa->write_character(static_cast(0xE0 + (value >> 14 & 0x0F))); + oa->write_character(static_cast(0xC0 + (value >> 8 & 0x3F))); + oa->write_character(static_cast(value)); + } + else if (value <= -11) + { + JSON_ASSERT(value >= -1930); + value = -(value + 11); + oa->write_character(static_cast(0xC2 + (value >> 6 & 0x1F))); + oa->write_character(static_cast(0xC0 + (value & 0x3F))); + } + else if (value <= -1) + { + JSON_ASSERT(value >= -10); + value = -(value + 1); + oa->write_character(static_cast(0xB8 + value)); + } + else if (value <= 39) + { + JSON_ASSERT(value >= 0); + oa->write_character(static_cast(0x90 + value)); + } + else if (value <= 3879) + { + JSON_ASSERT(value >= 40); + value -= 40; + oa->write_character(static_cast(0xC2 + (value >> 7 & 0x1F))); + oa->write_character(static_cast(value & 0x7F)); + } + else if (value <= 528167) + { + JSON_ASSERT(value >= 3880); + value -= 3880; + oa->write_character(static_cast(0xE0 + (value >> 15 & 0x0F))); + oa->write_character(static_cast(value >> 8 & 0x7F)); + oa->write_character(static_cast(value)); + } + else + { + JSON_ASSERT(value >= 528168); + JSON_ASSERT(value <= 67637031); + value -= 528168; + oa->write_character(static_cast(0xF0 + (value >> 23 & 0x17))); + oa->write_character(static_cast(value >> 16 & 0x7F)); + oa->write_character(static_cast(value >> 8)); + oa->write_character(static_cast(value)); + } + } + + static constexpr CharType get_bon8_float_prefix(float /*unused*/) + { + return to_char_type(0x8E); + } + + static constexpr CharType get_bon8_float_prefix(double /*unused*/) + { + return to_char_type(0x8F); + } + /////////////////////// // Utility functions // /////////////////////// @@ -15745,20 +16333,52 @@ class binary_writer #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wfloat-equal" #endif - if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && - static_cast(n) <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(n)) == static_cast(n)) + if (std::isnan(n) || std::isinf(n) || (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n))) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + switch (format) + { + case input_format_t::cbor: + oa->write_character(get_cbor_float_prefix(static_cast(n))); + break; + case input_format_t::msgpack: + oa->write_character(get_msgpack_float_prefix(static_cast(n))); + break; + case input_format_t::bon8: + oa->write_character(get_bon8_float_prefix(static_cast(n))); + break; + // LCOV_EXCL_START + case input_format_t::bson: + case input_format_t::json: + case input_format_t::ubjson: + default: + break; + // LCOV_EXCL_STOP + } write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + switch (format) + { + case input_format_t::cbor: + oa->write_character(get_cbor_float_prefix(n)); + break; + case input_format_t::msgpack: + oa->write_character(get_msgpack_float_prefix(n)); + break; + case input_format_t::bon8: + oa->write_character(get_bon8_float_prefix(n)); + break; + // LCOV_EXCL_START + case input_format_t::bson: + case input_format_t::json: + case input_format_t::ubjson: + default: + break; + // LCOV_EXCL_STOP + } write_number(n); } #ifdef __GNUC__ @@ -22275,6 +22895,29 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_bson(j); } + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static std::vector to_bon8(const basic_json& j) + { + std::vector result; + to_bon8(j, result); + return result; + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ + static void to_bon8(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bon8(j); + } + /// @brief create a JSON value from an input in CBOR format /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ template @@ -22549,6 +23192,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bon8(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bon8, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_bon8(InputType&&, const bool, const bool) + */ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bon8(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bon8, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + /// @} ////////////////////////// From 5927ddc3b24a5d014f464f4050f4729ecdc360ab Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 May 2022 20:08:44 +0200 Subject: [PATCH 25/32] :rotating_light: fix warnings --- include/nlohmann/detail/input/binary_reader.hpp | 4 ++-- include/nlohmann/detail/output/binary_writer.hpp | 1 + single_include/nlohmann/json.hpp | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 971781ca2..5246f69f4 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2794,7 +2794,7 @@ class binary_reader return false; } - if (len != std::size_t(-1)) + if (len != static_cast(-1)) { for (std::size_t i = 0; i < len; ++i) { @@ -2828,7 +2828,7 @@ class binary_reader if (len != 0) { string_t key; - if (len != std::size_t(-1)) + if (len != static_cast(-1)) { for (std::size_t i = 0; i < len; ++i) { diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index d14feb6be..6d7ced0b6 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2060,6 +2060,7 @@ class binary_writer case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: + case input_format_t::bjdata: default: break; // LCOV_EXCL_STOP diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 082c3c057..a59e6448b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11261,7 +11261,7 @@ class binary_reader return false; } - if (len != std::size_t(-1)) + if (len != static_cast(-1)) { for (std::size_t i = 0; i < len; ++i) { @@ -11295,7 +11295,7 @@ class binary_reader if (len != 0) { string_t key; - if (len != std::size_t(-1)) + if (len != static_cast(-1)) { for (std::size_t i = 0; i < len; ++i) { @@ -16352,6 +16352,7 @@ class binary_writer case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: + case input_format_t::bjdata: default: break; // LCOV_EXCL_STOP From a1e116f0152e3f141b5f852223d755596df42f35 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 May 2022 20:50:09 +0200 Subject: [PATCH 26/32] :white_check_mark: complete test suite --- tests/CMakeLists.txt | 2 +- tests/src/unit-binary_formats.cpp | 45 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1d5c78697..0df890ab4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -72,7 +72,7 @@ endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # avoid stack overflow, see https://github.com/nlohmann/json/issues/2955 - json_test_set_test_options("test-cbor;test-msgpack;test-ubjson;test-bjdata" LINK_OPTIONS /STACK:4000000) + json_test_set_test_options("test-cbor;test-msgpack;test-ubjson;test-bjdata;test-binary_formats" LINK_OPTIONS /STACK:4000000) endif() # disable exceptions for test-disabled_exceptions diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index d9f69ea2f..c83925089 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -49,6 +49,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) const auto ubjson_1_size = json::to_ubjson(j).size(); const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bjdata_1_size = json::to_bjdata(j).size(); + const auto bjdata_2_size = json::to_bjdata(j, true).size(); + const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); const auto bon8_size = json::to_bon8(j).size(); CHECK(json_size == 2090303); @@ -58,6 +61,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK(ubjson_1_size == 1112030); CHECK(ubjson_2_size == 1224148); CHECK(ubjson_3_size == 1169069); + CHECK(bjdata_1_size == 1112030); + CHECK(bjdata_2_size == 1224148); + CHECK(bjdata_3_size == 1224148); CHECK(bon8_size == 1055789); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); @@ -67,6 +73,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(53.199)); CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(58.563)); CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(55.928)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(53.199)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(58.563)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(58.563)); CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(50.509)); } @@ -82,6 +91,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) const auto ubjson_1_size = json::to_ubjson(j).size(); const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bjdata_1_size = json::to_bjdata(j).size(); + const auto bjdata_2_size = json::to_bjdata(j, true).size(); + const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); const auto bon8_size = json::to_bon8(j).size(); CHECK(json_size == 466906); @@ -91,6 +103,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK(ubjson_1_size == 426160); CHECK(ubjson_2_size == 430788); CHECK(ubjson_3_size == 430798); + CHECK(bjdata_1_size == 425342); + CHECK(bjdata_2_size == 429970); + CHECK(bjdata_3_size == 429970); CHECK(bon8_size == 391172); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); @@ -100,6 +115,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(91.273)); CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(92.264)); CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(92.266)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(91.097)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(92.089)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(92.089)); CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(83.779)); } @@ -115,6 +133,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) const auto ubjson_1_size = json::to_ubjson(j).size(); const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bjdata_1_size = json::to_bjdata(j).size(); + const auto bjdata_2_size = json::to_bjdata(j, true).size(); + const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); const auto bon8_size = json::to_bon8(j).size(); CHECK(json_size == 500299); @@ -124,6 +145,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK(ubjson_1_size == 391463); CHECK(ubjson_2_size == 434239); CHECK(ubjson_3_size == 425073); + CHECK(bjdata_1_size == 390781); + CHECK(bjdata_2_size == 433557); + CHECK(bjdata_3_size == 432964); CHECK(bon8_size == 317877); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); @@ -133,6 +157,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(78.245)); CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(86.795)); CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(84.963)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(78.109)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(86.659)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(86.541)); CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(63.537)); } @@ -148,6 +175,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) const auto ubjson_1_size = json::to_ubjson(j).size(); const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bjdata_1_size = json::to_bjdata(j).size(); + const auto bjdata_2_size = json::to_bjdata(j, true).size(); + const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); const auto bon8_size = json::to_bon8(j).size(); CHECK(json_size == 52508728); @@ -157,6 +187,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK(ubjson_1_size == 50710965); CHECK(ubjson_2_size == 51144830); CHECK(ubjson_3_size == 49861422); + CHECK(bjdata_1_size == 50710965); + CHECK(bjdata_2_size == 51144830); + CHECK(bjdata_3_size == 51144830); CHECK(bon8_size == 45942080); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); @@ -166,6 +199,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(96.576)); CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(97.402)); CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(94.958)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(96.576)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(97.402)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(97.402)); CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(87.494)); } @@ -181,6 +217,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) const auto ubjson_1_size = json::to_ubjson(j).size(); const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + const auto bjdata_1_size = json::to_bjdata(j).size(); + const auto bjdata_2_size = json::to_bjdata(j, true).size(); + const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); const auto bon8_size = json::to_bon8(j).size(); CHECK(json_size == 168677); @@ -189,6 +228,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK(ubjson_1_size == 148695); CHECK(ubjson_2_size == 150569); CHECK(ubjson_3_size == 150883); + CHECK(bjdata_1_size == 148695); + CHECK(bjdata_2_size == 150569); + CHECK(bjdata_3_size == 150569); CHECK(bon8_size == 144463); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); @@ -197,6 +239,9 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(88.153)); CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(89.264)); CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(89.450)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(88.153)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(89.264)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(89.264)); CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(85.644)); } } From b1e3165712fb342b7ab30831a672a71af249d3fa Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 May 2022 22:50:40 +0200 Subject: [PATCH 27/32] :rotating_light: fix warnings --- include/nlohmann/detail/output/binary_writer.hpp | 1 + tests/src/unit-bon8.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 6d7ced0b6..75582c0e9 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2084,6 +2084,7 @@ class binary_writer case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: + case input_format_t::bjdata: default: break; // LCOV_EXCL_STOP diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index 047e4e5af..6953adf0e 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -303,7 +303,7 @@ TEST_CASE("BON8") SECTION("-2147483649") { // cannot use -2147483649 directly, see https://developercommunity.visualstudio.com/t/-2147483648-c4146-error/141813#T-N229960 - json j = std::int64_t(-2147483647) - 2; + json j = static_cast(-2147483647) - 2; std::vector expected = {0x8D, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; const auto result = json::to_bon8(j); CHECK(result == expected); From ad662d15bcd98a1c7eed71ca0b590aba75e56c18 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 May 2022 07:10:03 +0200 Subject: [PATCH 28/32] :art: fix indendation --- single_include/nlohmann/json.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a59e6448b..0c4ff8a0b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -16376,6 +16376,7 @@ class binary_writer case input_format_t::bson: case input_format_t::json: case input_format_t::ubjson: + case input_format_t::bjdata: default: break; // LCOV_EXCL_STOP From 107e99ef1e23cec48ef3310cc67bb7d8d8e67def Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 13 Aug 2022 13:02:28 +0200 Subject: [PATCH 29/32] :memo: fix example --- docs/examples/to_bon8.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/examples/to_bon8.cpp b/docs/examples/to_bon8.cpp index 2b4e311b4..2f2634253 100644 --- a/docs/examples/to_bon8.cpp +++ b/docs/examples/to_bon8.cpp @@ -3,6 +3,7 @@ #include using json = nlohmann::json; +using namespace nlohmann::literals; int main() { From d360b7e396d16f6f22af5d900c6ddd15bd71d9d0 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 13 Aug 2022 13:11:37 +0200 Subject: [PATCH 30/32] :green_heart: fix unit test --- tests/src/unit-bson.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index a231c8cca..fa07e9d9c 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -10,6 +10,9 @@ #include using nlohmann::json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS +using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif #include #include From b116faa02df582d72badc2d906290d40c1a0b979 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 13 Aug 2022 13:29:11 +0200 Subject: [PATCH 31/32] :green_heart: fix unit test --- tests/src/unit-bon8.cpp | 3 +++ tests/src/unit-bson.cpp | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index 6953adf0e..bf3e99c9b 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -31,6 +31,9 @@ SOFTWARE. #include using nlohmann::json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS +using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif #include #include diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index fa07e9d9c..a231c8cca 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -10,9 +10,6 @@ #include using nlohmann::json; -#ifdef JSON_TEST_NO_GLOBAL_UDLS -using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) -#endif #include #include From f9003d169b286accbfecbabeaa57a459f83c8eeb Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 13 Aug 2022 13:36:08 +0200 Subject: [PATCH 32/32] :art: fix indentation --- tests/src/unit-bon8.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index bf3e99c9b..f69286af2 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -32,7 +32,7 @@ SOFTWARE. #include using nlohmann::json; #ifdef JSON_TEST_NO_GLOBAL_UDLS -using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) + using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif #include