From 408144ae73c9740f741e7a277c47b36447726c18 Mon Sep 17 00:00:00 2001 From: Qianqian Fang Date: Sat, 19 Feb 2022 11:32:03 -0500 Subject: [PATCH] simplify endian condition, format unit-bjdata --- .../nlohmann/detail/input/binary_reader.hpp | 5 +- .../nlohmann/detail/output/binary_writer.hpp | 5 +- single_include/nlohmann/json.hpp | 10 +- test/Makefile | 5 +- test/src/unit-bjdata.cpp | 110 ++++++++++++++++-- 5 files changed, 114 insertions(+), 21 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 988dffe96..b16ae5236 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2573,6 +2573,8 @@ class binary_reader @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ template bool get_number(const input_format_t format, NumberType& result) @@ -2588,8 +2590,7 @@ class binary_reader } // reverse byte order prior to conversion if necessary - if ((is_little_endian != InputIsLittleEndian && format != input_format_t::bjdata) || - (is_little_endian == InputIsLittleEndian && format == input_format_t::bjdata)) + if (is_little_endian != (InputIsLittleEndian || format == input_format_t::bjdata)) { vec[sizeof(NumberType) - i - 1] = static_cast(current); } diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 29c0c279e..f121ef667 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1600,7 +1600,8 @@ class binary_writer @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. - BJData is similar to UBJSON but uses little endian by default. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ template void write_number(const NumberType n) @@ -1610,7 +1611,7 @@ class binary_writer std::memcpy(vec.data(), &n, sizeof(NumberType)); // step 2: write array to output (with possible reordering) - if ((!is_bjdata && (is_little_endian != OutputIsLittleEndian)) || (is_bjdata && !is_little_endian)) + if (is_little_endian != (OutputIsLittleEndian || is_bjdata)) { // reverse byte order prior to conversion if necessary std::reverse(vec.begin(), vec.end()); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6b1a7dd41..1091ddf1a 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10949,6 +10949,8 @@ class binary_reader @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ template bool get_number(const input_format_t format, NumberType& result) @@ -10964,8 +10966,7 @@ class binary_reader } // reverse byte order prior to conversion if necessary - if ((is_little_endian != InputIsLittleEndian && format != input_format_t::bjdata) || - (is_little_endian == InputIsLittleEndian && format == input_format_t::bjdata)) + if (is_little_endian != (InputIsLittleEndian || format == input_format_t::bjdata)) { vec[sizeof(NumberType) - i - 1] = static_cast(current); } @@ -15387,7 +15388,8 @@ class binary_writer @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. - BJData is similar to UBJSON but uses little endian by default. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ template void write_number(const NumberType n) @@ -15397,7 +15399,7 @@ class binary_writer std::memcpy(vec.data(), &n, sizeof(NumberType)); // step 2: write array to output (with possible reordering) - if ((!is_bjdata && (is_little_endian != OutputIsLittleEndian)) || (is_bjdata && !is_little_endian)) + if (is_little_endian != (OutputIsLittleEndian || is_bjdata)) { // reverse byte order prior to conversion if necessary std::reverse(vec.begin(), vec.end()); diff --git a/test/Makefile b/test/Makefile index 2d08a8f7b..3a11ce7dd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -10,7 +10,7 @@ CXXFLAGS += -std=c++11 CPPFLAGS += -I ../single_include FUZZER_ENGINE = src/fuzzer-driver_afl.cpp -FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer +FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer fuzzers: $(FUZZERS) parse_afl_fuzzer: @@ -27,3 +27,6 @@ parse_msgpack_fuzzer: parse_ubjson_fuzzer: $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_ubjson.cpp -o $@ + +parse_bjdata_fuzzer: + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_bjdata.cpp -o $@ diff --git a/test/src/unit-bjdata.cpp b/test/src/unit-bjdata.cpp index eb6eec788..d23fd93e2 100644 --- a/test/src/unit-bjdata.cpp +++ b/test/src/unit-bjdata.cpp @@ -1034,7 +1034,7 @@ TEST_CASE("BJData") CHECK(json::parse("1.5") == json::from_bjdata(std::vector({'h', 0x00, 0x3e}))); CHECK(json::parse("65504.0") == json::from_bjdata(std::vector({'h', 0xff, 0x7b}))); } - + SECTION("errors") { json _; @@ -1043,7 +1043,7 @@ TEST_CASE("BJData") CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec0), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData number: unexpected end of input", json::parse_error); // missing the 2nd byte - std::vector vec1 = {'h', 0}; + std::vector vec1 = {'h', 0x00}; CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec1), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BJData number: unexpected end of input", json::parse_error); } } @@ -1776,7 +1776,7 @@ TEST_CASE("BJData") } } - SECTION("array with uint16_t elements") + SECTION("array with int16_t elements") { SECTION("size=false type=false") { @@ -1822,6 +1822,52 @@ TEST_CASE("BJData") } } + SECTION("array with uint16_t elements") + { + SECTION("size=false type=false") + { + json j(32768, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[32769] = ']'; // closing array + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j(32768, nullptr); + std::vector expected(j.size() + 5, 'Z'); // all null + expected[0] = '['; // opening array + expected[1] = '#'; // array size + expected[2] = 'u'; // int16 + expected[3] = 0x00; // 0x0101, first byte + expected[4] = 0x80; // 0x0101, second byte + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true") + { + json j(32768, nullptr); + std::vector expected = {'[', '$', 'Z', '#', 'u', 0x00, 0x80}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + SECTION("array with int32_t elements") { SECTION("size=false type=false") @@ -1869,6 +1915,36 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(result, true, false) == j); } } + + SECTION("array with uint32_t elements") + { + SECTION("size=true type=true") + { + json j(2147483648, nullptr); + std::vector expected = {'[', '$', 'Z', '#', 'm', 0x00, 0x00, 0x00, 0x80}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("array with uint64_t elements") + { + SECTION("size=true type=true") + { + json j(9223372036854775808ull, nullptr); + std::vector expected = {'[', '$', 'Z', '#', 'M', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } } SECTION("object") @@ -2613,25 +2689,35 @@ TEST_CASE("BJData") CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); CHECK(json::from_bjdata(v, true, false).is_discarded()); + std::vector vS0 = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(vS0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vS0), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vS0, true, false).is_discarded()); + std::vector vS = {'[', '$', 'i', '#', '[', '#', 'i', 2, 1, 2, 1}; CHECK_THROWS_AS(_ = json::from_bjdata(vS), json::parse_error&); CHECK_THROWS_WITH(_ = json::from_bjdata(vS), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: expected length type specification (U, i, I, l, L) after '#'; last byte: 0x01"); CHECK(json::from_bjdata(vS, true, false).is_discarded()); std::vector vT = {'[', '$', 'i', '#', '[', 'i', 2, 'i'}; - CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); - CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); - CHECK(json::from_bjdata(v, true, false).is_discarded()); + CHECK_THROWS_AS(_ = json::from_bjdata(vT), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vT), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vT, true, false).is_discarded()); + + std::vector vT0 = {'[', '$', 'i', '#', '[', 'i'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vT0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vT0), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vT0, true, false).is_discarded()); std::vector vm = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'm', 1, 0, 0, 0}; - CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); - CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); - CHECK(json::from_bjdata(v, true, false).is_discarded()); + CHECK_THROWS_AS(_ = json::from_bjdata(vm), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vm), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vm, true, false).is_discarded()); std::vector vM = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'M', 1, 0, 0, 0, 0, 0, 0, 0}; - CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); - CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); - CHECK(json::from_bjdata(v, true, false).is_discarded()); + CHECK_THROWS_AS(_ = json::from_bjdata(vM), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vM), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vM, true, false).is_discarded()); } SECTION("objects")