From 358970e3d0fe543dec46733c153d72c7ff20b2e3 Mon Sep 17 00:00:00 2001 From: Luca Date: Thu, 14 Apr 2022 13:00:52 +0200 Subject: [PATCH 1/3] fix iterator_input_adapter consuming input before it should --- .../nlohmann/detail/input/input_adapters.hpp | 29 +++++++++++++++---- single_include/nlohmann/json.hpp | 29 +++++++++++++++---- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index d196aec54..49d04395c 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -123,6 +123,7 @@ class input_stream_adapter }; #endif // JSON_NO_IO + // General-purpose iterator-based adapter. It might not be as fast as // theoretically possible for some containers, but it is extremely versatile. template @@ -132,30 +133,46 @@ class iterator_input_adapter using char_type = typename std::iterator_traits::value_type; iterator_input_adapter(IteratorType first, IteratorType last) - : current(std::move(first)), end(std::move(last)) + : current(std::move(first)), end(std::move(last)), current_has_been_consumed(false) {} typename std::char_traits::int_type get_character() { - if (JSON_HEDLEY_LIKELY(current != end)) + + if (JSON_HEDLEY_LIKELY(current_has_been_consumed)) { - auto result = std::char_traits::to_int_type(*current); std::advance(current, 1); - return result; } - return std::char_traits::eof(); + if (JSON_HEDLEY_LIKELY(current != end)) + { + current_has_been_consumed = true; + return std::char_traits::to_int_type(*current); + } + else + { + current_has_been_consumed = false; + return std::char_traits::eof(); + } + } private: IteratorType current; IteratorType end; + bool current_has_been_consumed = false; template friend struct wide_string_input_helper; - bool empty() const + bool empty() { + if (JSON_HEDLEY_LIKELY(current_has_been_consumed)) + { + std::advance(current, 1); + current_has_been_consumed = false; + } + return current == end; } }; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a29c52997..fdcf4e818 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5533,6 +5533,7 @@ class input_stream_adapter }; #endif // JSON_NO_IO + // General-purpose iterator-based adapter. It might not be as fast as // theoretically possible for some containers, but it is extremely versatile. template @@ -5542,30 +5543,46 @@ class iterator_input_adapter using char_type = typename std::iterator_traits::value_type; iterator_input_adapter(IteratorType first, IteratorType last) - : current(std::move(first)), end(std::move(last)) + : current(std::move(first)), end(std::move(last)), current_has_been_consumed(false) {} typename std::char_traits::int_type get_character() { - if (JSON_HEDLEY_LIKELY(current != end)) + + if (JSON_HEDLEY_LIKELY(current_has_been_consumed)) { - auto result = std::char_traits::to_int_type(*current); std::advance(current, 1); - return result; } - return std::char_traits::eof(); + if (JSON_HEDLEY_LIKELY(current != end)) + { + current_has_been_consumed = true; + return std::char_traits::to_int_type(*current); + } + else + { + current_has_been_consumed = false; + return std::char_traits::eof(); + } + } private: IteratorType current; IteratorType end; + bool current_has_been_consumed = false; template friend struct wide_string_input_helper; - bool empty() const + bool empty() { + if (JSON_HEDLEY_LIKELY(current_has_been_consumed)) + { + std::advance(current, 1); + current_has_been_consumed = false; + } + return current == end; } }; From f4a8d2d97fce0783a4c0cc08231cdc30c5dba135 Mon Sep 17 00:00:00 2001 From: Luca Date: Thu, 14 Apr 2022 14:35:55 +0200 Subject: [PATCH 2/3] tidying up code, no change to logic --- include/nlohmann/detail/input/input_adapters.hpp | 8 ++++---- single_include/nlohmann/json.hpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 49d04395c..e693f235b 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -158,14 +158,14 @@ class iterator_input_adapter } private: - IteratorType current; - IteratorType end; - bool current_has_been_consumed = false; + mutable IteratorType current; + const IteratorType end; + mutable bool current_has_been_consumed; template friend struct wide_string_input_helper; - bool empty() + bool empty() const { if (JSON_HEDLEY_LIKELY(current_has_been_consumed)) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index fdcf4e818..f4fd494df 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5568,14 +5568,14 @@ class iterator_input_adapter } private: - IteratorType current; - IteratorType end; - bool current_has_been_consumed = false; + mutable IteratorType current; + const IteratorType end; + mutable bool current_has_been_consumed; template friend struct wide_string_input_helper; - bool empty() + bool empty() const { if (JSON_HEDLEY_LIKELY(current_has_been_consumed)) { From 2614b81092e8afdc1c3eedc2fa53cd2c0ecf3e47 Mon Sep 17 00:00:00 2001 From: Luca Date: Mon, 20 Jun 2022 12:54:05 +0200 Subject: [PATCH 3/3] Added basic test for the applied fix --- tests/src/unit-consuming_stream.cpp | 126 ++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 tests/src/unit-consuming_stream.cpp diff --git a/tests/src/unit-consuming_stream.cpp b/tests/src/unit-consuming_stream.cpp new file mode 100644 index 000000000..7a700c348 --- /dev/null +++ b/tests/src/unit-consuming_stream.cpp @@ -0,0 +1,126 @@ + + + +#include "doctest_compatibility.h" + +#define JSON_TESTS_PRIVATE +#include +using nlohmann::json; + + +//This struct emulate an Input Stream, once a "char" is readed from it it is lost +struct EmulateStream +{ + using difference_type = std::ptrdiff_t; + using value_type = char; + using pointer = const char*; + using reference = const char&; + using iterator_category = std::input_iterator_tag; + + // Like a pop_left() -> consume one char from the underlying buffer + static char ConsumeChar(std::string* str) + { + const char c = str->front(); + str->erase(0, 1); + + return c; + } + + EmulateStream() + : + target{nullptr} + {} + + EmulateStream(std::string* target_) + : + target{target_}, + c{ConsumeChar(target_)} + {} + + EmulateStream& operator++() + { + c = ConsumeChar(target); + + return *this; + } + + bool operator!=(const EmulateStream& rhs) const + { + return rhs.target != target; + } + + reference operator*() const + { + return c; + } + + std::string* target; + char c; + +}; + +EmulateStream CreateBegin(std::string& tgt) +{ + return EmulateStream{&tgt}; +} + +EmulateStream CreateEnd(const std::string& tgt) +{ + return {}; +} + + + +TEST_CASE("consume only needed") +{ + + nlohmann::detail::json_sax_acceptor sax_parser; + + const std::string json_A = R"({ "key_A" : "value_A" })"; + const std::string json_B = R"({ "key_B" : "value_B" })"; + + std::string json_concat_AB = json_A + json_B; + + CHECK( + json::sax_parse( + CreateBegin(json_concat_AB), + CreateEnd(json_concat_AB), + &sax_parser, + nlohmann::detail::input_format_t::json, + false + ) + == true + ); + + + CHECK(json_concat_AB == json_B); + + + CHECK( + json::sax_parse( + CreateBegin(json_concat_AB), + CreateEnd(json_concat_AB), + &sax_parser, + nlohmann::detail::input_format_t::json, + false + ) + == true + ); + + + CHECK(json_concat_AB == ""); + CHECK(json_concat_AB.size() == 0); + + + CHECK( + json::sax_parse( + CreateBegin(json_concat_AB), + CreateEnd(json_concat_AB), + &sax_parser, + nlohmann::detail::input_format_t::json, + false + ) + == false + ); + +} \ No newline at end of file