Merge a416868818 into 58d6aa5831

README.md

@@ -1733,6 +1733,7 @@ I deeply appreciate the help of the following people.
 314. [Berkus Decker](https://github.com/berkus) fixed a typo in the README.
 315. [Illia Polishchuk](https://github.com/effolkronium) improved the CMake testing.
 316. [Ikko Ashimine](https://github.com/eltociear) fixed a typo.
+317. [Raphael Grimm](https://github.com/barcode) Added custom base classes as customization point and parser location information for sax parsers.
 
 Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone.
 

cmake/ci.cmake

@@ -636,7 +636,7 @@ add_custom_target(ci_test_valgrind
         -DJSON_BuildTests=ON -DJSON_Valgrind=ON
         -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_valgrind
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_valgrind
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_valgrind && ${CMAKE_CTEST_COMMAND} -L valgrind --parallel ${N} --output-on-failure
+    COMMAND cd ${PROJECT_BINARY_DIR}/build_valgrind && ${CMAKE_CTEST_COMMAND} -L valgrind --parallel ${N} --output-on-failure --timeout 10000
     COMMENT "Compile and test with Valgrind"
 )
 

docs/examples/sax_parse_with_src_location.cpp

@@ -0,0 +1,149 @@
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+// a simple event consumer that collects string representations of the passed
+// values and their source locations;
+// note inheriting from json::json_sax_t is not required, but can
+// help not to forget a required function
+class sax_event_consumer : public json::json_sax_t
+{
+  public:
+    std::vector<std::string> events;
+    std::size_t next_token_start_pos = 0;
+    std::size_t next_token_end_pos = 0;
+
+    void next_token_start(std::size_t pos)
+    {
+        next_token_start_pos = pos;
+    }
+
+    void next_token_end(std::size_t pos)
+    {
+        next_token_end_pos = pos;
+    }
+
+    std::string location_as_str() const
+    {
+        return "at=[" + std::to_string(next_token_start_pos) + "," + std::to_string(next_token_end_pos) + ")";
+    }
+
+    bool null() override
+    {
+        events.push_back("null(" + location_as_str() + ")");
+        return true;
+    }
+
+    bool boolean(bool val) override
+    {
+        events.push_back("boolean(val=" + std::string(val ? "true" : "false") + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool number_integer(number_integer_t val) override
+    {
+        events.push_back("number_integer(val=" + std::to_string(val) + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val) override
+    {
+        events.push_back("number_unsigned(val=" + std::to_string(val) + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& s) override
+    {
+        events.push_back("number_float(val=" + std::to_string(val) + ", s=" + s + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool string(string_t& val) override
+    {
+        events.push_back("string(val=" + val + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool start_object(std::size_t elements) override
+    {
+        events.push_back("start_object(elements=" + std::to_string(elements) + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool end_object() override
+    {
+        events.push_back("end_object(" + location_as_str() + ")");
+        return true;
+    }
+
+    bool start_array(std::size_t elements) override
+    {
+        events.push_back("start_array(elements=" + std::to_string(elements) + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool end_array() override
+    {
+        events.push_back("end_array(" + location_as_str() + ")");
+        return true;
+    }
+
+    bool key(string_t& val) override
+    {
+        events.push_back("key(val=" + val + ", " + location_as_str() + ")");
+        return true;
+    }
+
+    bool binary(json::binary_t& val) override
+    {
+        events.push_back("binary(val=[...], " + location_as_str() + ")");
+        return true;
+    }
+
+    bool parse_error(std::size_t position, const std::string& last_token, const json::exception& ex) override
+    {
+        events.push_back("parse_error(position=" + std::to_string(position) + ", last_token=" + last_token + ",\n            ex=" + std::string(ex.what()) + ")");
+        return false;
+    }
+};
+
+int main()
+{
+    // a JSON text
+    auto text = R"(
+    {
+        "Image": {
+            "Width":  800,
+            "Height": 600,
+            "Title":  "View from 15th Floor",
+            "Thumbnail": {
+                "Url":    "http://www.example.com/image/481989943",
+                "Height": 125,
+                "Width":  100
+            },
+            "Animated" : false,
+            "IDs": [116, 943, 234, -38793],
+            "DeletionDate": null,
+            "Distance": 12.723374634
+        }
+    }]
+    )";
+
+    // create a SAX event consumer object
+    sax_event_consumer sec;
+
+    // parse JSON
+    bool result = json::sax_parse(text, &sec);
+
+    // output the recorded events
+    for (auto& event : sec.events)
+    {
+        std::cout << event << "\n";
+    }
+
+    // output the result of sax_parse
+    std::cout << "\nresult: " << std::boolalpha << result << std::endl;
+}

docs/examples/sax_parse_with_src_location.output

@@ -0,0 +1,37 @@
+start_object(elements=18446744073709551615, at=[5,6))
+key(val=Image, at=[15,22))
+start_object(elements=18446744073709551615, at=[24,25))
+key(val=Width, at=[38,45))
+number_unsigned(val=800, at=[48,51))
+key(val=Height, at=[65,73))
+number_unsigned(val=600, at=[75,78))
+key(val=Title, at=[92,99))
+string(val=View from 15th Floor, at=[102,124))
+key(val=Thumbnail, at=[138,149))
+start_object(elements=18446744073709551615, at=[151,152))
+key(val=Url, at=[169,174))
+string(val=http://www.example.com/image/481989943, at=[179,219))
+key(val=Height, at=[237,245))
+number_unsigned(val=125, at=[247,250))
+key(val=Width, at=[268,275))
+number_unsigned(val=100, at=[278,281))
+end_object(at=[294,295))
+key(val=Animated, at=[309,319))
+boolean(val=false, at=[322,327))
+key(val=IDs, at=[341,346))
+start_array(elements=18446744073709551615, at=[348,349))
+number_unsigned(val=116, at=[349,352))
+number_unsigned(val=943, at=[354,357))
+number_unsigned(val=234, at=[359,362))
+number_integer(val=-38793, at=[364,370))
+end_array(at=[370,371))
+key(val=DeletionDate, at=[385,399))
+null(at=[401,405))
+key(val=Distance, at=[419,429))
+number_float(val=12.723375, s=12.723374634, at=[431,443))
+end_object(at=[452,453))
+end_object(at=[458,459))
+parse_error(position=460, last_token=12.723374634<U+000A>        }<U+000A>    }],
+            ex=[json.exception.parse_error.101] parse error at line 17, column 6: syntax error while parsing value - unexpected ']'; expected end of input)
+
+result: false

docs/examples/sax_parse_with_src_location_in_json.cpp

@@ -0,0 +1,336 @@
+#include <iomanip>
+#include <iostream>
+#include <nlohmann/json.hpp>
+#include <sstream>
+
+using json = nlohmann::json;
+
+// custom base class for the json node.
+// allows us to store metadata and add custom methods to each node
+struct token_start_stop
+{
+    nlohmann::position_t start{};
+    nlohmann::position_t stop{};
+
+    std::string start_pos_str() const
+    {
+        return "{l=" + std::to_string(start.lines_read) + ":c="
+               + std::to_string(start.chars_read_current_line) + "}";
+    }
+    std::string stop_pos_str() const
+    {
+        return "{l=" + std::to_string(stop.lines_read) + ":c=" + std::to_string(stop.chars_read_current_line) + "}";
+    }
+    std::string location_str() const
+    {
+        return "[" + start_pos_str() + ", " + stop_pos_str() + ")";
+    }
+};
+
+//json type using token_start_stop as base class
+using json_with_token_start_stop =
+    nlohmann::basic_json <
+    std::map,
+    std::vector,
+    std::string,
+    bool,
+    std::int64_t,
+    std::uint64_t,
+    double,
+    std::allocator,
+    nlohmann::adl_serializer,
+    std::vector<std::uint8_t>,
+    token_start_stop >;
+
+// a parser storing the lexer information for each node
+class sax_with_token_start_stop_metadata
+{
+  public:
+    using json = json_with_token_start_stop;
+    using number_integer_t = typename json::number_integer_t;
+    using number_unsigned_t = typename json::number_unsigned_t;
+    using number_float_t = typename json::number_float_t;
+    using string_t = typename json::string_t;
+    using binary_t = typename json::binary_t;
+
+    /*!
+    @param[in,out] r  reference to a JSON value that is manipulated while
+                       parsing
+    @param[in] allow_exceptions_  whether parse errors yield exceptions
+    */
+    explicit sax_with_token_start_stop_metadata(json& r, const bool allow_exceptions_ = true)
+        : root(r)
+        , ref_stack{}
+        , object_element{nullptr}
+        , errored{false}
+        , allow_exceptions(allow_exceptions_)
+        , start_stop{}
+    {}
+
+    void next_token_start(const nlohmann::position_t&  p)
+    {
+        start_stop.start = p;
+    }
+
+    void next_token_end(const nlohmann::position_t&  p)
+    {
+        start_stop.stop = p;
+    }
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool binary(binary_t& val)
+    {
+        handle_value(std::move(val));
+        return true;
+    }
+
+    bool start_object(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(json::value_t::object));
+        ref_stack.back()->start = start_stop.start;
+
+        if (len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size())
+        {
+            throw nlohmann::detail::out_of_range::create(408, nlohmann::detail::concat("excessive object size: ", std::to_string(len)), ref_stack.back());
+        }
+
+        return true;
+    }
+
+    bool key(string_t& val)
+    {
+        assert(!ref_stack.empty());
+        assert(ref_stack.back()->is_object());
+
+        // add null at given key and store the reference for later
+        object_element = &(*ref_stack.back())[val];
+        return true;
+    }
+
+    bool end_object()
+    {
+        assert(!ref_stack.empty());
+        assert(ref_stack.back()->is_object());
+
+        ref_stack.back()->stop = start_stop.stop;
+        ref_stack.pop_back();
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(json::value_t::array));
+        ref_stack.back()->start = start_stop.start;
+
+        if (len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size())
+        {
+            throw nlohmann::detail::out_of_range::create(408, nlohmann::detail::concat("excessive array size: ", std::to_string(len)), ref_stack.back());
+        }
+
+        return true;
+    }
+
+    bool end_array()
+    {
+        assert(!ref_stack.empty());
+        assert(ref_stack.back()->is_array());
+
+        ref_stack.back()->stop = start_stop.stop;
+        ref_stack.pop_back();
+        return true;
+    }
+
+    template<class Exception>
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const Exception& ex)
+    {
+        errored = true;
+        static_cast<void>(ex);
+        if (allow_exceptions)
+        {
+            throw ex;
+        }
+        return false;
+    }
+
+    bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+    */
+    template<typename Value>
+    json*
+    handle_value(Value&& v)
+    {
+        if (ref_stack.empty())
+        {
+            root = json(std::forward<Value>(v));
+            root.start = start_stop.start;
+            root.stop = start_stop.stop;
+            return &root;
+        }
+
+        assert(ref_stack.back()->is_array() || ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            auto& array_element = ref_stack.back()->emplace_back(std::forward<Value>(v));
+            array_element.start = start_stop.start;
+            array_element.stop = start_stop.stop;
+            return &array_element;
+        }
+
+        assert(ref_stack.back()->is_object());
+        assert(object_element);
+        *object_element = json(std::forward<Value>(v));
+        object_element->start = start_stop.start;
+        object_element->stop = start_stop.stop;
+        return object_element;
+    }
+
+    /// the parsed JSON value
+    json& root;
+    /// stack to model hierarchy of values
+    std::vector<json*> ref_stack{};
+    /// helper to hold the reference for the next object element
+    json* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+    /// start / stop information for the current token
+    token_start_stop start_stop{};
+};
+
+void dump(const json_with_token_start_stop& j, std::size_t indentlvl = 0)
+{
+    const std::string indent(indentlvl * 4, ' ');
+    switch (j.type())
+    {
+        case nlohmann::json::value_t::null:
+        {
+            std::cout << indent << "null(at=" << j.location_str() << ")\n";
+        }
+        break;
+        case nlohmann::json::value_t::object:
+        {
+            std::cout << indent << "object(size=" << j.size() << ", at=" << j.location_str() << ")\n";
+            for (const auto& elem : j.items())
+            {
+                dump(elem.value(), indentlvl + 1);
+            }
+        }
+        break;
+        case nlohmann::json::value_t::array:
+        {
+            std::cout << indent << "array(size=" << j.size() << ", at=" << j.location_str() << ")\n";
+            for (const auto& elem : j)
+            {
+                dump(elem, indentlvl + 1);
+            }
+        }
+        break;
+        case nlohmann::json::value_t::string:
+        {
+            std::cout << indent << "string(val=" << j.get<std::string>() << ", at=" << j.location_str() << ")\n";
+        }
+        break;
+        case nlohmann::json::value_t::boolean:
+        {
+            std::cout << indent << "boolean(val=" << j.get<bool>() << ", at=" << j.location_str() << ")\n";
+        }
+        break;
+        case nlohmann::json::value_t::number_integer:
+        {
+            std::cout << indent << "number_integer(val=" << j.get<std::int64_t>() << ", at=" << j.location_str() << ")\n";
+        }
+        break;
+        case nlohmann::json::value_t::number_unsigned:
+        {
+            std::cout << indent << "number_unsigned(val=" << j.get<std::uint64_t>() << ", at=" << j.location_str() << ")\n";
+        }
+        break;
+        case nlohmann::json::value_t::number_float:
+        {
+            std::cout << indent << "number_float(val=" << j.get<double>() << ", at=" << j.location_str() << ")\n";
+        }
+        break;
+        default:
+            throw std::runtime_error{"unexpected input"};
+    }
+}
+
+int main()
+{
+    // a JSON text
+    auto text = R"({
+        "Image": {
+            "Width":  800,
+            "Height": 600,
+            "Title":  "View from 15th Floor",
+            "Thumbnail": {
+                "Url":    "http://www.example.com/image/481989943",
+                "Height": 125,
+                "Width":  100
+            },
+            "Animated" : false,
+            "IDs": [116, 943, 234, -38793],
+            "DeletionDate": null,
+            "Distance": 12.723374634
+        }
+})";
+
+    // create a SAX parser object
+    json_with_token_start_stop parsed;
+    sax_with_token_start_stop_metadata sax{parsed};
+
+    // parse JSON
+    bool result = json::sax_parse(text, &sax);
+
+    // output the json data
+    dump(parsed);
+
+    // output the result of sax_parse
+    std::cout << "\nresult: " << std::boolalpha << result << std::endl;
+}

docs/examples/sax_parse_with_src_location_in_json.output

@@ -0,0 +1,19 @@
+object(size=1, at=[{l=0:c=0}, {l=15:c=1}))
+    object(size=8, at=[{l=1:c=17}, {l=14:c=9}))
+        boolean(val=0, at=[{l=10:c=25}, {l=10:c=30}))
+        null(at=[{l=12:c=28}, {l=12:c=32}))
+        number_float(val=12.7234, at=[{l=13:c=24}, {l=13:c=0}))
+        number_unsigned(val=600, at=[{l=3:c=22}, {l=3:c=25}))
+        array(size=4, at=[{l=11:c=19}, {l=11:c=42}))
+            number_unsigned(val=116, at=[{l=11:c=20}, {l=11:c=23}))
+            number_unsigned(val=943, at=[{l=11:c=25}, {l=11:c=28}))
+            number_unsigned(val=234, at=[{l=11:c=30}, {l=11:c=33}))
+            number_integer(val=-38793, at=[{l=11:c=35}, {l=11:c=41}))
+        object(size=3, at=[{l=5:c=25}, {l=9:c=13}))
+            number_unsigned(val=125, at=[{l=7:c=26}, {l=7:c=29}))
+            string(val=http://www.example.com/image/481989943, at=[{l=6:c=26}, {l=6:c=66}))
+            number_unsigned(val=100, at=[{l=8:c=26}, {l=8:c=0}))
+        string(val=View from 15th Floor, at=[{l=4:c=22}, {l=4:c=44}))
+        number_unsigned(val=800, at=[{l=2:c=22}, {l=2:c=25}))
+
+result: true

docs/mkdocs/docs/api/json_sax/index.md

@@ -37,8 +37,11 @@ processing the input.
 - [**start_array**](start_array.md) (_virtual_) - the beginning of an array was read
 - [**start_object**](start_object.md) (_virtual_) - the beginning of an object was read
 - [**string**](string.md) (_virtual_) - a string value was read
+- [**next_token_start**](next_token_start.md) - called to provide the start of the next element in the parsed input.
+- [**next_token_end**](next_token_end.md) - called to provide the end (one past convention) of the next element in the parsed input.
 
 ## Version history
 
 - Added in version 3.2.0.
 - Support for binary values (`binary_t`, `binary`) added in version 3.8.0.
+- Support for parser location information (`next_token_start`, `next_token_end`) added in version ???.???.???.

docs/mkdocs/docs/api/json_sax/next_token_end.md

@@ -0,0 +1,66 @@
+# <small>nlohmann::json_sax::</small>next_token_end
+
+Informs the sax parser about the end of the next element.
+There are two possible signatures for this method:
+
+1. 
+```cpp
+void next_token_end(std::size_t pos);
+```
+This version is called with the byte position after the next element ends.
+This version also works when parsing binary formats such as [msgpack](../basic_json/input_format_t.md).
+
+2. 
+```cpp
+void next_token_end(const nlohmann::position_t& p)
+```
+This version is called with the [detailed parser position information](../position_t/index.md) after the last character of the next element was parsed.
+This version only available when calling `nlohmann::json::sax_parse` with `nlohmann::json::input_format_t::json` and takes precedence.
+
+## Parameters
+1. 
+`pos` (in)
+:   Byte position one after the next elements last byte.
+2. 
+`p` (in)
+:   [Detailed parser position information](../position_t/index.md) after the last char of the next element was parsed.
+
+## Notes
+
+Implementing either version is optional, and no function is called if neither version of `next_token_end` is available in the sax parser.
+
+It is recommended, but not required, to also implement [next_token_start](next_token_start.md).
+
+## Examples
+
+??? example
+
+    The example below shows a SAX parser using the first version of this method to log the location.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location.output"
+    ```
+
+??? example
+
+    The example below shows a SAX parser using the second version of this method and
+    storing the location information in each json node using a [base class](../basic_json/json_base_class_t.md) for `nlohmann::json` as customization point.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location_in_json.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location_in_json.output"
+    ```
+## Version history
+
+- Added in version ???.???.???.

docs/mkdocs/docs/api/json_sax/next_token_start.md

@@ -0,0 +1,66 @@
+# <small>nlohmann::json_sax::</small>next_token_start
+
+Informs the sax parser about the start of the next element.
+There are two possible signatures for this method:
+
+1. 
+```cpp
+void next_token_start(std::size_t pos);
+```
+This version is called with the byte position where the next element starts.
+This version also works when parsing binary formats such as [msgpack](../basic_json/input_format_t.md).
+
+2. 
+```cpp
+void next_token_start(const nlohmann::position_t&  p)
+```
+This version is called with [detailed parser position information](../position_t/index.md).
+This version only available when calling `nlohmann::json::sax_parse` with `nlohmann::json::input_format_t::json` and takes precedence.
+
+## Parameters
+1. 
+`pos` (in)
+:   Byte position where the next element starts.
+2. 
+`p` (in)
+:   [Detailed parser position information](../position_t/index.md) after the first char of the next element was parsed.
+
+## Notes
+
+Implementing either version is optional, and no function is called if neither version of `next_token_start` is available in the sax parser.
+
+It is recommended, but not required, to also implement [next_token_end](next_token_end.md).
+
+## Examples
+
+??? example
+
+    The example below shows a SAX parser using the first version of this method to log the location.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location.output"
+    ```
+
+??? example
+
+    The example below shows a SAX parser using the second version of this method and
+    storing the location information in each json node using a [base class](../basic_json/json_base_class_t.md) for `nlohmann::json` as customization point.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location_in_json.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location_in_json.output"
+    ```
+## Version history
+
+- Added in version ???.???.???.

docs/mkdocs/docs/api/position_t/chars_read_current_line.md

@@ -0,0 +1,28 @@
+# <small>nlohmann::position_t::</small>chars_read_current_line
+
+```cpp
+std::size_t chars_read_current_line;
+```
+
+The number of characters read in the current line.
+
+## Examples
+
+??? example
+
+    The example below shows a SAX receiving the element bounds as `nlohmann::position_t` and
+    storing this location information in each json node using a [base class](../basic_json/json_base_class_t.md) for `nlohmann::json` as customization point.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location_in_json.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location_in_json.output"
+    ```
+
+## Version history
+
+- Moved from namespace `nlohmann::detail` to `nlohmann` in version ???.???.???.

docs/mkdocs/docs/api/position_t/chars_read_total.md

@@ -0,0 +1,28 @@
+# <small>nlohmann::position_t::</small>chars_read_total
+
+```cpp
+std::size_t chars_read_total;
+```
+
+The total number of characters read.
+
+## Examples
+
+??? example
+
+    The example below shows a SAX receiving the element bounds as `nlohmann::position_t` and
+    storing this location information in each json node using a [base class](../basic_json/json_base_class_t.md) for `nlohmann::json` as customization point.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location_in_json.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location_in_json.output"
+    ```
+
+## Version history
+
+- Moved from namespace `nlohmann::detail` to `nlohmann` in version ???.???.???.

docs/mkdocs/docs/api/position_t/index.md

@@ -0,0 +1,23 @@
+# <small>nlohmann::</small>position_t
+
+```cpp
+struct position_t;
+```
+
+This type represents the parsers position when parsing a json string using.
+This position can be retrieved when using a [sax parser](../json_sax/index.md) with the format `nlohmann::json::input_format_t::json`
+and implementing [next_token_start](../json_sax/next_token_start.md) or [next_token_end](../json_sax/next_token_end.md).
+
+## Member functions
+
+- [**operator size_t**](operator_size_t.md) - return the value of [chars_read_total](chars_read_total.md).
+
+## Member variables
+
+- [**chars_read_total**](chars_read_total.md) - The total number of characters read.
+- [**lines_read**](lines_read.md) - The number of lines read.
+- [**chars_read_current_line**](chars_read_current_line.md) - The number of characters read in the current line.
+
+## Version history
+
+- Moved from namespace `nlohmann::detail` to `nlohmann` in version ???.???.???.

docs/mkdocs/docs/api/position_t/lines_read.md

@@ -0,0 +1,28 @@
+# <small>nlohmann::position_t::</small>lines_read
+
+```cpp
+std::size_t lines_read;
+```
+
+The number of lines read.
+
+## Examples
+
+??? example
+
+    The example below shows a SAX receiving the element bounds as `nlohmann::position_t` and
+    storing this location information in each json node using a [base class](../basic_json/json_base_class_t.md) for `nlohmann::json` as customization point.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location_in_json.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location_in_json.output"
+    ```
+
+## Version history
+
+- Moved from namespace `nlohmann::detail` to `nlohmann` in version ???.???.???.

docs/mkdocs/docs/api/position_t/operator_size_t.md

@@ -0,0 +1,28 @@
+# <small>nlohmann::position_t::</small>
+
+```cpp
+constexpr operator size_t() const;
+```
+
+return the value of [chars_read_total](chars_read_total.md).
+
+## Examples
+
+??? example
+
+    The example below shows a SAX receiving the element bounds as `nlohmann::position_t` and
+    storing this location information in each json node using a [base class](../basic_json/json_base_class_t.md) for `nlohmann::json` as customization point.
+
+    ```cpp
+    --8<-- "examples/sax_parse_with_src_location_in_json.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/sax_parse_with_src_location_in_json.output"
+    ```
+
+## Version history
+
+- Moved from namespace `nlohmann::detail` to `nlohmann` in version ???.???.???.

docs/mkdocs/docs/features/parsing/sax_interface.md

@@ -67,6 +67,30 @@ To implement your own SAX handler, proceed as follows:
 
 Note the `sax_parse` function only returns a `#!cpp bool` indicating the result of the last executed SAX event. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`.
 
+## Element position information
+
+The position of a parsed element can be retrieved by implementing the optional methods [next_token_start](../../api/json_sax/next_token_start.md) and [next_token_end](../../api/json_sax/next_token_end.md).
+These methods will be called with the parser position before any of the other methods are called and can be used to retrieve the half open bounds (`[start, end)`) of a parsed element.
+
+These Methods come in two flavors:
+
+1. 
+```cpp
+void next_token_start(std::size_t pos);
+void next_token_end(std::size_t pos);
+```
+This flavor is called with the byte positions of each element and are available for any `nlohmann::json::input_format_t` passed to `nlohmann::json::sax_parse`.
+
+2. 
+```cpp
+void next_token_start(const nlohmann::position_t& p);
+void next_token_end(const nlohmann::position_t& p);
+```
+This flavor is called with the [detailed parser position information](../../api/position_t/index.md) of each element and are only available if `nlohmann::json::sax_parse` is called with `nlohmann::json::input_format_t::json`.
+Furthermore this flavor takes precedence over the first flavor.
+
+Depending on the required information it is possible for the SAX parser to implement all four or only one or none of these methods.
+
 ## See also
 
 - [json_sax](../../api/json_sax/index.md) - documentation of the SAX interface

docs/mkdocs/mkdocs.yml

@@ -250,6 +250,8 @@ nav:
         - 'start_array': api/json_sax/start_array.md
         - 'start_object': api/json_sax/start_object.md
         - 'string': api/json_sax/string.md
+        - 'next_token_start' : api/json_sax/next_token_start.md
+        - 'next_token_end' : api/json_sax/next_token_end.md
       - 'operator<<(basic_json)': api/operator_ltlt.md
       - 'operator<<(json_pointer)': api/operator_ltlt.md
       - 'operator>>(basic_json)': api/operator_gtgt.md
@@ -257,6 +259,12 @@ nav:
       - 'operator""_json_pointer': api/operator_literal_json_pointer.md
       - 'ordered_json': api/ordered_json.md
       - 'ordered_map': api/ordered_map.md
+      - position_t:
+        - 'Overview': api/position_t/index.md
+        - 'operator size_t': api/position_t/operator_size_t.md
+        - 'chars_read_total': api/position_t/chars_read_total.md
+        - 'lines_read': api/position_t/lines_read.md
+        - 'chars_read_current_line': api/position_t/chars_read_current_line.md
       - macros:
         - 'Overview': api/macros/index.md
         - 'JSON_ASSERT': api/macros/json_assert.md

include/nlohmann/detail/input/binary_reader.hpp

@@ -167,8 +167,9 @@ class binary_reader
     bool parse_bson_internal()
     {
         std::int32_t document_size{};
+        detail::sax_call_next_token_start_pos(sax, chars_read);
         get_number<std::int32_t, true>(input_format_t::bson, document_size);
-
+        detail::sax_call_next_token_end_pos(sax, chars_read);
         if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))
         {
             return false;
@@ -179,6 +180,7 @@ class binary_reader
             return false;
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
         return sax->end_object();
     }
 
@@ -276,6 +278,7 @@ class binary_reader
             case 0x01: // double
             {
                 double number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read, chars_read + sizeof(number));
                 return get_number<double, true>(input_format_t::bson, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
@@ -283,7 +286,10 @@ class binary_reader
             {
                 std::int32_t len{};
                 string_t value;
-                return get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value);
+                detail::sax_call_next_token_start_pos(sax, chars_read);
+                const bool result_get = get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_string(len, value);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->string(value);
             }
 
             case 0x03: // object
@@ -300,28 +306,35 @@ class binary_reader
             {
                 std::int32_t len{};
                 binary_t value;
-                return get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value);
+                detail::sax_call_next_token_start_pos(sax, chars_read);
+                const bool result_get =  get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_binary(len, value);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->binary(value);
             }
 
             case 0x08: // boolean
             {
+                detail::sax_call_next_token_start_end_pos(sax, chars_read, chars_read + 1);
                 return sax->boolean(get() != 0);
             }
 
             case 0x0A: // null
             {
+                detail::sax_call_next_token_start_end_pos(sax, chars_read);
                 return sax->null();
             }
 
             case 0x10: // int32
             {
                 std::int32_t value{};
-                return get_number<std::int32_t, true>(input_format_t::bson, value) && sax->number_integer(value);
+                detail::sax_call_next_token_start_end_pos(sax, chars_read, chars_read + sizeof(value));
+                return get_number<std::int32_t, true>(input_format_t::bson, value) &&  sax->number_integer(value);
             }
 
             case 0x12: // int64
             {
                 std::int64_t value{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read, chars_read + sizeof(value));
                 return get_number<std::int64_t, true>(input_format_t::bson, value) && sax->number_integer(value);
             }
 
@@ -360,14 +373,22 @@ class binary_reader
             }
 
             const std::size_t element_type_parse_position = chars_read;
+            if (!is_array)
+            {
+                detail::sax_call_next_token_start_pos(sax, chars_read);
+            }
             if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
             {
                 return false;
             }
 
-            if (!is_array && !sax->key(key))
+            if (!is_array)
             {
-                return false;
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                if (!sax->key(key))
+                {
+                    return false;
+                }
             }
 
             if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))
@@ -389,6 +410,7 @@ class binary_reader
     bool parse_bson_array()
     {
         std::int32_t document_size{};
+        detail::sax_call_next_token_start_end_pos(sax, chars_read, chars_read + sizeof(std::int32_t));
         get_number<std::int32_t, true>(input_format_t::bson, document_size);
 
         if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))
@@ -401,6 +423,7 @@ class binary_reader
             return false;
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
         return sax->end_array();
     }
 
@@ -450,29 +473,34 @@ class binary_reader
             case 0x15:
             case 0x16:
             case 0x17:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->number_unsigned(static_cast<number_unsigned_t>(current));
 
             case 0x18: // Unsigned integer (one-byte uint8_t follows)
             {
                 std::uint8_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
             }
 
             case 0x19: // Unsigned integer (two-byte uint16_t follows)
             {
                 std::uint16_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
             }
 
             case 0x1A: // Unsigned integer (four-byte uint32_t follows)
             {
                 std::uint32_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
             }
 
             case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
             {
                 std::uint64_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);
             }
 
@@ -501,29 +529,34 @@ class binary_reader
             case 0x35:
             case 0x36:
             case 0x37:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - current));
 
             case 0x38: // Negative integer (one-byte uint8_t follows)
             {
                 std::uint8_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
             {
                 std::uint16_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
             {
                 std::uint32_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
             {
                 std::uint64_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1)
                         - static_cast<number_integer_t>(number));
             }
@@ -560,7 +593,10 @@ class binary_reader
             case 0x5F: // Binary data (indefinite length)
             {
                 binary_t b;
-                return get_cbor_binary(b) && sax->binary(b);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_cbor_binary(b);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->binary(b);
             }
 
             // UTF-8 string (0x00..0x17 bytes follow)
@@ -595,7 +631,10 @@ class binary_reader
             case 0x7F: // UTF-8 string (indefinite length)
             {
                 string_t s;
-                return get_cbor_string(s) && sax->string(s);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_cbor_string(s);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->string(s);
             }
 
             // array (0x00..0x17 data items follow)
@@ -623,35 +662,51 @@ class binary_reader
             case 0x95:
             case 0x96:
             case 0x97:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return get_cbor_array(
                            conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu), tag_handler);
 
             case 0x98: // array (one-byte uint8_t for n follows)
             {
                 std::uint8_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_array(static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0x99: // array (two-byte uint16_t for n follow)
             {
                 std::uint16_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_array(static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0x9A: // array (four-byte uint32_t for n follow)
             {
                 std::uint32_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0x9B: // array (eight-byte uint64_t for n follow)
             {
                 std::uint64_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0x9F: // array (indefinite length)
+            {
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return get_cbor_array(static_cast<std::size_t>(-1), tag_handler);
+            }
 
             // map (0x00..0x17 pairs of data items follow)
             case 0xA0:
@@ -678,33 +733,47 @@ class binary_reader
             case 0xB5:
             case 0xB6:
             case 0xB7:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return get_cbor_object(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu), tag_handler);
 
             case 0xB8: // map (one-byte uint8_t for n follows)
             {
                 std::uint8_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_object(static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0xB9: // map (two-byte uint16_t for n follow)
             {
                 std::uint16_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_object(static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0xBA: // map (four-byte uint32_t for n follow)
             {
                 std::uint32_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0xBB: // map (eight-byte uint64_t for n follow)
             {
                 std::uint64_t len{};
-                return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_number(input_format_t::cbor, len);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);
             }
 
             case 0xBF: // map (indefinite length)
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return get_cbor_object(static_cast<std::size_t>(-1), tag_handler);
 
             case 0xC6: // tagged item
@@ -809,7 +878,10 @@ class binary_reader
                                 return parse_cbor_internal(true, tag_handler);
                         }
                         get();
-                        return get_cbor_binary(b) && sax->binary(b);
+                        detail::sax_call_next_token_start_pos(sax, chars_read);
+                        const bool result_get = get_cbor_binary(b);
+                        detail::sax_call_next_token_end_pos(sax, chars_read);
+                        return result_get && sax->binary(b);
                     }
 
                     default:                 // LCOV_EXCL_LINE
@@ -819,16 +891,20 @@ class binary_reader
             }
 
             case 0xF4: // false
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->boolean(false);
 
             case 0xF5: // true
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->boolean(true);
 
             case 0xF6: // null
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->null();
 
             case 0xF9: // Half-Precision Float (two-byte IEEE 754)
             {
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
                 const auto byte1_raw = get();
                 if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number")))
                 {
@@ -870,6 +946,7 @@ class binary_reader
                             return std::ldexp(mant + 1024, exp - 25);
                     }
                 }();
+                detail::sax_call_next_token_end_pos(sax, chars_read);
                 return sax->number_float((half & 0x8000u) != 0
                                          ? static_cast<number_float_t>(-val)
                                          : static_cast<number_float_t>(val), "");
@@ -878,12 +955,14 @@ class binary_reader
             case 0xFA: // Single-Precision Float (four-byte IEEE 754)
             {
                 float number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
             {
                 double number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::cbor, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
@@ -1127,6 +1206,7 @@ class binary_reader
             }
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read);
         return sax->end_array();
     }
 
@@ -1152,7 +1232,10 @@ class binary_reader
                 for (std::size_t i = 0; i < len; ++i)
                 {
                     get();
-                    if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key)))
+                    detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                    const bool result_get = get_cbor_string(key);
+                    detail::sax_call_next_token_end_pos(sax, chars_read);
+                    if (JSON_HEDLEY_UNLIKELY(!result_get || !sax->key(key)))
                     {
                         return false;
                     }
@@ -1168,7 +1251,10 @@ class binary_reader
             {
                 while (get() != 0xFF)
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key)))
+                    detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                    const bool result_get = get_cbor_string(key);
+                    detail::sax_call_next_token_end_pos(sax, chars_read);
+                    if (JSON_HEDLEY_UNLIKELY(!result_get || !sax->key(key)))
                     {
                         return false;
                     }
@@ -1182,6 +1268,7 @@ class binary_reader
             }
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read);
         return sax->end_object();
     }
 
@@ -1329,6 +1416,7 @@ class binary_reader
             case 0x7D:
             case 0x7E:
             case 0x7F:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->number_unsigned(static_cast<number_unsigned_t>(current));
 
             // fixmap
@@ -1348,6 +1436,7 @@ class binary_reader
             case 0x8D:
             case 0x8E:
             case 0x8F:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return get_msgpack_object(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));
 
             // fixarray
@@ -1367,6 +1456,7 @@ class binary_reader
             case 0x9D:
             case 0x9E:
             case 0x9F:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return get_msgpack_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));
 
             // fixstr
@@ -1407,16 +1497,22 @@ class binary_reader
             case 0xDB: // str 32
             {
                 string_t s;
-                return get_msgpack_string(s) && sax->string(s);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_msgpack_string(s);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->string(s);
             }
 
             case 0xC0: // nil
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->null();
 
             case 0xC2: // false
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->boolean(false);
 
             case 0xC3: // true
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->boolean(true);
 
             case 0xC4: // bin 8
@@ -1432,90 +1528,107 @@ class binary_reader
             case 0xD8: // fixext 16
             {
                 binary_t b;
-                return get_msgpack_binary(b) && sax->binary(b);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_msgpack_binary(b);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->binary(b);
             }
 
             case 0xCA: // float 32
             {
                 float number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             case 0xCB: // float 64
             {
                 double number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             case 0xCC: // uint 8
             {
                 std::uint8_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
             }
 
             case 0xCD: // uint 16
             {
                 std::uint16_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
             }
 
             case 0xCE: // uint 32
             {
                 std::uint32_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
             }
 
             case 0xCF: // uint 64
             {
                 std::uint64_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);
             }
 
             case 0xD0: // int 8
             {
                 std::int8_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
             }
 
             case 0xD1: // int 16
             {
                 std::int16_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
             }
 
             case 0xD2: // int 32
             {
                 std::int32_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
             }
 
             case 0xD3: // int 64
             {
                 std::int64_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format_t::msgpack, number) && sax->number_integer(number);
             }
 
             case 0xDC: // array 16
             {
                 std::uint16_t len{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(len));
                 return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast<std::size_t>(len));
             }
 
             case 0xDD: // array 32
             {
                 std::uint32_t len{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(len));
                 return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast<std::size_t>(len));
             }
 
             case 0xDE: // map 16
             {
                 std::uint16_t len{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(len));
                 return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast<std::size_t>(len));
             }
 
             case 0xDF: // map 32
             {
                 std::uint32_t len{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(len));
                 return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast<std::size_t>(len));
             }
 
@@ -1552,6 +1665,7 @@ class binary_reader
             case 0xFD:
             case 0xFE:
             case 0xFF:
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->number_integer(static_cast<std::int8_t>(current));
 
             default: // anything else
@@ -1782,6 +1896,7 @@ class binary_reader
             }
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read);
         return sax->end_array();
     }
 
@@ -1800,7 +1915,10 @@ class binary_reader
         for (std::size_t i = 0; i < len; ++i)
         {
             get();
-            if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key)))
+            detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+            const bool result_get = get_msgpack_string(key);
+            detail::sax_call_next_token_end_pos(sax, chars_read);
+            if (JSON_HEDLEY_UNLIKELY(!result_get || !sax->key(key)))
             {
                 return false;
             }
@@ -1812,6 +1930,7 @@ class binary_reader
             key.clear();
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read);
         return sax->end_object();
     }
 
@@ -2174,7 +2293,6 @@ class binary_reader
                             return true;
                         }
                     }
-
                     string_t key = "_ArraySize_";
                     if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size())))
                     {
@@ -2235,7 +2353,6 @@ class binary_reader
         bool is_ndarray = false;
 
         get_ignore_noop();
-
         if (current == '$')
         {
             result.second = get();  // must not ignore 'N', because 'N' maybe the type
@@ -2264,7 +2381,9 @@ class binary_reader
                                         exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr));
             }
 
+            // detail::sax_call_next_token_start_pos(sax, chars_read - 1);
             const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
+            //detail::sax_call_next_token_end_pos(sax, chars_read);
             if (input_format == input_format_t::bjdata && is_ndarray)
             {
                 if (inside_ndarray)
@@ -2279,7 +2398,9 @@ class binary_reader
 
         if (current == '#')
         {
+            //    detail::sax_call_next_token_start_pos(sax, chars_read - 1);
             const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
+            //   detail::sax_call_next_token_end_pos(sax, chars_read);
             if (input_format == input_format_t::bjdata && is_ndarray)
             {
                 return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,
@@ -2288,6 +2409,7 @@ class binary_reader
             return is_error;
         }
 
+        //   detail::sax_call_next_token_start_end_pos(sax, chars_read - 2, chars_read - 1);
         return true;
     }
 
@@ -2303,40 +2425,47 @@ class binary_reader
                 return unexpect_eof(input_format, "value");
 
             case 'T':  // true
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->boolean(true);
             case 'F':  // false
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->boolean(false);
 
             case 'Z':  // null
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
                 return sax->null();
 
             case 'U':
             {
                 std::uint8_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_unsigned(number);
             }
 
             case 'i':
             {
                 std::int8_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_integer(number);
             }
 
             case 'I':
             {
                 std::int16_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_integer(number);
             }
 
             case 'l':
             {
                 std::int32_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_integer(number);
             }
-
             case 'L':
             {
                 std::int64_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_integer(number);
             }
 
@@ -2347,6 +2476,7 @@ class binary_reader
                     break;
                 }
                 std::uint16_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_unsigned(number);
             }
 
@@ -2357,6 +2487,7 @@ class binary_reader
                     break;
                 }
                 std::uint32_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_unsigned(number);
             }
 
@@ -2367,11 +2498,13 @@ class binary_reader
                     break;
                 }
                 std::uint64_t number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_unsigned(number);
             }
 
             case 'h':
             {
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
                 if (input_format != input_format_t::bjdata)
                 {
                     break;
@@ -2417,25 +2550,30 @@ class binary_reader
                             return std::ldexp(mant + 1024, exp - 25);
                     }
                 }();
+                detail::sax_call_next_token_end_pos(sax, chars_read);
                 return sax->number_float((half & 0x8000u) != 0
                                          ? static_cast<number_float_t>(-val)
-                                         : static_cast<number_float_t>(val), "");
+                                         : static_cast<number_float_t>(val),
+                                         "");
             }
 
             case 'd':
             {
                 float number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             case 'D':
             {
                 double number{};
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read + sizeof(number));
                 return get_number(input_format, number) && sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             case 'H':
             {
+                // call to detail::sax_call_next_token_start_end_pos inside of the method
                 return get_ubjson_high_precision_number();
             }
 
@@ -2453,19 +2591,25 @@ class binary_reader
                                             exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr));
                 }
                 string_t s(1, static_cast<typename string_t::value_type>(current));
+                detail::sax_call_next_token_start_end_pos(sax, chars_read - 2, chars_read);
                 return sax->string(s);
             }
 
             case 'S':  // string
             {
                 string_t s;
-                return get_ubjson_string(s) && sax->string(s);
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_ubjson_string(s);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                return result_get && sax->string(s);
             }
 
             case '[':  // array
+                // call to detail::sax_call_next_token_start_end_pos inside of the method
                 return get_ubjson_array();
 
             case '{':  // object
+                // call to detail::sax_call_next_token_start_end_pos inside of the method
                 return get_ubjson_object();
 
             default: // anything else
@@ -2480,6 +2624,7 @@ class binary_reader
     */
     bool get_ubjson_array()
     {
+        detail::sax_call_next_token_start_pos(sax, chars_read - 1);
         std::pair<std::size_t, char_int_type> size_and_type;
         if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type)))
         {
@@ -2504,6 +2649,7 @@ class binary_reader
                                         exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr));
             }
 
+            detail::sax_call_next_token_end_pos(sax, chars_read);
             string_t type = it->second; // sax->string() takes a reference
             if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))
             {
@@ -2515,6 +2661,7 @@ class binary_reader
                 size_and_type.second = 'U';
             }
 
+            detail::sax_call_next_token_start_end_pos(sax, chars_read);
             key = "_ArrayData_";
             if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) ))
             {
@@ -2523,17 +2670,20 @@ class binary_reader
 
             for (std::size_t i = 0; i < size_and_type.first; ++i)
             {
+                // call to detail::sax_call_next_token_start_end_pos inside of the method
                 if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
                 {
                     return false;
                 }
             }
 
+            detail::sax_call_next_token_start_end_pos(sax, chars_read);
             return (sax->end_array() && sax->end_object());
         }
 
         if (size_and_type.first != npos)
         {
+            detail::sax_call_next_token_end_pos(sax, chars_read);
             if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first)))
             {
                 return false;
@@ -2545,6 +2695,7 @@ class binary_reader
                 {
                     for (std::size_t i = 0; i < size_and_type.first; ++i)
                     {
+                        // call to detail::sax_call_next_token_start_end_pos inside of the method
                         if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
                         {
                             return false;
@@ -2556,6 +2707,7 @@ class binary_reader
             {
                 for (std::size_t i = 0; i < size_and_type.first; ++i)
                 {
+                    // call to detail::sax_call_next_token_start_end_pos inside of the method
                     if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
                     {
                         return false;
@@ -2565,6 +2717,7 @@ class binary_reader
         }
         else
         {
+            detail::sax_call_next_token_end_pos(sax, chars_read - 1);
             if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))
             {
                 return false;
@@ -2580,6 +2733,7 @@ class binary_reader
             }
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
         return sax->end_array();
     }
 
@@ -2588,6 +2742,7 @@ class binary_reader
     */
     bool get_ubjson_object()
     {
+        detail::sax_call_next_token_start_pos(sax, chars_read - 1);
         std::pair<std::size_t, char_int_type> size_and_type;
         if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type)))
         {
@@ -2605,6 +2760,7 @@ class binary_reader
         string_t key;
         if (size_and_type.first != npos)
         {
+            detail::sax_call_next_token_end_pos(sax, chars_read - 1);
             if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first)))
             {
                 return false;
@@ -2614,7 +2770,10 @@ class binary_reader
             {
                 for (std::size_t i = 0; i < size_and_type.first; ++i)
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
+                    detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                    const bool result_get = get_ubjson_string(key);
+                    detail::sax_call_next_token_end_pos(sax, chars_read);
+                    if (JSON_HEDLEY_UNLIKELY(!result_get || !sax->key(key)))
                     {
                         return false;
                     }
@@ -2629,7 +2788,10 @@ class binary_reader
             {
                 for (std::size_t i = 0; i < size_and_type.first; ++i)
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
+                    detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                    const bool result_get = get_ubjson_string(key);
+                    detail::sax_call_next_token_end_pos(sax, chars_read);
+                    if (JSON_HEDLEY_UNLIKELY(!result_get || !sax->key(key)))
                     {
                         return false;
                     }
@@ -2643,6 +2805,7 @@ class binary_reader
         }
         else
         {
+            detail::sax_call_next_token_end_pos(sax, chars_read - 1);
             if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))
             {
                 return false;
@@ -2650,7 +2813,10 @@ class binary_reader
 
             while (current != '}')
             {
-                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))
+                detail::sax_call_next_token_start_pos(sax, chars_read - 1);
+                const bool result_get = get_ubjson_string(key, false);
+                detail::sax_call_next_token_end_pos(sax, chars_read);
+                if (JSON_HEDLEY_UNLIKELY(!result_get || !sax->key(key)))
                 {
                     return false;
                 }
@@ -2663,6 +2829,7 @@ class binary_reader
             }
         }
 
+        detail::sax_call_next_token_start_end_pos(sax, chars_read - 1, chars_read);
         return sax->end_object();
     }
 
@@ -2671,6 +2838,7 @@ class binary_reader
 
     bool get_ubjson_high_precision_number()
     {
+        detail::sax_call_next_token_start_pos(sax, chars_read - 1);
         // get size of following number string
         std::size_t size{};
         bool no_ndarray = true;
@@ -2691,6 +2859,7 @@ class binary_reader
             }
             number_vector.push_back(static_cast<char>(current));
         }
+        detail::sax_call_next_token_end_pos(sax, chars_read);
 
         // parse number string
         using ia_type = decltype(detail::input_adapter(number_vector));
@@ -2888,6 +3057,7 @@ class binary_reader
     {
         if (JSON_HEDLEY_UNLIKELY(current == std::char_traits<char_type>::eof()))
         {
+            detail::sax_call_next_token_end_pos(sax, chars_read);
             return sax->parse_error(chars_read, "<end of file>",
                                     parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr));
         }

include/nlohmann/detail/input/lexer.hpp

@@ -1506,13 +1506,13 @@ scan_number_done:
         while (current == ' ' || current == '\t' || current == '\n' || current == '\r');
     }
 
-    token_type scan()
+    bool scan_start()
     {
         // initially, skip the BOM
         if (position.chars_read_total == 0 && !skip_bom())
         {
             error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
-            return token_type::parse_error;
+            return false;
         }
 
         // read next character and ignore whitespace
@@ -1523,13 +1523,17 @@ scan_number_done:
         {
             if (!scan_comment())
             {
-                return token_type::parse_error;
+                return false;
             }
 
             // skip following whitespace
             skip_whitespace();
         }
+        return true;
+    }
 
+    token_type scan_end()
+    {
         switch (current)
         {
             // structural characters
@@ -1593,6 +1597,10 @@ scan_number_done:
                 return token_type::parse_error;
         }
     }
+    token_type scan()
+    {
+        return !scan_start() ? token_type::parse_error : scan_end();
+    }
 
   private:
     /// input adapter

include/nlohmann/detail/input/parser.hpp

@@ -76,8 +76,6 @@ class parser
         , m_lexer(std::move(adapter), skip_comments)
         , allow_exceptions(allow_exceptions_)
     {
-        // read first token
-        get_token();
     }
 
     /*!
@@ -98,7 +96,7 @@ class parser
             sax_parse_internal(&sdp);
 
             // in strict mode, input must be completely read
-            if (strict && (get_token() != token_type::end_of_input))
+            if (strict && (get_token(&sdp) != token_type::end_of_input))
             {
                 sdp.parse_error(m_lexer.get_position(),
                                 m_lexer.get_token_string(),
@@ -126,7 +124,7 @@ class parser
             sax_parse_internal(&sdp);
 
             // in strict mode, input must be completely read
-            if (strict && (get_token() != token_type::end_of_input))
+            if (strict && (get_token(&sdp) != token_type::end_of_input))
             {
                 sdp.parse_error(m_lexer.get_position(),
                                 m_lexer.get_token_string(),
@@ -164,7 +162,7 @@ class parser
         const bool result = sax_parse_internal(sax);
 
         // strict mode: next byte must be EOF
-        if (result && strict && (get_token() != token_type::end_of_input))
+        if (result && strict && (get_token(sax) != token_type::end_of_input))
         {
             return sax->parse_error(m_lexer.get_position(),
                                     m_lexer.get_token_string(),
@@ -185,6 +183,8 @@ class parser
         // value to avoid a goto (see comment where set to true)
         bool skip_to_state_evaluation = false;
 
+        // read first token
+        get_token(sax);
         while (true)
         {
             if (!skip_to_state_evaluation)
@@ -200,7 +200,7 @@ class parser
                         }
 
                         // closing } -> we are done
-                        if (get_token() == token_type::end_object)
+                        if (get_token(sax) == token_type::end_object)
                         {
                             if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))
                             {
@@ -222,7 +222,7 @@ class parser
                         }
 
                         // parse separator (:)
-                        if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
+                        if (JSON_HEDLEY_UNLIKELY(get_token(sax) != token_type::name_separator))
                         {
                             return sax->parse_error(m_lexer.get_position(),
                                                     m_lexer.get_token_string(),
@@ -233,7 +233,7 @@ class parser
                         states.push_back(false);
 
                         // parse values
-                        get_token();
+                        get_token(sax);
                         continue;
                     }
 
@@ -245,7 +245,7 @@ class parser
                         }
 
                         // closing ] -> we are done
-                        if (get_token() == token_type::end_array)
+                        if (get_token(sax) == token_type::end_array)
                         {
                             if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))
                             {
@@ -372,10 +372,10 @@ class parser
             if (states.back())  // array
             {
                 // comma -> next value
-                if (get_token() == token_type::value_separator)
+                if (get_token(sax) == token_type::value_separator)
                 {
                     // parse a new value
-                    get_token();
+                    get_token(sax);
                     continue;
                 }
 
@@ -405,10 +405,10 @@ class parser
             // states.back() is false -> object
 
             // comma -> next value
-            if (get_token() == token_type::value_separator)
+            if (get_token(sax) == token_type::value_separator)
             {
                 // parse key
-                if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string))
+                if (JSON_HEDLEY_UNLIKELY(get_token(sax) != token_type::value_string))
                 {
                     return sax->parse_error(m_lexer.get_position(),
                                             m_lexer.get_token_string(),
@@ -421,7 +421,7 @@ class parser
                 }
 
                 // parse separator (:)
-                if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
+                if (JSON_HEDLEY_UNLIKELY(get_token(sax) != token_type::name_separator))
                 {
                     return sax->parse_error(m_lexer.get_position(),
                                             m_lexer.get_token_string(),
@@ -429,7 +429,7 @@ class parser
                 }
 
                 // parse values
-                get_token();
+                get_token(sax);
                 continue;
             }
 
@@ -457,10 +457,19 @@ class parser
         }
     }
 
-    /// get next token from lexer
+    /// get next token from lexer and pass position info to sax (if it is accepted)
-    token_type get_token()
+    template<class SAX>
+    token_type get_token(SAX* sax)
     {
-        return last_token = m_lexer.scan();
+        if (!m_lexer.scan_start())
+        {
+            last_token = token_type::parse_error;
+            return token_type::parse_error;
+        }
+        detail::sax_call_next_token_start_pos(sax, m_lexer);
+        last_token = m_lexer.scan_end();
+        detail::sax_call_next_token_end_pos(sax, m_lexer);
+        return last_token;
     }
 
     std::string exception_message(const token_type expected, const std::string& context)

include/nlohmann/detail/input/position_t.hpp

@@ -13,9 +13,6 @@
 #include <nlohmann/detail/abi_macros.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
-namespace detail
-{
-
 /// struct to capture the start position of the current token
 struct position_t
 {
@@ -32,6 +29,4 @@ struct position_t
         return chars_read_total;
     }
 };
-
-}  // namespace detail
 NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/detail/meta/is_sax.hpp

@@ -15,10 +15,180 @@
 #include <nlohmann/detail/abi_macros.hpp>
 #include <nlohmann/detail/meta/detected.hpp>
 #include <nlohmann/detail/meta/type_traits.hpp>
+#include <nlohmann/detail/input/position_t.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
 {
+// helper struct to call sax->next_token_start
+//(we want this functionality as a type to ease passing it as template argument)
+struct sax_call_next_token_start_pos_direct
+{
+    template<typename SAX, typename...Ts>
+    static auto call(SAX* sax, Ts&& ...ts)
+    -> decltype(sax->next_token_start(std::forward<Ts>(ts)...))
+    {
+        sax->next_token_start(std::forward<Ts>(ts)...);
+    }
+};
+// helper struct to call sax->next_token_end
+// (we want this functionality as a type to ease passing it as template argument)
+struct sax_call_next_token_end_pos_direct
+{
+    template<typename SAX, typename...Ts>
+    static auto call(SAX* sax, Ts&& ...ts)
+    -> decltype(sax->next_token_end(std::forward<Ts>(ts)...))
+    {
+        sax->next_token_end(std::forward<Ts>(ts)...);
+    }
+};
+
+// dispatch the calls to next_token_start next_token_end
+// and drop the calls if the sax parser does not support these methods.
+//
+// DirectCaller can be set to one of sax_call_next_token_{start,end}_pos_direct to
+// determine which method is called
+template <typename DirectCaller, typename SAX, typename LexOrPos>
+struct sax_call_function
+{
+    // is the parameter a lexer or a byte position
+    static constexpr bool called_with_byte_pos = std::is_same<LexOrPos, std::size_t>::value;
+
+    template<typename SAX2, typename...Ts2>
+    using call_t = decltype(DirectCaller::call(std::declval<SAX2*>(), std::declval<Ts2>()...));
+
+    //the sax parser supports calls with a position
+    static constexpr bool detected_call_with_byte_pos =
+        is_detected_exact<void, call_t, SAX, std::size_t>::value;
+
+    //the sax parser supports calls with a lexer
+    static constexpr bool detected_call_with_lex_pos =
+        !called_with_byte_pos &&
+        is_detected_exact<void, call_t, SAX, position_t >::value;
+
+    //there either has to be a version accepting a lexer or a position
+    static constexpr bool valid = detected_call_with_byte_pos || detected_call_with_lex_pos;
+
+    //called with byte pos and byte pos is method supported -> pass data on
+    template<typename SaxT = SAX>
+    static typename std::enable_if <
+    std::is_same<SaxT, SAX>::value &&
+    valid &&
+    detected_call_with_byte_pos
+    >::type
+    call(SaxT* sax, std::size_t pos)
+    {
+        DirectCaller::call(sax, pos);
+    }
+
+    //the sax parser has no version of the method -> drop call
+    template<typename SaxT = SAX>
+    static typename std::enable_if <
+    std::is_same<SaxT, SAX>::value &&
+    !valid
+    >::type
+    call(SaxT* /*unused*/, const LexOrPos& /*unused*/) {}
+
+    //called with lex and lex pos method is supported -> call with position from lexer
+    // the start pos in the lexer is last read char -> chars_read_total-1
+    template<typename SaxT = SAX>
+    static typename std::enable_if <
+    std::is_same<SaxT, SAX>::value &&
+    valid &&
+    !called_with_byte_pos &&
+    detected_call_with_lex_pos &&
+    std::is_same<DirectCaller, sax_call_next_token_start_pos_direct>::value
+    >::type
+    call(SaxT* sax, const LexOrPos& lex)
+    {
+        JSON_ASSERT(lex.get_position().chars_read_total > 0);
+        JSON_ASSERT(lex.get_position().chars_read_current_line > 0);
+        //the lexer has already read the first char of the current element -> fix this
+        auto pos_copy = lex.get_position();
+        --pos_copy.chars_read_total;
+        --pos_copy.chars_read_current_line;
+        DirectCaller::call(sax, pos_copy);
+    }
+
+    //called with lex and lex pos method is supported -> pass data on
+    // the one past end pos in the lexer is the current index -> chars_read_total
+    template<typename SaxT = SAX>
+    static typename std::enable_if <
+    std::is_same<SaxT, SAX>::value &&
+    valid &&
+    !called_with_byte_pos &&
+    detected_call_with_lex_pos &&
+    std::is_same<DirectCaller, sax_call_next_token_end_pos_direct>::value
+    >::type
+    call(SaxT* sax, const LexOrPos& lex)
+    {
+        DirectCaller::call(sax, lex.get_position());
+    }
+
+    // called with lex and only byte pos method is supported -> call with byte position from lexer
+    // the start pos in the lexer is last read char -> chars_read_total-1
+    template<typename SaxT = SAX>
+    static typename std::enable_if <
+    std::is_same<SaxT, SAX>::value &&
+    valid &&
+    !called_with_byte_pos &&
+    !detected_call_with_lex_pos &&
+    std::is_same<DirectCaller, sax_call_next_token_start_pos_direct>::value
+    >::type
+    call(SaxT* sax, const LexOrPos& lex)
+    {
+        JSON_ASSERT(lex.get_position().chars_read_total > 0);
+        DirectCaller::call(sax, lex.get_position().chars_read_total - 1);
+    }
+
+    // called with lex and only byte pos method is supported -> call with byte position from lexer
+    // the one past end pos in the lexer is the current index -> chars_read_total
+    template<typename SaxT = SAX>
+    static typename std::enable_if <
+    std::is_same<SaxT, SAX>::value &&
+    valid &&
+    !called_with_byte_pos &&
+    !detected_call_with_lex_pos &&
+    std::is_same<DirectCaller, sax_call_next_token_end_pos_direct>::value
+    >::type
+    call(SaxT* sax, const LexOrPos& lex)
+    {
+        DirectCaller::call(sax, lex.get_position().chars_read_total);
+    }
+};
+
+//set the element start pos of a sax parser by calling any version of sax->next_token_start (if available)
+template<class SAX, class LexOrPos>
+void sax_call_next_token_start_pos(SAX* sax, const LexOrPos& lexOrPos)
+{
+    using call_t = sax_call_function<sax_call_next_token_start_pos_direct, SAX, LexOrPos>;
+    call_t::call(sax, lexOrPos);
+}
+//set the element end pos of a sax parser by calling any version of sax->next_token_end (if available)
+template<class SAX, class LexOrPos>
+void sax_call_next_token_end_pos(SAX* sax, const LexOrPos& lexOrPos)
+{
+    using call_t = sax_call_function<sax_call_next_token_end_pos_direct, SAX, LexOrPos>;
+    call_t::call(sax, lexOrPos);
+}
+//set the element start end pos of a sax parser by calling any version of
+// sax->next_token_start and sax->next_token_end (if available)
+template<class SAX, class LexOrPos1, class LexOrPos2>
+void sax_call_next_token_start_end_pos(SAX* sax, const LexOrPos1& lexOrPos1, const LexOrPos2& lexOrPos2)
+{
+    sax_call_next_token_start_pos(sax, lexOrPos1);
+    sax_call_next_token_end_pos(sax, lexOrPos2);
+}
+//set the element start end pos of a sax parser by calling any version of
+// sax->next_token_start and sax->next_token_end (if available)
+template<class SAX, class LexOrPos>
+void sax_call_next_token_start_end_pos(SAX* sax, const LexOrPos& lexOrPos)
+{
+    sax_call_next_token_start_pos(sax, lexOrPos);
+    sax_call_next_token_end_pos(sax, lexOrPos);
+}
+
+
 
 template<typename T>
 using null_function_t = decltype(std::declval<T&>().null());

tests/src/unit-sax-parser-store-source-location.cpp

@@ -0,0 +1,340 @@
+/*
+    __ _____ _____ _____
+ __|  |   __|     |   | |  JSON for Modern C++ (test suite)
+|  |  |__   |  |  | | | |  version 3.10.2
+|_____|_____|_____|_|___|  https://github.com/nlohmann/json
+
+Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+SPDX-License-Identifier: MIT
+Copyright (c) 2013-2019 Niels Lohmann <http://nlohmann.me>.
+
+Permission is hereby  granted, free of charge, to any  person obtaining a copy
+of this software and associated  documentation files (the "Software"), to deal
+in the Software  without restriction, including without  limitation the rights
+to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
+copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
+IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
+FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
+AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
+LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#include <iostream>
+#include <string>
+
+#include "doctest_compatibility.h"
+
+#include <nlohmann/json.hpp>
+
+//prototype to make -Wmissing-prototypes happy
+std::ostream& operator<<(std::ostream& out, const nlohmann::position_t& p);
+
+//test json parser with detailed line / col information as metadata
+
+struct token_start_stop
+{
+    nlohmann::position_t start{};
+    nlohmann::position_t stop{};
+};
+
+std::ostream& operator<<(std::ostream& out, const nlohmann::position_t& p)
+{
+    out << p.chars_read_total << '(' << p.lines_read << ':' << p.chars_read_current_line << ')';
+    return out;
+}
+
+using json_with_token_start_stop =
+    nlohmann::basic_json <
+    std::map,
+    std::vector,
+    std::string,
+    bool,
+    std::int64_t,
+    std::uint64_t,
+    double,
+    std::allocator,
+    nlohmann::adl_serializer,
+    std::vector<std::uint8_t>,
+    token_start_stop >;
+
+//adapted from detail::json_sax_dom_parser
+class sax_with_token_start_stop_metadata
+{
+  public:
+    using json = json_with_token_start_stop;
+    using number_integer_t = typename json::number_integer_t;
+    using number_unsigned_t = typename json::number_unsigned_t;
+    using number_float_t = typename json::number_float_t;
+    using string_t = typename json::string_t;
+    using binary_t = typename json::binary_t;
+
+    /*!
+    @param[in,out] r  reference to a JSON value that is manipulated while
+                       parsing
+    @param[in] allow_exceptions_  whether parse errors yield exceptions
+    */
+    explicit sax_with_token_start_stop_metadata(json& r, const bool allow_exceptions_ = true)
+        : root(r)
+        , object_element{nullptr} // NOLINT(modernize-use-default-member-init)
+        , errored{false} // NOLINT(modernize-use-default-member-init)
+        , allow_exceptions(allow_exceptions_)
+    {}
+
+    sax_with_token_start_stop_metadata(sax_with_token_start_stop_metadata&&) = delete;
+    sax_with_token_start_stop_metadata(const sax_with_token_start_stop_metadata&) = delete;
+    sax_with_token_start_stop_metadata& operator=(sax_with_token_start_stop_metadata&&) = delete;
+    sax_with_token_start_stop_metadata& operator=(const sax_with_token_start_stop_metadata&) = delete;
+
+    ~sax_with_token_start_stop_metadata() = default;
+
+    void next_token_start(const nlohmann::position_t& p)
+    {
+        start_stop.start = p;
+    }
+
+    void next_token_end(const nlohmann::position_t& p)
+    {
+        start_stop.stop = p;
+    }
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool binary(binary_t& val)
+    {
+        handle_value(std::move(val));
+        return true;
+    }
+
+    bool start_object(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(json::value_t::object));
+        ref_stack.back()->start = start_stop.start;
+
+        if (len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size())
+        {
+            throw nlohmann::detail::out_of_range::create(408, nlohmann::detail::concat("excessive object size: ", std::to_string(len)), ref_stack.back());
+        }
+
+        return true;
+    }
+
+    bool key(string_t& val)
+    {
+        assert(!ref_stack.empty());
+        assert(ref_stack.back()->is_object());
+
+        // add null at given key and store the reference for later
+        object_element = &(*ref_stack.back())[val];
+        return true;
+    }
+
+    bool end_object()
+    {
+        assert(!ref_stack.empty());
+        assert(ref_stack.back()->is_object());
+
+        ref_stack.back()->stop = start_stop.stop;
+        ref_stack.pop_back();
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(json::value_t::array));
+        ref_stack.back()->start = start_stop.start;
+
+        if (len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size())
+        {
+            throw nlohmann::detail::out_of_range::create(408, nlohmann::detail::concat("excessive array size: ", std::to_string(len)), ref_stack.back());
+        }
+
+        return true;
+    }
+
+    bool end_array()
+    {
+        assert(!ref_stack.empty());
+        assert(ref_stack.back()->is_array());
+
+        ref_stack.back()->stop = start_stop.stop;
+        ref_stack.pop_back();
+        return true;
+    }
+
+    template<class Exception>
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const Exception& ex)
+    {
+        errored = true;
+        static_cast<void>(ex);
+        if (allow_exceptions)
+        {
+            throw ex;
+        }
+        return false;
+    }
+
+    bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+    */
+    template<typename Value>
+    json*
+    handle_value(Value&& v)
+    {
+        if (ref_stack.empty())
+        {
+            root = json(std::forward<Value>(v));
+            root.start = start_stop.start;
+            root.stop = start_stop.stop;
+            return &root;
+        }
+
+        assert(ref_stack.back()->is_array() || ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            auto& array_element = ref_stack.back()->emplace_back(std::forward<Value>(v));
+            array_element.start = start_stop.start;
+            array_element.stop = start_stop.stop;
+            return &array_element;
+        }
+
+        assert(ref_stack.back()->is_object());
+        assert(object_element);
+        *object_element = json(std::forward<Value>(v));
+        object_element->start = start_stop.start;
+        object_element->stop = start_stop.stop;
+        return object_element;
+    }
+
+    /// the parsed JSON value
+    json& root;
+    /// stack to model hierarchy of values
+    std::vector<json*> ref_stack{};
+    /// helper to hold the reference for the next object element
+    json* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+    /// start / stop information for the current token
+    token_start_stop start_stop {};
+};
+
+TEST_CASE("parse-json-with-position-info")
+{
+    const std::string str =
+        /*line 0*/ R"({)"
+        "\n"
+        /*line 1*/ R"(  "array" : [)"
+        "\n"
+        /*line 2*/ R"(    14294967296,)"
+        "\n"
+        /*line 3*/ R"(    -1,)"
+        "\n"
+        /*line 4*/ R"(    true,)"
+        "\n"
+        /*line 5*/ R"(    4.2,)"
+        "\n"
+        /*line 6*/ R"(    null,)"
+        "\n"
+        /*line 7*/ R"(    "str")"
+        "\n"
+        /*line 8*/ R"(  ])"
+        "\n"
+        /*line 9*/ R"(})";
+    json_with_token_start_stop j;
+    sax_with_token_start_stop_metadata sax{j};
+    CHECK(nlohmann::json::sax_parse(str, &sax, nlohmann::json::input_format_t::json));
+    CHECK(j.start.lines_read == 0);
+    CHECK(j.start.chars_read_current_line == 0);
+
+    CHECK(j["array"].start.lines_read == 1);
+    CHECK(j["array"].start.chars_read_current_line == 12);
+
+    CHECK(j["array"][0].start.lines_read == 2);
+    CHECK(j["array"][0].start.chars_read_current_line == 4);
+    CHECK(j["array"][0].stop.lines_read == 2);
+    CHECK(j["array"][0].stop.chars_read_current_line == 15);
+
+    CHECK(j["array"][1].start.lines_read == 3);
+    CHECK(j["array"][1].start.chars_read_current_line == 4);
+    CHECK(j["array"][1].stop.lines_read == 3);
+    CHECK(j["array"][1].stop.chars_read_current_line == 6);
+
+    CHECK(j["array"][2].start.lines_read == 4);
+    CHECK(j["array"][2].start.chars_read_current_line == 4);
+    CHECK(j["array"][2].stop.lines_read == 4);
+    CHECK(j["array"][2].stop.chars_read_current_line == 8);
+
+    CHECK(j["array"][3].start.lines_read == 5);
+    CHECK(j["array"][3].start.chars_read_current_line == 4);
+    CHECK(j["array"][3].stop.lines_read == 5);
+    CHECK(j["array"][3].stop.chars_read_current_line == 7);
+
+    CHECK(j["array"][4].start.lines_read == 6);  //starts directly after last value....
+    CHECK(j["array"][4].start.chars_read_current_line == 4);
+    CHECK(j["array"][4].stop.lines_read == 6);
+    CHECK(j["array"][4].stop.chars_read_current_line == 8);
+
+    CHECK(j["array"][5].start.lines_read == 7);
+    CHECK(j["array"][5].start.chars_read_current_line == 4);
+    CHECK(j["array"][5].stop.lines_read == 7);
+    CHECK(j["array"][5].stop.chars_read_current_line == 9);
+
+    CHECK(j["array"].stop.lines_read == 8);
+    CHECK(j["array"].stop.chars_read_current_line == 3);
+
+    CHECK(j.stop.lines_read == 9);
+    CHECK(j.stop.chars_read_current_line == 1);
+}