From eb488bb4d9969eea53136515e4f90efb2785fdfc Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Aug 2021 20:54:02 +0200 Subject: [PATCH 1/4] :memo: add note for wstring handling --- README.md | 1 + doc/mkdocs/docs/home/faq.md | 41 ++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f972c211..b717f9bee 100644 --- a/README.md +++ b/README.md @@ -1612,6 +1612,7 @@ The library supports **Unicode input** as follows: - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors. - The strings stored in the library are UTF-8 encoded. When using the default string type (`std::string`), note that its length/size functions return the number of stored bytes rather than the number of characters or glyphs. - When you store strings with different encodings in the library, calling [`dump()`](https://nlohmann.github.io/json/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace` or `json::error_handler_t::ignore` are used as error handlers. +- To store wide strings (e.g., `std::wstring`), you need to convert them to a a UTF-8 encoded `std::string` before, see [an example](https://json.nlohmann.me/home/faq/#wide-string-handling). ### Comments in JSON diff --git a/doc/mkdocs/docs/home/faq.md b/doc/mkdocs/docs/home/faq.md index 23aa35a22..c6a283bc7 100644 --- a/doc/mkdocs/docs/home/faq.md +++ b/doc/mkdocs/docs/home/faq.md @@ -44,7 +44,7 @@ for objects. !!! question - - Can you add an option to ignore trailing commas? + Can you add an option to ignore trailing commas? This library does not support any feature which would jeopardize interoperability. @@ -70,6 +70,45 @@ The library supports **Unicode input** as follows: In most cases, the parser is right to complain, because the input is not UTF-8 encoded. This is especially true for Microsoft Windows where Latin-1 or ISO 8859-1 is often the standard encoding. +### Wide string handling + +!!! question + + Why are wide strings (e.g., `std::wstring`) dumped as arrays of numbers? + +As described [above](#parse-errors-reading-non-ascii-characters), the library assumes UTF-8 as encoding. To store a wide string, you need to change the encoding. + +!!! example + + ```cpp + #include // codecvt_utf8 + #include // wstring_convert + + // encoding function + std::string to_utf8(std::wstring& wide_string) + { + static std::wstring_convert> utf8_conv; + return utf8_conv.to_bytes(wide_string); + } + + json j; + std::wstring ws = L"車B1234 こんにちは"; + + j["original"] = ws; + j["encoded"] = to_utf8(ws); + + std::cout << j << std::endl; + ``` + + The result is: + + ```json + { + "encoded": "車B1234 こんにちは", + "original": [36554, 66, 49, 50, 51, 52, 32, 12371, 12435, 12395, 12385, 12399] + } + ``` + ## Exceptions ### Parsing without exceptions From 095aae19fecde12056ccc8280003c2adc6816cc5 Mon Sep 17 00:00:00 2001 From: Sven Fink Date: Tue, 3 Aug 2021 08:58:38 +0200 Subject: [PATCH 2/4] Supress -Wfloat-equal on intended float comparisions --- include/nlohmann/detail/output/binary_writer.hpp | 3 +++ single_include/nlohmann/json.hpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 24e7c1094..8a172056b 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1524,6 +1524,8 @@ class binary_writer void write_compact_float(const number_float_t n, detail::input_format_t format) { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)) @@ -1540,6 +1542,7 @@ class binary_writer : get_msgpack_float_prefix(n)); write_number(n); } +#pragma GCC diagnostic pop } public: diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 28ce55292..1e9ac7941 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -14825,6 +14825,8 @@ class binary_writer void write_compact_float(const number_float_t n, detail::input_format_t format) { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)) @@ -14841,6 +14843,7 @@ class binary_writer : get_msgpack_float_prefix(n)); write_number(n); } +#pragma GCC diagnostic pop } public: From 353d59717e8ff394fd2655c4abdb87020dfb6607 Mon Sep 17 00:00:00 2001 From: Sven Fink Date: Wed, 4 Aug 2021 08:34:53 +0200 Subject: [PATCH 3/4] Add more suppressions on float comparisons --- include/nlohmann/detail/conversions/to_chars.hpp | 3 +++ include/nlohmann/json.hpp | 3 +++ single_include/nlohmann/json.hpp | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index e904d10fa..e5c1d01f1 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -1066,6 +1066,8 @@ char* to_chars(char* first, const char* last, FloatType value) *first++ = '-'; } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" if (value == 0) // +-0 { *first++ = '0'; @@ -1074,6 +1076,7 @@ char* to_chars(char* first, const char* last, FloatType value) *first++ = '0'; return first; } +#pragma GCC diagnostic pop JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index cda764262..cd8152811 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -6278,6 +6278,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ friend bool operator==(const_reference lhs, const_reference rhs) noexcept { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" const auto lhs_type = lhs.type(); const auto rhs_type = rhs.type(); @@ -6342,6 +6344,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } return false; +#pragma GCC diagnostic pop } /*! diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1e9ac7941..cb2569c90 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15984,6 +15984,8 @@ char* to_chars(char* first, const char* last, FloatType value) *first++ = '-'; } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" if (value == 0) // +-0 { *first++ = '0'; @@ -15992,6 +15994,7 @@ char* to_chars(char* first, const char* last, FloatType value) *first++ = '0'; return first; } +#pragma GCC diagnostic pop JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); @@ -23366,6 +23369,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ friend bool operator==(const_reference lhs, const_reference rhs) noexcept { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" const auto lhs_type = lhs.type(); const auto rhs_type = rhs.type(); @@ -23430,6 +23435,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } return false; +#pragma GCC diagnostic pop } /*! From 3f5545f99b0000f8b5ae4df9e89d7b741aeff846 Mon Sep 17 00:00:00 2001 From: Sven Fink Date: Wed, 4 Aug 2021 09:14:45 +0200 Subject: [PATCH 4/4] Remove -Wfloat-equal suppressions on tests --- test/src/unit-cbor.cpp | 1 - test/src/unit-constructor1.cpp | 1 - test/src/unit-readme.cpp | 1 - test/src/unit-reference_access.cpp | 1 - test/src/unit-regression1.cpp | 1 - test/src/unit-regression2.cpp | 1 - 6 files changed, 6 deletions(-) diff --git a/test/src/unit-cbor.cpp b/test/src/unit-cbor.cpp index f0baec98e..d3b81d814 100644 --- a/test/src/unit-cbor.cpp +++ b/test/src/unit-cbor.cpp @@ -28,7 +28,6 @@ SOFTWARE. */ #include "doctest_compatibility.h" -DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") #include using nlohmann::json; diff --git a/test/src/unit-constructor1.cpp b/test/src/unit-constructor1.cpp index 884a1e6f1..09329ecd6 100644 --- a/test/src/unit-constructor1.cpp +++ b/test/src/unit-constructor1.cpp @@ -28,7 +28,6 @@ SOFTWARE. */ #include "doctest_compatibility.h" -DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") #define JSON_TESTS_PRIVATE #include diff --git a/test/src/unit-readme.cpp b/test/src/unit-readme.cpp index 7db7b4e6b..7da9c3bf5 100644 --- a/test/src/unit-readme.cpp +++ b/test/src/unit-readme.cpp @@ -28,7 +28,6 @@ SOFTWARE. */ #include "doctest_compatibility.h" -DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") #include using nlohmann::json; diff --git a/test/src/unit-reference_access.cpp b/test/src/unit-reference_access.cpp index 125c7ea4f..c0477d19d 100644 --- a/test/src/unit-reference_access.cpp +++ b/test/src/unit-reference_access.cpp @@ -28,7 +28,6 @@ SOFTWARE. */ #include "doctest_compatibility.h" -DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") #include using nlohmann::json; diff --git a/test/src/unit-regression1.cpp b/test/src/unit-regression1.cpp index 3d4a42fe1..cfb6885fd 100644 --- a/test/src/unit-regression1.cpp +++ b/test/src/unit-regression1.cpp @@ -28,7 +28,6 @@ SOFTWARE. */ #include "doctest_compatibility.h" -DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") // for some reason including this after the json header leads to linker errors with VS 2017... #include diff --git a/test/src/unit-regression2.cpp b/test/src/unit-regression2.cpp index fa5de3179..e58ae745f 100644 --- a/test/src/unit-regression2.cpp +++ b/test/src/unit-regression2.cpp @@ -28,7 +28,6 @@ SOFTWARE. */ #include "doctest_compatibility.h" -DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") // for some reason including this after the json header leads to linker errors with VS 2017... #include