External/json
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhance get_number() parsing (also eliminate Valgrind induced errors)

Browse Source
This commit is contained in:
Trevor Welsby 2016-01-18 19:10:51 +10:00
parent b40408a755
commit 09a4751ee7
3 changed files with 191 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
src/json.hpp
View File

@ -602,7 +602,7 @@ class basic_json
@sa @ref number_integer_t -- type for number values (integer) @sa @ref number_integer_t -- type for number values (integer)
@since version 1.0.0 @since version 2.0.0
*/ */
using number_unsigned_t = NumberUnsignedType; using number_unsigned_t = NumberUnsignedType;
@ -699,9 +699,9 @@ class basic_json
string, ///< string value string, ///< string value
boolean, ///< boolean value boolean, ///< boolean value
number_integer, ///< number value (integer) number_integer, ///< number value (integer)
number_unsigned,///< number value (unsigned integer)
number_float, ///< number value (floating-point) number_float, ///< number value (floating-point)
discarded, ///< discarded by the the parser callback function discarded ///< discarded by the the parser callback function
number_unsigned ///< number value (unsigned integer)
}; };
@ -1343,7 +1343,7 @@ class basic_json
@sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number
value (unsigned integer) from a compatible number type value (unsigned integer) from a compatible number type
@since version 1.0.0 @since version 2.0.0
*/ */
template<typename T, template<typename T,
typename std::enable_if< typename std::enable_if<
@ -1372,7 +1372,7 @@ class basic_json
@sa @ref basic_json(const number_unsigned_t) -- create a number value @sa @ref basic_json(const number_unsigned_t) -- create a number value
(unsigned) (unsigned)
@since version 1.0.0 @since version 2.0.0
*/ */
template<typename CompatibleNumberUnsignedType, typename template<typename CompatibleNumberUnsignedType, typename
std::enable_if< std::enable_if<
@ -2159,7 +2159,7 @@ class basic_json
/*! /*!
@brief return whether value is a number @brief return whether value is a number
This function returns true if the JSON value is a number. This includes This function returns true iff the JSON value is a number. This includes
both integer and floating-point values. both integer and floating-point values.
@return `true` if type is number (regardless whether integer, unsigned @return `true` if type is number (regardless whether integer, unsigned
@ -2185,7 +2185,7 @@ class basic_json
/*! /*!
@brief return whether value is an integer number @brief return whether value is an integer number
This function returns true if the JSON value is an integer or unsigned This function returns true iff the JSON value is an integer or unsigned
integer number. This excludes floating-point values. integer number. This excludes floating-point values.
@return `true` if type is an integer or unsigned integer number, `false` @return `true` if type is an integer or unsigned integer number, `false`
@ -2210,7 +2210,7 @@ class basic_json
/*! /*!
@brief return whether value is an unsigned integer number @brief return whether value is an unsigned integer number
This function returns true if the JSON value is an unsigned integer number. This function returns true iff the JSON value is an unsigned integer number.
This excludes floating-point and (signed) integer values. This excludes floating-point and (signed) integer values.
@return `true` if type is an unsigned integer number, `false` otherwise. @return `true` if type is an unsigned integer number, `false` otherwise.
@ -2222,7 +2222,7 @@ class basic_json
integer number integer number
@sa @ref is_number_float() -- check if value is a floating-point number @sa @ref is_number_float() -- check if value is a floating-point number
@since version 1.0.0 @since version 2.0.0
*/ */
bool is_number_unsigned() const noexcept bool is_number_unsigned() const noexcept
{ {
@ -2232,7 +2232,7 @@ class basic_json
/*! /*!
@brief return whether value is a floating-point number @brief return whether value is a floating-point number
This function returns true if the JSON value is a floating-point number. This function returns true iff the JSON value is a floating-point number.
This excludes integer and unsigned integer values. This excludes integer and unsigned integer values.
@return `true` if type is a floating-point number, `false` otherwise. @return `true` if type is a floating-point number, `false` otherwise.
@ -4837,16 +4837,15 @@ class basic_json
*/ */
friend bool operator<(const value_t lhs, const value_t rhs) friend bool operator<(const value_t lhs, const value_t rhs)
{ {
static constexpr std::array<uint8_t, 9> order = {{ static constexpr std::array<uint8_t, 8> order = {{
0, // null 0, // null
3, // object 3, // object
4, // array 4, // array
5, // string 5, // string
1, // boolean 1, // boolean
2, // integer 2, // integer
2, // unsigned
2, // float 2, // float
0, // filler for discarded (preserves existing value_t values)
2 // unsigned
} }
}; };
@ -7482,53 +7481,99 @@ basic_json_parser_64:
/*! /*!
@brief return number value for number tokens @brief return number value for number tokens
This function translates the last token into a floating point number. This function translates the last token into the most appropriate
The pointer m_start points to the beginning of the parsed number. We number type (either integer, unsigned integer or floating point),
pass this pointer to std::strtod which sets endptr to the first which is passed back to the caller via the result parameter. The pointer
character past the converted number. If this pointer is not the same as m_start points to the beginning of the parsed number. We first examine
m_cursor, then either more or less characters have been used during the the first character to determine the sign of the number and then pass
comparison. This can happen for inputs like "01" which will be treated this pointer to either std::strtoull (if positive) or std::strtoll
like number 0 followed by number 1. (if negative), both of which set endptr to the first character past the
converted number. If this pointer is not the same as m_cursor, then
either more or less characters have been used during the comparison.
@return the result of the number conversion or NAN if the conversion This can happen for inputs like "01" which will be treated like number 0
read past the current token. The latter case needs to be treated by the followed by number 1. This will also occur for valid floating point
caller function. inputs like "12e3" will be incorrectly read as 12. Numbers that are too
large or too small to be stored in the number_integer_t or
number_unsigned_t types will cause a range error (errno set to ERANGE).
In both cases (more/less characters read, or a range error) the pointer
is passed to std:strtod, which also sets endptr to the first character
past the converted number.
@throw std::range_error if passed value is out of range The resulting number_float_t is then cast to a number_integer_t or,
if positive, to a number_unsigned_t and compared to the original. If
there is no loss of precision then it is stored as a number_integer_t
or, if positive a number_unsigned_t, otherwise as a number_float_t.
A final comparison is made of endptr and if still not the same as
m_cursor a bad input is assumed and result parameter is set to NAN.
@param[out] result basic_json object to receive the number, or NAN if the
conversion read past the current token. The latter case needs to be
treated by the caller function.
*/ */
void get_number(basic_json& result) const void get_number(basic_json& result) const
{ {
typename string_t::value_type* endptr; typename string_t::value_type* endptr;
assert(m_start != nullptr); assert(m_start != nullptr);
// Parse it as an integer // Attempt to parse it as an integer - first checking for a negative number
if(*reinterpret_cast<typename string_t::const_pointer>(m_start) != '-') { if(*reinterpret_cast<typename string_t::const_pointer>(m_start) != '-')
// Unsigned {
result.m_value.number_unsigned = strtoull(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10); // Positive, parse with strtoull
result.m_value.number_unsigned = std::strtoull(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10);
result.m_type = value_t::number_unsigned; result.m_type = value_t::number_unsigned;
} }
else { else
// Signed {
result.m_value.number_integer = strtoll(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10); // Negative, parse with strtoll
result.m_value.number_integer = std::strtoll(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10);
result.m_type = value_t::number_integer; result.m_type = value_t::number_integer;
} }
// Parse it as a double // Check the end of the number was reached and no range error occurred
const auto float_val = strtold(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr); if(reinterpret_cast<lexer_char_t*>(endptr) != m_cursor || errno == ERANGE)
long double int_part; {
const auto frac_part = std::modf(float_val, &int_part); // Either the number won't fit in an integer (range error) or there was
// something else after the number, which could be an exponent
// Test if the double or integer is a better representation // Parse with strtod
if(!approx(frac_part, static_cast<long double>(0)) || result.m_value.number_float = std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr);
(result.m_type == value_t::number_unsigned && !approx(int_part, static_cast<long double>(result.m_value.number_unsigned))) ||
(result.m_type == value_t::number_integer && !approx(int_part, static_cast<long double>(result.m_value.number_integer)))) {
result.m_value.number_float = float_val;
result.m_type = value_t::number_float;
}
if(reinterpret_cast<lexer_char_t*>(endptr) != m_cursor) { // Check if it can be stored as an integer without loss of precision e.g. 1.2e3 = 1200
result.m_value.number_float = NAN; if (result.m_type == value_t::number_integer)
result.m_type = value_t::number_float; {
auto int_val = static_cast<number_integer_t>(result.m_value.number_float);
if (approx(result.m_value.number_float, static_cast<number_float_t>(int_val)))
{
// we would not lose precision -> return int
result.m_value.number_integer = int_val;
}
else
{
result.m_type = value_t::number_float;
}
}
else
{
auto int_val = static_cast<number_unsigned_t>(result.m_value.number_float);
if (approx(result.m_value.number_float, static_cast<number_float_t>(int_val)))
{
// we would not lose precision -> return int
result.m_value.number_unsigned = int_val;
}
else
{
result.m_type = value_t::number_float;
}
}
// Anything after the number is an error
if(reinterpret_cast<lexer_char_t*>(endptr) != m_cursor)
{
result.m_value.number_float = NAN;
result.m_type = value_t::number_float;
}
} }
} }

135
src/json.hpp.re2c
View File

@ -602,7 +602,7 @@ class basic_json
@sa @ref number_integer_t -- type for number values (integer) @sa @ref number_integer_t -- type for number values (integer)
@since version 1.0.0 @since version 2.0.0
*/ */
using number_unsigned_t = NumberUnsignedType; using number_unsigned_t = NumberUnsignedType;
@ -699,9 +699,9 @@ class basic_json
string, ///< string value string, ///< string value
boolean, ///< boolean value boolean, ///< boolean value
number_integer, ///< number value (integer) number_integer, ///< number value (integer)
number_unsigned,///< number value (unsigned integer)
number_float, ///< number value (floating-point) number_float, ///< number value (floating-point)
discarded, ///< discarded by the the parser callback function discarded ///< discarded by the the parser callback function
number_unsigned ///< number value (unsigned integer)
}; };
@ -1343,7 +1343,7 @@ class basic_json
@sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number
value (unsigned integer) from a compatible number type value (unsigned integer) from a compatible number type
@since version 1.0.0 @since version 2.0.0
*/ */
template<typename T, template<typename T,
typename std::enable_if< typename std::enable_if<
@ -1372,7 +1372,7 @@ class basic_json
@sa @ref basic_json(const number_unsigned_t) -- create a number value @sa @ref basic_json(const number_unsigned_t) -- create a number value
(unsigned) (unsigned)
@since version 1.0.0 @since version 2.0.0
*/ */
template<typename CompatibleNumberUnsignedType, typename template<typename CompatibleNumberUnsignedType, typename
std::enable_if< std::enable_if<
@ -2159,7 +2159,7 @@ class basic_json
/*! /*!
@brief return whether value is a number @brief return whether value is a number
This function returns true if the JSON value is a number. This includes This function returns true iff the JSON value is a number. This includes
both integer and floating-point values. both integer and floating-point values.
@return `true` if type is number (regardless whether integer, unsigned @return `true` if type is number (regardless whether integer, unsigned
@ -2185,7 +2185,7 @@ class basic_json
/*! /*!
@brief return whether value is an integer number @brief return whether value is an integer number
This function returns true if the JSON value is an integer or unsigned This function returns true iff the JSON value is an integer or unsigned
integer number. This excludes floating-point values. integer number. This excludes floating-point values.
@return `true` if type is an integer or unsigned integer number, `false` @return `true` if type is an integer or unsigned integer number, `false`
@ -2210,7 +2210,7 @@ class basic_json
/*! /*!
@brief return whether value is an unsigned integer number @brief return whether value is an unsigned integer number
This function returns true if the JSON value is an unsigned integer number. This function returns true iff the JSON value is an unsigned integer number.
This excludes floating-point and (signed) integer values. This excludes floating-point and (signed) integer values.
@return `true` if type is an unsigned integer number, `false` otherwise. @return `true` if type is an unsigned integer number, `false` otherwise.
@ -2222,7 +2222,7 @@ class basic_json
integer number integer number
@sa @ref is_number_float() -- check if value is a floating-point number @sa @ref is_number_float() -- check if value is a floating-point number
@since version 1.0.0 @since version 2.0.0
*/ */
bool is_number_unsigned() const noexcept bool is_number_unsigned() const noexcept
{ {
@ -2232,7 +2232,7 @@ class basic_json
/*! /*!
@brief return whether value is a floating-point number @brief return whether value is a floating-point number
This function returns true if the JSON value is a floating-point number. This function returns true iff the JSON value is a floating-point number.
This excludes integer and unsigned integer values. This excludes integer and unsigned integer values.
@return `true` if type is a floating-point number, `false` otherwise. @return `true` if type is a floating-point number, `false` otherwise.
@ -4837,16 +4837,15 @@ class basic_json
*/ */
friend bool operator<(const value_t lhs, const value_t rhs) friend bool operator<(const value_t lhs, const value_t rhs)
{ {
static constexpr std::array<uint8_t, 9> order = {{ static constexpr std::array<uint8_t, 8> order = {{
0, // null 0, // null
3, // object 3, // object
4, // array 4, // array
5, // string 5, // string
1, // boolean 1, // boolean
2, // integer 2, // integer
2, // unsigned
2, // float 2, // float
0, // filler for discarded (preserves existing value_t values)
2 // unsigned
} }
}; };
@ -7164,53 +7163,99 @@ class basic_json
/*! /*!
@brief return number value for number tokens @brief return number value for number tokens
This function translates the last token into a floating point number. This function translates the last token into the most appropriate
The pointer m_start points to the beginning of the parsed number. We number type (either integer, unsigned integer or floating point),
pass this pointer to std::strtod which sets endptr to the first which is passed back to the caller via the result parameter. The pointer
character past the converted number. If this pointer is not the same as m_start points to the beginning of the parsed number. We first examine
m_cursor, then either more or less characters have been used during the the first character to determine the sign of the number and then pass
comparison. This can happen for inputs like "01" which will be treated this pointer to either std::strtoull (if positive) or std::strtoll
like number 0 followed by number 1. (if negative), both of which set endptr to the first character past the
converted number. If this pointer is not the same as m_cursor, then
either more or less characters have been used during the comparison.
@return the result of the number conversion or NAN if the conversion This can happen for inputs like "01" which will be treated like number 0
read past the current token. The latter case needs to be treated by the followed by number 1. This will also occur for valid floating point
caller function. inputs like "12e3" will be incorrectly read as 12. Numbers that are too
large or too small to be stored in the number_integer_t or
number_unsigned_t types will cause a range error (errno set to ERANGE).
In both cases (more/less characters read, or a range error) the pointer
is passed to std:strtod, which also sets endptr to the first character
past the converted number.
@throw std::range_error if passed value is out of range The resulting number_float_t is then cast to a number_integer_t or,
if positive, to a number_unsigned_t and compared to the original. If
there is no loss of precision then it is stored as a number_integer_t
or, if positive a number_unsigned_t, otherwise as a number_float_t.
A final comparison is made of endptr and if still not the same as
m_cursor a bad input is assumed and result parameter is set to NAN.
@param[out] result basic_json object to receive the number, or NAN if the
conversion read past the current token. The latter case needs to be
treated by the caller function.
*/ */
void get_number(basic_json& result) const void get_number(basic_json& result) const
{ {
typename string_t::value_type* endptr; typename string_t::value_type* endptr;
assert(m_start != nullptr); assert(m_start != nullptr);
// Parse it as an integer // Attempt to parse it as an integer - first checking for a negative number
if(*reinterpret_cast<typename string_t::const_pointer>(m_start) != '-') { if(*reinterpret_cast<typename string_t::const_pointer>(m_start) != '-')
// Unsigned {
result.m_value.number_unsigned = strtoull(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10); // Positive, parse with strtoull
result.m_value.number_unsigned = std::strtoull(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10);
result.m_type = value_t::number_unsigned; result.m_type = value_t::number_unsigned;
} }
else { else
// Signed {
result.m_value.number_integer = strtoll(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10); // Negative, parse with strtoll
result.m_value.number_integer = std::strtoll(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr,10);
result.m_type = value_t::number_integer; result.m_type = value_t::number_integer;
} }
// Parse it as a double // Check the end of the number was reached and no range error occurred
const auto float_val = strtold(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr); if(reinterpret_cast<lexer_char_t*>(endptr) != m_cursor || errno == ERANGE)
long double int_part; {
const auto frac_part = std::modf(float_val, &int_part); // Either the number won't fit in an integer (range error) or there was
// something else after the number, which could be an exponent
// Test if the double or integer is a better representation // Parse with strtod
if(!approx(frac_part, static_cast<long double>(0)) || result.m_value.number_float = std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start),&endptr);
(result.m_type == value_t::number_unsigned && !approx(int_part, static_cast<long double>(result.m_value.number_unsigned))) ||
(result.m_type == value_t::number_integer && !approx(int_part, static_cast<long double>(result.m_value.number_integer)))) {
result.m_value.number_float = float_val;
result.m_type = value_t::number_float;
}
if(reinterpret_cast<lexer_char_t*>(endptr) != m_cursor) { // Check if it can be stored as an integer without loss of precision e.g. 1.2e3 = 1200
result.m_value.number_float = NAN; if (result.m_type == value_t::number_integer)
result.m_type = value_t::number_float; {
auto int_val = static_cast<number_integer_t>(result.m_value.number_float);
if (approx(result.m_value.number_float, static_cast<number_float_t>(int_val)))
{
// we would not lose precision -> return int
result.m_value.number_integer = int_val;
}
else
{
result.m_type = value_t::number_float;
}
}
else
{
auto int_val = static_cast<number_unsigned_t>(result.m_value.number_float);
if (approx(result.m_value.number_float, static_cast<number_float_t>(int_val)))
{
// we would not lose precision -> return int
result.m_value.number_unsigned = int_val;
}
else
{
result.m_type = value_t::number_float;
}
}
// Anything after the number is an error
if(reinterpret_cast<lexer_char_t*>(endptr) != m_cursor)
{
result.m_value.number_float = NAN;
result.m_type = value_t::number_float;
}
} }
} }

10
test/unit.cpp
View File

@ -9529,12 +9529,12 @@ TEST_CASE("parser class")
CHECK_THROWS_WITH(json::parser("01").parse(), "parse error - 0 is not a number"); CHECK_THROWS_WITH(json::parser("01").parse(), "parse error - 0 is not a number");
CHECK_THROWS_WITH(json::parser("--1").parse(), "parse error - unexpected '-'"); CHECK_THROWS_WITH(json::parser("--1").parse(), "parse error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("1.").parse(), "parse error - 1 is not a number"); CHECK_THROWS_WITH(json::parser("1.").parse(), "parse error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E").parse(), CHECK_THROWS_WITH(json::parser("1E").parse(),
"parse error - unexpected 'E'; expected end of input"); "parse error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E-").parse(), CHECK_THROWS_WITH(json::parser("1E-").parse(),
"parse error - unexpected 'E'; expected end of input"); "parse error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("1.E1").parse(), "parse error - 1 is not a number"); CHECK_THROWS_WITH(json::parser("1.E1").parse(), "parse error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("-1E").parse(), CHECK_THROWS_WITH(json::parser("-1E").parse(),
"parse error - unexpected 'E'; expected end of input"); "parse error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0E#").parse(), CHECK_THROWS_WITH(json::parser("-0E#").parse(),
@ -9576,18 +9576,18 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser("1E.").parse(), std::invalid_argument); CHECK_THROWS_AS(json::parser("1E.").parse(), std::invalid_argument);
CHECK_THROWS_AS(json::parser("1E/").parse(), std::invalid_argument); CHECK_THROWS_AS(json::parser("1E/").parse(), std::invalid_argument);
CHECK_THROWS_AS(json::parser("1E:").parse(), std::invalid_argument); CHECK_THROWS_AS(json::parser("1E:").parse(), std::invalid_argument);
CHECK_THROWS_WITH(json::parser("0.").parse(), "parse error - 0 is not a number"); CHECK_THROWS_WITH(json::parser("0.").parse(), "parse error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("-").parse(), "parse error - unexpected '-'"); CHECK_THROWS_WITH(json::parser("-").parse(), "parse error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("--").parse(), CHECK_THROWS_WITH(json::parser("--").parse(),
"parse error - unexpected '-'"); "parse error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("-0.").parse(), CHECK_THROWS_WITH(json::parser("-0.").parse(),
"parse error - -0 is not a number"); "parse error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("-.").parse(), CHECK_THROWS_WITH(json::parser("-.").parse(),
"parse error - unexpected '-'"); "parse error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("-:").parse(), CHECK_THROWS_WITH(json::parser("-:").parse(),
"parse error - unexpected '-'"); "parse error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("0.:").parse(), CHECK_THROWS_WITH(json::parser("0.:").parse(),
"parse error - 0 is not a number"); "parse error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("e.").parse(), CHECK_THROWS_WITH(json::parser("e.").parse(),
"parse error - unexpected 'e'"); "parse error - unexpected 'e'");
CHECK_THROWS_WITH(json::parser("1e.").parse(), CHECK_THROWS_WITH(json::parser("1e.").parse(),