Refactor unit test creation

Add functions for creating tests and to supply test- and
standard-specific build settings.

Raises minimum CMake version to 3.13 in test directory.

json_test_add_test_for(
    <file>
    MAIN <main>
    [CXX_STANDARDS <version_number>...] [FORCE])

Given a <file> unit-foo.cpp, produces

    test-foo_cpp<version_number>

if C++ standard <version_number> is supported by the compiler and
thesource file contains JSON_HAS_CPP_<version_number>.  Use FORCE to
create the test regardless of the file containing
JSON_HAS_CPP_<version_number>.  Test targets are linked against <main>.
CXX_STANDARDS defaults to "11".

json_test_set_test_options(
    all|<tests>
    [CXX_STANDARDS all|<args>...]
    [COMPILE_DEFINITIONS <args>...]
    [COMPILE_FEATURES <args>...]
    [COMPILE_OPTIONS <args>...]
    [LINK_LIBRARIES <args>...]
    [LINK_OPTIONS <args>...])

Supply test- and standard-specific build settings.
Specify multiple tests using a list e.g., "test-foo;test-bar".

Must be called BEFORE the test is created.
This commit is contained in:
Florian Albrechtskirchinger 2022-03-05 12:49:33 +01:00
parent 700b95f447
commit 00c4a4d066
3 changed files with 309 additions and 132 deletions

View File

@ -30,7 +30,13 @@ if (POLICY CMP0077)
cmake_policy(SET CMP0077 NEW)
endif ()
option(JSON_BuildTests "Build the unit tests when BUILD_TESTING is enabled." ${MAIN_PROJECT})
# VERSION_GREATER_EQUAL is not available in CMake 3.1
if(${MAIN_PROJECT} AND (${CMAKE_VERSION} VERSION_EQUAL 3.13 OR ${CMAKE_VERSION} VERSION_GREATER 3.13))
set(JSON_BuildTests_INIT ON)
else()
set(JSON_BuildTests_INIT OFF)
endif()
option(JSON_BuildTests "Build the unit tests when BUILD_TESTING is enabled." ${JSON_BuildTests_INIT})
option(JSON_CI "Enable CI build targets." OFF)
option(JSON_Diagnostics "Use extended diagnostic messages." OFF)
option(JSON_ImplicitConversions "Enable implicit conversions." ON)

204
cmake/test.cmake Normal file
View File

@ -0,0 +1,204 @@
set(_json_test_cmake_list_file ${CMAKE_CURRENT_LIST_FILE})
#############################################################################
# download test data
#############################################################################
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/download_test_data.cmake)
# test fixture to download test data
add_test(NAME "download_test_data" COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR}
--target download_test_data
)
set_tests_properties(download_test_data PROPERTIES FIXTURES_SETUP TEST_DATA)
if(JSON_Valgrind)
find_program(CMAKE_MEMORYCHECK_COMMAND valgrind)
message(STATUS "Executing test suite with Valgrind (${CMAKE_MEMORYCHECK_COMMAND})")
set(memcheck_command "${CMAKE_MEMORYCHECK_COMMAND} ${CMAKE_MEMORYCHECK_COMMAND_OPTIONS} --error-exitcode=1 --leak-check=full")
separate_arguments(memcheck_command)
endif()
#############################################################################
# detect standard support
#############################################################################
# C++11 is the minimum required
set(compiler_supports_cpp_11 TRUE)
foreach(feature ${CMAKE_CXX_COMPILE_FEATURES})
if (${feature} STREQUAL cxx_std_14)
set(compiler_supports_cpp_14 TRUE)
elseif (${feature} STREQUAL cxx_std_17)
set(compiler_supports_cpp_17 TRUE)
elseif (${feature} STREQUAL cxx_std_20)
set(compiler_supports_cpp_20 TRUE)
elseif (${feature} STREQUAL cxx_std_23)
set(compiler_supports_cpp_23 TRUE)
endif()
endforeach()
#############################################################################
# test functions
#############################################################################
#############################################################################
# json_test_set_test_options(
# all|<tests>
# [CXX_STANDARDS all|<args>...]
# [COMPILE_DEFINITIONS <args>...]
# [COMPILE_FEATURES <args>...]
# [COMPILE_OPTIONS <args>...]
# [LINK_LIBRARIES <args>...]
# [LINK_OPTIONS <args>...])
#
# Supply test- and standard-specific build settings.
# Specify multiple tests using a list e.g., "test-foo;test-bar".
#
# Must be called BEFORE the test is created.
#############################################################################
function(json_test_set_test_options tests)
cmake_parse_arguments(args "" ""
"CXX_STANDARDS;COMPILE_DEFINITIONS;COMPILE_FEATURES;COMPILE_OPTIONS;LINK_LIBRARIES;LINK_OPTIONS"
${ARGN})
if(NOT args_CXX_STANDARDS)
set(args_CXX_STANDARDS "all")
endif()
foreach(test ${tests})
if("${test}" STREQUAL "all")
set(test "")
endif()
foreach(cxx_standard ${args_CXX_STANDARDS})
if("${cxx_standard}" STREQUAL "all")
if("${test}" STREQUAL "")
message(FATAL_ERROR "Not supported. Change defaults in: ${_json_test_cmake_list_file}")
endif()
set(test_interface _json_test_interface_${test})
else()
set(test_interface _json_test_interface_${test}_cpp_${cxx_standard})
endif()
if(NOT TARGET ${test_interface})
add_library(${test_interface} INTERFACE)
endif()
target_compile_definitions(${test_interface} INTERFACE ${args_COMPILE_DEFINITIONS})
target_compile_features(${test_interface} INTERFACE ${args_COMPILE_FEATURES})
target_compile_options(${test_interface} INTERFACE ${args_COMPILE_OPTIONS})
target_link_libraries (${test_interface} INTERFACE ${args_LINK_LIBRARIES})
target_link_options(${test_interface} INTERFACE ${args_LINK_OPTIONS})
endforeach()
endforeach()
endfunction()
# for internal use by json_test_add_test_for()
function(_json_test_add_test test_name file main cxx_standard)
set(test_target ${test_name}_cpp${cxx_standard})
if(TARGET ${test_target})
message(FATAL_ERROR "Target ${test_target} has already been added.")
endif()
add_executable(${test_target} ${file})
target_link_libraries(${test_target} PRIVATE ${main})
# set and require C++ standard
set_target_properties(${test_target} PROPERTIES
CXX_STANDARD ${cxx_standard}
CXX_STANDARD_REQUIRED ON
)
# apply standard-specific build settings
if(TARGET _json_test_interface__cpp_${cxx_standard})
target_link_libraries(${test_target} PRIVATE _json_test_interface__cpp_${cxx_standard})
endif()
# apply test-specific build settings
if(TARGET _json_test_interface_${test_name})
target_link_libraries(${test_target} PRIVATE _json_test_interface_${test_name})
endif()
# apply test- and standard-specific build settings
if(TARGET _json_test_interface_${test_name}_cpp_${cxx_standard})
target_link_libraries(${test_target} PRIVATE
_json_test_interface_${test_name}_cpp_${cxx_standard}
)
endif()
if (JSON_FastTests)
add_test(NAME ${test_target}
COMMAND ${test_target} ${DOCTEST_TEST_FILTER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
else()
add_test(NAME ${test_target}
COMMAND ${test_target} ${DOCTEST_TEST_FILTER} --no-skip
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
endif()
set_tests_properties(${test_target} PROPERTIES LABELS "all" FIXTURES_REQUIRED TEST_DATA)
if(JSON_Valgrind)
add_test(NAME ${test_target}_valgrind
COMMAND ${memcheck_command} $<TARGET_FILE:${test_target}> ${DOCTEST_TEST_FILTER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
set_tests_properties(${test_target}_valgrind PROPERTIES
LABELS "valgrind" FIXTURES_REQUIRED TEST_DATA
)
endif()
endfunction()
#############################################################################
# json_test_add_test_for(
# <file>
# MAIN <main>
# [CXX_STANDARDS <version_number>...] [FORCE])
#
# Given a <file> unit-foo.cpp, produces
#
# test-foo_cpp<version_number>
#
# if C++ standard <version_number> is supported by the compiler and the
# source file contains JSON_HAS_CPP_<version_number>.
# Use FORCE to create the test regardless of the file containing
# JSON_HAS_CPP_<version_number>.
# Test targets are linked against <main>.
# CXX_STANDARDS defaults to "11".
#############################################################################
function(json_test_add_test_for file)
cmake_parse_arguments(args "FORCE" "MAIN" "CXX_STANDARDS" ${ARGN})
get_filename_component(file_basename ${file} NAME_WE)
string(REGEX REPLACE "unit-([^$]+)" "test-\\1" test_name ${file_basename})
if("${args_MAIN}" STREQUAL "")
message(FATAL_ERROR "Required argument MAIN <main> missing.")
endif()
if("${args_CXX_STANDARDS}" STREQUAL "")
set(args_CXX_STANDARDS 11)
endif()
file(READ ${file} file_content)
foreach(cxx_standard ${args_CXX_STANDARDS})
if(NOT compiler_supports_cpp_${cxx_standard})
continue()
endif()
# add unconditionally if C++11 (default) or forced
if(NOT ("${cxx_standard}" STREQUAL 11 OR args_FORCE))
string(FIND "${file_content}" JSON_HAS_CPP_${cxx_standard} has_cpp_found)
if(${has_cpp_found} EQUAL -1)
continue()
endif()
endif()
_json_test_add_test(${test_name} ${file} ${args_MAIN} ${cxx_standard})
endforeach()
endfunction()

View File

@ -1,163 +1,130 @@
cmake_minimum_required(VERSION 3.13)
option(JSON_Valgrind "Execute test suite with Valgrind." OFF)
option(JSON_FastTests "Skip expensive/slow tests." OFF)
# download test data
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/download_test_data.cmake)
set(JSON_TestStandards "" CACHE STRING "The list of standards to test explicitly.")
# test fixture to download test data
add_test(NAME "download_test_data" COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target download_test_data)
set_tests_properties(download_test_data PROPERTIES FIXTURES_SETUP TEST_DATA)
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/test.cmake)
if(JSON_Valgrind)
find_program(CMAKE_MEMORYCHECK_COMMAND valgrind)
message(STATUS "Executing test suite with Valgrind (${CMAKE_MEMORYCHECK_COMMAND})")
set(memcheck_command "${CMAKE_MEMORYCHECK_COMMAND} ${CMAKE_MEMORYCHECK_COMMAND_OPTIONS} --error-exitcode=1 --leak-check=full")
separate_arguments(memcheck_command)
#############################################################################
# override standard support
#############################################################################
# compiling json.hpp in C++14 mode fails with Clang <4.0
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.0)
unset(compiler_supports_cpp_14)
endif()
#############################################################################
# doctest library with the main function to speed up build
#############################################################################
add_library(doctest_main OBJECT src/unit.cpp)
set_target_properties(doctest_main PROPERTIES
COMPILE_DEFINITIONS "$<$<CXX_COMPILER_ID:MSVC>:_SCL_SECURE_NO_WARNINGS>"
COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:MSVC>:/EHsc;$<$<CONFIG:Release>:/Od>>"
)
if (${CMAKE_VERSION} VERSION_LESS "3.8.0")
target_compile_features(doctest_main PUBLIC cxx_range_for)
else()
target_compile_features(doctest_main PUBLIC cxx_std_11)
endif()
target_include_directories(doctest_main PRIVATE "thirdparty/doctest")
# https://stackoverflow.com/questions/2368811/how-to-set-warning-level-in-cmake
if(MSVC)
# Force to always compile with W4
if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]")
string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4")
endif()
# Disable warning C4566: character represented by universal-character-name '\uFF01' cannot be represented in the current code page (1252)
# Disable warning C4996: 'nlohmann::basic_json<std::map,std::vector,std::string,bool,int64_t,uint64_t,double,std::allocator,nlohmann::adl_serializer>::operator <<': was declared deprecated
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4566 /wd4996")
# https://github.com/nlohmann/json/issues/1114
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /bigobj")
endif()
#############################################################################
# one executable for each unit test file
#############################################################################
# check if compiler supports C++17
foreach(feature ${CMAKE_CXX_COMPILE_FEATURES})
if (${feature} STREQUAL cxx_std_17)
set(compiler_supports_cpp_17 TRUE)
endif()
endforeach()
# Clang only supports C++17 starting from Clang 5.0
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
unset(compiler_supports_cpp_17)
endif()
# MSVC 2015 (14.0) does not support C++17
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.1)
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.1)
unset(compiler_supports_cpp_17)
endif()
file(GLOB files src/unit-*.cpp)
# Clang C++20 support appears insufficient prior to Clang 9.0 (based on CI build failure)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
unset(compiler_supports_cpp_20)
endif()
# GCC started supporting C++20 features in 8.0 but a test for #3070 segfaults prior to 9.0
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
unset(compiler_supports_cpp_20)
endif()
foreach(file ${files})
get_filename_component(file_basename ${file} NAME_WE)
string(REGEX REPLACE "unit-([^$]+)" "test-\\1" testcase ${file_basename})
#############################################################################
# test_main library with shared code to speed up build and common settings
#############################################################################
add_executable(${testcase} $<TARGET_OBJECTS:doctest_main> ${file})
target_compile_definitions(${testcase} PRIVATE DOCTEST_CONFIG_SUPER_FAST_ASSERTS)
target_compile_options(${testcase} PRIVATE
$<$<CXX_COMPILER_ID:MSVC>:/EHsc;$<$<CONFIG:Release>:/Od>>
$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-Wno-deprecated;-Wno-float-equal>
$<$<CXX_COMPILER_ID:GNU>:-Wno-deprecated-declarations>
)
target_include_directories(${testcase} PRIVATE ${CMAKE_BINARY_DIR}/include thirdparty/doctest thirdparty/fifo_map)
target_link_libraries(${testcase} PRIVATE ${NLOHMANN_JSON_TARGET_NAME})
add_library(test_main OBJECT src/unit.cpp)
target_compile_definitions(test_main PUBLIC
DOCTEST_CONFIG_SUPER_FAST_ASSERTS
)
target_compile_features(test_main PRIVATE cxx_std_11)
target_compile_options(test_main PUBLIC
$<$<CXX_COMPILER_ID:MSVC>:/EHsc;$<$<CONFIG:Release>:/Od>>
# MSVC: Force to always compile with W4
# Disable warning C4566: character represented by universal-character-name '\uFF01'
# cannot be represented in the current code page (1252)
# Disable warning C4996: 'nlohmann::basic_json<...>::operator <<': was declared deprecated
$<$<CXX_COMPILER_ID:MSVC>:/W4 /wd4566 /wd4996>
# https://github.com/nlohmann/json/issues/1114
$<$<CXX_COMPILER_ID:MSVC>:/bigobj> $<$<BOOL:${MINGW}>:-Wa,-mbig-obj>
# add a copy with C++17 compilation
if (compiler_supports_cpp_17)
file(READ ${file} FILE_CONTENT)
string(FIND "${FILE_CONTENT}" "JSON_HAS_CPP_17" CPP_17_FOUND)
if(NOT ${CPP_17_FOUND} EQUAL -1)
add_executable(${testcase}_cpp17 $<TARGET_OBJECTS:doctest_main> ${file})
target_compile_definitions(${testcase}_cpp17 PRIVATE DOCTEST_CONFIG_SUPER_FAST_ASSERTS)
target_compile_options(${testcase}_cpp17 PRIVATE
$<$<CXX_COMPILER_ID:MSVC>:/EHsc;$<$<CONFIG:Release>:/Od>>
$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-Wno-deprecated;-Wno-float-equal>
$<$<CXX_COMPILER_ID:GNU>:-Wno-deprecated-declarations>
)
target_include_directories(${testcase}_cpp17 PRIVATE ${CMAKE_BINARY_DIR}/include thirdparty/doctest thirdparty/fifo_map)
target_link_libraries(${testcase}_cpp17 PRIVATE ${NLOHMANN_JSON_TARGET_NAME})
target_compile_features(${testcase}_cpp17 PRIVATE cxx_std_17)
$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-Wno-deprecated;-Wno-float-equal>
$<$<CXX_COMPILER_ID:GNU>:-Wno-deprecated-declarations>
)
target_include_directories(test_main PUBLIC
thirdparty/doctest
thirdparty/fifo_map
${PROJECT_BINARY_DIR}/include
)
target_link_libraries(test_main PUBLIC ${NLOHMANN_JSON_TARGET_NAME})
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0 AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0 AND NOT MINGW)
# fix for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90050
target_link_libraries(${testcase}_cpp17 PRIVATE stdc++fs)
endif()
#############################################################################
# define test- and standard-specific build settings
#############################################################################
if (JSON_FastTests)
add_test(NAME "${testcase}_cpp17"
COMMAND ${testcase}_cpp17 ${DOCTEST_TEST_FILTER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
else()
add_test(NAME "${testcase}_cpp17"
COMMAND ${testcase}_cpp17 ${DOCTEST_TEST_FILTER} --no-skip
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
endif()
set_tests_properties("${testcase}_cpp17" PROPERTIES LABELS "all" FIXTURES_REQUIRED TEST_DATA)
endif()
endif()
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0
AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0 AND NOT MINGW)
# fix for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90050
json_test_set_test_options(all CXX_STANDARDS 17 LINK_LIBRARIES stdc++fs)
endif()
if (JSON_FastTests)
add_test(NAME "${testcase}"
COMMAND ${testcase} ${DOCTEST_TEST_FILTER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
else()
add_test(NAME "${testcase}"
COMMAND ${testcase} ${DOCTEST_TEST_FILTER} --no-skip
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
endif()
set_tests_properties("${testcase}" PROPERTIES LABELS "all" FIXTURES_REQUIRED TEST_DATA)
if(JSON_Valgrind)
add_test(NAME "${testcase}_valgrind"
COMMAND ${memcheck_command} ${CMAKE_CURRENT_BINARY_DIR}/${testcase} ${DOCTEST_TEST_FILTER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
)
set_tests_properties("${testcase}_valgrind" PROPERTIES LABELS "valgrind" FIXTURES_REQUIRED TEST_DATA)
endif()
endforeach()
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# avoid stack overflow, see https://github.com/nlohmann/json/issues/2955
json_test_set_test_options("test-cbor;test-msgpack;test-ubjson" LINK_OPTIONS /STACK:4000000)
endif()
# disable exceptions for test-disabled_exceptions
target_compile_definitions(test-disabled_exceptions PUBLIC JSON_NOEXCEPTION)
json_test_set_test_options(test-disabled_exceptions COMPILE_DEFINITIONS JSON_NOEXCEPTION)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
target_compile_options(test-disabled_exceptions PUBLIC -fno-exceptions)
json_test_set_test_options(test-disabled_exceptions COMPILE_OPTIONS -fno-exceptions)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# disabled due to https://github.com/nlohmann/json/discussions/2824
#target_compile_options(test-disabled_exceptions PUBLIC /EH)
#target_compile_definitions(test-disabled_exceptions PUBLIC _HAS_EXCEPTIONS=0)
#json_test_set_test_options(test-disabled_exceptions COMPILE_DEFINITIONS _HAS_EXCEPTIONS=0 COMPILE_OPTIONS /EH)
endif()
# avoid stack overflow, see https://github.com/nlohmann/json/issues/2955
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set_property(TARGET test-cbor APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000")
set_property(TARGET test-msgpack APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000")
set_property(TARGET test-ubjson APPEND_STRING PROPERTY LINK_FLAGS " /STACK:4000000")
#############################################################################
# add unit tests
#############################################################################
if("${JSON_TestStandards}" STREQUAL "")
set(test_cxx_standards 11 14 17 20 23)
unset(test_force)
else()
set(test_cxx_standards ${JSON_TestStandards})
set(test_force FORCE)
endif()
# Print selected standards marking unavailable ones with brackets
set(msg_standards "")
foreach(cxx_standard ${test_cxx_standards})
if(compiler_supports_cpp_${cxx_standard})
list(APPEND msg_standards ${cxx_standard})
else()
list(APPEND msg_standards [${cxx_standard}])
endif()
endforeach()
string(JOIN " " msg_standards ${msg_standards})
set(msg "Testing standards: ${msg_standards}")
if(test_force)
string(APPEND msg " (forced)")
endif()
message(STATUS "${msg}")
# *DO* use json_test_set_test_options() above this line
file(GLOB files src/unit-*.cpp)
foreach(file ${files})
json_test_add_test_for(${file} MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force})
endforeach()
# *DO NOT* use json_test_set_test_options() below this line
#############################################################################
# Test the generated build configs
#############################################################################