diff --git a/doc/develop/test/ztest.rst b/doc/develop/test/ztest.rst index 48da60c12f..c96265910b 100644 --- a/doc/develop/test/ztest.rst +++ b/doc/develop/test/ztest.rst @@ -458,6 +458,30 @@ Example output for a failed macro from .. doxygengroup:: ztest_assert + +Expectations +============ + +These macros will continue test execution if the related expectation fails and subsequently fail the +test at the end of its execution. When an expectation fails, it will print the current file, line, +and function, alongside a reason for the failure and an optional message but continue executing the +test. If the config option:`CONFIG_ZTEST_ASSERT_VERBOSE` is 0, the expectations will only print the +file and line numbers, reducing the binary size of the test. + +Example output for a failed macro from:: + + zexpect_equal(buf->ref, 2, "Invalid refcount"); + zexpect_equal(buf->ref, 1337, "Invalid refcount"); + +.. code-block::none + + START - test_get_single_buffer + Expectation failed at main.c:62: test_get_single_buffer: Invalid refcount (buf->ref not equal to 2) + Expectation failed at main.c:63: test_get_single_buffer: Invalid refcount (buf->ref not equal to 1337) + FAIL - test_get_single_buffer in 0.0 seconds + +.. doxygengroup:: ztest_expect + Assumptions =========== diff --git a/subsys/testsuite/ztest/include/zephyr/ztest_assert.h b/subsys/testsuite/ztest/include/zephyr/ztest_assert.h index 4f7c4ecd4e..5b88357bd4 100644 --- a/subsys/testsuite/ztest/include/zephyr/ztest_assert.h +++ b/subsys/testsuite/ztest/include/zephyr/ztest_assert.h @@ -27,6 +27,7 @@ extern "C" { const char *ztest_relative_filename(const char *file); void ztest_test_fail(void); void ztest_test_skip(void); +void ztest_test_expect_fail(void); void ztest_skip_failed_assumption(void); #if CONFIG_ZTEST_ASSERT_VERBOSE == 0 @@ -56,6 +57,19 @@ static inline bool z_zassume_(bool cond, const char *file, int line) #define z_zassume(cond, default_msg, file, line, func, msg, ...) z_zassume_(cond, file, line) +static inline bool z_zexpect_(bool cond, const char *file, int line) +{ + if (cond == false) { + PRINT("\n Expectation failed at %s:%d\n", ztest_relative_filename(file), line); + ztest_test_expect_fail(); + return false; + } + + return true; +} + +#define z_zexpect(cond, default_msg, file, line, func, msg, ...) z_zexpect_(cond, file, line) + #else /* CONFIG_ZTEST_ASSERT_VERBOSE != 0 */ static inline bool z_zassert(bool cond, const char *default_msg, const char *file, int line, @@ -106,6 +120,30 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil return true; } +static inline bool z_zexpect(bool cond, const char *default_msg, const char *file, int line, + const char *func, const char *msg, ...) +{ + if (cond == false) { + va_list vargs; + + va_start(vargs, msg); + PRINT("\n Expectation failed at %s:%d: %s: %s\n", ztest_relative_filename(file), + line, func, default_msg); + vprintk(msg, vargs); + printk("\n"); + va_end(vargs); + ztest_test_expect_fail(); + return false; + } +#if CONFIG_ZTEST_ASSERT_VERBOSE == 2 + else { + PRINT("\n Expectation succeeded at %s:%d (%s)\n", ztest_relative_filename(file), + line, func); + } +#endif + return true; +} + #endif /* CONFIG_ZTEST_ASSERT_VERBOSE */ /** @@ -187,6 +225,36 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil #define zassume(cond, default_msg, ...) \ _zassume_va(cond, default_msg, COND_CODE_1(__VA_OPT__(1), (__VA_ARGS__), (NULL))) +/** + * @brief If @a cond is false, fail the test but continue its execution. + * + * You probably don't need to call this macro directly. You should + * instead use zexpect_{condition} macros below. + * + * @param cond Condition to check + * @param default_msg Message to print if @a cond is false + * @param msg Optional, can be NULL. Message to print if @a cond is false. + */ +#define _zexpect_base(cond, default_msg, msg, ...) \ + do { \ + bool _msg = (msg != NULL); \ + bool _ret = \ + z_zexpect(cond, _msg ? ("(" default_msg ")") : (default_msg), __FILE__, \ + __LINE__, __func__, _msg ? msg : "", ##__VA_ARGS__); \ + (void)_msg; \ + if (!_ret) { \ + /* If kernel but without multithreading return. */ \ + COND_CODE_1(KERNEL, (COND_CODE_1(CONFIG_MULTITHREADING, (), (return;))), \ + ()) \ + } \ + } while (0) + +#define _zexpect_va(cond, default_msg, msg, ...) \ + _zexpect_base(cond, default_msg, msg, ##__VA_ARGS__) + +#define zexpect(cond, default_msg, ...) \ + _zexpect_va(cond, default_msg, COND_CODE_1(__VA_OPT__(1), (__VA_ARGS__), (NULL))) + /** * @brief Assert that this function call won't be reached * @param ... Optional message and variables to print if the assertion fails @@ -476,6 +544,132 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil * @} */ +/** + * @defgroup ztest_expect Ztest expectation macros + * @ingroup ztest + * + * This module provides expectations when using Ztest. + * + * @{ + */ + +/** + * @brief Expect that @a cond is true, otherwise mark test as failed but continue its execution. + * + * @param cond Condition to check + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_true(cond, ...) zexpect(cond, #cond " is false", ##__VA_ARGS__) + +/** + * @brief Expect that @a cond is false, otherwise mark test as failed but continue its execution. + * + * @param cond Condition to check + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_false(cond, ...) zexpect(!(cond), #cond " is true", ##__VA_ARGS__) + +/** + * @brief Expect that @a cond is 0 (success), otherwise mark test as failed but continue its + * execution. + * + * @param cond Condition to check + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_ok(cond, ...) zexpect(!(cond), #cond " is non-zero", ##__VA_ARGS__) + +/** + * @brief Expect that @a ptr is NULL, otherwise mark test as failed but continue its execution. + * + * @param ptr Pointer to compare + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_is_null(ptr, ...) zexpect((ptr) == NULL, #ptr " is not NULL", ##__VA_ARGS__) + +/** + * @brief Expect that @a ptr is not NULL, otherwise mark test as failed but continue its execution. + * + * @param ptr Pointer to compare + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_not_null(ptr, ...) zexpect((ptr) != NULL, #ptr " is NULL", ##__VA_ARGS__) + +/** + * @brief Expect that @a a equals @a b, otherwise mark test as failed but continue its execution. + * expectation fails, the test will be marked as "skipped". + * + * @param a Value to compare + * @param b Value to compare + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_equal(a, b, ...) zexpect((a) == (b), #a " not equal to " #b, ##__VA_ARGS__) + +/** + * @brief Expect that @a a does not equal @a b, otherwise mark test as failed but continue its + * execution. + * + * @a a and @a b won't be converted and will be compared directly. + * + * @param a Value to compare + * @param b Value to compare + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_not_equal(a, b, ...) zexpect((a) != (b), #a " equal to " #b, ##__VA_ARGS__) + +/** + * @brief Expect that @a a equals @a b, otherwise mark test as failed but continue its execution. + * + * @a a and @a b will be converted to `void *` before comparing. + * + * @param a Value to compare + * @param b Value to compare + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_equal_ptr(a, b, ...) \ + zexpect((void *)(a) == (void *)(b), #a " not equal to " #b, ##__VA_ARGS__) + +/** + * @brief Expect that @a a is within @a b with delta @a d, otherwise mark test as failed but + * continue its execution. + * + * @param a Value to compare + * @param b Value to compare + * @param delta Difference between a and b + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_within(a, b, delta, ...) \ + zexpect(((a) >= ((b) - (delta))) && ((a) <= ((b) + (delta))), \ + #a " not within " #b " +/- " #delta, ##__VA_ARGS__) + +/** + * @brief Expect that @a a is greater than or equal to @a l and less + * than or equal to @a u, otherwise mark test as failed but continue its execution. + * + * @param a Value to compare + * @param lower Lower limit + * @param upper Upper limit + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_between_inclusive(a, lower, upper, ...) \ + zexpect(((a) >= (lower)) && ((a) <= (upper)), \ + #a " not between " #lower " and " #upper " inclusive", ##__VA_ARGS__) + +/** + * @brief Expect that 2 memory buffers have the same contents, otherwise mark test as failed but + * continue its execution. + * + * @param buf Buffer to compare + * @param exp Buffer with expected contents + * @param size Size of buffers + * @param ... Optional message and variables to print if the expectation fails + */ +#define zexpect_mem_equal(buf, exp, size, ...) \ + zexpect(memcmp(buf, exp, size) == 0, #buf " not equal to " #exp, ##__VA_ARGS__) + +/** + * @} + */ + #ifdef __cplusplus } #endif diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 1589681d28..cdfc03377e 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -16,6 +16,7 @@ #ifdef KERNEL static struct k_thread ztest_thread; #endif +static bool failed_expectation; #ifdef CONFIG_ZTEST_SHUFFLE #include @@ -366,6 +367,27 @@ void ztest_test_skip(void) } } +void ztest_test_expect_fail(void) +{ + failed_expectation = true; + + switch (phase) { + case TEST_PHASE_SETUP: + PRINT(" at %s function\n", get_friendly_phase_name(phase)); + break; + case TEST_PHASE_BEFORE: + case TEST_PHASE_TEST: + PRINT(" at %s function\n", get_friendly_phase_name(phase)); + break; + case TEST_PHASE_AFTER: + case TEST_PHASE_TEARDOWN: + case TEST_PHASE_FRAMEWORK: + PRINT(" ERROR: cannot fail in test phase '%s()', bailing\n", + get_friendly_phase_name(phase)); + longjmp(stack_fail, 1); + } +} + static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test, void *data) { int ret = TC_PASS; @@ -399,6 +421,11 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test } run_test_functions(suite, test, data); out: + if (failed_expectation) { + failed_expectation = false; + ret = TC_FAIL; + } + phase = TEST_PHASE_AFTER; if (test_result != ZTEST_RESULT_SUITE_FAIL) { if (suite->after != NULL) { @@ -494,6 +521,11 @@ void ztest_test_skip(void) } } +void ztest_test_expect_fail(void) +{ + failed_expectation = true; +} + void ztest_simple_1cpu_before(void *data) { ARG_UNUSED(data); @@ -582,8 +614,10 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test k_msleep(100); } - if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL) { + if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL || + failed_expectation) { ret = TC_FAIL; + failed_expectation = false; } else if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) { ret = TC_SKIP; } diff --git a/tests/ztest/zexpect/CMakeLists.txt b/tests/ztest/zexpect/CMakeLists.txt new file mode 100644 index 0000000000..6167adcab8 --- /dev/null +++ b/tests/ztest/zexpect/CMakeLists.txt @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +if(BOARD STREQUAL unit_testing) + find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(base) + + target_sources(testbinary PRIVATE src/main.c) +else() + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(base) + + if(CONFIG_CPLUSPLUS) + message(STATUS "adding main.cpp") + target_sources(app PRIVATE src/main.cpp) + else() + target_sources(app PRIVATE src/main.c) + + target_sources_ifdef(CONFIG_USERSPACE app PRIVATE src/main_userspace.c) + endif() +endif() diff --git a/tests/ztest/zexpect/prj.conf b/tests/ztest/zexpect/prj.conf new file mode 100644 index 0000000000..9228251051 --- /dev/null +++ b/tests/ztest/zexpect/prj.conf @@ -0,0 +1,2 @@ +CONFIG_ZTEST=y +CONFIG_ZTEST_NEW_API=y diff --git a/tests/ztest/zexpect/src/main.c b/tests/ztest/zexpect/src/main.c new file mode 100644 index 0000000000..492257af74 --- /dev/null +++ b/tests/ztest/zexpect/src/main.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2022 Google Inc + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include + +ZTEST_SUITE(expect, NULL, NULL, NULL, NULL, NULL); + +ZTEST_EXPECT_FAIL(expect, test_fail_later); +ZTEST(expect, test_fail_later) +{ + void *empty_ptr = NULL; + uint32_t val = 5; + + zexpect_equal(val, 2); + zexpect_not_equal(val, 5); + + zexpect_not_null(empty_ptr); + + zassert_true(true); +} + +ZTEST(expect, test_pass_expect_true) +{ + zexpect_true(true); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_true); +ZTEST(expect, test_fail_expect_true) +{ + zexpect_true(false); +} + +ZTEST(expect, test_expect_false) +{ + zexpect_false(false); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_false); +ZTEST(expect, test_fail_expect_false) +{ + zexpect_false(true); +} + +ZTEST(expect, test_expect_ok) +{ + zexpect_ok(0); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_ok); +ZTEST(expect, test_fail_expect_ok) +{ + zexpect_ok(5); +} + +ZTEST(expect, test_expect_is_null) +{ + void *ptr = NULL; + + zexpect_is_null(ptr); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_is_null); +ZTEST(expect, test_fail_expect_is_null) +{ + void *ptr = (void *)0x32137899; + + zexpect_is_null(ptr); +} + +ZTEST(expect, test_expect_not_null) +{ + void *ptr = (void *)0x91517141; + + zexpect_not_null(ptr); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_not_null); +ZTEST(expect, test_fail_expect_not_null) +{ + zexpect_not_null(NULL); +} + +ZTEST(expect, test_expect_equal) +{ + zexpect_equal(5, 5); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_equal); +ZTEST(expect, test_fail_expect_equal) +{ + zexpect_equal(5, 1); +} + +ZTEST(expect, test_expect_not_equal) +{ + zexpect_not_equal(5, 1); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_not_equal); +ZTEST(expect, test_fail_expect_not_equal) +{ + zexpect_not_equal(5, 5); +} + +ZTEST(expect, test_expect_equal_ptr) +{ + int v = 9; + int *a = &v; + int *b = &v; + + zexpect_equal_ptr(a, b); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_equal_ptr); +ZTEST(expect, test_fail_expect_equal_ptr) +{ + int v = 9; + int *a = &v; + int *b = NULL; + + zexpect_equal_ptr(a, b); +} + +ZTEST(expect, test_expect_within) +{ + zexpect_within(7, 5, 2); + zexpect_within(7, 7, 0); + zexpect_within(7, 7, 3); + zexpect_within(7, 7 + 3, 3); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_within); +ZTEST(expect, test_fail_expect_within) +{ + zexpect_within(7, 5, 1); +} + +ZTEST(expect, test_expect_between_inclusive) +{ + zexpect_between_inclusive(-5, -10, 0); + + zexpect_between_inclusive(5, 0, 10); + zexpect_between_inclusive(0, 0, 10); + zexpect_between_inclusive(10, 0, 10); +} + +ZTEST_EXPECT_FAIL(expect, test_fail_expect_between_inclusive); +ZTEST(expect, test_fail_expect_between_inclusive) +{ + zexpect_between_inclusive(-50, -20, 30); + + zexpect_between_inclusive(5, 6, 10); + zexpect_between_inclusive(5, 0, 4); + zexpect_between_inclusive(5, 0, 4); + zexpect_between_inclusive(5, 6, 10); +} diff --git a/tests/ztest/zexpect/src/main.cpp b/tests/ztest/zexpect/src/main.cpp new file mode 120000 index 0000000000..8a03e9439e --- /dev/null +++ b/tests/ztest/zexpect/src/main.cpp @@ -0,0 +1 @@ +main.c \ No newline at end of file diff --git a/tests/ztest/zexpect/testcase.yaml b/tests/ztest/zexpect/testcase.yaml new file mode 100644 index 0000000000..ad837a24f8 --- /dev/null +++ b/tests/ztest/zexpect/testcase.yaml @@ -0,0 +1,19 @@ +# Copyright (c) 2022 Google Inc +# SPDX-License-Identifier: Apache-2.0 + +common: + timeout: 15 + integration_platforms: + - native_posix +tests: + testing.ztest.expect: + integration_platforms: + - native_posix + testing.ztest.expect_cpp: + extra_configs: + - CONFIG_CPLUSPLUS=y + - CONFIG_LIB_CPLUSPLUS=y + integration_platforms: + - native_posix + testing.ztest.expect.unit: + type: unit