diff --git a/test/test_runner.c b/test/test_runner.c index 5ca397cd6c..901c6c86c4 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -30,7 +30,6 @@ void TestRunner_Battle(const struct Test *); static bool32 MgbaOpen_(void); static void MgbaExit_(u8 exitCode); -static s32 MgbaPuts_(const char *s); static s32 MgbaVPrintf_(const char *fmt, va_list va); static void Intr_Timer2(void); @@ -293,12 +292,6 @@ top: color = ""; } - if (gTestRunnerState.result == TEST_RESULT_PASS - && gTestRunnerState.result != gTestRunnerState.expectedResult) - { - MgbaPuts_("\e[31mPlease remove KNOWN_FAILING if this test intentionally PASSes\e[0m"); - } - switch (gTestRunnerState.result) { case TEST_RESULT_FAIL: @@ -313,7 +306,10 @@ top: } break; case TEST_RESULT_PASS: - result = "PASS"; + if (gTestRunnerState.result != gTestRunnerState.expectedResult) + result = "KNOWN_FAILING_PASS"; + else + result = "PASS"; break; case TEST_RESULT_ASSUMPTION_FAIL: result = "ASSUMPTION_FAIL"; @@ -341,7 +337,12 @@ top: } if (gTestRunnerState.result == TEST_RESULT_PASS) - MgbaPrintf_(":P%s%s\e[0m", color, result); + { + if (gTestRunnerState.result != gTestRunnerState.expectedResult) + MgbaPrintf_(":U%s%s\e[0m", color, result); + else + MgbaPrintf_(":P%s%s\e[0m", color, result); + } else if (gTestRunnerState.result == TEST_RESULT_ASSUMPTION_FAIL) MgbaPrintf_(":A%s%s\e[0m", color, result); else if (gTestRunnerState.result == TEST_RESULT_TODO) @@ -503,11 +504,6 @@ static void MgbaExit_(u8 exitCode) asm("swi 0x3" :: "r" (_exitCode)); } -static s32 MgbaPuts_(const char *s) -{ - return MgbaPrintf_("%s", s); -} - s32 MgbaPrintf_(const char *fmt, ...) { va_list va; diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index c2189ac0df..d4f39feb73 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -35,7 +35,7 @@ #define min(a, b) ((a) < (b) ? (a) : (b)) #define MAX_PROCESSES 32 // See also test/test.h -#define MAX_FAILED_TESTS_TO_LIST 100 +#define MAX_SUMMARY_TESTS_TO_LIST 50 #define MAX_TEST_LIST_BUFFER_LENGTH 256 #define ARRAY_COUNT(arr) (sizeof((arr)) / sizeof((arr)[0])) @@ -54,11 +54,13 @@ struct Runner char *output_buffer; int passes; int knownFails; + int knownFailsPassing; int todos; int assumptionFails; int fails; int results; - char failedTestNames[MAX_FAILED_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; + char failedTestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; + char knownFailingPassedTestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; }; static unsigned nrunners = 0; @@ -107,6 +109,11 @@ static void handle_read(int i, struct Runner *runner) case 'K': runner->knownFails++; goto add_to_results; + case 'U': + if (runner->knownFailsPassing < MAX_SUMMARY_TESTS_TO_LIST) + strcpy(runner->knownFailingPassedTestNames[runner->knownFailsPassing], runner->test_name); + runner->knownFailsPassing++; + goto add_to_results; case 'T': runner->todos++; goto add_to_results; @@ -114,7 +121,7 @@ static void handle_read(int i, struct Runner *runner) runner->assumptionFails++; goto add_to_results; case 'F': - if (runner->fails < MAX_FAILED_TESTS_TO_LIST) + if (runner->fails < MAX_SUMMARY_TESTS_TO_LIST) strcpy(runner->failedTestNames[runner->fails], runner->test_name); runner->fails++; add_to_results: @@ -519,12 +526,14 @@ int main(int argc, char *argv[]) int exit_code = 0; int passes = 0; int knownFails = 0; + int knownFailsPassing = 0; int todos = 0; int assumptionFails = 0; int fails = 0; int results = 0; - char failedTestNames[MAX_FAILED_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; + char failedTestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; + char knownFailingPassedTestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; for (int i = 0; i < nrunners; i++) { @@ -540,18 +549,25 @@ int main(int argc, char *argv[]) exit_code = WEXITSTATUS(wstatus); passes += runners[i].passes; knownFails += runners[i].knownFails; + for (int j = 0; j < runners[i].knownFailsPassing; j++) + { + if (j < MAX_SUMMARY_TESTS_TO_LIST) + strcpy(knownFailingPassedTestNames[fails], runners[i].knownFailingPassedTestNames[j]); + knownFailsPassing++; + } todos += runners[i].todos; assumptionFails += runners[i].assumptionFails; for (int j = 0; j < runners[i].fails; j++) { - if (j < MAX_FAILED_TESTS_TO_LIST) + if (j < MAX_SUMMARY_TESTS_TO_LIST) strcpy(failedTestNames[fails], runners[i].failedTestNames[j]); fails++; } results += runners[i].results; } - qsort(failedTestNames, min(fails, MAX_FAILED_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings); + qsort(failedTestNames, min(fails, MAX_SUMMARY_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings); + qsort(knownFailingPassedTestNames, min(fails, MAX_SUMMARY_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings); if (results == 0) { @@ -559,28 +575,42 @@ int main(int argc, char *argv[]) } else { + fprintf(stdout, "\n"); if (fails > 0) { - fprintf(stdout, "\n- Tests \e[31mFAILED\e[0m : %d Add TESTS='X' to run tests with the defined prefix.\n", fails); + fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d Add TESTS='X' to run tests with the defined prefix.\n", fails); for (int i = 0; i < fails; i++) { - if (i >= MAX_FAILED_TESTS_TO_LIST) + if (i >= MAX_SUMMARY_TESTS_TO_LIST) { - fprintf(stdout, " - \e[31mand %d more...\e[0m\n", fails - MAX_FAILED_TESTS_TO_LIST); + fprintf(stdout, " - \e[31mand %d more...\e[0m\n", fails - MAX_SUMMARY_TESTS_TO_LIST); break; } fprintf(stdout, " - \e[31m%s\e[0m.\n", failedTestNames[i]); } } - fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes); + if (knownFailsPassing > 0) + { + fprintf(stdout, "- \e[31mKNOWN_FAILING_PASSED\e[0m: %d \e[31mPlease remove KNOWN_FAILING if these tests intentionally PASS\e[0m\n", knownFailsPassing); + for (int i = 0; i < knownFailsPassing; i++) + { + if (i >= MAX_SUMMARY_TESTS_TO_LIST) + { + fprintf(stdout, " - \e[31mand %d more...\e[0m\n", knownFailsPassing - MAX_SUMMARY_TESTS_TO_LIST); + break; + } + fprintf(stdout, " - \e[31m%s\e[0m.\n", knownFailingPassedTestNames[i]); + } + } + fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes); if (knownFails > 0) - fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails); + fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails); if (todos > 0) - fprintf(stdout, "- Tests \e[33mTO_DO\e[0m: %d\n", todos); + fprintf(stdout, "- Tests \e[33mTO_DO\e[0m: %d\n", todos); if (assumptionFails > 0) - fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails); + fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails); - fprintf(stdout, "- Tests \e[34mTOTAL\e[0m: %d\n", results); + fprintf(stdout, "- Tests \e[34mTOTAL\e[0m: %d\n", results); } fprintf(stdout, "\n");