atiran/sovereignx
3
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added passing "KNOWN_FAILING" to test summary (#4063)

Browse source 
* Added passing "KNOWN_FAILING" to test summary

* Added test names for passing KNOWN_FAILING to summary

* Removed unused function
This commit is contained in:
Eduardo Quezada D'Ottone 2024-01-29 17:01:50 -03:00 committed by GitHub
parent 127a7e9600
commit a2a38a57e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 54 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
test/test_runner.c
View file

@ -30,7 +30,6 @@ void TestRunner_Battle(const struct Test *);
static bool32 MgbaOpen_(void);
static void MgbaExit_(u8 exitCode);
static s32 MgbaPuts_(const char *s);
static s32 MgbaVPrintf_(const char *fmt, va_list va);
static void Intr_Timer2(void);
@ -293,12 +292,6 @@ top:
color = "";
}
if (gTestRunnerState.result == TEST_RESULT_PASS
&& gTestRunnerState.result != gTestRunnerState.expectedResult)
{
MgbaPuts_("\e[31mPlease remove KNOWN_FAILING if this test intentionally PASSes\e[0m");
}
switch (gTestRunnerState.result)
{
case TEST_RESULT_FAIL:
@ -313,7 +306,10 @@ top:
}
break;
case TEST_RESULT_PASS:
result = "PASS";
if (gTestRunnerState.result != gTestRunnerState.expectedResult)
result = "KNOWN_FAILING_PASS";
else
result = "PASS";
break;
case TEST_RESULT_ASSUMPTION_FAIL:
result = "ASSUMPTION_FAIL";
@ -341,7 +337,12 @@ top:
}
if (gTestRunnerState.result == TEST_RESULT_PASS)
MgbaPrintf_(":P%s%s\e[0m", color, result);
{
if (gTestRunnerState.result != gTestRunnerState.expectedResult)
MgbaPrintf_(":U%s%s\e[0m", color, result);
else
MgbaPrintf_(":P%s%s\e[0m", color, result);
}
else if (gTestRunnerState.result == TEST_RESULT_ASSUMPTION_FAIL)
MgbaPrintf_(":A%s%s\e[0m", color, result);
else if (gTestRunnerState.result == TEST_RESULT_TODO)
@ -503,11 +504,6 @@ static void MgbaExit_(u8 exitCode)
asm("swi 0x3" :: "r" (_exitCode));
}
static s32 MgbaPuts_(const char *s)
{
return MgbaPrintf_("%s", s);
}
s32 MgbaPrintf_(const char *fmt, ...)
{
va_list va;

58
tools/mgba-rom-test-hydra/main.c
View file

@ -35,7 +35,7 @@
#define min(a, b) ((a) < (b) ? (a) : (b))
#define MAX_PROCESSES 32 // See also test/test.h
#define MAX_FAILED_TESTS_TO_LIST 100
#define MAX_SUMMARY_TESTS_TO_LIST 50
#define MAX_TEST_LIST_BUFFER_LENGTH 256
#define ARRAY_COUNT(arr) (sizeof((arr)) / sizeof((arr)[0]))
@ -54,11 +54,13 @@ struct Runner
char *output_buffer;
int passes;
int knownFails;
int knownFailsPassing;
int todos;
int assumptionFails;
int fails;
int results;
char failedTestNames[MAX_FAILED_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH];
char failedTestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH];
char knownFailingPassedTestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH];
};
static unsigned nrunners = 0;
@ -107,6 +109,11 @@ static void handle_read(int i, struct Runner *runner)
case 'K':
runner->knownFails++;
goto add_to_results;
case 'U':
if (runner->knownFailsPassing < MAX_SUMMARY_TESTS_TO_LIST)
strcpy(runner->knownFailingPassedTestNames[runner->knownFailsPassing], runner->test_name);
runner->knownFailsPassing++;
goto add_to_results;
case 'T':
runner->todos++;
goto add_to_results;
@ -114,7 +121,7 @@ static void handle_read(int i, struct Runner *runner)
runner->assumptionFails++;
goto add_to_results;
case 'F':
if (runner->fails < MAX_FAILED_TESTS_TO_LIST)
if (runner->fails < MAX_SUMMARY_TESTS_TO_LIST)
strcpy(runner->failedTestNames[runner->fails], runner->test_name);
runner->fails++;
add_to_results:
@ -519,12 +526,14 @@ int main(int argc, char *argv[])
int exit_code = 0;
int passes = 0;
int knownFails = 0;
int knownFailsPassing = 0;
int todos = 0;
int assumptionFails = 0;
int fails = 0;
int results = 0;
char failedTestNames[MAX_FAILED_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH];
char failedTestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH];
char knownFailingPassedTestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH];
for (int i = 0; i < nrunners; i++)
{
@ -540,18 +549,25 @@ int main(int argc, char *argv[])
exit_code = WEXITSTATUS(wstatus);
passes += runners[i].passes;
knownFails += runners[i].knownFails;
for (int j = 0; j < runners[i].knownFailsPassing; j++)
{
if (j < MAX_SUMMARY_TESTS_TO_LIST)
strcpy(knownFailingPassedTestNames[fails], runners[i].knownFailingPassedTestNames[j]);
knownFailsPassing++;
}
todos += runners[i].todos;
assumptionFails += runners[i].assumptionFails;
for (int j = 0; j < runners[i].fails; j++)
{
if (j < MAX_FAILED_TESTS_TO_LIST)
if (j < MAX_SUMMARY_TESTS_TO_LIST)
strcpy(failedTestNames[fails], runners[i].failedTestNames[j]);
fails++;
}
results += runners[i].results;
}
qsort(failedTestNames, min(fails, MAX_FAILED_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings);
qsort(failedTestNames, min(fails, MAX_SUMMARY_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings);
qsort(knownFailingPassedTestNames, min(fails, MAX_SUMMARY_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings);
if (results == 0)
{
@ -559,28 +575,42 @@ int main(int argc, char *argv[])
}
else
{
fprintf(stdout, "\n");
if (fails > 0)
{
fprintf(stdout, "\n- Tests \e[31mFAILED\e[0m : %d Add TESTS='X' to run tests with the defined prefix.\n", fails);
fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d Add TESTS='X' to run tests with the defined prefix.\n", fails);
for (int i = 0; i < fails; i++)
{
if (i >= MAX_FAILED_TESTS_TO_LIST)
if (i >= MAX_SUMMARY_TESTS_TO_LIST)
{
fprintf(stdout, " - \e[31mand %d more...\e[0m\n", fails - MAX_FAILED_TESTS_TO_LIST);
fprintf(stdout, " - \e[31mand %d more...\e[0m\n", fails - MAX_SUMMARY_TESTS_TO_LIST);
break;
}
fprintf(stdout, " - \e[31m%s\e[0m.\n", failedTestNames[i]);
}
}
fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes);
if (knownFailsPassing > 0)
{
fprintf(stdout, "- \e[31mKNOWN_FAILING_PASSED\e[0m: %d \e[31mPlease remove KNOWN_FAILING if these tests intentionally PASS\e[0m\n", knownFailsPassing);
for (int i = 0; i < knownFailsPassing; i++)
{
if (i >= MAX_SUMMARY_TESTS_TO_LIST)
{
fprintf(stdout, " - \e[31mand %d more...\e[0m\n", knownFailsPassing - MAX_SUMMARY_TESTS_TO_LIST);
break;
}
fprintf(stdout, " - \e[31m%s\e[0m.\n", knownFailingPassedTestNames[i]);
}
}
fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes);
if (knownFails > 0)
fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails);
fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails);
if (todos > 0)
fprintf(stdout, "- Tests \e[33mTO_DO\e[0m: %d\n", todos);
fprintf(stdout, "- Tests \e[33mTO_DO\e[0m: %d\n", todos);
if (assumptionFails > 0)
fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails);
fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails);
fprintf(stdout, "- Tests \e[34mTOTAL\e[0m: %d\n", results);
fprintf(stdout, "- Tests \e[34mTOTAL\e[0m: %d\n", results);
}
fprintf(stdout, "\n");