mirror of
https://github.com/linux-apfs/apfstests.git
synced 2026-05-01 15:01:44 -07:00
06dc7f12ad
Currently a test appears to pass even if it leaves a corrupt filesystem behind, or a splat in the system logs that should not be there. While the test is internally tracked as failed (and the summary reports it as failed) the per-test output exits with a success and so emits a completion time before the post-test checks are run by the test harness. Rework the check code to report post-test check failures as specific test failures rather than as separate failure line items in the overall harness output. Reworking where we emit the errors this also allows us to include the post-test filesystem checking in the test runtime. This is currently not accounted to the test and can be substantial. Hence the real elapsed time of each test is not accurately reflected in the time stats being reported and so regressions in filesystem checking performance go unnoticed. Changing the output reporting requires a complete reworking of the main test check loop. It's a bunch of spaghetti at the moment because it has post test reporting code at the end of the loop which must run regardless of the test result. By moving the post test reporting to the start of the next loop iteration, we can clean up the code substantially by using continue directives where appropriate. Also, for cases where we haven't run the test or it's already been marked as failed, don't bother running the filesystem/dmesg checks for failure as we're already going to report the test as failed. This touches almost all of the loop, so get rid of the remaining 4 space indents inside the loop while moving all this code around. [Eryu: fixed wrong test seq name issue in xUnit report when test hit "continue" in the check loop, e.g. notrun, with Dave ACKing the fix] Signed-Off-By: Dave Chinner <dchinner@redhat.com> Reviewed-by: Eryu Guan <guaneryu@gmail.com> Signed-off-by: Eryu Guan <guaneryu@gmail.com>
186 lines
4.8 KiB
Plaintext
186 lines
4.8 KiB
Plaintext
#
|
|
# Reports generator funcitons lives here
|
|
#
|
|
|
|
# List of xfstests's enviroment variables to include reports
|
|
## TODO automate list population inside common/conf
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST SECTION"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST FSTYP"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST PLATFORM"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST MKFS_OPTIONS"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST MOUNT_OPTIONS"
|
|
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST HOST_OPTIONS"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST CHECK_OPTIONS"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST XFS_MKFS_OPTIONS"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST TIME_FACTOR"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST LOAD_FACTOR"
|
|
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DIR"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DEV"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_DEV"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_MNT"
|
|
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST OVL_UPPER"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST OVL_LOWER"
|
|
REPORT_ENV_LIST="$REPORT_ENV_LIST OVL_WORK"
|
|
|
|
encode_xml()
|
|
{
|
|
cat -v | \
|
|
sed -e 's/&/\&/g' \
|
|
-e 's/>/\>/g' \
|
|
-e 's/</\</g' \
|
|
-e "s/'/\'/g" \
|
|
-e 's/"/\"/g'
|
|
}
|
|
|
|
#
|
|
# Xunit format report functions
|
|
_xunit_add_property()
|
|
{
|
|
local name="$1"
|
|
local value="${!name}"
|
|
|
|
if [ ! -z "$value" ]; then
|
|
echo -e "\t\t<property name=\"$name\" value=\"$value\"/>" >> $REPORT_DIR/result.xml
|
|
fi
|
|
}
|
|
_xunit_make_section_report()
|
|
{
|
|
# xfstest:section ==> xunit:testsuite
|
|
local sect_name=$section
|
|
local sect_time=`expr $sect_stop - $sect_start`
|
|
local n_total=`expr $n_try + $n_notrun`
|
|
|
|
if [ $sect_name == '-no-sections-' ]; then
|
|
sect_name='global'
|
|
fi
|
|
local report=$tmp.report.xunit.$sect_name.xml
|
|
# Header
|
|
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $REPORT_DIR/result.xml
|
|
local dtime=`echo $date_time| tr " " 'T'`
|
|
local stats="failures=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\""
|
|
local hw_info="hostname=\"$HOST\" timestamp=\"$dtime\" "
|
|
echo "<testsuite name=\"xfstests\" $stats $hw_info >" >> $REPORT_DIR/result.xml
|
|
|
|
# Properties
|
|
echo -e "\t<properties>" >> $REPORT_DIR/result.xml
|
|
for p in $REPORT_ENV_LIST;do
|
|
_xunit_add_property "$p"
|
|
done
|
|
echo -e "\t</properties>" >> $REPORT_DIR/result.xml
|
|
cat $tmp.report.xunit.$sect_name.xml >> $REPORT_DIR/result.xml
|
|
echo "</testsuite>" >> $REPORT_DIR/result.xml
|
|
echo "Xunit report: $REPORT_DIR/result.xml"
|
|
}
|
|
|
|
_xunit_make_testcase_report()
|
|
{
|
|
local test_seq="$1"
|
|
local test_status="$2"
|
|
local test_time=`expr $stop - $start`
|
|
local strip="$SRC_DIR/"
|
|
local test_name=${test_seq#$strip}
|
|
local sect_name=$section
|
|
|
|
# TODO: other places may also win if no-section mode will be named like 'default/global'
|
|
if [ $sect_name == '-no-sections-' ]; then
|
|
sect_name='global'
|
|
|
|
fi
|
|
local report=$tmp.report.xunit.$sect_name.xml
|
|
|
|
echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report
|
|
case $test_status in
|
|
"pass")
|
|
;;
|
|
"notrun")
|
|
if [ -f $seqres.notrun ]; then
|
|
local msg=`cat $seqres.notrun | encode_xml`
|
|
echo -e "\t\t<skipped message=\"$msg\" />" >> $report
|
|
else
|
|
echo -e "\t\t<skipped/>" >> $report
|
|
fi
|
|
;;
|
|
"list")
|
|
echo -e "\t\t<skipped/>" >> $report
|
|
;;
|
|
"fail")
|
|
if [ -z "$_err_msg" ]; then
|
|
_err_msg="Test $sequm failed, reason unknown"
|
|
fi
|
|
echo -e "\t\t<failure message=\"$_err_msg\" type=\"TestFail\" />" >> $report
|
|
if [ -s $seqres.full ]; then
|
|
echo -e "\t\t<system-out>" >> $report
|
|
printf '<![CDATA[\n' >>$report
|
|
cat $seqres.full | tr -dc '[:print:][:space:]' | encode_xml >>$report
|
|
printf ']]>\n' >>$report
|
|
echo -e "\t\t</system-out>" >> $report
|
|
fi
|
|
if [ -f $seqres.dmesg ]; then
|
|
echo -e "\t\t<system-err>" >> $report
|
|
printf '<![CDATA[\n' >>$report
|
|
cat $seqres.dmesg | tr -dc '[:print:][:space:]' | encode_xml >>$report
|
|
printf ']]>\n' >>$report
|
|
echo -e "\t\t</system-err>" >> $report
|
|
elif [ -s $seqres.out.bad ]; then
|
|
echo -e "\t\t<system-err>" >> $report
|
|
printf '<![CDATA[\n' >>$report
|
|
$diff $test_seq.out $seqres.out.bad | encode_xml >>$report
|
|
printf ']]>\n' >>$report
|
|
echo -e "\t\t</system-err>" >> $report
|
|
fi
|
|
;;
|
|
*)
|
|
echo -e "\t\t<failure message=\"Unknown test_status=$test_status\" type=\"TestFail\"/>" >> $report
|
|
;;
|
|
esac
|
|
echo -e "\t</testcase>" >> $report
|
|
}
|
|
|
|
|
|
#
|
|
# Common report generator entry points
|
|
_make_section_report()
|
|
{
|
|
for report in $REPORT_LIST; do
|
|
case "$report" in
|
|
"xunit")
|
|
_xunit_make_section_report "$test_status"
|
|
;;
|
|
*)
|
|
_dump_err "format '$report' is not supported"
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
_make_testcase_report()
|
|
{
|
|
local test_seq="$1"
|
|
local test_status="$2"
|
|
for report in $REPORT_LIST; do
|
|
case "$report" in
|
|
"xunit")
|
|
_xunit_make_testcase_report "$test_seq" "$test_status"
|
|
;;
|
|
*)
|
|
_dump_err "report format '$report' is not supported"
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
_assert_report_list() {
|
|
for report in $REPORT_LIST; do
|
|
case "$report" in
|
|
"xunit")
|
|
;;
|
|
*)
|
|
_fatal "report format '$report' is not supported"
|
|
;;
|
|
esac
|
|
done
|
|
}
|