Files
apfstests/common/report
T
Dave Chinner 06dc7f12ad check: fail tests if check/dmesg are not clean
Currently a test appears to pass even if it leaves a corrupt
filesystem behind, or a splat in the system logs that should not be
there. While the test is internally tracked as failed (and the
summary reports it as failed) the per-test output exits with a
success and so emits a completion time before the post-test checks
are run by the test harness.  Rework the check code to report
post-test check failures as specific test failures rather than as
separate failure line items in the overall harness output.

Reworking where we emit the errors this also allows us to include
the post-test filesystem checking in the test runtime. This is
currently not accounted to the test and can be substantial. Hence
the real elapsed time of each test is not accurately reflected in
the time stats being reported and so regressions in filesystem
checking performance go unnoticed.

Changing the output reporting requires a complete reworking of the
main test check loop. It's a bunch of spaghetti at the moment
because it has post test reporting code at the end of the loop which
must run regardless of the test result.  By moving the post test
reporting to the start of the next loop iteration, we can clean up
the code substantially by using continue directives where
appropriate.

Also, for cases where we haven't run the test or it's already been
marked as failed, don't bother running the filesystem/dmesg checks
for failure as we're already going to report the test as failed.

This touches almost all of the loop, so get rid of the remaining
4 space indents inside the loop while moving all this code around.

[Eryu: fixed wrong test seq name issue in xUnit report when test hit
"continue" in the check loop, e.g. notrun, with Dave ACKing the fix]

Signed-Off-By: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Eryu Guan <guaneryu@gmail.com>
Signed-off-by: Eryu Guan <guaneryu@gmail.com>
2018-06-04 13:04:22 +08:00

186 lines
4.8 KiB
Plaintext

#
# Reports generator funcitons lives here
#
# List of xfstests's enviroment variables to include reports
## TODO automate list population inside common/conf
REPORT_ENV_LIST="$REPORT_ENV_LIST SECTION"
REPORT_ENV_LIST="$REPORT_ENV_LIST FSTYP"
REPORT_ENV_LIST="$REPORT_ENV_LIST PLATFORM"
REPORT_ENV_LIST="$REPORT_ENV_LIST MKFS_OPTIONS"
REPORT_ENV_LIST="$REPORT_ENV_LIST MOUNT_OPTIONS"
REPORT_ENV_LIST="$REPORT_ENV_LIST HOST_OPTIONS"
REPORT_ENV_LIST="$REPORT_ENV_LIST CHECK_OPTIONS"
REPORT_ENV_LIST="$REPORT_ENV_LIST XFS_MKFS_OPTIONS"
REPORT_ENV_LIST="$REPORT_ENV_LIST TIME_FACTOR"
REPORT_ENV_LIST="$REPORT_ENV_LIST LOAD_FACTOR"
REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DIR"
REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DEV"
REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_DEV"
REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_MNT"
REPORT_ENV_LIST="$REPORT_ENV_LIST OVL_UPPER"
REPORT_ENV_LIST="$REPORT_ENV_LIST OVL_LOWER"
REPORT_ENV_LIST="$REPORT_ENV_LIST OVL_WORK"
encode_xml()
{
cat -v | \
sed -e 's/&/\&amp;/g' \
-e 's/>/\&gt;/g' \
-e 's/</\&lt;/g' \
-e "s/'/\&apos;/g" \
-e 's/"/\&quot;/g'
}
#
# Xunit format report functions
_xunit_add_property()
{
local name="$1"
local value="${!name}"
if [ ! -z "$value" ]; then
echo -e "\t\t<property name=\"$name\" value=\"$value\"/>" >> $REPORT_DIR/result.xml
fi
}
_xunit_make_section_report()
{
# xfstest:section ==> xunit:testsuite
local sect_name=$section
local sect_time=`expr $sect_stop - $sect_start`
local n_total=`expr $n_try + $n_notrun`
if [ $sect_name == '-no-sections-' ]; then
sect_name='global'
fi
local report=$tmp.report.xunit.$sect_name.xml
# Header
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $REPORT_DIR/result.xml
local dtime=`echo $date_time| tr " " 'T'`
local stats="failures=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\""
local hw_info="hostname=\"$HOST\" timestamp=\"$dtime\" "
echo "<testsuite name=\"xfstests\" $stats $hw_info >" >> $REPORT_DIR/result.xml
# Properties
echo -e "\t<properties>" >> $REPORT_DIR/result.xml
for p in $REPORT_ENV_LIST;do
_xunit_add_property "$p"
done
echo -e "\t</properties>" >> $REPORT_DIR/result.xml
cat $tmp.report.xunit.$sect_name.xml >> $REPORT_DIR/result.xml
echo "</testsuite>" >> $REPORT_DIR/result.xml
echo "Xunit report: $REPORT_DIR/result.xml"
}
_xunit_make_testcase_report()
{
local test_seq="$1"
local test_status="$2"
local test_time=`expr $stop - $start`
local strip="$SRC_DIR/"
local test_name=${test_seq#$strip}
local sect_name=$section
# TODO: other places may also win if no-section mode will be named like 'default/global'
if [ $sect_name == '-no-sections-' ]; then
sect_name='global'
fi
local report=$tmp.report.xunit.$sect_name.xml
echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report
case $test_status in
"pass")
;;
"notrun")
if [ -f $seqres.notrun ]; then
local msg=`cat $seqres.notrun | encode_xml`
echo -e "\t\t<skipped message=\"$msg\" />" >> $report
else
echo -e "\t\t<skipped/>" >> $report
fi
;;
"list")
echo -e "\t\t<skipped/>" >> $report
;;
"fail")
if [ -z "$_err_msg" ]; then
_err_msg="Test $sequm failed, reason unknown"
fi
echo -e "\t\t<failure message=\"$_err_msg\" type=\"TestFail\" />" >> $report
if [ -s $seqres.full ]; then
echo -e "\t\t<system-out>" >> $report
printf '<![CDATA[\n' >>$report
cat $seqres.full | tr -dc '[:print:][:space:]' | encode_xml >>$report
printf ']]>\n' >>$report
echo -e "\t\t</system-out>" >> $report
fi
if [ -f $seqres.dmesg ]; then
echo -e "\t\t<system-err>" >> $report
printf '<![CDATA[\n' >>$report
cat $seqres.dmesg | tr -dc '[:print:][:space:]' | encode_xml >>$report
printf ']]>\n' >>$report
echo -e "\t\t</system-err>" >> $report
elif [ -s $seqres.out.bad ]; then
echo -e "\t\t<system-err>" >> $report
printf '<![CDATA[\n' >>$report
$diff $test_seq.out $seqres.out.bad | encode_xml >>$report
printf ']]>\n' >>$report
echo -e "\t\t</system-err>" >> $report
fi
;;
*)
echo -e "\t\t<failure message=\"Unknown test_status=$test_status\" type=\"TestFail\"/>" >> $report
;;
esac
echo -e "\t</testcase>" >> $report
}
#
# Common report generator entry points
_make_section_report()
{
for report in $REPORT_LIST; do
case "$report" in
"xunit")
_xunit_make_section_report "$test_status"
;;
*)
_dump_err "format '$report' is not supported"
;;
esac
done
}
_make_testcase_report()
{
local test_seq="$1"
local test_status="$2"
for report in $REPORT_LIST; do
case "$report" in
"xunit")
_xunit_make_testcase_report "$test_seq" "$test_status"
;;
*)
_dump_err "report format '$report' is not supported"
;;
esac
done
}
_assert_report_list() {
for report in $REPORT_LIST; do
case "$report" in
"xunit")
;;
*)
_fatal "report format '$report' is not supported"
;;
esac
done
}