generic: add an fs-verity stress test

Add a stress test for fs-verity.  This tests enabling fs-verity on
multiple files concurrently with concurrent readers on those files
(with reads occurring before, during, and after the fs-verity
enablement), while fsstress is also running on the same filesystem.

I haven't seen any failures from running this on ext4 and f2fs.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Eryu Guan <guaneryu@gmail.com>
Signed-off-by: Eryu Guan <guaneryu@gmail.com>

tests/generic/579

@@ -0,0 +1,115 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright 2019 Google LLC
+#
+# FS QA Test generic/579
+#
+# Stress test for fs-verity.  This tests enabling fs-verity on multiple files
+# concurrently with concurrent readers on those files (with reads occurring
+# before, during, and after the fs-verity enablement), while fsstress is also
+# running on the same filesystem.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	# Stop all subprocesses.
+	$KILLALL_PROG -q $FSSTRESS_PROG
+	touch $tmp.done
+	wait
+
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/verity
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch_verity
+_require_command "$KILLALL_PROG" killall
+
+_scratch_mkfs_verity &>> $seqres.full
+_scratch_mount
+
+fsv_file_size=10000000
+nproc_enabler=$((4 * LOAD_FACTOR))
+nproc_reader=$((6 * LOAD_FACTOR))
+nproc_stress=$((3 * LOAD_FACTOR))
+runtime=$((20 * TIME_FACTOR))
+
+# Create the test files and start the fs-verity enabler processes.
+for ((proc = 0; proc < nproc_enabler; proc++)); do
+	orig_file=$SCRATCH_MNT/orig$proc
+	fsv_file=$SCRATCH_MNT/fsv$proc
+	head -c $fsv_file_size /dev/urandom > $orig_file
+	(
+		while [ ! -e $tmp.done ]; do
+			rm -f $fsv_file
+			cp $orig_file $fsv_file
+			_fsv_enable $fsv_file
+			# Give the readers some time to read from the file.
+			sleep 0.$((RANDOM % 100))
+		done
+	) &
+done
+
+# Start the reader processes.
+for ((proc = 0; proc < nproc_reader; proc++)); do
+	(
+		while [ ! -e $tmp.done ]; do
+			# Choose a random file for each iteration, so that
+			# sometimes multiple processes read from the same file.
+			i=$((RANDOM % nproc_enabler))
+			orig_file=$SCRATCH_MNT/orig$i
+			fsv_file=$SCRATCH_MNT/fsv$i
+
+			# After the copy from $orig_file to $fsv_file has
+			# completed, the contents of these two files should
+			# match, regardless of whether verity has been enabled
+			# or not yet (or is currently being enabled).
+			cmp $orig_file $fsv_file |& _filter_scratch | \
+				grep -v "SCRATCH_MNT/fsv$i: No such file or directory" | \
+				grep -v "EOF on SCRATCH_MNT/fsv$i"
+
+			_fsv_measure $fsv_file 2>&1 >/dev/null | \
+				grep -v "No such file or directory" | \
+				grep -v "No data available"
+		done
+	) &
+done
+
+# Start a process that occasionally runs 'sync && drop_caches'.  This makes more
+# reads go through fs-verity for real, rather than just returning pagecache.
+(
+	while [ ! -e $tmp.done ]; do
+		sleep 2.$((RANDOM % 100))
+		sync && echo 3 > /proc/sys/vm/drop_caches
+	done
+) &
+
+# Start the fsstress processes.
+$FSSTRESS_PROG $FSSTRESS_AVOID -p $nproc_stress -l 0 -d $SCRATCH_MNT/stressdir \
+	>> $seqres.full 2>&1 &
+
+# Run for a while.
+sleep $runtime
+
+echo "Silence is golden"
+
+# success, all done
+status=0
+exit

tests/generic/579.out

@@ -0,0 +1,2 @@
+QA output created by 579
+Silence is golden

tests/generic/group

@@ -581,3 +581,4 @@
 576 auto quick verity encrypt
 577 auto quick verity
 578 auto quick rw clone
+579 auto stress verity