#! /bin/bash
# FS QA Test No. 475
#
# Test log recovery with repeated (simulated) disk failures.  We kick
# off fsstress on the scratch fs, then switch out the underlying device
# with dm-error to see what happens when the disk goes down.  Having
# taken down the fs in this manner, remount it and repeat.  This test
# is a Good Enough (tm) simulation of our internal multipath failure
# testing efforts.
#
#-----------------------------------------------------------------------
# Copyright (c) 2017 Oracle, Inc.  All Rights Reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write the Free Software Foundation,
# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
#-----------------------------------------------------------------------
#

seq=`basename $0`
seqres=$RESULT_DIR/$seq
echo "QA output created by $seq"

here=`pwd`
tmp=/tmp/$$
status=1	# failure is the default!
trap "_cleanup; exit \$status" 0 1 2 3 15

_cleanup()
{
	cd /
	rm -f $tmp.*
	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
	_dmerror_unmount
	_dmerror_cleanup
}

# get standard environment, filters and checks
. ./common/rc
. ./common/dmerror

# Modify as appropriate.
_supported_fs generic
_supported_os Linux

_require_scratch
_require_dm_target error
_require_command "$KILLALL_PROG" "killall"

rm -f $seqres.full

echo "Silence is golden."

_scratch_mkfs >> $seqres.full 2>&1
_require_metadata_journaling $SCRATCH_DEV
_dmerror_init
_dmerror_mount

for i in $(seq 1 $((50 * TIME_FACTOR)) ); do
	($FSSTRESS_PROG $FSSTRESS_AVOID -d $SCRATCH_MNT -n 999999 -p $((LOAD_FACTOR * 4)) >> $seqres.full &) \
		> /dev/null 2>&1

	# purposely include 0 second sleeps to test shutdown immediately after
	# recovery
	sleep $((RANDOM % 3))

	# Loading error table without "--nolockfs" option. Because "--nolockfs"
	# won't freeze fs, then some running I/Os may cause XFS to shutdown
	# prematurely. That's not what we want to test.
	_dmerror_load_error_table

	ps -e | grep fsstress > /dev/null 2>&1
	while [ $? -eq 0 ]; do
		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
		wait > /dev/null 2>&1
		ps -e | grep fsstress > /dev/null 2>&1
	done

	# Mount again to replay log after loading working table, so we have a
	# consistent XFS after test.
	_dmerror_unmount || _fail "unmount failed"
	_dmerror_load_working_table
	_dmerror_mount || _fail "mount failed"
done

# success, all done
status=0
exit