Rewrote test 068, test to to reproduce xfs_freeze hang under filsystem load.

Merge of master-melb:xfs-cmds:21434a by kenmcd.

  Rewrote test 068. Test case to reproduce xfs_freeze hang under filsystem load.
This commit is contained in:
Andrew Jones
2005-02-09 02:49:23 +00:00
parent f1d5abef99
commit 6bf85bc85e
3 changed files with 145 additions and 255 deletions
+82 -223
View File
@@ -1,99 +1,49 @@
#! /bin/sh #! /bin/sh
# FS QA Test No. 068 # FSQA Test No. 068
# #
# Test Linux LVM snapshot creation # Test case to reproduce xfs_freeze hang under filsystem load.
# # The fail case for this test is a hang on an xfs_freeze.
# The timing and placement of kills and waits is particularily sensitive.
# Don't change them unless you want to spend some time getting it right again.
# #
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
# Copyright (c) 2000, 2002 Silicon Graphics, Inc. All Rights Reserved. # Copyright (c) 2000-2005 Silicon Graphics, Inc. All Rights Reserved.
# # This program is free software; you can redistribute it and/or modify
# This program is free software; you can redistribute it and/or modify it # it under the terms of the GNU General Public License as published by
# under the terms of version 2 of the GNU General Public License as # the Free Software Foundation; either version 2 of the License, or
# published by the Free Software Foundation. # (at your option) any later version.
# #
# This program is distributed in the hope that it would be useful, but # This program is distributed in the hope that it will be useful,
# WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# # GNU General Public License for more details.
# Further, this software is distributed without any warranty that it is #
# free of the rightful claim of any third person regarding infringement # You should have received a copy of the GNU General Public License
# or the like. Any license provided herein, whether implied or # along with this program; if not, write to the Free Software
# otherwise, applies only to this software file. Patent licenses, if # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# any, provided herein do not apply to combinations of this program with # USA
# other software, or any other product whatsoever. #
# # Contact information: Silicon Graphics, Inc., 1500 Crittenden Lane,
# You should have received a copy of the GNU General Public License along # Mountain View, CA 94043, USA, or: http://www.sgi.com
# with this program; if not, write the Free Software Foundation, Inc., 59
# Temple Place - Suite 330, Boston MA 02111-1307, USA.
#
# Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
# Mountain View, CA 94043, or:
#
# http://www.sgi.com
#
# For further information regarding this notice, see:
#
# http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
# #
# creator # creator
owner=freemyer@NorcrossGroup.com owner=ajones@sgi.com
seq=`basename $0` seq=`basename $0`
echo "QA output created by $seq" echo "QA output created by $seq"
here=`pwd` here=`pwd`
tmp=/tmp/$$ tmp=/tmp/$$
mkdir "$tmp" status=0 # success is the default!
status=1 # failure is the default!
# Either "dd" or "fsstress" can be used to generate io load. ITERATIONS=10
# Setting both to "FALSE" tests with no io load.
GENERATE_IO_LOAD_VIA_DD=TRUE
GENERATE_IO_LOAD_VIA_FSSTRESS=TRUE
DELAY_BETWEEN_ITERATIONS=10
#ITERATIONS=10000
ITERATIONS=2
VG=/dev/VGscratch
SCRATCH_SNAP_MNT="$tmp/scratch_snap"
# Only needed if running by hand, ie. check sets these
#SCRATCH_LVM_DEV=/dev/xxxx
#SCRATCH_MNT=/scratch
_cleanup() _cleanup()
{ {
echo Cleanup beginning cd /
_cleanup_testdir
rm "$tmp/running" trap 0 1 2 3 15
xfs_freeze -u "$SCRATCH_MNT" exit $status
sleep 10 # Give the dd loop time to finish
# Comment out unless needed.
# If needed, wrap with logic to ensure the FS is mounted.
# Kill off any other possible stray stragglers that may be out there.
# There should not be any.
#fuser -k -m "$SCRATCH_SNAP_MNT/dummy" >/dev/null 2>&1
#fuser -k -m "$SCRATCH_MNT/dummy" >/dev/null 2>&1
wait
umount "$SCRATCH_SNAP_MNT" > /dev/null 2>&1
rmdir "$SCRATCH_SNAP_MNT" > /dev/null 2>&1
umount "$SCRATCH_MNT" > /dev/null 2>&1
lvremove -f "$VG/scratch_snap" > /dev/null 2>&1
lvremove -f "$VG/scratch" > /dev/null 2>&1
#TODO vgremove "$VG"
rm -rf "$tmp" # if we ever use tmp files
trap 0 1 2 3 15
exit $status
} }
trap "_cleanup" 0 1 2 3 15 trap "_cleanup" 0 1 2 3 15
@@ -105,162 +55,71 @@ trap "_cleanup" 0 1 2 3 15
# real QA test starts here # real QA test starts here
_supported_fs xfs _supported_fs xfs
_supported_os Linux _supported_os Linux IRIX
if [ -e "$SCRATCH_SNAP_MNT" ]; then _require_scratch
rmdir "$SCRATCH_SNAP_MNT" || _notrun "Cannot rmdir $SCRATCH_SNAP_MNT"
fi
mkdir "$SCRATCH_SNAP_MNT"
#Verify we have the lvm user tools echo "*** init FS"
[ -x /sbin/lvcreate ] || _notrun "lvcreate utility is not installed in /sbin"
[ -x /sbin/lvremove ] || _notrun "lvremove utility is not installed in /sbin"
# If the above fails for LVM 2, the below may be useful notes rm -f $seq.full
# try "lvm vgdisplay --version" and look for a > 1.9x version number, eg, umount $SCRATCH_DEV >/dev/null 2>&1
# echo "*** MKFS ***" >>$seq.full
# LVM version: 1.95.10-cvs (2002-05-31) echo "" >>$seq.full
# Library version: 0.96.03-ioctl-cvs (2002-06-27) _scratch_mkfs_xfs >>$seq.full 2>&1 \
# Driver version: 1.0.3 || _fail "mkfs failed"
_scratch_mount >>$seq.full 2>&1 \
|| _fail "mount failed"
touch $tmp.running
LVM=false # start fsstress loop in a background block
{
STRESS_DIR="$SCRATCH_MNT/fsstress_test_dir"
mkdir "$STRESS_DIR"
# Check if LVM 1 is in the kernel procs=2
if grep lvm /proc/devices > /dev/null; then LVM=true; fi nops=200
while [ -f "$tmp.running" ]
do
# -w ensures that the only ops are ones which cause write I/O
ltp/fsstress -d $STRESS_DIR -w -p $procs -n $nops $FSSTRESS_AVOID \
> /dev/null 2>&1
done
# Check if LVM 2 is in the kernel rm -r $STRESS_DIR/*
if grep device-mapper /proc/devices > /dev/null; then LVM=true; fi rmdir $STRESS_DIR
} &
# Check if EVMS is in the kernel i=0
#TODO # I don't know how to do this one. ITERATIONS=`expr $ITERATIONS - 1`
[ "$LVM" = false ] && _notrun "LVM is not present in the running kernel." echo | tee -a $seq.full
while [ $i -le $ITERATIONS ]
# Create a PV set from the scratch partition
#TODO # (I don't know if this is needed.and it is dangerous because it intentionally deletes the partition table!!!
#TODO # dd if=/dev/zero of="$SCRATCH_LVM_DEV" bs=512 count=1
#TODO # pvcreate "$SCRATCH_LVM_DEV"
# Create a VG from the PV
#TODO # vgcreate "$VG" "$SCRATCH_LVM_DEV"
# Create a LV in the VG to snapshot
#TODO # lvcreate -L 2G -n scratch "$VG"
# Mount the LV
mkdir "$SCRATCH_MNT"
mount "$VG/scratch" "$SCRATCH_MNT"
touch "$tmp/running"
if [ "$GENERATE_IO_LOAD_VIA_DD" != FALSE ]
then
# Create a large 64 Meg zero filled file on the LV
dd if=/dev/zero of=$SCRATCH_MNT/dummy bs=64k count=1000 >/dev/null 2>&1
# Setup infinite loop copying the large file, thus generating heavy I/O
touch "$tmp/running"
while [ -f "$tmp/running" ]
do
dd if="$SCRATCH_MNT/dummy" of="$SCRATCH_MNT/junk" bs=64k \
> /dev/null 2>&1
# This forces metadata updates the next time around
rm "$SCRATCH_MNT/junk"
sync
done
fi &
if [ "$GENERATE_IO_LOAD_VIA_FSSTRESS" != FALSE ]
then
STRESS_DIR="$SCRATCH_MNT/fsstress_test_dir"
mkdir "$STRESS_DIR"
while [ -f "$tmp/running" ]
do
# -n 10 makes this take about 10 seconds,
# This allows the loop to end shortly after $tmp/running
# is deleted
$here/ltp/fsstress -d "$STRESS_DIR" -n 10 > /dev/null 2>&1
sync
done
rm -rf "$STRESS_DIR"
fi &
ii=1
while [ $ii -le "$ITERATIONS" ]
do do
echo "*** iteration: $i" | tee -a $seq.full
echo "*** freezing \$SCRATCH_MNT" | tee -a $seq.full
xfs_freeze -f "$SCRATCH_MNT" | tee -a $seq.full
[ $? != 0 ] && echo xfs_freeze -f "$SCRATCH_MNT" failed | \
tee -a $seq.full
sleep 2
# Useful if running interactively, but not from the xfs test scripts echo "*** thawing \$SCRATCH_MNT" | tee -a $seq.full
#echo $ii xfs_freeze -u "$SCRATCH_MNT" | tee -a $seq.full
[ $? != 0 ] && echo xfs_freeze -u "$SCRATCH_MNT" failed | \
# If the VFS lock patch is present, the calls to xfs_freeze tee -a $seq.full
# are redundant, but should cause no problems sleep 2
# OPTIONAL
xfs_freeze -f "$SCRATCH_MNT"
[ $? != 0 ] && echo xfs_freeze -f "$SCRATCH_MNT" failed
(
lvcreate --snapshot --size 1G --name scratch_snap "$VG/scratch"\
> /dev/null 2>&1
ret=$?
[ $ret != 0 ] && \
echo "Snapshot $SCRATCH_MNT create failed, status=$ret"
) &
SNAPSHOT_shell_pid=$!
# If the Snapshot has not completed in ten minutes, kill it
(
# I have NOT figured out how to kill the sleep 600 before
# it exits naturally.
# This does not cause a problem, but it clutters the ps table.
sleep 600
# The kill $TIMEOUT_shell_pid keeps the below from occuring
echo Snapshot Lockup Occured on loop $ii
xfs_freeze -u "$SCRATCH_MNT"
kill $$
) &
TIMEOUT_shell_pid=$!
wait "$SNAPSHOT_shell_pid"
exec 2> /dev/null # Send the shells stderr to /dev/null
kill "$TIMEOUT_shell_pid" # Cancel the timeout
wait "$TIMEOUT_shell_pid" # This causes consistent shell
# notification for some unknown reason
exec 2>&1 # Put it back to the same as stdout
# If the VFS lock patch is present, the calls to xfs_freeze
# are redundant, but should cause no problems
# OPTIONAL
xfs_freeze -u "$SCRATCH_MNT"
[ $? != 0 ] && echo xfs_freeze -u "$SCRATCH_MNT" failed
# MANDANTORY (end)
mount -t xfs -o ro,nouuid "$VG/scratch_snap" "$SCRATCH_SNAP_MNT"
[ $? -eq 0 ] || echo mount for "$SCRATCH_SNAP_MNT" failed
umount "$SCRATCH_SNAP_MNT"
[ $? -eq 0 ] || echo umount for "$SCRATCH_SNAP_MNT" failed
lvremove -f "$VG/scratch_snap" > /dev/null 2>&1
[ $? -eq 0 ] || echo lvremove for "$VG/scratch_snap" failed
ii=`expr $ii + 1`
sleep "$DELAY_BETWEEN_ITERATIONS"
# The VG seems to need time to stabalize between snapshots
# With LVM 1.0.3 and XFS 1.1, I have tried this at 3600 seconds
# and still had failures
echo | tee -a $seq.full
i=`expr $i + 1`
done done
# success, all done # stop fsstress iterations
echo SUCCESS, COMPLETED ALL ITERATIONS WITH NO TIME OUTS!!!!!!!!!!!! rm $tmp.running
status=0
_cleanup # wait for fsstresses to finish
exit 1 # _cleanup should exit, so we should never get here. wait
_check_scratch_fs
exit 1
+42 -2
View File
@@ -1,3 +1,43 @@
QA output created by 068 QA output created by 068
SUCCESS, COMPLETED ALL ITERATIONS WITH NO TIME OUTS!!!!!!!!!!!! *** init FS
Cleanup beginning
*** iteration: 0
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 1
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 2
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 3
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 4
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 5
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 6
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 7
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 8
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
*** iteration: 9
*** freezing $SCRATCH_MNT
*** thawing $SCRATCH_MNT
+21 -30
View File
@@ -1,34 +1,25 @@
#! /bin/sh #! /bin/sh
# #
# Copyright (c) 2000-2004 Silicon Graphics, Inc. All Rights Reserved. #-----------------------------------------------------------------------
# # Copyright (c) 2000-2005 Silicon Graphics, Inc. All Rights Reserved.
# This program is free software; you can redistribute it and/or modify it # This program is free software; you can redistribute it and/or modify
# under the terms of version 2 of the GNU General Public License as # it under the terms of the GNU General Public License as published by
# published by the Free Software Foundation. # the Free Software Foundation; either version 2 of the License, or
# # (at your option) any later version.
# This program is distributed in the hope that it would be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of # This program is distributed in the hope that it will be useful,
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # but WITHOUT ANY WARRANTY; without even the implied warranty of
# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Further, this software is distributed without any warranty that it is # GNU General Public License for more details.
# free of the rightful claim of any third person regarding infringement #
# or the like. Any license provided herein, whether implied or # You should have received a copy of the GNU General Public License
# otherwise, applies only to this software file. Patent licenses, if # along with this program; if not, write to the Free Software
# any, provided herein do not apply to combinations of this program with # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# other software, or any other product whatsoever. # USA
# #
# You should have received a copy of the GNU General Public License along # Contact information: Silicon Graphics, Inc., 1500 Crittenden Lane,
# with this program; if not, write the Free Software Foundation, Inc., 59 # Mountain View, CA 94043, USA, or: http://www.sgi.com
# Temple Place - Suite 330, Boston MA 02111-1307, USA. #-----------------------------------------------------------------------
#
# Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
# Mountain View, CA 94043, or:
#
# http://www.sgi.com
#
# For further information regarding this notice, see:
#
# http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
# #
# #
# Make a new test # Make a new test
@@ -99,7 +90,7 @@ cat <<End-of-File >$id
# what am I here for? # what am I here for?
# #
#----------------------------------------------------------------------- #-----------------------------------------------------------------------
# Copyright (c) 2000-2004 Silicon Graphics, Inc. All Rights Reserved. # Copyright (c) 2000-2005 Silicon Graphics, Inc. All Rights Reserved.
# #
# This program is free software; you can redistribute it and/or modify it # This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as # under the terms of version 2 of the GNU General Public License as