gecko/security/nss/tests/qa_stat

#! /bin/sh 
########################################################################
#
# /u/sonmi/bin/qa_stat - /u/svbld/bin/init/nss/qa_stat
#
# this script is supposed to automatically run QA for NSS on all required
# Unix platforms
#
# parameters
# ----------
#    nssversion (supported: 30b, 31, tip)
#    builddate  (default - today)
#
# options
# -------
#    -y answer all questions with y - use at your own risk...ignores warnings
#    -s silent (only usefull with -y)
#    -h, -? - you guessed right - displays this text
#    -d debug
#    -f <filename> - write the (error)output to filename
#    -m <mailinglist> - send filename to mailinglist (csl) only useful
#        with -f
#    -cron equivalient to -y -s -d -f $RESULTDIR/$HOST.qa_stat
#
########################################################################

O_OPTIONS=ON

TBX_EXIT=49          # in case we are running on a tinderbox build, any
                          # early exit needs to return an error
if [ -z "$O_TBX" -o "$O_TBX" = "OFF" ] ; then
    if [ -z "$O_LOCAL" -o "$O_LOCAL" = "OFF" ] ; then
        . `dirname $0`/header
    fi
fi
Debug "Sourced header O_TBX=$O_TBX O_LOCAL=$O_LOCAL"
TBX_EXIT=48
EARLY_EXIT=TRUE

URL="cindercone.red.iplanet.com"

DOCDIR=/u/sonmi/doc

HTML_ERRORCOLOR=\"#FF0000\"
HTML_ERRORMSG=Failed

HTML_MISSINGCOLOR=\"#FFFFCC\"
HTML_MISSINGMSG=Missing

HTML_INCOMPLETECOLOR=$HTML_MISSINGCOLOR
HTML_INCOMPLETEMSG=Incomplete

HTML_PASSEDCOLOR=\"#66FF99\"
HTML_PASSEDMSG=Passed

# this file is used to deal with hanging rsh - a new shell is started 
# for each rsh, and a function is called after it is finished - they
# communicate with this file

RSH_FILE=$TMP/rsh.$$
echo >$RSH_FILE
TMPFILES="$TMPFILES $WARNINGLIST $RSH_FILE "
RSH_WAIT_TIME=80 #maximum time allowed for the 2 rsh to finish...
#TOTAL_TESTS=106
TOTAL_TESTS=252 #tip
#TOTAL_TESTS=244 #3.4
#TOTAL_TESTS=123 #3.3.2
BCT_TOTAL_TESTS=122 #3.2.2
#TOTAL_TESTS=133 #tip

Debug "NTDIST $NTDIST"
Debug "UXDIST $UXDIST"
Debug "TESTSCRIPTDIR $TESTSCRIPTDIR"
Debug "RESULTDIR $RESULTDIR"

############################### watch_rsh ##############################
# local shell function, deals with a hanging rsh (kills it...)
# this function is started as a backgroundprocess before the rsh is started,
# and writes info to the RSH_FILE, after the rsh is finished it writes finish
# info to the same file (this time called as a function, forground). 
# the backgroundprocess stays around for RSH_WAIT_TIME, if then the finish 
# information is not there attempts to kill the rsh
#
# watch_rsh start qa_computername &
# watch_rsh stop qa_computername 
#
########################################################################
watch_rsh()
{
    case $1 in
        start)
            echo "$2 started" >>$RSH_FILE
            sleep $RSH_WAIT_TIME
            O_ALWAYS_YES=ON # may modify global flags because this is a 
                            # forked off bg process - kill_by_name otherwise
                            # will ask the user if it really should be killed 
            grep "$2 finished" $RSH_FILE >/dev/null || kill_by_name "rsh $2"
            exit
            ;;
        stop)
            echo "$2 finished" >>$RSH_FILE
            ;;
    esac
}

############################### find_qa_systems ########################
# local shell function, tries to determine the QA operating system
# works remotely, and for Windows machines
########################################################################
find_qa_systems()
{
for QA_SYS in `ls $RESULTDIR | grep '\.1$' | sed -e "s/\..*//" | sort -u`
do
    NO_RSH="FALSE"
    QA_OS=""
    QA_RHVER=""
    IS_64=""
    IS_WIN=""

    grep OS-LINE ${RESULTDIR}/${QA_SYS}.nssqa >/dev/null && NO_RSH=TRUE

    if [ "$NO_RSH" = "TRUE" ]
    then

        QA_OS=`grep OS-LINE ${RESULTDIR}/${QA_SYS}.nssqa | sort -u | sed \
            -e "s/.*-OS-LINE: /${QA_SYS}/"`
        QA_OS_STRING=`echo $QA_OS | sed -e "s/^[_ ]//" -e "s/ /_/g"`
        echo $QA_OS_STRING >>$PLATFORMLIST
        if [ "$O_SILENT" != ON ] ; then
            echo $QA_OS
        fi

        #grep OS-LINE ${RESULTDIR}/${QA_SYS}.nssqa | sort -u | sed \
             #-e "s/.*-OS-LINE: /${QA_SYS}_/" >>$PLATFORMLIST
        #if [ "$O_SILENT" != ON ] ; then
            #grep OS-LINE ${RESULTDIR}/${QA_SYS}.nssqa | sort -u | sed \
                 #-e "s/.*-OS-LINE:/${QA_SYS}/" 
        #fi
    else
        REM_SYSNAME=$QA_SYS
        watch_rsh start $REM_SYSNAME &
        qa_stat_get_sysinfo $QA_SYS
        watch_rsh stop $REM_SYSNAME 
        echo $QA_OS_STRING >>$PLATFORMLIST
                          # use later for missing list
    fi
done

}

################################### qa_stat_init ##########################
# local shell function, sets the name of the resultfile to:
#    <filename> if option -f <filename>
#    $RESULTDIR/result if write permission 
#        (mozilla/tests_results/security/result)
#    $HOME/resultNSS${NSSVER}-${BUILDDATE} if no write permission in $RESULTDIR
########################################################################
qa_stat_init()
{
    if [ $O_FILE = ON -a $O_CRON = OFF ]    # if -f was specified write there 
    then
        RFILE=$FILENAME    
    else
        RFILE=${RESULTDIR}/result.$$
        if [ ! -w $RESULTDIR ]
        then
            RFILE=$HOME/resultNSS${NSSVER}-${BUILDDATE}.$$
            Debug "Using alternate resultfile $RFILE"
        #elif [ $O_CRON = ON ]
        #then
             ##find ${RESULTDIR} -exec chmod a+rw {} \;    #FIXME - umask 
                            ##doesn't seem to work - this is a tmp workaround
        fi
    
        if [ ! -x $RESULTDIR -o ! -r  $RESULTDIR -o ! -w $RESULTDIR ]
        then
            glob_usage "$RESULTDIR does not have the right permissions `ls -l $RESULTDIR`"
        fi
        if [ -d $RESULTDIR ]
        then
            cd $RESULTDIR
        else
            glob_usage "$RESULTDIR does not exist"
        fi
    fi

    ERRORLIST=${RFILE}.E
    PLATFORMLIST=${RFILE}.P
    PERFLIST=${RFILE}.PE
    TMP_HTML_FILE=${RFILE}.html
    HTML_FILE=${RESULTDIR}/result.html
    WARNINGLIST=${RFILE}.W
    BCMISSINGLIST=${RFILE}.BCM
    BCERRORLIST=${RFILE}.BCE
    TMPFILE=${RFILE}.T
    ML_FILE=${RFILE}.ML

    TMPFILES="$TMPFILES $TMPFILE"
    TMPFILES="$TMPFILES $ERRORLIST $PLATFORMLIST $PERFLIST $WARNINGLIST \
       $BCMISSINGLIST $BCERRORLIST $ML_FILE" #FIXME uncomment

    FILENAME=$RFILE        #we might want to mail it...later switch to html file
    O_FILE="ON"

    rm $ERRORLIST $PLATFORMLIST $PERFLIST $WARNINGLIST \
       $BCMISSINGLIST $BCERRORLIST $TMP_HTML_FILE  2>/dev/null
    touch $ERRORLIST $PLATFORMLIST $PERFLIST $WARNINGLIST \
       $BCMISSINGLIST $BCERRORLIST $TMP_HTML_FILE  2>/dev/null

    if [  $O_WIN = "ON" -a "$O_TBX" = "ON" ] ; then
        HTML_PATH="http://${URL}${UX_D0}/nss${NSSVER}/tinderbox/tests_results/security/`basename $RESULTDIR`"
    else
        HTML_PATH="http://${URL}${RESULTDIR}"
    fi
    HREF_TMP_HTML_FILE="${HTML_PATH}/`basename $HTML_FILE`"

    write_qa_header_html >$TMP_HTML_FILE
}

################################# html_footer #########################
# local shell function, writes end of the html body
#######################################################################
write_qa_header_html()
{
echo 'Subject: QA Report ' $NSSVER $BUILDDATE '
From: sonmi@iplanet.com
Reply-To: sonmi@iplanet.com
Content-Type: text/html; charset=us-ascii
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<html>
<head>
   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
   <meta name="GENERATOR" content="Mozilla/4.7 [en] (X11; U; SunOS 5.8 sun4u) [N
etscape]">
</head>
<body>
<br>
&nbsp;
<br>&nbsp;
<h2>
<a href="http://tinderbox.mozilla.org/showbuilds.cgi?tree=NSS">Tinderbox</a
><br>
<a href="http://cindercone.red.iplanet.com/share/builds/mccrel3/nss/nsstip/tinderbox/tests_results/security/">Tinderbox QA&nbsp;result</a><br>
<a href="ftp://ftp.mozilla.org/pub/security/nss/daily_qa">Mozilla Daily NSS QA&nbsp;result</a></h2>
&nbsp;

&nbsp;
<br>&nbsp;
<center>
<h1>
<a NAME="Top"></a><b><font size=+2>QA&nbsp;Results</font></b></h1></center>


<table BORDER WIDTH="100%" NOSAVE >
<tr>
<td>&nbsp;<b><font size=+1>Build-OS and version</font></b></td>
<td><b><font size=+1>QA-OS</font></b></td>
<td><b><font size=+1>Systemname</font></b></td>
<td><b><font size=+1>P/F</font></b></td>
<td><b><font size=+1>result</font></b></td>
<td><b><font size=+1>output</font></b></td>
<td><b><font size=+1>errors</font></b></td>
<td><b><font size=+1>QA time / #</font></b></td>
</tr>
'
}

################################# html_footer #########################
# local shell function, writes end of the html body
#######################################################################
html_footer()
{
  echo '</body>'
  echo '</html>'
}

################################# setQAsysvars #########################
# local shell function, sets system specific variables
########################################################################
setQAsysvars()
{
    if [ "$MACHINE" != "0" ]
    then
        MACHINE=`echo $MACHINE | sed -e 's/^bct.//g'`
        TESTDATE=`ls -ld $MACHINE | awk '{ print $6, $7, $8 }'`
        TESTNUMBER=`echo $MACHINE | sed -e 's/.*\.//'`
        SYSNAME=`echo $MACHINE | sed -e 's/\..*//'`
        Debug "SYSNAME= $SYSNAME"

        if [  "$O_TBX" = "ON" -o "$O_LOCAL" = "ON" ] ; then
            QA_SYS_OS=$QA_OS
        else
            QA_SYS_OS=`grep $SYSNAME $PLATFORMLIST |
                sed -e 's/
//' | \
                sort | uniq | sed  -e "s/$SYSNAME//" \
                -e "s/^_//" | sort | uniq`
        fi
        Debug "QA_SYS_OS= $QA_SYS_OS"
    fi
    BUILD_SYS=`echo $BUILDPLATFORM | sed -e 's/\.OBJ//' -e 's/_DBG/ Debug/' \
            -e 's/_OPT/ Optimized/'  -e 's/_64/ 64bit/' -e 's/_glibc_PTH//' \
            -e 's/_/ /'`
    Debug "BUILD_SYS=$BUILD_SYS"
    if [ -f "${RESULTDIR}/${MACHINE}/results.html" ] ; then
        RESULT="${HTML_PATH}/${MACHINE}/results.html"
    else
        RESULT="0"
    fi
    if [ -f "${RESULTDIR}/bct/${MACHINE}/results.html" ] ; then
        BCB_RESULT="${HTML_PATH}/bct/${MACHINE}/results.html"
    else
      BCB_RESULT="0"
    fi

    if [ -f "${RESULTDIR}/${MACHINE}/output.log" ] ; then
        LOG="${HTML_PATH}/${MACHINE}/output.log"
    else
        LOG="0"
    fi
    if [ -f "${RESULTDIR}/bct/${MACHINE}/output.log" ] ; then
        BCB_LOG="${HTML_PATH}/bct/${MACHINE}/output.log"
    else
        BCB_LOG="0"
    fi
}

################################# html_line() #########################
# local shell function, writes a line in the html table
########################################################################
html_line()
{
  echo '<tr NOSAVE>'
  echo '<td NOSAVE>'$BUILD_SYS'</td>'
  echo ''
  if [ "$QA_SYS_OS" != "0" ] ; then
      echo '<td NOSAVE>'$QA_SYS_OS'</td>'
  else
      echo '<td></td>'
  fi
  echo ''
  if [ "$SYSNAME" != "0" ] ; then
      echo '<td>'$SYSNAME'</td>'
  else
      echo '<td></td>'
  fi
  #echo '<td>'$SYSNAME $TESTNUMBER $TESTDATE'</td>'
  echo ''
  # hopefully we never run more different tests on a tinderbox build...
  # on win some shells can not handle exit codes greater then 52 (64???)
  # so for very early exits the codes are set 50-45, for failures later
  # in the process the higher the number, the more failures
  if [ "$O_TBX" = "ON" -a "$TBX_EXIT" -gt 45 ] ; then
      TBX_EXIT=0
  fi
  if [ "$1" = "failed" ]
  then
      TBX_EXIT=`expr $TBX_EXIT + 1`
      echo '<td BGCOLOR='$HTML_ERRORCOLOR' NOSAVE><b>'$HTML_ERRORMSG'</b></td>'
  elif [ "$1" = "passed" ]
  then
      echo '<td BGCOLOR='$HTML_PASSEDCOLOR' NOSAVE>'$HTML_PASSEDMSG'</td>'
  elif [ "$1" = "incomplete" ]
  then
      TBX_EXIT=`expr $TBX_EXIT + 1`
      echo '<td BGCOLOR='$HTML_INCOMPLETECOLOR' NOSAVE>'$HTML_INCOMPLETEMSG'</td>'
  else
      TBX_EXIT=`expr $TBX_EXIT + 1`
      echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'    
  fi
  if [ "$CURRENT_TABLE" != "BC" ] ; then
      if [ "$RESULT" = "0" ] ; then
          echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'
      else
          echo '<td>&nbsp;<a href="'$RESULT'">result</a>&nbsp;</td>'
      fi
      if [ "$LOG" = "0" ] ; then
          echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'
      else
          echo '<td>&nbsp;<a href="'$LOG'">log</a>&nbsp;</td>'
      fi
      if [ "$1" = "failed" ] ; then
          echo '<td>&nbsp;<a href="'${HREF_TMP_HTML_FILE}'#errorlist">error</a>&nbsp;</td>'
      else
          echo '<td></td>'
      fi
  else
     #<td><b><font size=+1>errors</font></b></td>
     #<td><b><font size=+1>P/F</font></b></td>
     #<td><b><font size=+1>P/F</font></b></td>

     #echo '<td><b><font size=+1>All Current</font></b></td>'
     #echo '<td><b><font size=+1>old dlls</font></b></td>'
     #echo '<td><b><font size=+1>old executables</font></b></td>'
      #if [ "$RESULT" != "0" -a "$LOG" != "0" ] ; then
          #echo '<td><a href="'$RESULT'">result</a>, <a href="'$LOG'">log</td>'
      #elif [ "$RESULT" = "0" -a "$LOG" != "0" ] ; then
          #echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE><a href="'$LOG'">log</a></td>'
      #elif [ "$RESULT" != "0" -a "$LOG" = "0" ] ; then
          #echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE><a href="'$RESULT'">result</a></td>'
      #else
          #echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'
      #fi
      #if [ "$BCB_RESULT" != "0" -a "$BCB_LOG" != "0" ] ; then
          #echo '<td><a href="'$BCB_RESULT'">result</a>, <a href="'$BCB_LOG'"> log</td>'
      #elif [ "$BCB_RESULT" = "0" -a "$BCB_LOG" != "0" ] ; then
          #echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE><a href="'$BCB_LOG'">log</a></td>'
      #elif [ "$BCB_RESULT" != "0" -a "$BCB_LOG" = "0" ] ; then
          #echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE><a href="'$BCB_RESULT'">result</a></td>'
      #else
          #echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'
      #fi
      if [ "$BCB_RESULT" = "0" ] ; then
          echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'
      else
          echo '<td>&nbsp;<a href="'$BCB_RESULT'">result</a>&nbsp;</td>'
      fi
      if [ "$BCB_LOG" = "0" ] ; then
          echo '<td BGCOLOR='$HTML_MISSINGCOLOR' NOSAVE>'$HTML_MISSINGMSG'</td>'
      else
          echo '<td>&nbsp;<a href="'$BCB_LOG'">log</a>&nbsp;</td>'
      fi
  fi
  echo '<td>'$TESTDATE $TESTNUMBER'</td>'
  echo '</tr>'
}

################################# qa_errorlist #########################
# local shell function, finds problems in the previously run QA
# linux:the gnu grep, on Linux can output 10 lines above and 3 lines below 
# the errormessage
########################################################################
qa_errorlist()
{
    grep "bgcolor=red" ${MACHINES_TO_CHECK}*/results.html | 
        sed -e 's/.results.html:<TR><TD>/ /' -e 's/<[^>]*>/ /g'
    grep 'cache hits; .* cache misses, .* cache not reusable' \
        ${MACHINES_TO_CHECK}*/output.log | 
        grep strsclnt |
        grep -v '0 cache hits; 0 cache misses, 0 cache not reusable' | 
        grep -v ' cache hits; 1 cache misses, 0 cache not reusable'
    for logfile in ${MACHINES_TO_CHECK}*/output.log; do
        grep -vi "write to SSL socket" $logfile |
            grep -vi "HDX PR_Read returned error" |
            grep -vi "no error" |
            grep -vi "12285" |
            grep -i  $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP error 
            #grep -vi "5938" |  needed for -v option
            #grep -vi "HDX PR_Read hit EOF" | 
        grep  -vi "write to SSL socket" $logfile |
            grep -vi "peer cannot verify" |
            grep -vi "error" |
            grep -vi "fatal" |
            grep -vi "TCP Connection aborted" |
            grep -vi "TCP connection reset" |
            grep  $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP  -i failed
    done
    grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "segmentation violation" \
        ${MACHINES_TO_CHECK}*/output.log
    grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "memory fault" \
        ${MACHINES_TO_CHECK}*/output.log
    grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "bus error" \
        ${MACHINES_TO_CHECK}*/output.log
    grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "core dumped" \
        ${MACHINES_TO_CHECK}*/output.log
    grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP fatal \
        ${MACHINES_TO_CHECK}*/output.log
    grep -i  $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP -i\
        "PKCS12 decode not verified"   ${MACHINES_TO_CHECK}*/output.log

    if [ -n "${MACHINES_TO_CHECK}" ] ; then
        find ${MACHINES_TO_CHECK}* -name core -print 2>/dev/null |
            grep -v bct 
    else
        find . -name core -print 2>/dev/null |
            grep -v bct 
    fi
}

tbx_missing_platforms ()
{
    QA_MISSING="QA report missing"
    MACHINE="0"

    if [ "$QA_OS_STRING" = "WINNT4.0" ] ; then
        QA_OS_STRING="Windows-NT-4.0"
    fi
    for BUILDPLATFORM in `grep $QA_OS_STRING $TESTSCRIPTDIR/platformlist.tbx`
    do
        if [ "$BUILDPLATFORM" != "$QA_OS_STRING" ] ; then
            Debug  "BUILDPLATFORM = $BUILDPLATFORM QA_OS_STRING = $QA_OS_STRING"
            grep $BUILDPLATFORM ${MACHINES_TO_CHECK}*/results.html \
                >/dev/null || {
                setQAsysvars
                html_line missing >>$TMP_HTML_FILE
            }
        fi
    done
}

############################ platform _list ###########################
# local shell function, generate pass/fail information for each Platform
########################################################################
platformlist()
{
    grep Platform ${MACHINES_TO_CHECK}*/results.html | 
        sed -e 's/.results.html:<H4>Platform: /---/' \
            -e 's/<BR>//' >$TMPFILE
    # this is done a little complicated to avoid starting a subshell in 
    # a while read that gets the input from a pipeline, and variables set 
    #in or underneath this function get unset after done...
    for  MB in `cat $TMPFILE` ; do
        MACHINE=`echo $MB | sed -e "s/---.*//"`
        BUILDPLATFORM=`echo $MB | sed -e "s/.*---//"`
        grep "${MACHINE}[^0-9]" $ERRORLIST >/dev/null
        ret=$?
        setQAsysvars
        if [ $ret -eq 0 ]
        then
            echo "Failed $MACHINE $BUILDPLATFORM" >>$RFILE
            html_line failed >>$TMP_HTML_FILE
        else
            echo "Passed $MACHINE $BUILDPLATFORM" >>$RFILE
            html_line passed >>$TMP_HTML_FILE
        fi
    done 
}

############################ missing_platforms ###########################
# local shell function, finds out if we ran on all required platforms
########################################################################
missing_platforms()
{
    QA_MISSING="QA report missing"
    MACHINE="0"
    SYSNAME="0"
    QA_SYS_OS="0"

    for BUILDPLATFORM in `cat $TESTSCRIPTDIR/platformlist`
    do
        grep $BUILDPLATFORM $PLATFORMLIST > /dev/null || {
            setQAsysvars
            html_line missing >>$TMP_HTML_FILE
        }
    done
}

############################ incomplete_results ###########################
# local shell function, finds out if all qa runs were complete
########################################################################
incomplete_results ()
{
    
    for w in `ls ${MACHINES_TO_CHECK}*/results.html`
    do
      grep bgcolor=red $w || {
        PASSED_LINES=""
        PASSED_LINES=`grep bgcolor=lightGreen $w | wc -l`
        if [ -n "$PASSED_LINES" -a "$PASSED_LINES" -lt "$TOTAL_TESTS" ] ; then
            BUILDPLATFORM=`grep Platform $w | sed -e 's/<H4>Platform:/    /'                   -e 's/<BR>//'` 
            MACHINE=`echo $w | sed -e "s/.results.html//"`
            #MACHINE=`echo $w | sed -e "s/\.[0-9]*.results.html//"`
            setQAsysvars
            html_line incomplete >>$TMP_HTML_FILE
        elif [ "$PASSED_LINES" -gt "$TOTAL_TESTS" ] ; then
            echo "WARNING - more tests than expected on $w ($PASSED_LINES)" >>$WARNINGLIST
	fi
      }
    done
}

qa_stat_table()
{
    echo '&nbsp;'
    echo '<br>&nbsp;'
    echo '<center>'
    echo '<h1>'
    echo '<a NAME="'$1'"></a>'$1'</h1></center>'
    echo '&nbsp;'
    echo '<table BORDER WIDTH="100%" NOSAVE >'
    echo '<tr NOSAVE>'
}

############################### psaperf ########################
# local shell function, copies results of the daily performance test
# into a table in the QA report
########################################################################
rsaperf()
{
    grep RSAPERF */output.log | grep -v "_DBG" > $PERFLIST
    
    qa_stat_table "Performance list"

    echo '<td NOSAVE><b><font size=+1>Build-OS and version</font></b></td>'
    echo '<td><b><font size=+1>Systemname</font></b></td>'
    echo '<td><b><font size=+1># of iterations</font></b></td>'
    echo '<td><b><font size=+1>average for one op</font></b></td>'
    echo '<td><b><font size=+1>Total</font></b></td>'
    echo '<td><b><font size=+1>QA time / #</font></b></td>'
    echo '</tr>'
    cat $PERFLIST | 
    while read MACHINE BUILDPLATFORM no_iter t1 t2 total total_unit t3 \
               t4 t5 average average_unit
    do
        #caution subshell, variables local to this loop
        BUILD_SYS=`echo $BUILDPLATFORM | sed -e 's/\.OBJ//' \
            -e 's/_DBG/ Debug/' \
            -e 's/_OPT/ Optimized/'  -e 's/_64/ 64bit/' -e 's/_glibc_PTH//' \
            -e 's/_/ /'`
        TESTNUMBER=`echo $MACHINE | sed -e 's/[^\.]*\.//' -e 's/\/.*//'`
        MACHINE=`echo $MACHINE | sed -e 's/\..*//'`
        TESTDATE=`ls -ld ${MACHINE}.${TESTNUMBER} | awk '{ print $6, $7, $8 }'`
        echo '<tr>'
        echo '<td>'$BUILD_SYS'</td>'
        echo ''
        echo '<td>'$MACHINE'</td>'
        echo ''
        echo '<td>'$no_iter'</td>'
        echo ''
        echo '<td>'$average' '$average_unit'</td>'
        echo ''
        echo '<td>'$total' '$total_unit'</td>'
        echo ''
        echo '<td>'$TESTDATE $TESTNUMBER'</td>'
        echo ''
        echo '</tr>'
    done
    echo '</table>'
}

############################### qa_stat_cleanup ########################
# local shell function, finishes html file, sets variables for global Exit
########################################################################
qa_stat_cleanup()
{

    html_footer >>$TMP_HTML_FILE

    O_DEBUG=OFF

    EARLY_EXIT=FALSE
    cp $TMP_HTML_FILE $HTML_FILE
    FILENAME=$HTML_FILE        #we might want to mail it...
    Exit
}


############################### bc_test ########################
# local shell function, evaluates the results of the backward u
# compatibility tests
########################################################################
bc_header()
{
CURRENT_TABLE="BC"   #so html_line can determine which fields to write
  
  qa_stat_table "Backward Compatibility Test"
  echo '<td NOSAVE><b><font size=+1>Build-OS and version</font></b></td>'
  echo '<td><b><font size=+1>QA-OS</font></b></td>'
  echo '<td><b><font size=+1>Systemname</font></b></td>'
  echo '<td><b><font size=+1>P/F</font></b></td>'
  #echo '<td><b><font size=+1>All Current</font></b></td>'
  #echo '<td><b><font size=+1>backward comp. test</font></b></td>'
  echo '<td><b><font size=+1>result</font></b></td>'
  echo '<td><b><font size=+1>output</font></b></td>'
  echo '<td><b><font size=+1>QA time / #</font></b></td>'
  echo '</tr>'

}

old_bc_test()
{
CURRENT_TABLE="BC"   #so html_line can determine which fields to write
  
  qa_stat_table "Backward Compatibility Test"
  echo '<td NOSAVE><b><font size=+1>Build-OS and version</font></b></td>'
  echo '<td><b><font size=+1>QA-OS</font></b></td>'
  echo '<td><b><font size=+1>Systemname</font></b></td>'
  echo '<td><b><font size=+1>P/F</font></b></td>'
  #echo '<td><b><font size=+1>All Current</font></b></td>'
  #echo '<td><b><font size=+1>backward comp. test</font></b></td>'
  echo '<td><b><font size=+1>result</font></b></td>'
  echo '<td><b><font size=+1>output</font></b></td>'
  echo '<td><b><font size=+1>QA time / #</font></b></td>'
  echo '</tr>'

  for w in `ls */results.html`
  do
      TMP_RESULT="`dirname $w`/results.tmp"
      TMP_BC_RESULT="`dirname bct/$w`/results.tmp"
      rm $TMP_RESULT $TMP_BC_RESULT 2>/dev/null
      cat $w | sed -e 's/<[^>]*>//g'  -e 's/ /_/g' \
                   -e 's/signtool_-[vw]/signtool_-vw/' |
               grep '_[PF]a[si][sl]ed' >$TMP_RESULT
      cat bct/$w   | sed -e 's/<[^>]*>//g'  -e 's/ /_/g' \
                         -e 's/signtool_-[vw]/signtool_-vw/' |
                     grep '_[PF]a[si][sl]ed' >$TMP_BC_RESULT
      diff $TMP_RESULT $TMP_BC_RESULT    2>>$BCMISSINGLIST |
           grep -v "Create_objsign_cert_.signtool_-G.*Passed" |
           grep -v "porting_Alice.s_email_cert" |
           grep -v "^[0-9,cad]*$" | grep -v "^---$" | grep -v "^---.$" |
           grep -v "Can.t_run_pk12util_tests_for_NSS_3.2"  >/dev/null && (
                echo "$w differs" >> $BCMISSINGLIST
                echo "========================================="
                echo "diff $w bct/$w"
                echo "========================================="
                diff $TMP_RESULT $TMP_BC_RESULT 2>&1 |
                     grep -v "Create_objsign_cert_.signtool_-G.*Passed" |
                     grep -v "porting_Alice.s_email_cert" |
                     grep -v "Can.t_run_pk12util_tests_for_NSS_3.2"  
           )  2>&1 >>$BCERRORLIST

      #diff -b $w bct/$w  2>>$BCMISSINGLIST | 
           #grep -v "Create objsign cert .signtool -G.*Passed" |
           #grep -v "Listing signed files in jar .signtool -v.*Passed" |
           #grep -v "Listing signed files in jar .signtool -w.*Passed" |
           #grep -v "backward compatibility" |
           #grep -v "Can.t run pk12util tests for NSS 3.2" |
           #grep -v "porting Alice.s email cert " |
           #grep -v "^---$" | grep -v "^[<> ] $" | 
           #grep -v "^---.$" | grep -v "^[<> ] .$" | 
           #grep -v '< </BODY></HTML>' |
           #grep -v "^[0-9,cad]*$" 2>>$BCMISSINGLIST >/dev/null &&  (
                #echo "$w differs" >> $BCMISSINGLIST
                #echo "========================================="
                #echo "diff $w bct/$w"
                #echo "========================================="
                #diff -b $w bct/$w 2>&1 | 
                     #grep -v "Listing signed files in jar .signtool -v.*Passed" |
                     #grep -v "Listing signed files in jar .signtool -w.*Passed" |
                     #grep -v "backward compatibility" |
                     #grep -v "Can.t run pk12util tests for NSS 3.2" |
                     #grep -v "porting Alice.s email cert " |
                     #grep -v "^---$" | grep -v "^[<> ] $" | 
                     #grep -v "^---.$" | grep -v "^[<> ] .$" | 
                     #grep -v '< </BODY></HTML>' |
                     #grep -v "^[0-9,cad]*$" \
           #)  2>&1 >>$BCERRORLIST
      rm $TMP_RESULT $TMP_BC_RESULT 2>/dev/null
  done
  rm $ERRORLIST
  cat $BCMISSINGLIST | sed -e "s/^diff: bc_...s.//" \
                    -e "s/.results.html.*/\/results.html/" | 
                sort -u > $ERRORLIST

  platformlist  
  echo '</table>' >>$TMP_HTML_FILE

  head -200 $BCERRORLIST | sed -e 's/<[^>]*>//g' -e "s/^/<br>/" 
}

bc_test()
{
CURRENT_TABLE="BC"   #so html_line can determine which fields to write
  
  qa_stat_table "Backward Compatibility Test"
  echo '<td NOSAVE><b><font size=+1>Build-OS and version</font></b></td>'
  echo '<td><b><font size=+1>QA-OS</font></b></td>'
  echo '<td><b><font size=+1>Systemname</font></b></td>'
  echo '<td><b><font size=+1>P/F</font></b></td>'
  #echo '<td><b><font size=+1>All Current</font></b></td>'
  #echo '<td><b><font size=+1>backward comp. test</font></b></td>'
  echo '<td><b><font size=+1>result</font></b></td>'
  echo '<td><b><font size=+1>output</font></b></td>'
  echo '<td><b><font size=+1>QA time / #</font></b></td>'
  echo '</tr>'

set -x
  for w in `ls */results.html`
  do
      BCT_DIR=`dirname "bct/$w"`
      BCT_RESULT="bct/$w"
      BCT_LOG="$BCT_DIR/output.log"
      grep "bgcolor=red" $BCT_RESULT | 
           sed -e 's/.results.html:<TR><TD>/ /' -e 's/<[^>]*>/ /g'
      grep 'cache hits; .* cache misses, .* cache not reusable' \
        $BCT_LOG |
        grep -v selfserv |
        grep -v '0 cache hits; 1 cache misses, 0 cache not reusable' |
        grep -v '0 cache hits; 0 cache misses, 0 cache not reusable' |
        grep -v ' cache hits; 1 cache misses, 0 cache not reusable'
      grep -vi "write to SSL socket" $BCT_LOG |
        grep -vi "HDX PR_Read returned error" |
        grep -vi "no error" |
        grep -vi "12285" |
        grep -i  $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP error
      grep  -vi "write to SSL socket" $BCT_LOG |
        grep -vi "peer cannot verify" |
        grep -vi "TCP Connection aborted" |
        grep -vi "error" |
        grep -vi "fatal" |
        grep -vi "TCP connection reset" |
        grep  $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP  -i failed $BCT_LOG
      grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "segmentation violation"  $BCT_LOG
      grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "memory fault" $BCT_LOG
      grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "bus error" $BCT_LOG
      grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP "core dumped" $BCT_LOG
      grep -i $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP fatal  $BCT_LOG
      grep -i  $BEFORE_CONTEXT_GREP $AFTER_CONTEXT_GREP -i "PKCS12 decode not verified"   $BCT_LOG
      find ${BTC_DIR} -name core -print

  done 2>&1 >>$BCERRORLIST
  rm $ERRORLIST
  cat $BCMISSINGLIST | sed -e "s/^diff: bc_...s.//" \
                    -e "s/.results.html.*/\/results.html/" | 
                sort -u > $ERRORLIST

  platformlist  
  echo '</table>' >>$TMP_HTML_FILE

  head -200 $BCERRORLIST | sed -e 's/<[^>]*>//g' -e "s/^/<br>/" 
}


############################### bc_test ########################
# local shell function, evaluates the results of the backward u
# compatibility tests
# move the whole function to old to tests a new solution
########################################################################
bc_test_old()
{
CURRENT_TABLE="BC"   #so html_line can determine which fields to write
  
  qa_stat_table "Backward Compatibility Test"
  echo '<td NOSAVE><b><font size=+1>Build-OS and version</font></b></td>'
  echo '<td><b><font size=+1>QA-OS</font></b></td>'
  echo '<td><b><font size=+1>Systemname</font></b></td>'
  echo '<td><b><font size=+1>P/F</font></b></td>'
  #echo '<td><b><font size=+1>All Current</font></b></td>'
  #echo '<td><b><font size=+1>backward comp. test</font></b></td>'
  echo '<td><b><font size=+1>result</font></b></td>'
  echo '<td><b><font size=+1>output</font></b></td>'
  echo '<td><b><font size=+1>QA time / #</font></b></td>'
  echo '</tr>'

  for w in `ls */results.html`
  do
      diff -b $w bct/$w  2>>$BCMISSINGLIST | 
           grep -v "Create objsign cert .signtool -G.*Passed" |
           grep -v "Listing signed files in jar .signtool -v.*Passed" |
           grep -v "Listing signed files in jar .signtool -w.*Passed" |
           grep -v "backward compatibility" |
           grep -v "Can.t run pk12util tests for NSS 3.2" |
           grep -v "porting Alice.s email cert " |
           grep -v "^---$" | grep -v "^[<> ] $" | 
           grep -v "^---.$" | grep -v "^[<> ] .$" | 
           grep -v '< </BODY></HTML>' |
           grep -v "^[0-9,cad]*$" 2>>$BCMISSINGLIST >/dev/null &&  (
                echo "$w differs" >> $BCMISSINGLIST
                echo "========================================="
                echo "diff $w bct/$w"
                echo "========================================="
                diff -b $w bct/$w 2>&1 | 
                     grep -v "Listing signed files in jar .signtool -v.*Passed" |
                     grep -v "Listing signed files in jar .signtool -w.*Passed" |
                     grep -v "backward compatibility" |
                     grep -v "Can.t run pk12util tests for NSS 3.2" |
                     grep -v "porting Alice.s email cert " |
                     grep -v "^---$" | grep -v "^[<> ] $" | 
                     grep -v "^---.$" | grep -v "^[<> ] .$" | 
                     grep -v '< </BODY></HTML>' |
                     grep -v "^[0-9,cad]*$" \
           )  2>&1 >>$BCERRORLIST
  done
  rm $ERRORLIST
  cat $BCMISSINGLIST | sed -e "s/^diff: bc_...s.//" \
                    -e "s/.results.html.*/\/results.html/" | 
                sort -u > $ERRORLIST

  platformlist  
  echo '</table>' >>$TMP_HTML_FILE

  head -200 $BCERRORLIST | sed -e 's/<[^>]*>//g' -e "s/^/<br>/" 

}

############################### tbx_main ########################
# local shell function, tinderbox variation of the qa status script
########################################################################
tbx_main()
{
    TBX_EXIT=47
    qa_stat_get_sysinfo # find out the OS we are running and all required tests
                        # on this OS
    
    MACHINES_TO_CHECK=$HOST  #`uname -n` only search the local tests for errors
    qa_errorlist > $ERRORLIST        # 
    platformlist 
    #tbx_missing_platforms  #temp. taken out until we find a better way to
    #determine if all necessary QA ran - right now we run different 
    #tinderboxes on one machine
    incomplete_results 
    echo '</table>' >>$TMP_HTML_FILE
    echo '<a NAME="errorlist"></a>' >> $TMP_HTML_FILE
    cat $ERRORLIST  | sed -e "s/^/<br>/" >>$TMP_HTML_FILE

}

############################### qa_stat_main ########################
# local shell function, main flow of the qa status script
########################################################################
qa_stat_main()
{
    find_qa_systems 2>/dev/null
    MACHINES_TO_CHECK=""   # check all founf qa runs
    qa_errorlist > $ERRORLIST
    platformlist 
    missing_platforms 
    incomplete_results 
    echo '</table>' >>$TMP_HTML_FILE
    echo '<a NAME="errorlist"></a>' >> $TMP_HTML_FILE
    cat $ERRORLIST  | sed -e "s/^/<br>/" >>$TMP_HTML_FILE
    cat $WARNINGLIST 2>/dev/null | sed -e "s/^/<br>/" >>$TMP_HTML_FILE 2>/dev/null
    rsaperf >>$TMP_HTML_FILE
    bc_header >>$TMP_HTML_FILE
    MACHINES_TO_CHECK="bct/"
    TOTAL_TESTS=$BCT_TOTAL_TESTS
    BEFORE_CONTEXT_GREP="" #WORKAROUND - errors in one outputlog within the first 
    AFTER_CONTEXT_GREP=""  # or last lines will show up in the next/previos file
    qa_errorlist > $ERRORLIST
    platformlist 
    missing_platforms 
    incomplete_results 
    echo '</table>' >>$TMP_HTML_FILE
    echo '<a NAME="errorlist"></a>' >> $TMP_HTML_FILE
    cat $ERRORLIST  | sed -e "s/^/<br>/" >>$TMP_HTML_FILE
    cat $WARNINGLIST 2>/dev/null | sed -e "s/^/<br>/" >>$TMP_HTML_FILE 2>/dev/null
    #bc_test >>$TMP_HTML_FILE
}

CURRENT_TABLE="Standard"
qa_stat_init

if [  "$O_TBX" = "ON" -o "$O_LOCAL" = "ON" ] ; then
    tbx_main
else
    qa_stat_main
fi

qa_stat_cleanup