gecko/modules/oji/tests/script/autorun.pl

741 lines
21 KiB
Prolog
Executable File

#!/usr/bin/perl
#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Sun Microsystems, Inc.
# Portions created by the Initial Developer are Copyright (C) 1999
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****
# Created By: Raju Pallath
# Creation Date: Aug 2nd 1999
#
# Ported by: Konstantin S. Ermakov
# Ported date: Aug 24th 1999
#
# This script is used to invoke all test case for OJI API
# through apprunner and to recored their results
#
#
# Attach Perl Libraries
###################################################################
use POSIX ":sys_wait_h"; #used by waitpid
use Cwd;
use File::Copy;
#use Win32::Process;
#use IO::Handle;
###################################################################
########################
# #
# User's settings #
# #
########################
#Mozilla's executable
$MOZILLA_EXECUTABLE="mozilla-bin";
if ($^O =~ /Win32/i) {
$MOZILLA_EXECUTABLE="mozilla.exe";
}
#sometimes we need to specify additional parameters for mozilla
$ADDITIONAL_PARAMETERS="-P default";
# time in seconds after which the apprunner has to be killed.
# by default the apprunner will be up for so much time regardless of
# whether over or not. User can either decrease it or increase it.
#
$DELAY_FACTOR = 50;
$FULL_TEST_LIST="OJITestsList.lst";
$TEST_RESULTS="OJITestResults.txt";
$TEST_DESCRIPTIONS="../OJIAPITestsDescr.html";
$TARGET_WINDOW="Test Descriptions";
$curdir = cwd();
$DEFAULT_TEST_URL="file:///$curdir";
# time period in seconds of periodically checking: is the apprunner still alive
$DELAY_OF_CYCLE = 1;
##############################################################
##############################################################
##############################################################
$DOCROOT = $ENV{"TEST_URL"} ? $ENV{"TEST_URL"} : $DEFAULT_TEST_URL;
# delimiter for logfile
$delimiter="###################################################\n";
$nosort=0;
##################################################################
# Usage
##################################################################
sub usage() {
print "\n";
print "##################################################################\n";
print " perl autorun.pl [ -help ] [ -genhtml [ ALL/PASS/FAILED/<test_type> ] ] [ -compare [ <output_file> <file1> <file2> ] ] [ -t <test_case> ]\n";
print "\n";
print " -help This message.\n\n";
print " -compare <output_file> <file1> <file2> \n";
print " Compares two test result files and generates the diff files\n";
print " <output_file> the resulting diff files in text (.txt)\n";
print " and HTML (.html) formats\n";
print " <file1(2)> files to be compared (optional parameters)\n";
print " should contain test results in text format\n";
print " (generated by autorun.pl as BWTest.txt);\n";
print " if ommited the default values are used: \n";
print " <file1> => log/BWTest.txt \n";
print " <file2> => log/BWTest.txt.bak \n\n";
print " -genhtml <option> Generates report in HTML format (without tests execution).\n";
print " ALL Generates report for all executed tests. \n";
print " PASS Generates report for passed tests\n";
print " FAILED Generates report for failed tests\n";
print " <test_type> Generates report for the specific <test_type>\n";
print " <test_type> = JNIEnv/LCM/ThreadManager/JVMManager\n";
print " If specified all other parameters are ignored\n\n";
print " -t <test case> Executes the specified <test_case>.\n";
print " test_case example: JVMManager_CreateProxyJNI_1\n";
print "\n";
print "##################################################################\n";
print "\n";
}
##################################################################
# Title display
##################################################################
sub title() {
print "\n";
print "################################################\n";
print " Automated Execution of OJI API TestSuite\n";
print "################################################\n";
print "\n\n";
}
##################################################################
#
# check which tests to run. XML/HTML or both
#
##################################################################
sub checkRun() {
$runtype = "0";
while( true ) {
print "Run 1) HTML Test suite.\n";
print " 2) XML Test suite.\n";
print " 3) BOTH HTML and XML.\n";
print "Enter choice (1-3) :\n";
$runtype = getc;
if(( $runtype ne "1" ) &&
( $runtype ne "2" ) &&
( $runtype ne "3" ) )
{
print "Invaid choice. Range is from 1-3...\n";
print "\n";
next;
} else {
last;
}
}
}
#########################################################################
#
# Append table entries to Output HTML File
#
#########################################################################
sub appendEntries() {
print LOGHTML "<tr><td></td><td></td></tr><td></td>\n";
print LOGHTML "<tr><td></td><td></td></tr><td></td>\n";
print LOGHTML "<tr bgcolor=\"#FF6666\"><td>Test Status (XML)</td><td>Result</td><td>Comment</td></tr>\n";
}
#########################################################################
#
# Construct Output HTML file Header
#
#########################################################################
sub constructHTMLHeader() {
print LOGHTML "<html><head><title>\n";
print LOGHTML "OJI API Test Status\n";
print LOGHTML "</title></head><body bgcolor=\"white\">\n";
print LOGHTML "<center><h1>\n";
print LOGHTML "OJI API Automated TestRun Results\n";
$date = localtime;
print LOGHTML "</h1><h2>", $date;
print LOGHTML "</h2></center>\n";
print LOGHTML "<hr noshade>";
print LOGHTML "<table bgcolor=\"lightgreen\" border=1 cellpadding=10>\n";
print LOGHTML "<tr bgcolor=\"lightblue\">\n";
print LOGHTML "<td>Test Case</td>\n";
print LOGHTML "<td>Result</td>\n";
print LOGHTML "<td>Comment</td></tr>\n";
}
#########################################################################
#
# Construct Output HTML file indicating status of each run
#
#########################################################################
sub constructHTMLBody() {
open( MYLOG, $LOGTXT ) || print("WARNING: can't open log file $LOGTXT: $!\n");
@all_lines = <MYLOG>;
close( MYLOG );
my $line;
my $f_cnt = 0;
my $p_cnt = 0;
foreach $line ( ($nosort) ? @all_lines : sort @all_lines ) {
# avoid linebreaks
chop $line;
if ($line eq "" || $line =~ /^\0*$/ || $line =~ /^\s*$/) {
next;
}
$comment = "---";
# assuming that all lines are kind'of 'aaa=bbb'
($class, $status) = split /\: /, $line;
if ($status =~ /(.*?) \((.*?)\)$/) {
$status = $1;
$comment = $2 ? $2 : "---";
}
if ($status =~ /FAIL/) {
$status = "<font color=\"red\">".$status;
$status = $status."<\/font>";
$f_cnt++;
} elsif ($status =~ /PASS/) {
$p_cnt++;
}
if ($genparam eq "ALL" || ($genparam ne "" && $line =~ /$genparam/)) {
print LOGHTML "<tr><td><a target=\"$TARGET_WINDOW\" href=\"$TEST_DESCRIPTIONS#$class\">",$class,"</a></td><td>",$status,"</td><td>",$comment,"</td></tr>\n";
}
}
if ($p_cnt+$f_cnt) {
$pp = sprintf "%.2f", $p_cnt/($p_cnt+$f_cnt)*100;
$pf = sprintf "%.2f", $f_cnt/($p_cnt+$f_cnt)*100;
}
print LOGHTML "<tr><td colspan=3>Total: $p_cnt($pp\%) tests passed and $f_cnt($pf\%) tests failed.</td></tr>\n";
}
#########################################################################
#
# Construct Output HTML file Footer
#
#########################################################################
sub constructHTMLFooter() {
print LOGHTML "</table></body></html>\n";
}
#########################################################################
#
# Construct Output HTML file indicating status of each run
#
#########################################################################
sub constructHTML() {
constructHTMLHeader();
constructHTMLBody();
constructHTMLFooter();
}
#########################################################################
#
# Construct LogFile Header. The Log file is always appended with entries
#
#########################################################################
sub constructLogHeader() {
print LOGFILE "\n";
print LOGFILE "\n";
print LOGFILE $delimiter;
$date = localtime;
print LOGFILE "Logging Test Run on $date ...\n";
print LOGFILE $delimiter;
print LOGFILE "\n";
print "All Log Entries are maintained in LogFile $LOGFILE\n";
print "\n";
}
#########################################################################
#
# Construct LogFile Footer.
#
#########################################################################
sub constructLogFooter() {
print "\n";
print LOGFILE $delimiter;
$date = localtime;
print LOGFILE "End of Logging Test $date ...\n";
print LOGFILE $delimiter;
print "\n";
}
########################################################################
#
# Construct Log String
#
########################################################################
sub constructLogString {
my $logstring = shift(@_);
print LOGFILE "$logstring\n";
print "$logstring\n";
}
########################################################################
#
# Safely append to file : open, append, close.
#
########################################################################
sub safeAppend {
my $file = shift(@_);
my $line = shift(@_);
open (FILE, ">>$file") or die ("Cann't open $file");
print FILE $line;
close FILE;
}
#######################################################################
#
# Running Test case under Win32
#
#######################################################################
sub RunTestCaseWin {
# win32 specific values
# STILL_ALIVE is a constant defined in winbase.h and indicates what
# process still alive
$STILL_ALIVE = 0x103;
do 'Win32/Process.pm';
open(SAVEOUT, ">&STDOUT" );
open(SAVEERR, ">&STDERR" );
open(STDOUT, ">$testlog" ) || die "Can't redirect stdout";
open(STDERR, ">&STDOUT" ) || die "Can't dup STDOUT";
select STDERR; $|=1; select STDOUT; $|=1;
Win32::Process::Create($ProcessObj,
"$mozhome/$MOZILLA_EXECUTABLE",
"$mozhome/$MOZILLA_EXECUTABLE $ADDITIONAL_PARAMETERS $DOCFILE",
1,
NORMAL_PRIORITY_CLASS,
"$mozhome" ) || die "cann't start apprunner";
close(STDOUT);
close(STDERR);
open(STDOUT, ">&SAVEOUT");
open(STDERR, ">&SAVEERR");
$crashed = 0;
$cnt = 0;
while (true) {
sleep($DELAY_OF_CYCLE);
system("$curdir\\killer.exe");
$ProcessObj->GetExitCode($exit_code);
if ($exit_code != $STILL_ALIVE ) {
$crashed = 1;
$logstr = "Test FAILED...";
constructLogString "$logstr";
$logstr = "Mozilla terminated with exit code $exit_code.";
constructLogString "$logstr";
$logstr = "Check ErrorLog File $testlog ";
constructLogString "$logstr";
constructLogString "========================================\n";
safeAppend $LOGTXT, "$testcase: FAILED (Mozilla crashed with exitcode $exit_code)\n";
last;
}
$cnt += $DELAY_OF_CYCLE;
if ( $cnt >= $DELAY_FACTOR ) {
$ProcessObj->Kill(0);
last;
}
} # while with sleep
$crashed;
}
#######################################################################
#
# Running Test case under Unix
#
#######################################################################
sub RunTestCaseUnix {
$exit_code = 0;
$pid = 0;
open(SAVEOUT, ">&STDOUT" );
open(SAVEERR, ">&STDERR" );
open(STDOUT, ">$testlog" ) || die "Can't redirect stdout";
open(STDERR, ">&STDOUT" ) || die "Can't dup STDOUT";
select STDERR; $|=1; select STDOUT; $|=1;
if (!($pid = fork())) {
#this is a child process - we should start mozilla in it
# the second exec to overide any shell parsing
# If argument list had any metacharacter (like ~) then it is pass
# to /bin/sh -c as a separate process. To avoid that we have the
# second exec.
exec(" exec $mozhome/$MOZILLA_EXECUTABLE $ADDITIONAL_PARAMETERS $DOCFILE");
close(STDOUT);
close(STDERR);
open(STDOUT, ">&SAVEOUT");
open(STDERR, ">&SAVEERR");
die "ERROR: Can't start mozilla: $!\n";
}
#this is a parent process - we should control mozilla execution from it
close(STDOUT);
close(STDERR);
open(STDOUT, ">&SAVEOUT");
open(STDERR, ">&SAVEERR");
$cnt = 0;
$crashed = 0;
print("Mozilla's pid: $pid\n");
while (true) {
sleep($DELAY_OF_CYCLE);
#print "Try call waitpid ...\n";
if (waitpid($pid,&WNOHANG)) {
$crashed = 1;
$logstr = "Test FAILED...";
constructLogString "$logstr";
$logstr = "Mozilla terminated by signal ".($? & 127)." with exitcode ".($? >> 8);
constructLogString "$logstr";
$logstr = "Check ErrorLog File $testlog ";
constructLogString "$logstr";
constructLogString "========================================\n";
safeAppend $LOGTXT, "$testcase: FAILED (Mozilla terminated by signal ".($? & 127)." with exitcode ".($? >> 8).")\n";
last;
}
$cnt += $DELAY_OF_CYCLE;
if ( $cnt >= $DELAY_FACTOR ) {
kill(9, $pid);
#now we should take exitcode to avoid zombie appearance
sleep(1);
waitpid($pid,&WNOHANG);
last;
}
} # while with sleep
$crashed;
}
sub compareResults {
$out_fn = shift;
$fn1 = shift;
$fn2 = shift;
if (!defined($out_fn)) {
$out_fn = "$curdir/log/diff";
}
if (!defined($fn2)) {
$fn2 = "$curdir/log/BWTest.txt.bak";
}
if (!defined($fn1)) {
$fn1 = "$curdir/log/BWTest.txt";
}
(-f $fn1 && open(IN1, "<$fn1")) || die "Can't open file $fn1: $!\n";
(-f $fn2 && open(IN2, "<$fn2")) || die "Can't open file $fn2: $!\n";
open(OUT_TXT, ">$out_fn".".txt") || die "Can't open output file $out_fn.txt: $!\n";
$genparam = "ALL";
$nosort = 1;
my %res1;
my %line1;
my %res2;
my $ln = 0;
my $j=0;
while($line = <IN1>) {
$ln ++;
chop $line;
if ($line =~ /(.*): (.*)/) {
$res1{$1} = $2;
$line1{$1} = $line;
} else {
die "File format error in file $fn1, line $ln: $line\n";
}
}
$ln = 0;
while($line = <IN2>) {
$ln ++;
chop $line;
if ($line =~ /(.*): (.*)/) {
if (defined($res1{$1})) {
if ($res1{$1} ne $2) {
print OUT_TXT $line1{$1}."\n";
print OUT_TXT $line."\n";
}
delete $res1{$1};
} else {
$new_in_2[$j++] = $line;
#file2 contain extra tests (that file1 doesn't contain)
#file1 can also contain extra tests (that file2 doen't contain)
#do we need to report about it ?
}
} else {
die "File format error in file $fn2, line $ln: $line\n";
}
}
#report about extra tests in file1
@remained_keys = keys %res1;
if ($#remained_keys >= 0) {
print OUT_TXT " : EXTRA (TESTS IN $fn1:)\n";
foreach $key (@remained_keys) {
print OUT_TXT "$line1{$key}\n";
}
}
#report about extra tests in file2
if ($j > 0) {
print OUT_TXT " : EXTRA (TESTS IN $fn2:)\n";
for($i=0; $i<$j; $i++) {
print OUT_TXT "$new_in_2[$i]\n";
}
}
if (<OUT_TXT>) {
close OUT_TXT;
$LOGTXT = "$out_fn".".txt";
open(LOGHTML, ">$out_fn.html") || die "Can't open output file $out_fn.html: $!\n";
constructHTML();
close LOGHTML;
} else {
close OUT_TXT;
print "Results are equivalent !\n";
}
close IN1;
close IN2;
}
##################################################################
# main
##################################################################
title;
$curdir = cwd();
# Prepare file names
$x=`mkdir -p $curdir/log`;
$LOGFILE = "$curdir/log/BWTestRun.log";
$LOGTXT = "$curdir/log/BWTest.txt";
$LOGHTML = "$curdir/log/BWTest.html";
# process command-line parameters
# and check for valid usage
$testparam = "";
$genparam = "";
if ($ARGV[0] =~ /help/ || $ARGV[0] =~ /\?/) {
usage();
exit(0);
}
if ($ARGV[0] =~ "-genhtml") {
$genparam = $ARGV[1];
if ($genparam eq "") {
$genparam = "ALL";
}
open( LOGHTML, ">$LOGHTML" ) or die("Can't open HTML file $LOGHTML ($!)...\n");
print("Generating result HTML page ...\n");
constructHTML();
exit(0);
}
if ($ARGV[0] eq "-compare") {
compareResults($ARGV[1], $ARGV[2], $ARGV[3]);
exit(0);
}
if ( $#ARGV >= 0 ) {
if ( $ARGV[0] != "-t" ) {
usage;
die;
} elsif ($ARGV[1] ne "") {
$testparam = $ARGV[1];
} else {
usage();
die;
}
}
## Default behaviour is to generate the HTML file
if ($genparam eq "") {
$genparam = "ALL";
}
$mozhome = $ENV{"MOZILLA_FIVE_HOME"};
if ( $mozhome eq "" ) {
print "MOZILLA_FIVE_HOME is not set. Please set it and rerun this script....\n";
die;
}
if ( ! -f "$mozhome/$MOZILLA_EXECUTABLE" ) {
print "Could not find $MOZILLA_EXECUTABLE in MOZILLA_FIVE_HOME.\n";
print "Please check your setting...\n";
die;
}
# Here must come a piece of code, that determinates
# apprunner instance, removes core, but there's no
# core under win32
# Backup existing .lst file
if ( ! -f "$curdir/$FULL_TEST_LIST" ) {
print "Can't find list of OJI API tests to be run.";
die;
}
$id=$$;
# check if output text file of previous run exists.
# if so, then save it as .bak
#
if ( -f "$LOGTXT" ) {
$newfile="$LOGTXT.bak";
rename $LOGTXT, $newfile;
}
# check if output html file of previous run exists.
# if so, then save it as .bak
#
if ( -f "$LOGHTML" ) {
$newfile="$LOGHTML.bak";
rename $LOGHTML, $newfile;
}
# construct DOCFILE
$DOCFILE = "$DOCROOT/test.html";
$runcnt = 1;
$filename = "$curdir/$FULL_TEST_LIST";
# Prepare log streams
open( LOGHTML, ">$LOGHTML" ) or die("Can't open HTML file...\n");
open( LOGFILE, ">>$LOGFILE" ) or die("Can't open LOG file...\n");
select LOGFILE; $| = 1; select STDOUT;
constructLogHeader();
$LST_OUT = "$mozhome/OJITests.lst";
$currcnt = 0;
while (true) {
open( file, $filename ) or die("Can't open $filename...\n");
while( $line = <file> ) {
chop $line;
if ( $testparam ne "" ) {
$testcase = $testparam;
} else {
$testcase = $line;
}
if ( $testcase eq "" || $testcase =~ /^\s*#/ ) {
next;
}
open(LST_OUT, ">$LST_OUT") or die ("Can't open LST_OUT file...\n");
print LST_OUT $testcase;
close(LST_OUT);
chdir( $mozhome );
#deleting old result ...
unlink("$mozhome/$TEST_RESULTS");
$logstr="Running TestCase $testcase....";
constructLogString "========================================";
constructLogString "$logstr";
($nom) = ($testcase =~ /([^\.]*)$/);
$testlog = "$curdir/log/$nom.$id.log";
if ($^O =~ /Win32/i) {
$crashed = RunTestCaseWin();
} else {
$crashed = RunTestCaseUnix();
}
if (!$crashed) {
if (!open (TEST_RES, "$mozhome/$TEST_RESULTS") || ($logstr = <TEST_RES>) =~ /^\s*$/) {
$logstr = "$testcase: FAILED (undefined result)";
}
close TEST_RES;
chomp $logstr;
constructLogString "$logstr";
constructLogString "========================================\n";
safeAppend "$LOGTXT", "$logstr\n";
}
( $testparam eq "" ) || last;
} # while ( $line
( ++$currcnt < $runcnt ) || last;
} # while(true)
constructLogFooter;
constructHTML();
if (-f "$curdir/log/BWTest.txt.bak") {
print("Old result file found => generating diff files: \nTXT:->$curdir/log/diff.txt \nHTML:->$curdir/log/diff.html");
compareResults();
}
chdir($curdir);