2007-03-22 10:30:00 -07:00
#
# Updates step. Generates binary update (MAR) files as well as AUS config
# snippets.
#
package Bootstrap::Step::Updates ;
2007-07-12 10:54:14 -07:00
2007-03-22 10:30:00 -07:00
use Bootstrap::Step ;
use Bootstrap::Config ;
2007-10-11 10:28:33 -07:00
use Bootstrap::Util qw( CvsCatfile SyncNightlyDirToStaging GetLocaleManifest ) ;
2007-07-12 10:54:14 -07:00
2007-03-22 10:30:00 -07:00
use File::Find qw( find ) ;
2007-07-12 10:54:14 -07:00
use POSIX qw( strftime ) ;
2007-03-22 10:30:00 -07:00
use MozBuild::Util qw( MkdirWithPath ) ;
2007-07-12 10:54:14 -07:00
2007-03-22 10:30:00 -07:00
@ ISA = ( "Bootstrap::Step" ) ;
sub Execute {
my $ this = shift ;
my $ config = new Bootstrap:: Config ( ) ;
my $ product = $ config - > Get ( var = > 'product' ) ;
2007-08-10 16:30:01 -07:00
my $ logDir = $ config - > Get ( sysvar = > 'logDir' ) ;
2007-04-30 13:55:21 -07:00
my $ oldVersion = $ config - > Get ( var = > 'oldVersion' ) ;
2007-03-22 10:30:00 -07:00
my $ version = $ config - > Get ( var = > 'version' ) ;
my $ mozillaCvsroot = $ config - > Get ( var = > 'mozillaCvsroot' ) ;
my $ mofoCvsroot = $ config - > Get ( var = > 'mofoCvsroot' ) ;
my $ updateDir = $ config - > Get ( var = > 'updateDir' ) ;
my $ patcherConfig = $ config - > Get ( var = > 'patcherConfig' ) ;
my $ patcherToolsRev = $ config - > Get ( var = > 'patcherToolsRev' ) ;
my $ versionedUpdateDir = catfile ( $ updateDir , $ product . '-' . $ version ) ;
# Create updates area.
if ( not - d $ versionedUpdateDir ) {
MkdirWithPath ( dir = > $ versionedUpdateDir )
or die ( "Cannot mkdir $versionedUpdateDir: $!" ) ;
}
# check out patcher
$ this - > Shell (
cmd = > 'cvs' ,
cmdArgs = > [ '-d' , $ mozillaCvsroot , 'co' , '-d' , 'patcher' ,
2007-04-30 13:55:21 -07:00
CvsCatfile ( 'mozilla' , 'tools' , 'patcher' ) ] ,
2007-03-22 10:30:00 -07:00
logFile = > catfile ( $ logDir , 'updates_patcher-checkout.log' ) ,
dir = > $ versionedUpdateDir ,
) ;
# check out utilities
$ this - > Shell (
cmd = > 'cvs' ,
cmdArgs = > [ '-d' , $ mozillaCvsroot , 'co' , '-d' , 'MozBuild' ,
2007-04-30 13:55:21 -07:00
CvsCatfile ( 'mozilla' , 'tools' , 'release' , 'MozBuild' ) ] ,
2007-03-22 10:30:00 -07:00
logFile = > catfile ( $ logDir , 'updates_patcher-utils-checkout.log' ) ,
dir = > catfile ( $ versionedUpdateDir , 'patcher' ) ,
) ;
# config lives in private repo
$ this - > Shell (
cmd = > 'cvs' ,
cmdArgs = > [ '-d' , $ mofoCvsroot , 'co' , '-d' , 'config' ,
2007-04-30 13:55:21 -07:00
CvsCatfile ( 'release' , 'patcher' , $ patcherConfig ) ] ,
2007-03-22 10:30:00 -07:00
logFile = > catfile ( $ logDir , 'updates_patcher-config-checkout.log' ) ,
dir = > $ versionedUpdateDir ,
) ;
# build tools
my $ originalCvsrootEnv = $ ENV { 'CVSROOT' } ;
$ ENV { 'CVSROOT' } = $ mozillaCvsroot ;
$ this - > Shell (
cmd = > './patcher2.pl' ,
cmdArgs = > [ '--build-tools' , '--tools-revision=' . $ patcherToolsRev ,
'--app=' . $ product ,
'--config=../config/' . $ patcherConfig ] ,
logFile = > catfile ( $ logDir , 'updates_patcher-build-tools.log' ) ,
dir = > catfile ( $ versionedUpdateDir , 'patcher' ) ,
) ;
if ( $ originalCvsrootEnv ) {
$ ENV { 'CVSROOT' } = $ originalCvsrootEnv ;
}
# download complete MARs
$ this - > Shell (
cmd = > './patcher2.pl' ,
cmdArgs = > [ '--download' , '--app=' . $ product ,
'--config=../config/' . $ patcherConfig ] ,
logFile = > catfile ( $ logDir , 'updates_patcher-download.log' ) ,
dir = > catfile ( $ versionedUpdateDir , 'patcher' ) ,
) ;
# Create partial patches and snippets
$ this - > Shell (
cmd = > './patcher2.pl' ,
cmdArgs = > [ '--create-patches' , '--app=' . $ product ,
'--config=../config/' . $ patcherConfig ] ,
logFile = > catfile ( $ logDir , 'updates_patcher-create-patches.log' ) ,
dir = > catfile ( $ versionedUpdateDir , 'patcher' ) ,
timeout = > 18000 ,
) ;
### quick verification
2007-07-12 10:54:14 -07:00
my $ fullUpdateDir = catfile ( $ versionedUpdateDir , 'patcher' , 'temp' ,
$ product , $ oldVersion . '-' . $ version ) ;
$ snippetErrors = undef ; # evil (??) global to get results from callbacks
# ensure that there are only test channels in aus2.test dir
File::Find:: find ( \ & TestAusCallback , catfile ( $ fullUpdateDir , "aus2.test" ) ) ;
2007-03-22 10:30:00 -07:00
2007-07-12 10:54:14 -07:00
# ensure that there are only beta channels in beta dir (if that exists)
if ( - d catfile ( $ fullUpdateDir , "aus2.beta" ) ) {
File::Find:: find ( \ & BetaAusCallback , catfile ( $ fullUpdateDir , "aus2.beta" ) ) ;
File::Find:: find ( \ & ReleaseAusCallback , catfile ( $ fullUpdateDir , "aus2" ) ) ;
}
# otherwise allow beta and release in aus2 dir
else {
File::Find:: find ( \ & ReleaseBetaAusCallback , catfile ( $ fullUpdateDir , "aus2" ) ) ;
}
if ( $ snippetErrors ) {
$ snippetErrors =~ s!$fullUpdateDir/!!g ;
die ( "Execute: Snippets failed location checks: $snippetErrors\n" ) ;
}
2007-03-22 10:30:00 -07:00
}
sub Verify {
my $ this = shift ;
my $ config = new Bootstrap:: Config ( ) ;
2007-08-10 16:30:01 -07:00
my $ logDir = $ config - > Get ( sysvar = > 'logDir' ) ;
2007-03-22 10:30:00 -07:00
my $ version = $ config - > Get ( var = > 'version' ) ;
my $ mozillaCvsroot = $ config - > Get ( var = > 'mozillaCvsroot' ) ;
my $ verifyDir = $ config - > Get ( var = > 'verifyDir' ) ;
my $ product = $ config - > Get ( var = > 'product' ) ;
2007-04-05 11:13:42 -07:00
my $ verifyConfig = $ config - > Get ( sysvar = > 'verifyConfig' ) ;
2007-03-22 10:30:00 -07:00
# Create verification area.
my $ verifyDirVersion = catfile ( $ verifyDir , $ product . '-' . $ version ) ;
MkdirWithPath ( dir = > $ verifyDirVersion )
or die ( "Could not mkdir $verifyDirVersion: $!" ) ;
foreach my $ dir ( 'updates' , 'common' ) {
$ this - > Shell (
cmd = > 'cvs' ,
cmdArgs = > [ '-d' , $ mozillaCvsroot , 'co' , '-d' , $ dir ,
2007-04-30 13:55:21 -07:00
CvsCatfile ( 'mozilla' , 'testing' , 'release' , $ dir ) ] ,
2007-03-22 10:30:00 -07:00
logFile = > catfile ( $ logDir ,
'updates_verify_checkout-' . $ dir . '.log' ) ,
dir = > $ verifyDirVersion ,
) ;
}
2007-10-11 10:28:33 -07:00
$ this - > BumpVerifyConfig ( ) ;
2007-03-22 10:30:00 -07:00
# Customize updates.cfg to contain the channels you are interested in
# testing.
my $ verifyLog = catfile ( $ logDir , 'updates_verify.log' ) ;
$ this - > Shell (
cmd = > './verify.sh' ,
cmdArgs = > [ '-c' , $ verifyConfig ] ,
logFile = > $ verifyLog ,
dir = > catfile ( $ verifyDirVersion , 'updates' ) ,
timeout = > 36000 ,
) ;
$ this - > CheckLog (
log = > $ verifyLog ,
notAllowed = > '^FAIL' ,
) ;
}
2007-07-12 10:54:14 -07:00
# locate snippets for which the channel doesn't end in test
2007-03-22 10:30:00 -07:00
sub TestAusCallback {
my $ dir = $ File:: Find:: name ;
2007-07-12 10:54:14 -07:00
if ( ( $ dir =~ /\.txt/ ) and
( not $ dir =~ /\/\w*test\/(partial|complete)\.txt$/ ) ) {
$ snippetErrors . = "\nNon-test: $dir" ;
}
}
# locate snippets for which the channel isn't beta
sub BetaAusCallback {
my $ dir = $ File:: Find:: name ;
if ( ( $ dir =~ /\.txt/ ) and
( not $ dir =~ /\/beta\/(partial|complete)\.txt$/ ) ) {
$ snippetErrors . = "\nNon-beta: $dir" ;
}
}
# locate snippets for which the channel isn't release
sub ReleaseAusCallback {
my $ dir = $ File:: Find:: name ;
if ( ( $ dir =~ /\.txt/ ) and
( not $ dir =~ /\/release\/(partial|complete)\.txt$/ ) ) {
$ snippetErrors . = "\nNon-release: $dir" ;
}
}
# locate snippets for which the channel isn't release or beta
sub ReleaseBetaAusCallback {
my $ dir = $ File:: Find:: name ;
if ( ( $ dir =~ /\.txt/ ) and
( not $ dir =~ /\/(release|beta)\/(partial|complete)\.txt$/ ) ) {
$ snippetErrors . = "\nNon-release: $dir" ;
}
}
2007-10-08 13:34:45 -07:00
sub PermissionsAusCallback {
my $ dir = $ File:: Find:: name ;
if ( - f $ dir ) {
chmod ( 0644 , $ dir ) or die ( "Couldn't chmod $dir to 644: $!" ) ;
} elsif ( - d $ dir ) {
chmod ( 0775 , $ dir ) or die ( "Couldn't chmod $dir to 775: $!" ) ;
}
}
2007-10-11 10:28:33 -07:00
sub BumpVerifyConfig {
my $ this = shift ;
my $ config = new Bootstrap:: Config ( ) ;
my $ osname = $ config - > SystemInfo ( var = > 'osname' ) ;
my $ product = $ config - > Get ( var = > 'product' ) ;
my $ oldVersion = $ config - > Get ( var = > 'oldVersion' ) ;
my $ version = $ config - > Get ( var = > 'version' ) ;
my $ rc = $ config - > Get ( var = > 'rc' ) ;
my $ appName = $ config - > Get ( var = > 'appName' ) ;
my $ mozillaCvsroot = $ config - > Get ( var = > 'mozillaCvsroot' ) ;
my $ verifyDir = $ config - > Get ( var = > 'verifyDir' ) ;
2007-10-12 09:26:35 -07:00
my $ ausServer = $ config - > Get ( var = > 'ausServer' ) ;
my $ externalStagingServer = $ config - > Get ( var = > 'externalStagingServer' ) ;
2007-10-11 10:28:33 -07:00
my $ verifyConfig = $ config - > Get ( sysvar = > 'verifyConfig' ) ;
my $ logDir = $ config - > Get ( sysvar = > 'logDir' ) ;
my $ verifyDirVersion = catfile ( $ verifyDir , $ product . '-' . $ version ) ;
my $ configFile = catfile ( $ verifyDirVersion , 'updates' , $ verifyConfig ) ;
# NOTE - channel is hardcoded to betatest
my $ channel = 'betatest' ;
# grab os-specific buildID file on FTP
my $ candidateDir = CvsCatfile ( $ config - > GetFtpNightlyDir ( ) , $ oldVersion . '-candidates' , 'rc' . $ rc ) . '/' ;
my $ buildID = $ this - > GetBuildIDFromFTP ( os = > $ osname , releaseDir = > $ candidateDir ) ;
my $ buildTarget = undef ;
my $ releaseFile = undef ;
my $ nightlyFile = undef ;
my $ ftpOsname = undef ;
if ( $ osname eq 'linux' ) {
$ buildTarget = 'Linux_x86-gcc3' ;
$ platform = 'linux' ;
$ ftpOsname = 'linux-i686' ;
$ releaseFile = $ product . '-' . $ oldVersion . '.tar.gz' ;
$ nightlyFile = $ product . '-' . $ version . '.%locale%.linux-i686.tar.gz' ;
} elsif ( $ osname eq 'macosx' ) {
$ buildTarget = 'Darwin_Universal-gcc3' ;
$ platform = 'osx' ;
$ ftpOsname = 'mac' ;
$ releaseFile = ucfirst ( $ product ) . ' ' . $ oldVersion . '.dmg' ;
$ nightlyFile = $ product . '-' . $ version . '.%locale%.' . $ osname . '.mac.dmg' ;
} elsif ( $ osname eq 'win32' ) {
$ buildTarget = 'WINNT_x86-msvc' ;
$ platform = 'win32' ;
$ ftpOsname = 'win32' ;
$ releaseFile = ucfirst ( $ product ) . ' Setup ' . $ oldVersion . '.exe' ;
$ nightlyFile = $ product . '-' . $ version . '.%locale%.' . $ osname . '.installer.exe' ;
} else {
die ( "ASSERT: unknown OS $osname" ) ;
}
open ( FILE , "< $configFile" ) or die ( "Could not open file $configFile: $!" ) ;
my @ origFile = <FILE> ;
close ( FILE ) or die ( "Could not close file $configFile: $!" ) ;
if ( $ origFile [ 0 ] =~ $ oldVersion ) {
$ this - > Log ( 'msg' = > "verifyConfig $configFile already bumped" ) ;
return ;
}
# remove "from" and "to" vars from @origFile
my @ strippedFile = ( ) ;
for ( my $ i = 0 ; $ i < scalar ( @ origFile ) ; $ i + + ) {
my $ line = $ origFile [ $ i ] ;
$ line =~ s/from.*$// ;
$ strippedFile [ $ i ] = $ line ;
}
my $ localeFileTag = uc ( $ product ) . '_' . $ oldVersion . '_RELEASE' ;
$ localeFileTag =~ s/\./_/g ;
my $ localeManifest = GetLocaleManifest ( app = > $ appName ,
cvsroot = > $ mozillaCvsroot ,
tag = > $ localeFileTag ) ;
my @ locales = ( ) ;
foreach my $ locale ( keys ( % { $ localeManifest } ) ) {
foreach my $ allowedPlatform ( @ { $ localeManifest - > { $ locale } } ) {
if ( $ allowedPlatform eq $ platform ) {
push ( @ locales , $ locale ) ;
}
}
}
# add data for latest release
my @ data = ( "# $oldVersion $osname\n" ,
2007-10-11 10:38:00 -07:00
'release="' . $ oldVersion . '" product="' . ucfirst ( $ product ) .
2007-10-11 10:28:33 -07:00
'" platform="' . $ buildTarget . '" build_id="' . $ buildID .
'" locales="' . join ( ' ' , sort ( @ locales ) ) . '" channel="' .
$ channel . '" from="/' . $ product . '/releases/' .
$ oldVersion . '/' . $ ftpOsname . '/%locale%/' . $ releaseFile .
2007-10-12 09:26:35 -07:00
'" aus_server="' . $ ausServer . '" ftp_server="' .
$ externalStagingServer . '" to="/' . $ product . '/nightly/' .
$ version . '-candidates/rc' . $ rc . '/' . $ nightlyFile . '"' .
"\n" ) ;
2007-10-11 10:28:33 -07:00
open ( FILE , "> $configFile" ) or die ( "Could not open file $configFile: $!" ) ;
print FILE @ data ;
print FILE @ strippedFile ;
close ( FILE ) or die ( "Could not close file $configFile: $!" ) ;
$ this - > Shell (
cmd = > 'cvs' ,
cmdArgs = > [ '-d' , $ mozillaCvsroot ,
'ci' , '-m' ,
'"Automated configuration bump, release for '
. $ product . ' ' . $ version . "rc$rc" . '"' ,
$ verifyConfig ] ,
logFile = > catfile ( $ logDir , 'update_verify-checkin.log' ) ,
dir = > catfile ( $ verifyDirVersion , 'updates' )
) ;
}
2007-07-12 10:54:14 -07:00
sub Push {
my $ this = shift ;
my $ config = new Bootstrap:: Config ( ) ;
2007-08-10 16:30:01 -07:00
my $ logDir = $ config - > Get ( sysvar = > 'logDir' ) ;
2007-07-12 10:54:14 -07:00
my $ product = $ config - > Get ( var = > 'product' ) ;
my $ version = $ config - > Get ( var = > 'version' ) ;
my $ rc = $ config - > Get ( var = > 'rc' ) ;
my $ oldVersion = $ config - > Get ( var = > 'oldVersion' ) ;
2007-08-20 17:32:16 -07:00
my $ stagingUser = $ config - > Get ( var = > 'stagingUser' ) ;
my $ stagingServer = $ config - > Get ( var = > 'stagingServer' ) ;
2007-07-12 10:54:14 -07:00
my $ ausUser = $ config - > Get ( var = > 'ausUser' ) ;
my $ ausServer = $ config - > Get ( var = > 'ausServer' ) ;
my $ updateDir = $ config - > Get ( var = > 'updateDir' ) ;
my $ pushLog = catfile ( $ logDir , 'updates_push.log' ) ;
my $ fullUpdateDir = catfile ( $ updateDir , $ product . '-' . $ version ,
'patcher' , 'temp' , $ product ,
$ oldVersion . '-' . $ version ) ;
my $ candidateDir = $ config - > GetFtpCandidateDir ( bitsUnsigned = > 0 ) ;
# push partial mar files up to ftp server
2007-10-08 13:34:45 -07:00
my $ marsDir = catfile ( 'ftp' , $ product , 'nightly' ,
$ version . '-candidates' , 'rc' . $ rc ) . '/' ;
chmod ( 0644 , glob ( catfile ( $ fullUpdateDir , $ marsDir , "*partial.mar" ) ) )
or die ( "Couldn't chmod a partial mar to 644: $!" ) ;
2007-07-12 10:54:14 -07:00
$ this - > Shell (
cmd = > 'rsync' ,
cmdArgs = > [ '-av' , '-e' , 'ssh' ,
'--include=*partial.mar' ,
'--exclude=*' ,
2007-10-08 13:34:45 -07:00
$ marsDir ,
2007-08-20 17:32:16 -07:00
$ stagingUser . '@' . $ stagingServer . ':' . $ candidateDir ] ,
2007-07-12 10:54:14 -07:00
dir = > $ fullUpdateDir ,
logFile = > $ pushLog ,
) ;
# push update snippets to AUS server
2007-10-12 12:51:06 -07:00
my $ pushDir = strftime ( "%Y%m%d" , localtime ) . '-' . ucfirst ( $ product ) .
'-' . $ version ;
2007-07-12 10:54:14 -07:00
my $ targetPrefix = CvsCatfile ( '/opt' , 'aus2' , 'snippets' , 'staging' ,
2007-10-12 12:51:06 -07:00
$ pushDir ) ;
2007-07-12 10:54:14 -07:00
$ config - > Set ( var = > 'ausDeliveryDir' , value = > $ targetPrefix ) ;
my @ snippetDirs = glob ( catfile ( $ fullUpdateDir , "aus2*" ) ) ;
2007-10-08 13:34:45 -07:00
File::Find:: find ( \ & PermissionsAusCallback , @ snippetDirs ) ;
2007-07-12 10:54:14 -07:00
foreach $ dir ( @ snippetDirs ) {
my $ targetDir = $ targetPrefix ;
if ( $ dir =~ /aus2\.(.*)$/ ) {
$ targetDir . = '-' . $ 1 ;
}
$ this - > Shell (
cmd = > 'rsync' ,
cmdArgs = > [ '-av' ,
'-e' , 'ssh -i ' . catfile ( $ ENV { 'HOME' } , '.ssh' , 'aus' ) ,
$ dir . '/' ,
$ ausUser . '@' . $ ausServer . ':' . $ targetDir ] ,
logFile = > $ pushLog ,
) ;
2007-03-22 10:30:00 -07:00
}
2007-10-01 20:00:18 -07:00
SyncNightlyDirToStaging ( ) ;
2007-10-12 12:51:06 -07:00
# Push test channels live
$ this - > Shell (
cmd = > 'ssh' ,
cmdArgs = > [ '-i ' . catfile ( $ ENV { 'HOME' } , '.ssh' , 'aus' ) ,
$ ausUser . '@' . $ ausServer ,
'/home/cltbld/bin/pushsnip' , $ pushDir . '-test' ] ,
logFile = > $ pushLog ,
) ;
2007-03-22 10:30:00 -07:00
}
sub Announce {
my $ this = shift ;
my $ config = new Bootstrap:: Config ( ) ;
my $ product = $ config - > Get ( var = > 'product' ) ;
my $ version = $ config - > Get ( var = > 'version' ) ;
2007-07-12 10:54:14 -07:00
my $ ausDeliveryDir = $ config - > Get ( var = > 'ausDeliveryDir' ) ;
2007-03-22 10:30:00 -07:00
$ this - > SendAnnouncement (
subject = > "$product $version update step finished" ,
2007-10-12 12:51:06 -07:00
message = > "$product $version updates finished. Partial mars were copied to the candidates dir, and the snippets in $ausDeliveryDir-test were pushed live." ,
2007-03-22 10:30:00 -07:00
) ;
}
1 ;