bug 531135 - rewrite build-list.pl in Python to fix race conditions with locking. r=bsmedberg

config/Makefile.in

@@ -163,6 +163,7 @@ PYUNITS := \
   unit-nsinstall.py \
   unit-printprereleasesuffix.py \
   unit-JarMaker.py \
+  unit-buildlist.py \
   $(NULL)
 
 check:: check-python-modules check-jar-mn

config/build-list.pl

@@ -1,114 +0,0 @@
-#!env perl
-
-#
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Christopher Seawood <cls@seawood.org>
-#
-# Alternatively, the contents of this file may be used under the terms of
-# either of the GNU General Public License Version 2 or later (the "GPL"),
-# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
-#
-# ***** END LICENSE BLOCK *****
-
-#
-# A generic script to add entries to a file 
-# if the entry does not already exist
-# 
-# Usage: $0 [-l] <filename> <entry> [<entry> <entry>]
-#
-#   -l do not attempt flock the file.
-
-use Fcntl qw(:DEFAULT :flock);
-use Getopt::Std;
-use mozLock;
-
-sub usage() {
-    print "$0 [-l] <filename> <entry>\n";
-    exit(1);
-}
-
-$nofilelocks = 0;
-
-getopts("l");
-
-$nofilelocks = 1 if defined($::opt_l);
-
-$file = shift;
-
-undef @entrylist;
-while (defined($entry = shift)) {
-    push @entrylist, $entry;
-}
-
-$lockfile = $file . ".lck";
-
-# touch the file if it doesn't exist
-if ( ! -e "$file") {
-    $now = time;
-    utime $now, $now, $file;
-}
-
-# This needs to be atomic
-mozLock($lockfile) unless $nofilelocks;
-
-# Read entire file into mem
-undef @inbuf;
-if ( -e "$file" ) {
-    open(IN, "$file") || die ("$file: $!\n");
-    binmode(IN);
-    while ($tmp = <IN>) {
-	chomp($tmp);
-	push @inbuf, $tmp;
-    }
-    close(IN);
-}
-
-undef @outbuf;
-# Add each entry to file if it's not already there
-foreach $entry (@entrylist) {
-    push @outbuf, $entry if (!grep(/^$entry$/, @inbuf));
-}
-
-$count = $#outbuf + 1;
-
-# Append new entry to file
-if ($count) {
-    open(OUT, ">>$file") || die ("$file: $!\n");
-    binmode(OUT);
-    foreach $entry (@outbuf) {
-	print OUT "$entry\n";
-    }
-    close(OUT);
-}
-
-mozUnlock($lockfile) unless $nofilelocks;
-
-exit(0);

config/buildlist.py

@@ -0,0 +1,73 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla build system.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Ted Mielczarek <ted.mielczarek@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisiwons above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'''A generic script to add entries to a file 
+if the entry does not already exist.
+
+Usage: buildlist.py <filename> <entry> [<entry> ...]
+'''
+
+import sys
+import os
+from utils import lockFile
+
+def addEntriesToListFile(listFile, entries):
+  """Given a file |listFile| containing one entry per line,
+  add each entry in |entries| to the file, unless it is already
+  present."""
+  lock = lockFile(listFile + ".lck")
+  try:
+    if os.path.exists(listFile):
+      f = open(listFile)
+      existing = set([x.strip() for x in f.readlines()])
+      f.close()
+    else:
+      existing = set()
+    f = open(listFile, 'a')
+    for e in entries:
+      if e not in existing:
+        f.write("%s\n" % e)
+        existing.add(e)
+    f.close()
+  finally:
+    lock = None
+
+if __name__ == '__main__':
+  if len(sys.argv) < 3:
+    print >>sys.stderr, "Usage: buildlist.py <list file> <entry> [<entry> ...]"
+    sys.exit(1)
+  addEntriesToListFile(sys.argv[1], sys.argv[2:])

config/rules.mk

@@ -155,7 +155,7 @@ SOLO_FILE ?= $(error Specify a test filename in SOLO_FILE when using check-inter
 
 libs::
 	$(foreach dir,$(XPCSHELL_TESTS),$(_INSTALL_TESTS))
-	$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl \
+	$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py \
           $(testxpcobjdir)/all-test-dirs.list \
           $(addprefix $(MODULE)/,$(XPCSHELL_TESTS))
 
@@ -824,13 +824,13 @@ ifdef LIBRARY_NAME
 ifdef EXPORT_LIBRARY
 ifdef IS_COMPONENT
 ifdef BUILD_STATIC_LIBS
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_COMPS) $(STATIC_LIBRARY_NAME)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_COMPS) $(STATIC_LIBRARY_NAME)
 ifdef MODULE_NAME
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_COMP_NAMES) $(MODULE_NAME)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_COMP_NAMES) $(MODULE_NAME)
 endif
 endif # BUILD_STATIC_LIBS
 else  # !IS_COMPONENT
-	$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_LIBS) $(STATIC_LIBRARY_NAME)
+	$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_LIBS) $(STATIC_LIBRARY_NAME)
 endif # IS_COMPONENT
 endif # EXPORT_LIBRARY
 endif # LIBRARY_NAME
@@ -890,7 +890,7 @@ ifdef SHARED_LIBRARY
 ifdef IS_COMPONENT
 	$(INSTALL) $(IFLAGS2) $(SHARED_LIBRARY) $(FINAL_TARGET)/components
 	$(ELF_DYNSTR_GC) $(FINAL_TARGET)/components/$(SHARED_LIBRARY)
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_TARGET)/components/components.list $(SHARED_LIBRARY)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_TARGET)/components/components.list $(SHARED_LIBRARY)
 ifdef BEOS_ADDON_WORKAROUND
 	( cd $(FINAL_TARGET)/components && $(CC) -nostart -o $(SHARED_LIBRARY).stub $(SHARED_LIBRARY) )
 endif
@@ -1811,7 +1811,7 @@ ifdef EXTRA_COMPONENTS
 libs:: $(EXTRA_COMPONENTS)
 ifndef NO_DIST_INSTALL
 	$(INSTALL) $(IFLAGS1) $^ $(FINAL_TARGET)/components
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_TARGET)/components/components.list $(notdir $^)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_TARGET)/components/components.list $(notdir $^)
 endif
 
 endif
@@ -1826,7 +1826,7 @@ ifndef NO_DIST_INSTALL
 	  dest=$(FINAL_TARGET)/components/$${fname}; \
 	  $(RM) -f $$dest; \
 	  $(PYTHON) $(topsrcdir)/config/Preprocessor.py $(DEFINES) $(ACDEFINES) $(XULPPFLAGS) $$i > $$dest; \
-	  $(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_TARGET)/components/components.list $$fname; \
+	  $(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_TARGET)/components/components.list $$fname; \
 	done
 endif
 

config/tests/unit-buildlist.py

@@ -0,0 +1,80 @@
+import unittest
+
+import os, sys, os.path, time
+from tempfile import mkdtemp
+from shutil import rmtree
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+from buildlist import addEntriesToListFile
+
+class TestBuildList(unittest.TestCase):
+  """
+  Unit tests for buildlist.py
+  """
+  def setUp(self):
+    self.tmpdir = mkdtemp()
+
+  def tearDown(self):
+    rmtree(self.tmpdir)
+
+  # utility methods for tests
+  def touch(self, file, dir=None):
+    if dir is None:
+      dir = self.tmpdir
+    f = os.path.join(dir, file)
+    open(f, 'w').close()
+    return f
+
+  def assertFileContains(self, filename, l):
+    """Assert that the lines in the file |filename| are equal
+    to the contents of the list |l|, in order."""
+    l = l[:]
+    f = open(filename, 'r')
+    lines = [line.rstrip() for line in f.readlines()]
+    f.close()
+    for line in lines:
+      self.assert_(len(l) > 0, "ran out of expected lines! (expected '%s', got '%s')" % (l, lines))
+      self.assertEqual(line, l.pop(0))
+    self.assert_(len(l) == 0, "not enough lines in file! (expected '%s', got '%s'" % (l, lines))
+
+  def test_basic(self):
+    "Test that addEntriesToListFile works when file doesn't exist."
+    testfile = os.path.join(self.tmpdir, "test.list")
+    l = ["a", "b", "c"]
+    addEntriesToListFile(testfile, l)
+    self.assertFileContains(testfile, l)
+    # ensure that attempting to add the same entries again doesn't change it
+    addEntriesToListFile(testfile, l)
+    self.assertFileContains(testfile, l)
+
+  def test_append(self):
+    "Test adding new entries."
+    testfile = os.path.join(self.tmpdir, "test.list")
+    l = ["a", "b", "c"]
+    addEntriesToListFile(testfile, l)
+    self.assertFileContains(testfile, l)
+    l2 = ["x","y","z"]
+    addEntriesToListFile(testfile, l2)
+    l.extend(l2)
+    self.assertFileContains(testfile, l)
+
+  def test_append_some(self):
+    "Test adding new entries mixed with existing entries."
+    testfile = os.path.join(self.tmpdir, "test.list")
+    l = ["a", "b", "c"]
+    addEntriesToListFile(testfile, l)
+    self.assertFileContains(testfile, l)
+    addEntriesToListFile(testfile, ["a", "x", "c", "z"])
+    self.assertFileContains(testfile, ["a", "b", "c", "x", "z"])
+
+  def test_add_multiple(self):
+    """Test that attempting to add the same entry multiple times results in
+    only one entry being added."""
+    testfile = os.path.join(self.tmpdir, "test.list")
+    addEntriesToListFile(testfile, ["a","b","a","a","b"])
+    self.assertFileContains(testfile, ["a","b"])
+    addEntriesToListFile(testfile, ["c","a","c","b","c"])
+    self.assertFileContains(testfile, ["a","b","c"])
+
+if __name__ == '__main__':
+  unittest.main()

js/src/config/rules.mk

@@ -155,7 +155,7 @@ SOLO_FILE ?= $(error Specify a test filename in SOLO_FILE when using check-inter
 
 libs::
 	$(foreach dir,$(XPCSHELL_TESTS),$(_INSTALL_TESTS))
-	$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl \
+	$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py \
           $(testxpcobjdir)/all-test-dirs.list \
           $(addprefix $(MODULE)/,$(XPCSHELL_TESTS))
 
@@ -824,13 +824,13 @@ ifdef LIBRARY_NAME
 ifdef EXPORT_LIBRARY
 ifdef IS_COMPONENT
 ifdef BUILD_STATIC_LIBS
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_COMPS) $(STATIC_LIBRARY_NAME)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_COMPS) $(STATIC_LIBRARY_NAME)
 ifdef MODULE_NAME
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_COMP_NAMES) $(MODULE_NAME)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_COMP_NAMES) $(MODULE_NAME)
 endif
 endif # BUILD_STATIC_LIBS
 else  # !IS_COMPONENT
-	$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_LIBS) $(STATIC_LIBRARY_NAME)
+	$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_LIBS) $(STATIC_LIBRARY_NAME)
 endif # IS_COMPONENT
 endif # EXPORT_LIBRARY
 endif # LIBRARY_NAME
@@ -890,7 +890,7 @@ ifdef SHARED_LIBRARY
 ifdef IS_COMPONENT
 	$(INSTALL) $(IFLAGS2) $(SHARED_LIBRARY) $(FINAL_TARGET)/components
 	$(ELF_DYNSTR_GC) $(FINAL_TARGET)/components/$(SHARED_LIBRARY)
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_TARGET)/components/components.list $(SHARED_LIBRARY)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_TARGET)/components/components.list $(SHARED_LIBRARY)
 ifdef BEOS_ADDON_WORKAROUND
 	( cd $(FINAL_TARGET)/components && $(CC) -nostart -o $(SHARED_LIBRARY).stub $(SHARED_LIBRARY) )
 endif
@@ -1811,7 +1811,7 @@ ifdef EXTRA_COMPONENTS
 libs:: $(EXTRA_COMPONENTS)
 ifndef NO_DIST_INSTALL
 	$(INSTALL) $(IFLAGS1) $^ $(FINAL_TARGET)/components
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_TARGET)/components/components.list $(notdir $^)
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_TARGET)/components/components.list $(notdir $^)
 endif
 
 endif
@@ -1826,7 +1826,7 @@ ifndef NO_DIST_INSTALL
 	  dest=$(FINAL_TARGET)/components/$${fname}; \
 	  $(RM) -f $$dest; \
 	  $(PYTHON) $(topsrcdir)/config/Preprocessor.py $(DEFINES) $(ACDEFINES) $(XULPPFLAGS) $$i > $$dest; \
-	  $(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_TARGET)/components/components.list $$fname; \
+	  $(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_TARGET)/components/components.list $$fname; \
 	done
 endif
 

layout/tools/reftest/Makefile.in

@@ -89,7 +89,7 @@ copy-harness: $(_HARNESS_FILES)
 	(cd $(DIST)/xpi-stage && tar $(TAR_CREATE_FLAGS) - reftest) | (cd $(_DEST_DIR) && tar -xf -)
 	$(INSTALL) $(DIST)/bin/components/httpd.js $(_DEST_DIR)/reftest/components
 # need to get httpd.js into components.list so it loads
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(_DEST_DIR)/reftest/components/components.list httpd.js
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(_DEST_DIR)/reftest/components/components.list httpd.js
 	$(INSTALL) $(DIST)/bin/components/test_necko.xpt $(_DEST_DIR)/reftest/components
 
 PKG_STAGE = $(DIST)/test-package-stage

toolkit/xre/Makefile.in

@@ -176,10 +176,7 @@ include $(topsrcdir)/config/rules.mk
 
 ifdef BUILD_STATIC_LIBS
 export::
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_COMP_NAMES) Apprunner
+	@$(PYTHON) $(MOZILLA_DIR)/config/buildlist.py $(FINAL_LINK_COMP_NAMES) Apprunner
-#	embedding/browser/gtk/src/Makefile.in sucks! we need to add an empty line to 
-# FINAL_LINK_COMPS to keep the two lists in sync :-(
-	@$(PERL) -I$(MOZILLA_DIR)/config $(MOZILLA_DIR)/config/build-list.pl $(FINAL_LINK_COMPS) ""
 endif
 
 LOCAL_INCLUDES += \