No bug, DONTBUILD. Updates to the static rooting analysis, including transition from Makefile to python script.

Although this patch contains some updates to Makefile.in, I am no longer using it at all. I now run analyze.py for better control, though note that it depends on loading in some configuration settings that are hardcoded to my environment.

This patch also contains a number of updates to the annotations.

--HG--
extra : rebase_source : ebd4deb590fb9fde4532bdf45214ffca117e1c3a

js/src/devtools/rootAnalysis/Makefile.in

@@ -51,25 +51,36 @@ src_body.xdb src_comp.xdb: run_complete
 
 callgraph.txt: src_body.xdb src_comp.xdb computeCallgraph.js
 	@echo Started computation of $@ at $$(date)
-	$(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeCallgraph.js > $@
+	$(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeCallgraph.js > $@.tmp
+	mv $@.tmp $@
 	@echo Finished computation of $@ at $$(date)
 
 gcFunctions.txt: callgraph.txt computeGCFunctions.js annotations.js
 	@echo Started computation of $@ at $$(date)
-	$(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCFunctions.js ./callgraph.txt > $@
+	$(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCFunctions.js ./callgraph.txt > $@.tmp
+	mv $@.tmp $@
 	@echo Finished computation of $@ at $$(date)
 
+gcFunctions.lst: gcFunctions.txt
+	perl -lne 'print $$1 if /^GC Function: (.*)/' gcFunctions.txt > $@
+
+suppressedFunctions.lst: gcFunctions.txt
+	perl -lne 'print $$1 if /^Suppressed Function: (.*)/' gcFunctions.txt > $@
+
 gcTypes.txt: src_comp.xdb computeGCTypes.js annotations.js
 	@echo Started computation of $@ at $$(date)
-	$(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCTypes.js > $@
+	$(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCTypes.js > $@.tmp
+	mv $@.tmp $@
 	@echo Finished computation of $@ at $$(date)
 
 allFunctions.txt: src_body.xdb
 	@echo Started computation of $@ at $$(date)
-	time $(SIXGILL)/bin/xdbkeys $^ > $@
+	time $(SIXGILL)/bin/xdbkeys $^ > $@.tmp
+	mv $@.tmp $@
 	@echo Finished computation of $@ at $$(date)
 
-rootingHazards.txt: gcFunctions.txt gcTypes.txt analyzeRoots.js annotations.js gen-hazards.sh
+rootingHazards.txt: gcFunctions.lst suppressedFunctions.lst gcTypes.txt analyzeRoots.js annotations.js gen-hazards.sh
 	@echo Started computation of $@ at $$(date)
-	time env JS=$(JS) ANALYZE="$(ANALYSIS_SCRIPT_DIR)/analyzeRoots.js" SIXGILL="$(SIXGILL)" "$(ANALYSIS_SCRIPT_DIR)/gen-hazards.sh" $(JOBS) > $@
+	time env JS=$(JS) ANALYZE="$(ANALYSIS_SCRIPT_DIR)/analyzeRoots.js" SIXGILL="$(SIXGILL)" "$(ANALYSIS_SCRIPT_DIR)/gen-hazards.sh" $(JOBS) > $@.tmp
+	mv $@.tmp $@
 	@echo Finished computation of $@ at $$(date)

js/src/devtools/rootAnalysis/analyze.py

@@ -0,0 +1,182 @@
+#!/usr/bin/python
+
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Runs the static rooting analysis
+"""
+
+from subprocess import Popen
+import subprocess
+import os
+import argparse
+import sys
+
+def env(config):
+    e = os.environ
+    e['PATH'] = '%s:%s/bin' % (e['PATH'], config['sixgill'])
+    e['XDB'] = '%(sixgill)s/bin/xdb.so' % config
+    e['SOURCE_ROOT'] = e['TARGET']
+    return e
+
+def fill(command, config):
+    try:
+        return tuple(s % config for s in command)
+    except:
+        print("Substitution failed:")
+        for fragment in command:
+            try:
+                fragment % config
+            except:
+                print("  %s" % fragment)
+        raise Hell
+
+def generate_hazards(config, outfilename):
+    jobs = []
+    for i in range(config['jobs']):
+        command = fill(('%(js)s',
+                        '%(analysis_scriptdir)s/analyzeRoots.js',
+                        '%(gcFunctions_list)s',
+                        '%(suppressedFunctions_list)s',
+                        '%(gcTypes)s',
+                        str(i+1), '%(jobs)s',
+                        'tmp.%s' % (i+1,)),
+                       config)
+        outfile = 'rootingHazards.%s' % (i+1,)
+        output = open(outfile, 'w')
+        print(' '.join(command) + ' > ' + outfile)
+        jobs.append((command, Popen(command, stdout=output, env=env(config))))
+
+    final_status = 0
+    while jobs:
+        pid, status = os.wait()
+        jobs = [ job for job in jobs if job[1].pid != pid ]
+        final_status = final_status or status
+
+    if final_status:
+        raise subprocess.CalledProcessError(final_status, 'analyzeRoots.js')
+
+    with open(outfilename, 'w') as output:
+        command = ['cat'] + [ 'rootingHazards.%s' % (i+1,) for i in range(config['jobs']) ]
+        print(' '.join(command) + ' > ' + outfilename)
+        subprocess.call(command, stdout=output)
+
+JOBS = { 'dbs':
+             (('%(CWD)s/run_complete',
+               '--foreground',
+               '--build-root=%(objdir)s',
+               '--work=dir=work',
+               '-b', '%(sixgill)s/bin',
+               '--buildcommand=%(buildcommand)s',
+               '.'),
+              None),
+
+         'callgraph':
+             (('%(js)s', '%(analysis_scriptdir)s/computeCallgraph.js'),
+              'callgraph.txt'),
+
+         'gcFunctions':
+             (('%(js)s', '%(analysis_scriptdir)s/computeGCFunctions.js', '%(callgraph)s'),
+              'gcFunctions.txt'),
+
+         'gcFunctions_list':
+             (('perl', '-lne', 'print $1 if /^GC Function: (.*)/', '%(gcFunctions)s'),
+              'gcFunctions.lst'),
+
+         'suppressedFunctions_list':
+             (('perl', '-lne', 'print $1 if /^Suppressed Function: (.*)/', '%(gcFunctions)s'),
+              'suppressedFunctions.lst'),
+
+         'gcTypes':
+             (('%(js)s', '%(analysis_scriptdir)s/computeGCTypes.js',),
+              'gcTypes.txt'),
+
+         'allFunctions':
+             (('%(sixgill)s/bin/xdbkeys', 'src_body.xdb',),
+              'allFunctions.txt'),
+
+         'hazards':
+             (generate_hazards, 'rootingHazards.txt')
+         }
+
+def run_job(name, config):
+    command, outfilename = JOBS[name]
+    print("Running " + name + " to generate " + str(outfilename))
+    if hasattr(command, '__call__'):
+        command(config, outfilename)
+    else:
+        command = fill(command, config)
+        print(' '.join(command))
+        temp = '%s.tmp' % name
+        with open(temp, 'w') as output:
+            subprocess.check_call(command, stdout=output, env=env(config))
+        if outfilename is not None:
+            os.rename(temp, outfilename)
+
+config = { 'CWD': os.path.dirname(__file__) }
+
+defaults = [ '%s/defaults.py' % config['CWD'] ]
+
+for default in defaults:
+    try:
+        execfile(default, config)
+    except:
+        pass
+
+data = config.copy()
+
+parser = argparse.ArgumentParser(description='Statically analyze build tree for rooting hazards.')
+parser.add_argument('target', metavar='TARGET', type=str, nargs='?',
+                    help='run starting from this target')
+parser.add_argument('--jobs', '-j', default=4, metavar='JOBS', type=int,
+                    help='number of simultaneous analyzeRoots.js jobs')
+parser.add_argument('--list', const=True, nargs='?', type=bool,
+                    help='display available targets')
+parser.add_argument('--buildcommand', '--build', '-b', type=str, nargs='?',
+                    help='command to build the tree being analyzed')
+parser.add_argument('--tag', '-t', type=str, nargs='?',
+                    help='name of job, also sets build command to "build.<tag>"')
+
+args = parser.parse_args()
+data.update(vars(args))
+
+if args.tag and not args.buildcommand:
+    args.buildcommand="build.%s" % args.tag
+
+if args.buildcommand:
+    data['buildcommand'] = args.buildcommand
+elif 'BUILD' in os.environ:
+    data['buildcommand'] = os.environ['BUILD']
+else:
+    data['buildcommand'] = 'make -j4 -s'
+
+targets = [ 'dbs',
+            'callgraph',
+            'gcTypes',
+            'gcFunctions',
+            'gcFunctions_list',
+            'suppressedFunctions_list',
+            'allFunctions',
+            'hazards' ]
+
+if args.list:
+    for target in targets:
+        command, outfilename = JOBS[target]
+        if outfilename:
+            print("%s -> %s" % (target, outfilename))
+        else:
+            print(target)
+    sys.exit(0)
+
+for target in targets:
+    command, outfilename = JOBS[target]
+    data[target] = outfilename
+
+if args.target:
+    targets = targets[targets.index(args.target):]
+
+for target in targets:
+    run_job(target, data)

js/src/devtools/rootAnalysis/analyzeRoots.js

@@ -6,35 +6,33 @@ load('utility.js');
 load('annotations.js');
 load('suppressedPoints.js');
 
-var sourceRoot = null;
+var sourceRoot = (environment['SOURCE_ROOT'] || '') + '/'
 
 var functionName;
 var functionBodies;
 
 if (typeof arguments[0] != 'string' || typeof arguments[1] != 'string')
-    throw "Usage: analyzeRoots.js <gcFunctions.txt> <gcTypes.txt> [start end [tmpfile]]";
+    throw "Usage: analyzeRoots.js <gcFunctions.lst> <suppressedFunctions.lst> <gcTypes.txt> [start end [tmpfile]]";
 
 var gcFunctionsFile = arguments[0];
-var gcTypesFile = arguments[1];
-var batch = arguments[2]|0;
-var numBatches = (arguments[3]|0) || 1;
-var tmpfile = arguments[4] || "tmp.txt";
+var suppressedFunctionsFile = arguments[1];
+var gcTypesFile = arguments[2];
+var batch = arguments[3]|0;
+var numBatches = (arguments[4]|0) || 1;
+var tmpfile = arguments[5] || "tmp.txt";
 
 var gcFunctions = {};
-var suppressedFunctions = {};
-assert(!system("grep 'GC Function' " + gcFunctionsFile + " > tmp.txt"));
-var text = snarf("tmp.txt").split('\n');
+var text = snarf("gcFunctions.lst").split('\n');
 assert(text.pop().length == 0);
 for (var line of text) {
-    match = /GC Function: (.*)/.exec(line);
-    gcFunctions[match[1]] = true;
+    gcFunctions[line] = true;
 }
-assert(!system("grep 'Suppressed Function' " + arguments[0] + " > tmp.txt"));
-text = snarf("tmp.txt").split('\n');
+
+var suppressedFunctions = {};
+var text = snarf("suppressedFunctions.lst").split('\n');
 assert(text.pop().length == 0);
 for (var line of text) {
-    match = /Suppressed Function: (.*)/.exec(line);
-    suppressedFunctions[match[1]] = true;
+    suppressedFunctions[line] = true;
 }
 text = null;
 
@@ -494,22 +492,6 @@ var each = Math.floor(N/numBatches);
 var start = minStream + each * (batch - 1);
 var end = Math.min(minStream + each * batch - 1, maxStream);
 
-// Find the source tree
-for (let nameIndex = minStream; nameIndex <= maxStream; nameIndex++) {
-    var name = xdb.read_key(nameIndex);
-    functionName = name.readString();
-    var data = xdb.read_entry(name);
-    functionBodies = JSON.parse(data.readString());
-    let filename = functionBodies[0].Location[0].CacheString;
-    let match = /(.*)\bjs\/src\//.exec(filename);
-    if (match) {
-        sourceRoot = match[1];
-        printErr("sourceRoot = " + sourceRoot);
-        break;
-    }
-}
-assert(typeof(sourceRoot) == "string");
-
 for (var nameIndex = start; nameIndex <= end; nameIndex++) {
     var name = xdb.read_key(nameIndex);
     functionName = name.readString();

js/src/devtools/rootAnalysis/annotations.js

@@ -9,8 +9,11 @@ var ignoreIndirectCalls = {
     "__conv" : true,
     "__convf" : true,
     "prerrortable.c:callback_newtable" : true,
+    "mozalloc_oom.cpp:void (* gAbortHandler)(size_t)" : true,
+    "JSObject* js::GetWeakmapKeyDelegate(JSObject*)" : true, // FIXME: mark with AutoAssertNoGC instead
 };
 
+
 function indirectCallCannotGC(caller, name)
 {
     if (name in ignoreIndirectCalls)
@@ -26,10 +29,6 @@ function indirectCallCannotGC(caller, name)
     if (/CallDestroyScriptHook/.test(caller))
         return true;
 
-    // hooks called deep inside utility libraries.
-    if (name == "_malloc_message")
-        return true;
-
     return false;
 }
 
@@ -43,13 +42,15 @@ var ignoreClasses = {
     "PRIOMethods": true,
     "XPCOMFunctions" : true, // I'm a little unsure of this one
     "_MD_IOVector" : true,
-    "PRIOMethods" : true,
 };
 
 var ignoreCallees = {
     "js::Class.trace" : true,
     "js::Class.finalize" : true,
     "JSRuntime.destroyPrincipals" : true,
+    "nsISupports.AddRef" : true,
+    "nsISupports.Release" : true, // makes me a bit nervous; this is a bug but can happen
+    "nsAXPCNativeCallContext.GetJSContext" : true,
 };
 
 function fieldCallCannotGC(csu, fullfield)
@@ -92,18 +93,21 @@ function ignoreEdgeUse(edge, variable)
     return false;
 }
 
-var ignoreFunctions = [
-    "ptio.c:pt_MapError",
-    "PR_ExplodeTime",
-    "PR_ErrorInstallTable"
-];
+var ignoreFunctions = {
+    "ptio.c:pt_MapError" : true,
+    "PR_ExplodeTime" : true,
+    "PR_ErrorInstallTable" : true,
+    "PR_SetThreadPrivate" : true
+};
 
 function ignoreGCFunction(fun)
 {
-    for (var i = 0; i < ignoreFunctions.length; i++) {
-        if (fun == ignoreFunctions[i])
-            return true;
-    }
+    if (fun in ignoreFunctions)
+        return true;
+
+    // Templatized function
+    if (fun.indexOf("void nsCOMPtr<T>::Assert_NoQueryNeeded()") >= 0)
+        return true;
 
     // XXX modify refillFreeList<NoGC> to not need data flow analysis to understand it cannot GC.
     if (/refillFreeList/.test(fun) && /\(js::AllowGC\)0u/.test(fun))

js/src/devtools/rootAnalysis/gen-hazards.sh

@@ -5,7 +5,7 @@ set -e
 JOBS="$1"
 
 for j in $(seq $JOBS); do
-    env PATH=$PATH:$SIXGILL/bin XDB=$SIXGILL/bin/xdb.so $JS $ANALYZE gcFunctions.txt gcTypes.txt $j $JOBS tmp.$j > rootingHazards.$j &
+    env PATH=$PATH:$SIXGILL/bin XDB=$SIXGILL/bin/xdb.so $JS $ANALYZE gcFunctions.lst suppressedFunctions.lst gcTypes.txt $j $JOBS tmp.$j > rootingHazards.$j &
 done
 
 wait

js/src/devtools/rootAnalysis/run_complete

@@ -53,6 +53,8 @@ my $old_dir = "";
 # run in the foreground
 my $foreground;
 
+my $builder = "make -j4";
+
 GetOptions("build-root|b=s" => \$build_dir,
            "poll-file=s" => \$poll_file,
            "work-dir=s" => \$WORKDIR,
@@ -61,11 +63,12 @@ GetOptions("build-root|b=s" => \$build_dir,
            "annotations-file|annotations|a=s" => \$ann_file,
            "old-dir|old=s" => \$old_dir,
            "foreground!" => \$foreground,
+	   "buildcommand=s" => \$builder,
            )
     or die;
 
 if (not -d $build_dir) {
-    die "Need build directory: $build_dir\n";
+    mkdir($build_dir);
 }
 if ($old_dir ne "" && not -d $old_dir) {
     die "Old directory '$old_dir' does not exist\n";
@@ -75,10 +78,6 @@ $WORKDIR ||= "sixgill-work";
 $poll_file ||= "$WORKDIR/poll.file";
 $build_dir ||= "$WORKDIR/js-inbound-xgill";
 
-if (not -d $build_dir) {
-    die "Build root '$build_dir' does not exist\n";
-}
-
 if (!defined $SIXGILL_BIN) {
     chomp(my $path = `which xmanager`);
     if ($path) {
@@ -100,11 +99,7 @@ sub clean_project {
 
 # code to build the project from $build_dir.
 sub build_project {
-    if ($ENV{BUILD}) {
-        return system($ENV{BUILD});
-    } else {
-        return system("make -j4");
-    }
+    return system($builder);
 }
 
 # commands to start the various xgill binaries. timeouts can be specified
@@ -241,7 +236,7 @@ sub run_build
 
         # build is finished, the complete run can resume.
         # return value only useful if --foreground
-        exit $? || $exit_status;
+        exit($? || $exit_status);
     }
 
     # this is the complete process, wait for the build to finish.