From 2c15ee71688d3b205cb8336ccf16d075255e8e39 Mon Sep 17 00:00:00 2001 From: Steve Fink Date: Thu, 25 Apr 2013 09:16:37 -0700 Subject: [PATCH] No bug, DONTBUILD. Updates to the static rooting analysis, including transition from Makefile to python script. Although this patch contains some updates to Makefile.in, I am no longer using it at all. I now run analyze.py for better control, though note that it depends on loading in some configuration settings that are hardcoded to my environment. This patch also contains a number of updates to the annotations. --HG-- extra : rebase_source : ebd4deb590fb9fde4532bdf45214ffca117e1c3a --- js/src/devtools/rootAnalysis/Makefile.in | 23 ++- js/src/devtools/rootAnalysis/analyze.py | 182 +++++++++++++++++++ js/src/devtools/rootAnalysis/analyzeRoots.js | 44 ++--- js/src/devtools/rootAnalysis/annotations.js | 32 ++-- js/src/devtools/rootAnalysis/gen-hazards.sh | 2 +- js/src/devtools/rootAnalysis/run_complete | 17 +- 6 files changed, 237 insertions(+), 63 deletions(-) create mode 100644 js/src/devtools/rootAnalysis/analyze.py diff --git a/js/src/devtools/rootAnalysis/Makefile.in b/js/src/devtools/rootAnalysis/Makefile.in index 809b5913c2d..d1aa349a3d1 100644 --- a/js/src/devtools/rootAnalysis/Makefile.in +++ b/js/src/devtools/rootAnalysis/Makefile.in @@ -51,25 +51,36 @@ src_body.xdb src_comp.xdb: run_complete callgraph.txt: src_body.xdb src_comp.xdb computeCallgraph.js @echo Started computation of $@ at $$(date) - $(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeCallgraph.js > $@ + $(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeCallgraph.js > $@.tmp + mv $@.tmp $@ @echo Finished computation of $@ at $$(date) gcFunctions.txt: callgraph.txt computeGCFunctions.js annotations.js @echo Started computation of $@ at $$(date) - $(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCFunctions.js ./callgraph.txt > $@ + $(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCFunctions.js ./callgraph.txt > $@.tmp + mv $@.tmp $@ @echo Finished computation of $@ at $$(date) +gcFunctions.lst: gcFunctions.txt + perl -lne 'print $$1 if /^GC Function: (.*)/' gcFunctions.txt > $@ + +suppressedFunctions.lst: gcFunctions.txt + perl -lne 'print $$1 if /^Suppressed Function: (.*)/' gcFunctions.txt > $@ + gcTypes.txt: src_comp.xdb computeGCTypes.js annotations.js @echo Started computation of $@ at $$(date) - $(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCTypes.js > $@ + $(CALL_JS) $(ANALYSIS_SCRIPT_DIR)/computeGCTypes.js > $@.tmp + mv $@.tmp $@ @echo Finished computation of $@ at $$(date) allFunctions.txt: src_body.xdb @echo Started computation of $@ at $$(date) - time $(SIXGILL)/bin/xdbkeys $^ > $@ + time $(SIXGILL)/bin/xdbkeys $^ > $@.tmp + mv $@.tmp $@ @echo Finished computation of $@ at $$(date) -rootingHazards.txt: gcFunctions.txt gcTypes.txt analyzeRoots.js annotations.js gen-hazards.sh +rootingHazards.txt: gcFunctions.lst suppressedFunctions.lst gcTypes.txt analyzeRoots.js annotations.js gen-hazards.sh @echo Started computation of $@ at $$(date) - time env JS=$(JS) ANALYZE="$(ANALYSIS_SCRIPT_DIR)/analyzeRoots.js" SIXGILL="$(SIXGILL)" "$(ANALYSIS_SCRIPT_DIR)/gen-hazards.sh" $(JOBS) > $@ + time env JS=$(JS) ANALYZE="$(ANALYSIS_SCRIPT_DIR)/analyzeRoots.js" SIXGILL="$(SIXGILL)" "$(ANALYSIS_SCRIPT_DIR)/gen-hazards.sh" $(JOBS) > $@.tmp + mv $@.tmp $@ @echo Finished computation of $@ at $$(date) diff --git a/js/src/devtools/rootAnalysis/analyze.py b/js/src/devtools/rootAnalysis/analyze.py new file mode 100644 index 00000000000..f39d3491b9b --- /dev/null +++ b/js/src/devtools/rootAnalysis/analyze.py @@ -0,0 +1,182 @@ +#!/usr/bin/python + +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Runs the static rooting analysis +""" + +from subprocess import Popen +import subprocess +import os +import argparse +import sys + +def env(config): + e = os.environ + e['PATH'] = '%s:%s/bin' % (e['PATH'], config['sixgill']) + e['XDB'] = '%(sixgill)s/bin/xdb.so' % config + e['SOURCE_ROOT'] = e['TARGET'] + return e + +def fill(command, config): + try: + return tuple(s % config for s in command) + except: + print("Substitution failed:") + for fragment in command: + try: + fragment % config + except: + print(" %s" % fragment) + raise Hell + +def generate_hazards(config, outfilename): + jobs = [] + for i in range(config['jobs']): + command = fill(('%(js)s', + '%(analysis_scriptdir)s/analyzeRoots.js', + '%(gcFunctions_list)s', + '%(suppressedFunctions_list)s', + '%(gcTypes)s', + str(i+1), '%(jobs)s', + 'tmp.%s' % (i+1,)), + config) + outfile = 'rootingHazards.%s' % (i+1,) + output = open(outfile, 'w') + print(' '.join(command) + ' > ' + outfile) + jobs.append((command, Popen(command, stdout=output, env=env(config)))) + + final_status = 0 + while jobs: + pid, status = os.wait() + jobs = [ job for job in jobs if job[1].pid != pid ] + final_status = final_status or status + + if final_status: + raise subprocess.CalledProcessError(final_status, 'analyzeRoots.js') + + with open(outfilename, 'w') as output: + command = ['cat'] + [ 'rootingHazards.%s' % (i+1,) for i in range(config['jobs']) ] + print(' '.join(command) + ' > ' + outfilename) + subprocess.call(command, stdout=output) + +JOBS = { 'dbs': + (('%(CWD)s/run_complete', + '--foreground', + '--build-root=%(objdir)s', + '--work=dir=work', + '-b', '%(sixgill)s/bin', + '--buildcommand=%(buildcommand)s', + '.'), + None), + + 'callgraph': + (('%(js)s', '%(analysis_scriptdir)s/computeCallgraph.js'), + 'callgraph.txt'), + + 'gcFunctions': + (('%(js)s', '%(analysis_scriptdir)s/computeGCFunctions.js', '%(callgraph)s'), + 'gcFunctions.txt'), + + 'gcFunctions_list': + (('perl', '-lne', 'print $1 if /^GC Function: (.*)/', '%(gcFunctions)s'), + 'gcFunctions.lst'), + + 'suppressedFunctions_list': + (('perl', '-lne', 'print $1 if /^Suppressed Function: (.*)/', '%(gcFunctions)s'), + 'suppressedFunctions.lst'), + + 'gcTypes': + (('%(js)s', '%(analysis_scriptdir)s/computeGCTypes.js',), + 'gcTypes.txt'), + + 'allFunctions': + (('%(sixgill)s/bin/xdbkeys', 'src_body.xdb',), + 'allFunctions.txt'), + + 'hazards': + (generate_hazards, 'rootingHazards.txt') + } + +def run_job(name, config): + command, outfilename = JOBS[name] + print("Running " + name + " to generate " + str(outfilename)) + if hasattr(command, '__call__'): + command(config, outfilename) + else: + command = fill(command, config) + print(' '.join(command)) + temp = '%s.tmp' % name + with open(temp, 'w') as output: + subprocess.check_call(command, stdout=output, env=env(config)) + if outfilename is not None: + os.rename(temp, outfilename) + +config = { 'CWD': os.path.dirname(__file__) } + +defaults = [ '%s/defaults.py' % config['CWD'] ] + +for default in defaults: + try: + execfile(default, config) + except: + pass + +data = config.copy() + +parser = argparse.ArgumentParser(description='Statically analyze build tree for rooting hazards.') +parser.add_argument('target', metavar='TARGET', type=str, nargs='?', + help='run starting from this target') +parser.add_argument('--jobs', '-j', default=4, metavar='JOBS', type=int, + help='number of simultaneous analyzeRoots.js jobs') +parser.add_argument('--list', const=True, nargs='?', type=bool, + help='display available targets') +parser.add_argument('--buildcommand', '--build', '-b', type=str, nargs='?', + help='command to build the tree being analyzed') +parser.add_argument('--tag', '-t', type=str, nargs='?', + help='name of job, also sets build command to "build."') + +args = parser.parse_args() +data.update(vars(args)) + +if args.tag and not args.buildcommand: + args.buildcommand="build.%s" % args.tag + +if args.buildcommand: + data['buildcommand'] = args.buildcommand +elif 'BUILD' in os.environ: + data['buildcommand'] = os.environ['BUILD'] +else: + data['buildcommand'] = 'make -j4 -s' + +targets = [ 'dbs', + 'callgraph', + 'gcTypes', + 'gcFunctions', + 'gcFunctions_list', + 'suppressedFunctions_list', + 'allFunctions', + 'hazards' ] + +if args.list: + for target in targets: + command, outfilename = JOBS[target] + if outfilename: + print("%s -> %s" % (target, outfilename)) + else: + print(target) + sys.exit(0) + +for target in targets: + command, outfilename = JOBS[target] + data[target] = outfilename + +if args.target: + targets = targets[targets.index(args.target):] + +for target in targets: + run_job(target, data) diff --git a/js/src/devtools/rootAnalysis/analyzeRoots.js b/js/src/devtools/rootAnalysis/analyzeRoots.js index 0bf6709eac6..e52ccb39d1c 100644 --- a/js/src/devtools/rootAnalysis/analyzeRoots.js +++ b/js/src/devtools/rootAnalysis/analyzeRoots.js @@ -6,35 +6,33 @@ load('utility.js'); load('annotations.js'); load('suppressedPoints.js'); -var sourceRoot = null; +var sourceRoot = (environment['SOURCE_ROOT'] || '') + '/' var functionName; var functionBodies; if (typeof arguments[0] != 'string' || typeof arguments[1] != 'string') - throw "Usage: analyzeRoots.js [start end [tmpfile]]"; + throw "Usage: analyzeRoots.js [start end [tmpfile]]"; var gcFunctionsFile = arguments[0]; -var gcTypesFile = arguments[1]; -var batch = arguments[2]|0; -var numBatches = (arguments[3]|0) || 1; -var tmpfile = arguments[4] || "tmp.txt"; +var suppressedFunctionsFile = arguments[1]; +var gcTypesFile = arguments[2]; +var batch = arguments[3]|0; +var numBatches = (arguments[4]|0) || 1; +var tmpfile = arguments[5] || "tmp.txt"; var gcFunctions = {}; -var suppressedFunctions = {}; -assert(!system("grep 'GC Function' " + gcFunctionsFile + " > tmp.txt")); -var text = snarf("tmp.txt").split('\n'); +var text = snarf("gcFunctions.lst").split('\n'); assert(text.pop().length == 0); for (var line of text) { - match = /GC Function: (.*)/.exec(line); - gcFunctions[match[1]] = true; + gcFunctions[line] = true; } -assert(!system("grep 'Suppressed Function' " + arguments[0] + " > tmp.txt")); -text = snarf("tmp.txt").split('\n'); + +var suppressedFunctions = {}; +var text = snarf("suppressedFunctions.lst").split('\n'); assert(text.pop().length == 0); for (var line of text) { - match = /Suppressed Function: (.*)/.exec(line); - suppressedFunctions[match[1]] = true; + suppressedFunctions[line] = true; } text = null; @@ -494,22 +492,6 @@ var each = Math.floor(N/numBatches); var start = minStream + each * (batch - 1); var end = Math.min(minStream + each * batch - 1, maxStream); -// Find the source tree -for (let nameIndex = minStream; nameIndex <= maxStream; nameIndex++) { - var name = xdb.read_key(nameIndex); - functionName = name.readString(); - var data = xdb.read_entry(name); - functionBodies = JSON.parse(data.readString()); - let filename = functionBodies[0].Location[0].CacheString; - let match = /(.*)\bjs\/src\//.exec(filename); - if (match) { - sourceRoot = match[1]; - printErr("sourceRoot = " + sourceRoot); - break; - } -} -assert(typeof(sourceRoot) == "string"); - for (var nameIndex = start; nameIndex <= end; nameIndex++) { var name = xdb.read_key(nameIndex); functionName = name.readString(); diff --git a/js/src/devtools/rootAnalysis/annotations.js b/js/src/devtools/rootAnalysis/annotations.js index 4b8ae82ef14..7227f399894 100644 --- a/js/src/devtools/rootAnalysis/annotations.js +++ b/js/src/devtools/rootAnalysis/annotations.js @@ -9,8 +9,11 @@ var ignoreIndirectCalls = { "__conv" : true, "__convf" : true, "prerrortable.c:callback_newtable" : true, + "mozalloc_oom.cpp:void (* gAbortHandler)(size_t)" : true, + "JSObject* js::GetWeakmapKeyDelegate(JSObject*)" : true, // FIXME: mark with AutoAssertNoGC instead }; + function indirectCallCannotGC(caller, name) { if (name in ignoreIndirectCalls) @@ -26,10 +29,6 @@ function indirectCallCannotGC(caller, name) if (/CallDestroyScriptHook/.test(caller)) return true; - // hooks called deep inside utility libraries. - if (name == "_malloc_message") - return true; - return false; } @@ -43,13 +42,15 @@ var ignoreClasses = { "PRIOMethods": true, "XPCOMFunctions" : true, // I'm a little unsure of this one "_MD_IOVector" : true, - "PRIOMethods" : true, }; var ignoreCallees = { "js::Class.trace" : true, "js::Class.finalize" : true, "JSRuntime.destroyPrincipals" : true, + "nsISupports.AddRef" : true, + "nsISupports.Release" : true, // makes me a bit nervous; this is a bug but can happen + "nsAXPCNativeCallContext.GetJSContext" : true, }; function fieldCallCannotGC(csu, fullfield) @@ -92,18 +93,21 @@ function ignoreEdgeUse(edge, variable) return false; } -var ignoreFunctions = [ - "ptio.c:pt_MapError", - "PR_ExplodeTime", - "PR_ErrorInstallTable" -]; +var ignoreFunctions = { + "ptio.c:pt_MapError" : true, + "PR_ExplodeTime" : true, + "PR_ErrorInstallTable" : true, + "PR_SetThreadPrivate" : true +}; function ignoreGCFunction(fun) { - for (var i = 0; i < ignoreFunctions.length; i++) { - if (fun == ignoreFunctions[i]) - return true; - } + if (fun in ignoreFunctions) + return true; + + // Templatized function + if (fun.indexOf("void nsCOMPtr::Assert_NoQueryNeeded()") >= 0) + return true; // XXX modify refillFreeList to not need data flow analysis to understand it cannot GC. if (/refillFreeList/.test(fun) && /\(js::AllowGC\)0u/.test(fun)) diff --git a/js/src/devtools/rootAnalysis/gen-hazards.sh b/js/src/devtools/rootAnalysis/gen-hazards.sh index 840f9618ab0..7007969a141 100755 --- a/js/src/devtools/rootAnalysis/gen-hazards.sh +++ b/js/src/devtools/rootAnalysis/gen-hazards.sh @@ -5,7 +5,7 @@ set -e JOBS="$1" for j in $(seq $JOBS); do - env PATH=$PATH:$SIXGILL/bin XDB=$SIXGILL/bin/xdb.so $JS $ANALYZE gcFunctions.txt gcTypes.txt $j $JOBS tmp.$j > rootingHazards.$j & + env PATH=$PATH:$SIXGILL/bin XDB=$SIXGILL/bin/xdb.so $JS $ANALYZE gcFunctions.lst suppressedFunctions.lst gcTypes.txt $j $JOBS tmp.$j > rootingHazards.$j & done wait diff --git a/js/src/devtools/rootAnalysis/run_complete b/js/src/devtools/rootAnalysis/run_complete index 22bcf1bf4b9..2e46afc3d0e 100755 --- a/js/src/devtools/rootAnalysis/run_complete +++ b/js/src/devtools/rootAnalysis/run_complete @@ -53,6 +53,8 @@ my $old_dir = ""; # run in the foreground my $foreground; +my $builder = "make -j4"; + GetOptions("build-root|b=s" => \$build_dir, "poll-file=s" => \$poll_file, "work-dir=s" => \$WORKDIR, @@ -61,11 +63,12 @@ GetOptions("build-root|b=s" => \$build_dir, "annotations-file|annotations|a=s" => \$ann_file, "old-dir|old=s" => \$old_dir, "foreground!" => \$foreground, + "buildcommand=s" => \$builder, ) or die; if (not -d $build_dir) { - die "Need build directory: $build_dir\n"; + mkdir($build_dir); } if ($old_dir ne "" && not -d $old_dir) { die "Old directory '$old_dir' does not exist\n"; @@ -75,10 +78,6 @@ $WORKDIR ||= "sixgill-work"; $poll_file ||= "$WORKDIR/poll.file"; $build_dir ||= "$WORKDIR/js-inbound-xgill"; -if (not -d $build_dir) { - die "Build root '$build_dir' does not exist\n"; -} - if (!defined $SIXGILL_BIN) { chomp(my $path = `which xmanager`); if ($path) { @@ -100,11 +99,7 @@ sub clean_project { # code to build the project from $build_dir. sub build_project { - if ($ENV{BUILD}) { - return system($ENV{BUILD}); - } else { - return system("make -j4"); - } + return system($builder); } # commands to start the various xgill binaries. timeouts can be specified @@ -241,7 +236,7 @@ sub run_build # build is finished, the complete run can resume. # return value only useful if --foreground - exit $? || $exit_status; + exit($? || $exit_status); } # this is the complete process, wait for the build to finish.