mirror of
https://github.com/izzy2lost/2ship2harkinian-Android.git
synced 2026-06-19 01:20:08 -07:00
Remove decomp-permuter (#447)
* Nuke decomp permuter * Add decomp permuter and mips2c to gitignore
This commit is contained in:
Anghelo Carvajal
@@ -1,45 +0,0 @@
|
||||
name: Systray
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- 'src/net/cmd/systray/*'
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- 'src/net/cmd/systray/*'
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: Build on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-16.04
|
||||
binary: permuter-systray-linux
|
||||
- os: windows-latest
|
||||
binary: permuter-systray.exe
|
||||
- os: macos-latest
|
||||
binary: permuter-systray-macos
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
|
||||
- name: Install gtk3
|
||||
if: ${{ matrix.os == 'ubuntu-16.04' }}
|
||||
run: sudo apt-get install libgtk-3-dev libappindicator3-dev
|
||||
|
||||
- name: Setup Go environment
|
||||
uses: actions/setup-go@v2.1.3
|
||||
|
||||
- name: Build
|
||||
run: go build -o ${{ matrix.binary }} -ldflags "-s -w" tray.go
|
||||
working-directory: src/net/cmd/systray/
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ matrix.binary }}
|
||||
path: src/net/cmd/systray/${{ matrix.binary }}
|
||||
@@ -1,11 +0,0 @@
|
||||
*.o
|
||||
*.s
|
||||
*.c
|
||||
*.py[cod]
|
||||
.mypy_cache/
|
||||
.cache/
|
||||
__pycache__/
|
||||
!test/*.c
|
||||
/nonmatchings
|
||||
.vscode/
|
||||
pah.conf
|
||||
@@ -1,12 +0,0 @@
|
||||
; DO NOT EDIT (unless you know what you are doing)
|
||||
;
|
||||
; This subdirectory is a git "subrepo", and this file is maintained by the
|
||||
; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
|
||||
;
|
||||
[subrepo]
|
||||
remote = https://github.com/simonlindholm/decomp-permuter.git
|
||||
branch = main
|
||||
commit = a20bac9422b6d8adbf7c06473c2ae3c3fee16be5
|
||||
parent = 2668eec556c01fa2f4c16a203c93c208dc03e639
|
||||
method = merge
|
||||
cmdver = 0.4.3
|
||||
@@ -1,6 +0,0 @@
|
||||
repos:
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 20.8b1
|
||||
hooks:
|
||||
- id: black
|
||||
language_version: python3.6
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 Simon Lindholm and contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,120 +0,0 @@
|
||||
# Decomp permuter
|
||||
|
||||
Automatically permutes C files to better match a target binary. The permuter has two modes of operation:
|
||||
- Random: purely at random, introduce temporary variables for values, change types, put statements on the same line...
|
||||
- Manual: test all combinations of user-specified variations, using macros like `PERM_GENERAL(a = b ? c : d;, if (b) a = c; else a = d;)` to try both specified alternatives.
|
||||
|
||||
The modes can also be combined, by using the `PERM_RANDOMIZE` macro.
|
||||
|
||||
[<img src="https://asciinema.org/a/232846.svg" height="300">](https://asciinema.org/a/232846)
|
||||
|
||||
The main target for the tool is MIPS code compiled by old compilers (IDO, possibly GCC).
|
||||
Getting it to work on other architectures shouldn't be too hard, however.
|
||||
https://github.com/laqieer/decomp-permuter-arm has an ARM port.
|
||||
|
||||
## Usage
|
||||
|
||||
`./permuter.py directory/` runs the permuter; see below for the meaning of the directory.
|
||||
Pass `-h` to see possible flags. `-j` is suggested (enables multi-threaded mode).
|
||||
|
||||
You'll first need to install a couple of prerequisites: `python3 -m pip install pycparser pynacl toml` (also `dataclasses` if on Python 3.6 or below)
|
||||
|
||||
The permuter expects as input one or more directory containing:
|
||||
- a .c file with a single function,
|
||||
- a .o file to match,
|
||||
- a .sh file that compiles the .c file.
|
||||
|
||||
For projects with a properly configured makefile, you should be able to set these up by running
|
||||
```
|
||||
./import.py <path/to/file.c> <path/to/file.s>
|
||||
```
|
||||
where file.c contains the function to be permuted, and file.s is its assembly in a self-contained file.
|
||||
Otherwise, see USAGE.md for more details.
|
||||
|
||||
For projects using Ninja instead of Make, add a `permuter_settings.toml` in the root or `tools/` directory of the project:
|
||||
```toml
|
||||
build_system = "ninja"
|
||||
```
|
||||
Then `import.py` should work as expected if `build.ninja` is at the root of the project.
|
||||
|
||||
The .c file may be modified with any of the following macros which affect manual permutation:
|
||||
|
||||
- `PERM_GENERAL(a, b, ...)` expands to any of `a`, `b`, ...
|
||||
- `PERM_VAR(a, b)` sets the meta-variable `a` to `b`, `PERM_VAR(a)` expands to the meta-variable `a`.
|
||||
- `PERM_RANDOMIZE(code)` expands to `code`, but allows randomization within that region. Multiple regions may be specified.
|
||||
- `PERM_LINESWAP(lines)` expands to a permutation of the ordered set of non-whitespace lines (split by `\n`). Each line must contain zero or more complete C statements. (For incomplete statements use `PERM_LINESWAP_TEXT`, which is slower because it has to repeatedly parse C code.)
|
||||
- `PERM_INT(lo, hi)` expands to an integer between `lo` and `hi` (which must be constants).
|
||||
- `PERM_IGNORE(code)` expands to `code`, without passing it through the C parser library (pycparser)/randomizer. This can be used to avoid parse errors for non-standard C, e.g. `asm` blocks.
|
||||
- `PERM_PRETEND(code)` expands to `code` for the purpose of the C parser/randomizer, but gets removed afterwards. This can be used together with `PERM_IGNORE` to enable the permuter to deal with input it isn't designed for (e.g. inline functions, C++, non-code).
|
||||
- `PERM_ONCE([key,] code)` expands to either `code` or to nothing, such that each unique key gets expanded exactly once. `key` defaults to `code`. For example, `PERM_ONCE(a;) b; PERM_ONCE(a;)` expands to either `a; b;` or `b; a;`.
|
||||
|
||||
Arguments are split by a commas, exluding commas inside parenthesis. `(,)` is a special escape sequence that resolves to `,`.
|
||||
|
||||
Nested macros are allowed, so e.g.
|
||||
```
|
||||
PERM_VAR(delayed, )
|
||||
PERM_GENERAL(stmt;, PERM_VAR(delayed, stmt;))
|
||||
...
|
||||
PERM_VAR(delayed)
|
||||
```
|
||||
is an alternative way of writing `PERM_ONCE`.
|
||||
|
||||
## permuter@home
|
||||
|
||||
The permuter supports a distributed mode, where people can donate processor power to your permuter runs to speed them up.
|
||||
To use this, pass `-J` when running `permuter.py` and follow the instructions.
|
||||
You will need to be granted access by someone who is already connected to a permuter network.
|
||||
|
||||
To allow others to use your computer for permuter runs, do the following:
|
||||
|
||||
- install Docker (used for sandboxing and to ensure a consistent environment)
|
||||
- if on Linux, add yourself to the Docker group: `sudo usermod -aG docker $USER`
|
||||
- install required packages: `python3 -m pip install docker`
|
||||
- open a terminal, and run `./pah.py run-server` to start the server.
|
||||
There are a few required arguments (e.g. how many cores to use), see `--help` for more details.
|
||||
|
||||
Please be aware that being in the Docker group implies (password-less) sudo rights.
|
||||
You can avoid that for your personal account by running the permuter under a separate user.
|
||||
Unfortunately, there is currently no way to run a sandboxed permuter server without sudo rights. 😢
|
||||
|
||||
Anyone who is granted access to permuter@home can run a server.
|
||||
|
||||
To set up a new permuter network, see [src/net/controller/README.md](./src/net/controller/README.md).
|
||||
|
||||
## FAQ
|
||||
|
||||
**What do the scores mean?** The scores are computed by taking diffs of objdump'd .o
|
||||
files, and giving different penalties for lines that are the same/use the same
|
||||
instruction/are reordered/don't match at all. 0 means the function matches fully.
|
||||
Stack positions are ignored unless --stack-diffs is passed (but beware that the
|
||||
permuter is currently quite bad at resolving stack differences). For more details,
|
||||
see scorer.py. It's far from a perfect system, and should probably be tweaked to
|
||||
look at e.g. the register diff graph.
|
||||
|
||||
**What sort of non-matchings are the permuter good at?** It's generally best towards
|
||||
the end, when mostly regalloc changes remain. If there are reorderings or functional
|
||||
changes, it's often easy to resolve those by hand, and neither the scorer nor the
|
||||
randomizer tends to play well with them.
|
||||
|
||||
**Should I use this instead of trying to match code by hand?** No, but it can be a good
|
||||
complement. PERM macros can be used to quickly test lots of variations of a function at
|
||||
once, in cases where there are interactions between several parts of a function.
|
||||
The randomization mode often finds lots of nonsensical changes that improve regalloc
|
||||
"by accident"; it's up to you to pick out the ones that look sensible. If none do,
|
||||
it can still be useful to know which parts of the function need to be changed to get the
|
||||
code nearer to matching. Having made one of the improvements, and the function can then be
|
||||
permuted again, to find further possible improvements.
|
||||
|
||||
## Helping out
|
||||
|
||||
There's tons of room for helping out with the permuter!
|
||||
Many more randomization passes could be added, the scoring function is far from optimal,
|
||||
the permuter could be made easier to use, etc. etc. The GitHub Issues list has some ideas.
|
||||
|
||||
Ideally, `mypy permuter.py` and `./run-tests.sh` should succeed with no errors, and files
|
||||
formatted with `black`. To setup a pre-commit hook for black, run:
|
||||
```
|
||||
pip install pre-commit black
|
||||
pre-commit install
|
||||
```
|
||||
PRs that skip this are still welcome, however.
|
||||
@@ -1,25 +0,0 @@
|
||||
This file describes how to manually set up a directory for use with the permuter.
|
||||
**You probably don't need to do this!** In normal circumstances, `./import.py`
|
||||
does all this for you. See README.md for more details.
|
||||
|
||||
* create a directory that will contain all of the input files for the invokation
|
||||
* put a compile command into `<dir>/compile.sh` (see e.g. `compile_example.sh`; it will be invoked as `./compile.sh input.c -o output.o`)
|
||||
* `gcc -E -P -I header_dir -D'__attribute__(x)=' orig_c_file.c > <dir>/base.c`
|
||||
* `python3 strip_other_fns.py <dir>/base.c func_name`
|
||||
* put asm for `func_name` into `<dir>/target.s`, with the following header:
|
||||
|
||||
```asm
|
||||
.set noat
|
||||
.set noreorder
|
||||
.set gp=64
|
||||
.macro glabel label
|
||||
.global \label
|
||||
.type \label, @function
|
||||
\label:
|
||||
.endm
|
||||
```
|
||||
* `mips-linux-gnu-as -march=vr4300 -mabi=32 <dir>/target.s -o <dir>/target.o`
|
||||
* optional sanity checks:
|
||||
- `<dir>/compile.sh <dir>/base.c -o <dir>/base.o`
|
||||
- `./diff.sh <dir>/target.o <dir>/base.o`
|
||||
* `./permuter.py <dir>`
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/bin/bash
|
||||
mips-linux-gnu-gcc -O2 "$@"
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $# < 2 ]]; then
|
||||
echo "Usage: $0 orig.o new.o [flags]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f $1 -o ! -f $2 ]; then
|
||||
echo Source files not readable
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INPUT1="$1"
|
||||
INPUT2="$2"
|
||||
shift
|
||||
shift
|
||||
wdiff -n <(python3 ./src/objdump.py "$INPUT1" "$@") <(python3 ./src/objdump.py "$INPUT2" "$@") | colordiff | less -Ric
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,27 +0,0 @@
|
||||
[mypy]
|
||||
check_untyped_defs = True
|
||||
disallow_any_generics = False
|
||||
disallow_incomplete_defs = True
|
||||
disallow_subclassing_any = True
|
||||
disallow_untyped_calls = True
|
||||
disallow_untyped_decorators = True
|
||||
disallow_untyped_defs = True
|
||||
no_implicit_optional = True
|
||||
warn_redundant_casts = True
|
||||
warn_return_any = True
|
||||
warn_unused_ignores = True
|
||||
mypy_path = stubs
|
||||
python_version = 3.7
|
||||
files = import.py, pah.py, permuter.py, src/net/evaluator.py
|
||||
|
||||
[mypy-nacl.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-pystray.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-docker.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-PIL.*]
|
||||
ignore_missing_imports = True
|
||||
@@ -1,4 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from src.net.cmd.main import main
|
||||
|
||||
main()
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from src.main import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,9 +0,0 @@
|
||||
# Optional configuration file for import.py. Put it in the root or in tools/
|
||||
# of the repo you are importing from.
|
||||
|
||||
build_system = "ninja"
|
||||
|
||||
[preserve_macros]
|
||||
"g[DS]P.*" = "void"
|
||||
"gDma.*" = "void"
|
||||
"_SHIFTL" = "unsigned int"
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
python3 -m unittest discover -s test/
|
||||
# python3 -m pytest test/
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $1 < 2 ]]; then
|
||||
echo "Usage: $0 output_dir"
|
||||
echo "Ex: $0 nonmatchings/func_80000000"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d $1 ]]; then
|
||||
echo "Argument must be a directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
find $1 -name score.txt -exec echo -n {}\ \; -exec cat {} \; | sort -rnk2
|
||||
@@ -1,303 +0,0 @@
|
||||
"""Functions and classes for dealing with types in a C AST.
|
||||
|
||||
They make a number of simplifying assumptions:
|
||||
- const and volatile doesn't matter.
|
||||
- arithmetic promotes all int-like types to 'int'.
|
||||
- no two variables can have the same name, even across functions.
|
||||
|
||||
For the purposes of the randomizer these restrictions are acceptable."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Union, Dict, Set, List
|
||||
|
||||
from pycparser import c_ast
|
||||
from pycparser.c_ast import ArrayDecl, TypeDecl, PtrDecl, FuncDecl, IdentifierType
|
||||
|
||||
Type = Union[PtrDecl, ArrayDecl, TypeDecl, FuncDecl]
|
||||
SimpleType = Union[PtrDecl, TypeDecl]
|
||||
|
||||
StructUnion = Union[c_ast.Struct, c_ast.Union]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TypeMap:
|
||||
typedefs: Dict[str, Type] = field(default_factory=dict)
|
||||
fn_ret_types: Dict[str, Type] = field(default_factory=dict)
|
||||
var_types: Dict[str, Type] = field(default_factory=dict)
|
||||
struct_defs: Dict[str, StructUnion] = field(default_factory=dict)
|
||||
|
||||
|
||||
def basic_type(name: Union[str, List[str]]) -> TypeDecl:
|
||||
names = [name] if isinstance(name, str) else name
|
||||
idtype = IdentifierType(names=names)
|
||||
return TypeDecl(declname=None, quals=[], type=idtype)
|
||||
|
||||
|
||||
def pointer(type: Type) -> Type:
|
||||
return PtrDecl(quals=[], type=type)
|
||||
|
||||
|
||||
def resolve_typedefs(type: Type, typemap: TypeMap) -> Type:
|
||||
while (
|
||||
isinstance(type, TypeDecl)
|
||||
and isinstance(type.type, IdentifierType)
|
||||
and len(type.type.names) == 1
|
||||
and type.type.names[0] in typemap.typedefs
|
||||
):
|
||||
type = typemap.typedefs[type.type.names[0]]
|
||||
return type
|
||||
|
||||
|
||||
def pointer_decay(type: Type, typemap: TypeMap) -> SimpleType:
|
||||
real_type = resolve_typedefs(type, typemap)
|
||||
if isinstance(real_type, ArrayDecl):
|
||||
return PtrDecl(quals=[], type=real_type.type)
|
||||
if isinstance(real_type, FuncDecl):
|
||||
return PtrDecl(quals=[], type=type)
|
||||
if isinstance(real_type, TypeDecl) and isinstance(real_type.type, c_ast.Enum):
|
||||
return basic_type("int")
|
||||
assert not isinstance(
|
||||
type, (ArrayDecl, FuncDecl)
|
||||
), "resolve_typedefs can't hide arrays/functions"
|
||||
return type
|
||||
|
||||
|
||||
def get_decl_type(decl: c_ast.Decl) -> Type:
|
||||
"""For a Decl that declares a variable (and not just a struct/union/enum),
|
||||
return its type."""
|
||||
assert decl.name is not None
|
||||
assert isinstance(decl.type, (PtrDecl, ArrayDecl, FuncDecl, TypeDecl))
|
||||
return decl.type
|
||||
|
||||
|
||||
def deref_type(type: Type, typemap: TypeMap) -> Type:
|
||||
type = resolve_typedefs(type, typemap)
|
||||
assert isinstance(type, (ArrayDecl, PtrDecl)), "dereferencing non-pointer"
|
||||
return type.type
|
||||
|
||||
|
||||
def struct_member_type(struct: StructUnion, field_name: str, typemap: TypeMap) -> Type:
|
||||
if not struct.decls:
|
||||
assert (
|
||||
struct.name in typemap.struct_defs
|
||||
), f"Accessing field {field_name} of undefined struct {struct.name}"
|
||||
struct = typemap.struct_defs[struct.name]
|
||||
assert struct.decls, "struct_defs never points to an incomplete type"
|
||||
for decl in struct.decls:
|
||||
if isinstance(decl, c_ast.Decl):
|
||||
if decl.name == field_name:
|
||||
return get_decl_type(decl)
|
||||
if decl.name == None and isinstance(decl.type, (c_ast.Struct, c_ast.Union)):
|
||||
try:
|
||||
return struct_member_type(decl.type, field_name, typemap)
|
||||
except AssertionError:
|
||||
pass
|
||||
|
||||
assert False, f"No field {field_name} in struct {struct.name}"
|
||||
|
||||
|
||||
def expr_type(node: c_ast.Node, typemap: TypeMap) -> Type:
|
||||
def rec(sub_expr: c_ast.Node) -> Type:
|
||||
return expr_type(sub_expr, typemap)
|
||||
|
||||
if isinstance(node, c_ast.Assignment):
|
||||
return rec(node.lvalue)
|
||||
if isinstance(node, c_ast.StructRef):
|
||||
lhs_type = rec(node.name)
|
||||
if node.type == "->":
|
||||
lhs_type = deref_type(lhs_type, typemap)
|
||||
struct_type = resolve_typedefs(lhs_type, typemap)
|
||||
assert isinstance(struct_type, TypeDecl)
|
||||
assert isinstance(
|
||||
struct_type.type, (c_ast.Struct, c_ast.Union)
|
||||
), f"struct deref of non-struct {struct_type.declname}"
|
||||
return struct_member_type(struct_type.type, node.field.name, typemap)
|
||||
if isinstance(node, c_ast.Cast):
|
||||
return node.to_type.type
|
||||
if isinstance(node, c_ast.Constant):
|
||||
if node.type == "string":
|
||||
return pointer(basic_type("char"))
|
||||
if node.type == "char":
|
||||
return basic_type("int")
|
||||
return basic_type(node.type.split(" "))
|
||||
if isinstance(node, c_ast.ID):
|
||||
return typemap.var_types[node.name]
|
||||
if isinstance(node, c_ast.UnaryOp):
|
||||
if node.op in ["p++", "p--", "++", "--"]:
|
||||
return rec(node.expr)
|
||||
if node.op == "&":
|
||||
return pointer(rec(node.expr))
|
||||
if node.op == "*":
|
||||
subtype = rec(node.expr)
|
||||
return deref_type(subtype, typemap)
|
||||
if node.op in ["-", "+"]:
|
||||
subtype = pointer_decay(rec(node.expr), typemap)
|
||||
if allowed_basic_type(subtype, typemap, ["double"]):
|
||||
return basic_type("double")
|
||||
if allowed_basic_type(subtype, typemap, ["float"]):
|
||||
return basic_type("float")
|
||||
if node.op in ["sizeof", "-", "+", "~", "!"]:
|
||||
return basic_type("int")
|
||||
assert False, f"unknown unary op {node.op}"
|
||||
if isinstance(node, c_ast.BinaryOp):
|
||||
lhs_type = pointer_decay(rec(node.left), typemap)
|
||||
rhs_type = pointer_decay(rec(node.right), typemap)
|
||||
if node.op in [">>", "<<"]:
|
||||
return lhs_type
|
||||
if node.op in ["<", "<=", ">", ">=", "==", "!=", "&&", "||"]:
|
||||
return basic_type("int")
|
||||
if node.op in "&|^%":
|
||||
return basic_type("int")
|
||||
real_lhs = resolve_typedefs(lhs_type, typemap)
|
||||
real_rhs = resolve_typedefs(rhs_type, typemap)
|
||||
if node.op in "+-":
|
||||
lptr = isinstance(real_lhs, PtrDecl)
|
||||
rptr = isinstance(real_rhs, PtrDecl)
|
||||
if lptr or rptr:
|
||||
if lptr and rptr:
|
||||
assert node.op != "+", "pointer + pointer"
|
||||
return basic_type("int")
|
||||
if lptr:
|
||||
return lhs_type
|
||||
assert node.op == "+", "int - pointer"
|
||||
return rhs_type
|
||||
if node.op in "*/+-":
|
||||
assert isinstance(real_lhs, TypeDecl)
|
||||
assert isinstance(real_rhs, TypeDecl)
|
||||
assert isinstance(real_lhs.type, IdentifierType)
|
||||
assert isinstance(real_rhs.type, IdentifierType)
|
||||
if "double" in real_lhs.type.names + real_rhs.type.names:
|
||||
return basic_type("double")
|
||||
if "float" in real_lhs.type.names + real_rhs.type.names:
|
||||
return basic_type("float")
|
||||
return basic_type("int")
|
||||
if isinstance(node, c_ast.FuncCall):
|
||||
expr = node.name
|
||||
if isinstance(expr, c_ast.ID):
|
||||
if expr.name not in typemap.fn_ret_types:
|
||||
raise Exception(f"Called function {expr.name} is missing a prototype")
|
||||
return typemap.fn_ret_types[expr.name]
|
||||
else:
|
||||
fptr_type = resolve_typedefs(rec(expr), typemap)
|
||||
if isinstance(fptr_type, PtrDecl):
|
||||
fptr_type = fptr_type.type
|
||||
fptr_type = resolve_typedefs(fptr_type, typemap)
|
||||
assert isinstance(fptr_type, FuncDecl), "call to non-function"
|
||||
return fptr_type.type
|
||||
if isinstance(node, c_ast.ExprList):
|
||||
return rec(node.exprs[-1])
|
||||
if isinstance(node, c_ast.ArrayRef):
|
||||
subtype = rec(node.name)
|
||||
return deref_type(subtype, typemap)
|
||||
if isinstance(node, c_ast.TernaryOp):
|
||||
return rec(node.iftrue)
|
||||
assert False, f"Unknown expression node type: {node}"
|
||||
|
||||
|
||||
def decayed_expr_type(expr: c_ast.Node, typemap: TypeMap) -> SimpleType:
|
||||
return pointer_decay(expr_type(expr, typemap), typemap)
|
||||
|
||||
|
||||
def same_type(
|
||||
type1: Type, type2: Type, typemap: TypeMap, allow_similar: bool = False
|
||||
) -> bool:
|
||||
while True:
|
||||
type1 = resolve_typedefs(type1, typemap)
|
||||
type2 = resolve_typedefs(type2, typemap)
|
||||
if isinstance(type1, ArrayDecl) and isinstance(type2, ArrayDecl):
|
||||
type1 = type1.type
|
||||
type2 = type2.type
|
||||
continue
|
||||
if isinstance(type1, PtrDecl) and isinstance(type2, PtrDecl):
|
||||
type1 = type1.type
|
||||
type2 = type2.type
|
||||
continue
|
||||
if isinstance(type1, TypeDecl) and isinstance(type2, TypeDecl):
|
||||
sub1 = type1.type
|
||||
sub2 = type2.type
|
||||
if isinstance(sub1, c_ast.Struct) and isinstance(sub2, c_ast.Struct):
|
||||
return sub1.name == sub2.name
|
||||
if isinstance(sub1, c_ast.Union) and isinstance(sub2, c_ast.Union):
|
||||
return sub1.name == sub2.name
|
||||
if (
|
||||
allow_similar
|
||||
and isinstance(sub1, (IdentifierType, c_ast.Enum))
|
||||
and isinstance(sub2, (IdentifierType, c_ast.Enum))
|
||||
):
|
||||
# All int-ish types are similar (except void, but whatever)
|
||||
return True
|
||||
if isinstance(sub1, c_ast.Enum) and isinstance(sub2, c_ast.Enum):
|
||||
return sub1.name == sub2.name
|
||||
if isinstance(sub1, IdentifierType) and isinstance(sub2, IdentifierType):
|
||||
return sorted(sub1.names) == sorted(sub2.names)
|
||||
return False
|
||||
|
||||
|
||||
def allowed_basic_type(
|
||||
type: SimpleType, typemap: TypeMap, allowed_types: List[str]
|
||||
) -> bool:
|
||||
"""Check if a type resolves to a basic type with one of the allowed_types
|
||||
keywords in it."""
|
||||
base_type = resolve_typedefs(type, typemap)
|
||||
if not isinstance(base_type, c_ast.TypeDecl):
|
||||
return False
|
||||
if not isinstance(base_type.type, c_ast.IdentifierType):
|
||||
return False
|
||||
if all(x not in base_type.type.names for x in allowed_types):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_typemap(ast: c_ast.FileAST) -> TypeMap:
|
||||
ret = TypeMap()
|
||||
for item in ast.ext:
|
||||
if isinstance(item, c_ast.Typedef):
|
||||
ret.typedefs[item.name] = item.type
|
||||
if isinstance(item, c_ast.FuncDef):
|
||||
assert item.decl.name is not None, "cannot define anonymous function"
|
||||
assert isinstance(item.decl.type, FuncDecl)
|
||||
ret.fn_ret_types[item.decl.name] = item.decl.type.type
|
||||
if isinstance(item, c_ast.Decl) and isinstance(item.type, FuncDecl):
|
||||
assert item.name is not None, "cannot define anonymous function"
|
||||
ret.fn_ret_types[item.name] = item.type.type
|
||||
defined_function_decls: Set[c_ast.Decl] = set()
|
||||
|
||||
class Visitor(c_ast.NodeVisitor):
|
||||
def visit_Struct(self, struct: c_ast.Struct) -> None:
|
||||
if struct.decls and struct.name is not None:
|
||||
ret.struct_defs[struct.name] = struct
|
||||
# Do not visit decls of this struct
|
||||
|
||||
def visit_Union(self, union: c_ast.Union) -> None:
|
||||
if union.decls and union.name is not None:
|
||||
ret.struct_defs[union.name] = union
|
||||
# Do not visit decls of this union
|
||||
|
||||
def visit_Decl(self, decl: c_ast.Decl) -> None:
|
||||
if decl.name is not None:
|
||||
ret.var_types[decl.name] = get_decl_type(decl)
|
||||
if not isinstance(decl.type, FuncDecl) or decl in defined_function_decls:
|
||||
# Do not visit declarations in parameter lists of functions
|
||||
# other than our own.
|
||||
self.visit(decl.type)
|
||||
|
||||
def visit_Enumerator(self, enumerator: c_ast.Enumerator) -> None:
|
||||
ret.var_types[enumerator.name] = basic_type("int")
|
||||
|
||||
def visit_FuncDef(self, fn: c_ast.FuncDef) -> None:
|
||||
if fn.decl.name is not None:
|
||||
ret.var_types[fn.decl.name] = get_decl_type(fn.decl)
|
||||
defined_function_decls.add(fn.decl)
|
||||
self.generic_visit(fn)
|
||||
|
||||
Visitor().visit(ast)
|
||||
return ret
|
||||
|
||||
|
||||
def set_decl_name(decl: c_ast.Decl) -> None:
|
||||
name = decl.name
|
||||
assert name is not None
|
||||
type = get_decl_type(decl)
|
||||
while not isinstance(type, TypeDecl):
|
||||
type = type.type
|
||||
type.declname = name
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,99 +0,0 @@
|
||||
import copy
|
||||
from dataclasses import dataclass, field
|
||||
import functools
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from pycparser import c_ast as ca
|
||||
|
||||
from .compiler import Compiler
|
||||
from .randomizer import Randomizer
|
||||
from .scorer import Scorer
|
||||
from .perm.perm import EvalState
|
||||
from .perm.ast import apply_ast_perms
|
||||
from .helpers import try_remove
|
||||
from .profiler import Profiler
|
||||
from . import ast_util
|
||||
|
||||
|
||||
@dataclass
|
||||
class CandidateResult:
|
||||
"""Represents the result of scoring a candidate, and is sent from child to
|
||||
parent processes, or server to client with p@h."""
|
||||
|
||||
score: int
|
||||
hash: Optional[str]
|
||||
source: Optional[str]
|
||||
profiler: Optional[Profiler] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Candidate:
|
||||
"""
|
||||
Represents a AST candidate created from a source which can be randomized
|
||||
(possibly multiple times), compiled, and scored.
|
||||
"""
|
||||
|
||||
ast: ca.FileAST
|
||||
|
||||
fn_index: int
|
||||
rng_seed: int
|
||||
randomizer: Randomizer
|
||||
score_value: Optional[int] = field(init=False, default=None)
|
||||
score_hash: Optional[str] = field(init=False, default=None)
|
||||
_cache_source: Optional[str] = field(init=False, default=None)
|
||||
|
||||
@staticmethod
|
||||
@functools.lru_cache(maxsize=16)
|
||||
def _cached_shared_ast(
|
||||
source: str, fn_name: str
|
||||
) -> Tuple[ca.FuncDef, int, ca.FileAST]:
|
||||
ast = ast_util.parse_c(source)
|
||||
orig_fn, fn_index = ast_util.extract_fn(ast, fn_name)
|
||||
ast_util.normalize_ast(orig_fn, ast)
|
||||
return orig_fn, fn_index, ast
|
||||
|
||||
@staticmethod
|
||||
def from_source(
|
||||
source: str, eval_state: EvalState, fn_name: str, rng_seed: int
|
||||
) -> "Candidate":
|
||||
# Use the same AST for all instances of the same original source, but
|
||||
# with the target function deeply copied. Since we never change the
|
||||
# AST outside of the target function, this is fine, and it saves us
|
||||
# performance (deepcopy is really slow).
|
||||
orig_fn, fn_index, ast = Candidate._cached_shared_ast(source, fn_name)
|
||||
ast = copy.copy(ast)
|
||||
ast.ext = copy.copy(ast.ext)
|
||||
fn_copy = copy.deepcopy(orig_fn)
|
||||
ast.ext[fn_index] = fn_copy
|
||||
apply_ast_perms(fn_copy, eval_state)
|
||||
return Candidate(
|
||||
ast=ast,
|
||||
fn_index=fn_index,
|
||||
rng_seed=rng_seed,
|
||||
randomizer=Randomizer(rng_seed),
|
||||
)
|
||||
|
||||
def randomize_ast(self) -> None:
|
||||
self.randomizer.randomize(self.ast, self.fn_index)
|
||||
self._cache_source = None
|
||||
|
||||
def get_source(self) -> str:
|
||||
if self._cache_source is None:
|
||||
self._cache_source = ast_util.to_c(self.ast)
|
||||
return self._cache_source
|
||||
|
||||
def compile(self, compiler: Compiler, show_errors: bool = False) -> Optional[str]:
|
||||
source: str = self.get_source()
|
||||
return compiler.compile(source, show_errors=show_errors)
|
||||
|
||||
def score(self, scorer: Scorer, o_file: Optional[str]) -> CandidateResult:
|
||||
self.score_value = None
|
||||
self.score_hash = None
|
||||
try:
|
||||
self.score_value, self.score_hash = scorer.score(o_file)
|
||||
finally:
|
||||
if o_file:
|
||||
try_remove(o_file)
|
||||
return CandidateResult(
|
||||
score=self.score_value, hash=self.score_hash, source=self.get_source()
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user