spark2014/scripts/testgen.py

import os
import sys
import json
import argparse
import subprocess


def run_spark(project_file, filename, line):
    print("Running gnatprove")
    subprocess.run(
        [
            "gnatprove",
            f"-P{project_file}",
            "-j0",
            "--output=oneline",
            "-u",
            "--counterexamples=on",
            "--check-counterexamples=on",
            "--level=2",
            f"--limit-subp={filename}:{line}",
        ]
    )


def run_gnattest(project_name, filename, line):
    print(f"Running gnattest at {(filename)}:{str(line)}")
    subprocess.run(
        [
            "gnattest",
            f"-P{project_name}",
            filename,
            f"--gen-test-subprograms={filename}:{str(line)}",
            "--gen-test-vectors",
            "--gen-test-num=1",
        ]
    )


def get_hash(project_name, filename, line):
    print(
        f"Retrieving {project_name} subprogram's hash16 at "
        + f"{(filename)}:{str(line)}"
    )

    command = ["gnattest", f"-P{project_name}", f"--dump-subp-hash={filename}:{line}"]

    process = subprocess.run(command, capture_output=True, text=True)

    return process.stdout.strip("\n").strip("\t")


def get_values_from_spark_JSON(JSON_file, source_file, source_line):
    print(f"Retrieving counter-example's values from {os.path.abspath(JSON_file)}.")
    try:
        with open(JSON_file, "r") as f:
            data = json.load(f)

            proofs = data.get("proof", {})
            for proof in proofs:

                cntexmp_value = proof.get("cntexmp_value", {})

                subp_file = cntexmp_value.get("subp_file")
                subp_line = cntexmp_value.get("subp_line")

                if subp_file == source_file and str(subp_line) == source_line:
                    return cntexmp_value.get("inputs", {})

        return None

    except FileNotFoundError:
        print(f"Error: filename  {JSON_file}  not found.")
        return None
    except json.JSONDecodeError:
        print(f"Error: Invalid JSON format in {JSON_file}.")
        return None
    except Exception as e:
        print(f"An unexpected error occurred: {e}")
        return None


def get_subprogram(JSON_file):
    print(
        "Retrieving subprogram's declaration location from "
        + f"{os.path.abspath(JSON_file)}"
    )
    try:
        with open(JSON_file, "r") as f:
            data = json.load(f)

            flows = data.get("flow", {})

            for flow in flows:
                if flow.get("rule", {}) == "SUBPROGRAM_TERMINATION":

                    filename = flow.get("file", {})
                    line = flow.get("line", {})

                    return [filename, line]

            sys.exit("error: subprogram not found.")

    except FileNotFoundError:
        print(f"Error: filename  {JSON_file}  not found.")
        return None
    except json.JSONDecodeError:
        print(f"Error: Invalid JSON format in {JSON_file}.")
        return None
    except Exception as e:
        print(f"An unexpected error occurred: {e}")
        return None


def replace_values(JSON_file, subp_hash, values):
    print(f"Replacing gnattest values by SPARK's in {os.path.abspath(JSON_file)}.")
    try:
        with open(JSON_file, "r") as f:
            data = json.load(f)

            subprogram = data.get(subp_hash, {})

            if subprogram == {}:
                sys.exit("error: hashes do not match")  # This should never happen

            test_vectors = data.get(subp_hash, {}).get("test_vectors", {})

            param_vectors = test_vectors[0].get("param_values", {})

            for value in values:
                name = value.get("name", {})
                val = value.get("value", {})

                for param in param_vectors:
                    if param.get("name", {}) == name:
                        param["value"] = val

        with open(JSON_file, "w") as f:
            json.dump(data, f, indent=4)

    except FileNotFoundError:
        print(f"Error: filename {JSON_file} not found.")
        return None
    except json.JSONDecodeError:
        print(f"Error: Invalid JSON format in {JSON_file}.")
        return None


def generate_tests(project_name, filename, line):
    print("Generating tests")
    subprocess.run(
        [
            "gnattest",
            f"-P{project_name}",
            f"--gen-test-subprograms={filename}:{str(line)}",
        ]
    )


def run(project_file, filename, line):
    project_path = os.path.dirname(os.path.realpath(project_file))

    os.chdir(project_path)

    # SPARK step

    run_spark(project_file, filename, line)

    spark_filenames = []

    for file in os.listdir(os.path.join(project_path, "obj/gnatprove/")):
        if file.endswith(".spark"):
            spark_filenames.append(file)

    if spark_filenames == []:
        sys.exit("could not find generated .spark file.")

    ce_values = None
    for spark_filename in spark_filenames:
        ce_values = get_values_from_spark_JSON(
            os.path.join(project_path, "obj/gnatprove/", spark_filename), filename, line
        )
        if ce_values is not None:
            break

    if ce_values is None:
        sys.exit("no counter-example at " + filename + ":" + line)

    print(f"counter-example values: {str(ce_values)}")

    # gnattest step

    [project_name, _] = [s for s in project_file.split(".") if s.strip()]

    hash_value = get_hash(project_name, os.path.join("src/", filename), line)

    run_gnattest(project_name, filename, line)

    replace_values(
        os.path.join(
            project_path,
            "obj",
            "gnattest",
            "tests",
            "JSON_Tests",
            f"{project_name}.json",
        ),
        hash_value,
        ce_values,
    )

    generate_tests(project_name, filename, line)

    print(
        "tests have been generated and can be found at "
        + f"{os.path.abspath(project_path + "/obj/gnattest/tests/")}"
    )

    # run tests

    os.chdir("obj/gnattest/harness")

    subprocess.run("make")

    subprocess.run(["./test_runner", f"--routines={filename}:{str(line)}"])


def main():
    parser = argparse.ArgumentParser(
        prog="Testgen",
        description="Generate a ADA test harness from gnatprove's counter-exemples",
    )

    parser.add_argument(
        "-p",
        help="Path to the project file",
        required=True,
        type=str,
        dest="project_file",
    )

    parser.add_argument(
        "-s",
        help="file:line of the subprogram to generate tests from",
        required=True,
        type=str,
        dest="position",
    )

    args = parser.parse_args()

    project_path = args.project_file
    [filename, line] = [s for s in args.position.split(":") if s.strip()]

    run(project_path, filename, line)


if __name__ == "__main__":
    main()