langkit/testsuite/testsuite.py

#! /usr/bin/env python

"""
Usage::

    testsuite.py [OPTIONS]

Run the Langkit testsuite.
"""

import glob
import os
import shutil
import subprocess
import sys

from e3.testsuite import Testsuite

import drivers.adalog_driver
import drivers.langkit_support_driver
import drivers.lkt_build_and_run_driver
import drivers.lkt_compile_driver
import drivers.lkt_parse_driver
import drivers.lkt_toolbox_driver
import drivers.lkt_unparse_driver
import drivers.python_driver


def add_to_path(env_var, directory):
    """
    Add the given directory to the `env_var` path.
    """
    path = os.environ.get(env_var)
    path = (
        "{}{}{}".format(directory, os.path.pathsep, path) if path else env_var
    )
    os.environ[env_var] = path


class LangkitTestsuite(Testsuite):
    tests_subdir = "tests"
    test_driver_map = {
        "adalog": drivers.adalog_driver.AdalogDriver,
        "langkit_support": drivers.langkit_support_driver.LangkitSupportDriver,
        "lkt": drivers.lkt_toolbox_driver.LktToolboxDriver,
        "lkt_build_and_run": (
            drivers.lkt_build_and_run_driver.LktBuildAndRunDriver
        ),
        "lkt_compile": drivers.lkt_compile_driver.LktCompileDriver,
        "lkt_parse": drivers.lkt_parse_driver.LktParseDriver,
        "lkt_unparse": drivers.lkt_unparse_driver.LktUnparseDriver,
        "python": drivers.python_driver.PythonDriver,
    }

    def add_options(self, parser):
        parser.add_argument(
            "--with-langkit-python",
            help="If provided, use as the Python interpreter in scripts that"
            " use Langkit modules.",
        )
        parser.add_argument(
            "--with-python",
            help="If provided, use as the Python interpreter in testcases.",
        )

        parser.add_argument(
            "--coverage",
            "-C",
            action="store_true",
            help="Enable computation of code coverage for Langkit and"
            " Langkit_Support. This requires coverage.py and"
            " GNATcoverage.",
        )

        parser.add_argument(
            "--valgrind",
            action="store_true",
            help="Run tests with Valgrind to check memory issues.",
        )

        parser.add_argument(
            "--no-auto-path",
            action="store_true",
            help="Do not automatically add Langkit to PYTHONPATH. This is"
            " useful to test that Langkit was properly installed in the"
            " environment that runs the testsuite.",
        )

        parser.add_argument(
            "--disable-ocaml",
            action="store_true",
            help="Disable testcases that require OCaml (they are enabled by"
            " default).",
        )

        parser.add_argument(
            "--disable-java",
            action="store_true",
            help="Disable testcases that require Java (they are enabled by"
            " default).",
        )
        parser.add_argument(
            "--disable-native-image",
            action="store_true",
            help="Disable testcases that require native-image (they are"
            " enabled by default).",
        )
        parser.add_argument(
            "--maven-local-repo",
            help="Specify the Maven repository to use, default one is the"
            " user's repository (~/.m2).",
        )
        parser.add_argument(
            "--maven-executable",
            help="Specify the Maven executable to use. The default one is"
            ' "mvn".',
        )

        parser.add_argument(
            "--disable-gdb",
            action="store_true",
            help="Disable testcases that require GDB (they are enabled by"
            " default).",
        )

        parser.add_argument(
            "--restricted-env",
            action="store_true",
            help="Skip testcases that cannot run in a restricted environment"
            " (need for non-standard Python packages).",
        )

        #
        # Convenience options for developpers
        #

        parser.add_argument(
            "--disable-tear-up-builds",
            "-B",
            action="store_true",
            help="Disable the automatic build of Langkit_Support during the"
            " testsuite tear_up step. This is used to speed up successive"
            " testsuite runs during development.",
        )
        parser.add_argument(
            "--pretty-print",
            action="store_true",
            help="Pretty-print generated source code.",
        )

        # Tests update
        parser.add_argument(
            "--rewrite",
            "-r",
            action="store_true",
            help="Rewrite test baselines according to current output.",
        )

    def set_up(self):
        super().set_up()

        args = self.main.args

        self.env.rewrite_baselines = args.rewrite
        self.env.control_condition_env = {
            "restricted_env": args.restricted_env,
            "has_ocaml": not args.disable_ocaml,
            "has_java": not args.disable_java,
            "has_native_image": not args.disable_native_image,
            "has_gdb": not args.disable_gdb,
            "os": self.env.build.os.name,
            "valgrind": args.valgrind,
        }

        if args.coverage:
            # Create a directory that we'll use to:
            #
            #   1) collect coverage data for each testcase;
            #   2) generate the HTML report.
            self.env.coverage_dir = os.path.join(self.output_dir, "coverage")
            self.env.langkit_support_coverage_dir = os.path.join(
                self.env.coverage_dir, "langkit_support"
            )
            os.mkdir(self.env.coverage_dir)
            os.mkdir(self.env.langkit_support_coverage_dir)

        def run(name, args):
            p = subprocess.run(
                args,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                encoding="utf-8",
            )
            if p.returncode != 0:
                raise RuntimeError(
                    "{} build failed (GPRbuild returned {})\n{}".format(
                        name, p.returncode, p.stdout
                    )
                )

        # Build Langkit_Support so that each testcase does not try to build it
        # in parallel. To achieve this, we build+install it with all library
        # kinds, and then update the environment so that testcases can assume
        # it is installed.
        if not args.disable_tear_up_builds:
            langkit_root_dir = os.path.join(self.root_dir, "..")
            self.env.langkit_support_project_file = os.path.join(
                langkit_root_dir, "langkit", "support", "langkit_support.gpr"
            )
            install_prefix = os.path.join(self.working_dir, "install")
            if os.path.exists(install_prefix):
                shutil.rmtree(install_prefix)

            gargs = ["-p", "-P", self.env.langkit_support_project_file]
            cargs = ["-cargs", "-O0", "-g", "-gnatwae"]
            if args.coverage:
                gargs.append("--subdirs=gnatcov")
                cargs.extend(["-fdump-scos", "-fpreserve-control-flow"])

            for build in ("static", "relocatable"):
                run(
                    f"Langkit support - build {build}",
                    ["gprbuild"]
                    + gargs
                    + ["-XLIBRARY_TYPE={}".format(build)]
                    + cargs,
                )

                run(
                    f"Langkit support - install {build}",
                    [
                        "gprinstall",
                        "-P",
                        self.env.langkit_support_project_file,
                        "-p",
                        "--sources-subdir=include/langkit_support",
                        "-XLIBRARY_TYPE={}".format(build),
                        "--prefix={}".format(install_prefix),
                        "--build-var=LIBRARY_TYPE",
                        "--build-name={}".format(build),
                    ],
                )

            # Make the installed library available to all testcases
            add_to_path("PATH", os.path.join(install_prefix, "bin"))
            add_to_path("LD_LIBRARY_PATH", os.path.join(install_prefix, "lib"))
            add_to_path(
                "GPR_PROJECT_PATH",
                os.path.join(install_prefix, "share", "gpr"),
            )

            # On GNU/Linux, the OCaml and Java bindings need to have access
            # to the sigsegv_handler shared object. Build it and add it to
            # the environment.
            if (
                not args.disable_ocaml or not args.disable_java
            ) and self.env.build.os.name == "linux":
                sigsegv_handler_dir = os.path.join(
                    langkit_root_dir, "sigsegv_handler"
                )
                sigsegv_handler_prj = os.path.join(
                    sigsegv_handler_dir, "langkit_sigsegv_handler.gpr"
                )
                run(
                    "Langkit_SIGSEGV_Handler",
                    ["gprbuild", "-p", "-P", sigsegv_handler_prj],
                )
                add_to_path(
                    "LD_LIBRARY_PATH", os.path.join(sigsegv_handler_dir, "lib")
                )

        # If Java is enabled and there is a Maven local repo, export it
        if not args.disable_java:
            if args.maven_executable:
                os.environ["MAVEN_EXECUTABLE"] = args.maven_executable
            if args.maven_local_repo:
                os.environ["MAVEN_LOCAL_REPO"] = args.maven_local_repo

    def adjust_dag_dependencies(self, dag):
        # Crude assumption that should do the work.
        #
        # If there are more tests than testsuite jobs, assume running tests in
        # parallel will keep all cores busy most of the time, so disable
        # parallelism inside each tests. This is necessary to avoid exceeding
        # our job limit in full testsuite runs.
        #
        # Otherwise, assume we are running only a handful of tests, in which
        # case we want to run them as fast as possible (for dev convenience):
        # forward testsuite parallelism inside each test.
        self.env.inner_jobs = (
            1
            if len(dag.vertex_data) >= self.main.args.jobs
            else self.main.args.jobs
        )

    def tear_down(self):
        if self.main.args.coverage:
            # Consolidate coverage data for each testcase and generate both a
            # sumary textual report on the standard output and a detailed HTML
            # report.
            langkit_cov_files = glob.glob(
                os.path.join(self.env.coverage_dir, "*.coverage")
            )

            lksp_cov_files = glob.glob(
                os.path.join(self.env.coverage_dir, "*.trace")
            )

            if langkit_cov_files:
                rcfile = os.path.join(self.root_dir, "..", "coverage.ini")
                for argv in (
                    ["combine"] + langkit_cov_files,
                    ["report"],
                    ["html", "-d.", "--title=Langkit coverage report"],
                ):
                    if argv[0] != "combine":
                        argv.append("--rcfile=" + rcfile)
                    subprocess.check_call(
                        ["coverage"] + argv, cwd=self.env.coverage_dir
                    )

                html_index = os.path.join(self.env.coverage_dir, "index.html")
                assert os.path.exists(html_index)
                print(
                    "Detailed HTML coverage report for Langkit available at:"
                    " {}".format(html_index)
                )
            else:
                print(
                    "No test exercised Langkit: no coverage report to"
                    " produce"
                )

            if lksp_cov_files:
                subprocess.check_call(
                    [
                        "gnatcov",
                        "coverage",
                        "-P",
                        self.env.langkit_support_project_file,
                        "--level=stmt+decision",
                        "--annotate=dhtml",
                        "--output-dir",
                        self.env.langkit_support_coverage_dir,
                        "--subdirs=gnatcov",
                    ]
                    + lksp_cov_files
                )
                print(
                    "HTML coverage report for Langkit_Support: {}".format(
                        os.path.join(
                            self.env.langkit_support_coverage_dir, "index.html"
                        )
                    )
                )
            else:
                print(
                    "No test exercised Langkit_Support: no coverage report"
                    " to produce"
                )

        super().tear_down()


if __name__ == "__main__":
    sys.exit(LangkitTestsuite().testsuite_main())