langkit/testsuite/python_support/utils.py

from __future__ import annotations

import dataclasses
import os
import os.path as P
import shlex
import shutil
import subprocess
import sys
import traceback
from typing import List, Optional, Set

import langkit
import langkit.compile_context
from langkit.compile_context import (
    CacheCollectionConf, CompileCtx, LibraryEntity, UnparseScript
)
from langkit.diagnostics import DiagnosticError, Diagnostics, WarningSet
from langkit.libmanage import ManageScript

from drivers.valgrind import valgrind_cmd


python_support_dir = P.dirname(P.abspath(__file__))
c_support_dir = P.join(python_support_dir, "..", "c_support")


Diagnostics.blacklisted_paths.append(python_support_dir)


default_warning_set = WarningSet()

# We don't want to be forced to provide dummy docs for nodes and public
# properties in testcases.
default_warning_set.disable(WarningSet.undocumented_nodes)
default_warning_set.disable(WarningSet.undocumented_public_properties)

pretty_print = bool(int(os.environ.get('LANGKIT_PRETTY_PRINT', '0')))

project_template = """
with "libfoolang";

project Gen is
    for Languages use ({languages});
    for Source_Dirs use ({source_dirs});
    for Object_Dir use "obj";
    for Main use ({main_sources});

    package Compiler is
        for Default_Switches ("Ada") use
          ("-g", "-O0", "-gnata", "-gnatwae", "-gnatyg");
        for Default_Switches ("C") use
          ("-g", "-O0", "-Wall", "-W", "-Werror", "-pedantic");
    end Compiler;

    package Binder is
        for Switches ("Ada") use ("-Es");
    end Binder;
end Gen;
"""


@dataclasses.dataclass
class Main:
    source_file: str
    """
    Basename of the main source file.
    """

    args: list[str] = dataclasses.field(default_factory=list)
    """
    Arguments to pass to this main when running it.
    """

    @property
    def label(self) -> str:
        """
        Return a representation of this main that is suitable to include in
        test baselines.
        """
        return " ".join([self.source_file] + self.args)

    @property
    def unit_name(self) -> str:
        """
        Return the name of the main source file without extension.
        """
        return os.path.splitext(self.source_file)[0]

    @classmethod
    def parse(cls, value: str) -> Main:
        """
        Create a Main instance from a shell-encoded list of arguments.
        """
        argv = shlex.split(value)
        return cls(argv[0], argv[1:])


valgrind_enabled = bool(os.environ.get('VALGRIND_ENABLED'))
jobs = int(os.environ.get('LANGKIT_JOBS', '1'))


# Determine where to find the root directory for Langkit sources
langkit_root = os.environ.get('LANGKIT_ROOT_DIR')
if not langkit_root:
    test_dir = P.dirname(P.abspath(__file__))
    testsuite_dir = P.dirname(test_dir)
    langkit_root = P.dirname(testsuite_dir)


# When unparsing the concrete syntax, name of the file to write
unparse_destination = 'concrete_syntax.lkt'
unparse_script = ('to:{},import:lexer_example,grammar,nodes'
                  .format(unparse_destination))
unparse_all_script = 'to:{},lexer,grammar,nodes'.format(unparse_destination)


def prepare_context(grammar=None, lexer=None, lkt_file=None,
                    warning_set=default_warning_set,
                    default_unit_provider=None, symbol_canonicalizer=None,
                    show_property_logging=False, types_from_lkt=False,
                    lkt_semantic_checks=False,
                    case_insensitive: bool = False,
                    version: Optional[str] = None,
                    build_date: Optional[str] = None,
                    standalone: bool = False,
                    property_exceptions: Set[str] = set(),
                    generate_unparser: bool = False,
                    default_unparsing_config: str | None = None,
                    cache_coll_conf: Optional[CacheCollectionConf] = None):
    """
    Create a compile context and prepare the build directory for code
    generation.

    :param langkit.parsers.Grammar grammar: The language grammar to use for
        this context.

    :param langkit.lexer.Lexer lexer: The language lexer to use for this
        context.

    :param str|None lkt_file: If provided, file from which to read the Lkt
        language spec.

    :param WarningSet warning_set: Set of warnings to emit.

    :param langkit.compile_context.LibraryEntity|None default_unit_provider:
        Default unit provider to use for this context, if any.

    :param langkit.compile_context.LibraryEntity|None symbol_canonicalizer:
        Symbol canonicalizer to use for this context, if any.

    :param bool show_property_logging: See CompileCtx.show_property_logging.

    :param bool types_from_lkt: See CompileCtx.types_from_lkt.

    :param case_insensitive: See CompileCtx's constructor.

    :param version: See CompileCtx's constructor.

    :param build_date: See CompileCtx's constructor.

    :param standalone: See CompileCtx's constructor.

    :param generate_unparser: See CompileCtx's constructor.

    :param default_unparsing_config: See the homonym CompileCtx constructor
        argument.

    :param cache_coll_conf: See CompileCtx's ``cache_collection_conf``
        constructor argument.
    """

    # Have a clean build directory
    if P.exists('build'):
        shutil.rmtree('build')
    os.mkdir('build')

    # Try to emit code
    ctx = CompileCtx(
        lang_name='Foo', short_name='foo', lexer=lexer, grammar=grammar,
        default_unit_provider=default_unit_provider,
        symbol_canonicalizer=symbol_canonicalizer,
        show_property_logging=show_property_logging,
        lkt_file=lkt_file,
        types_from_lkt=types_from_lkt,
        lkt_semantic_checks=lkt_semantic_checks,
        case_insensitive=case_insensitive,
        version=version,
        build_date=build_date,
        standalone=standalone,
        property_exceptions=property_exceptions,
        generate_unparser=generate_unparser,
        default_unparsing_config=default_unparsing_config,
        cache_collection_conf=cache_coll_conf,
    )
    ctx.warnings = warning_set
    ctx.pretty_print = pretty_print

    return ctx


def emit_and_print_errors(grammar=None, lexer=None, lkt_file=None,
                          warning_set=default_warning_set,
                          generate_unparser=False, symbol_canonicalizer=None,
                          unparse_script=None,
                          version=None,
                          build_date=None,
                          explicit_passes_triggers={},
                          lkt_semantic_checks=False,
                          types_from_lkt: bool = False):
    """
    Compile and emit code the given set of arguments. Return the compile
    context if this was successful, None otherwise.

    :param langkit.parsers.Grammar grammar_fn: The language grammar to use.

    :param langkit.lexer.Lexer lexer: The lexer to use along with the grammar.
        Use `lexer_example.foo_lexer` if left to None.

    :param str|None lkt_file: If provided, file from which to read the Lkt
        language spec.

    :param WarningSet warning_set: Set of warnings to emit.

    :param bool generate_unparser: Whether to generate unparser.

    :param langkit.compile_context.LibraryEntity|None symbol_canonicalizer:
        Symbol canoncalizes to use for this context, if any.

    :rtype: None|langkit.compile_context.CompileCtx

    :param None|str unparse_script: Script to unparse the language spec.

    :param version: See CompileCtx's constructor.

    :param build_date: See CompileCtx's constructor.

    :param types_from_lkt: See CompileCtx.types_from_lkt.

    :param property_exceptions: See CompileCtx's constructor.
    """

    try:
        ctx = prepare_context(
            grammar,
            lexer,
            lkt_file,
            warning_set,
            symbol_canonicalizer=symbol_canonicalizer,
            types_from_lkt=types_from_lkt,
            lkt_semantic_checks=lkt_semantic_checks,
            version=version,
            build_date=build_date,
            generate_unparser=generate_unparser,
        )
        ctx.create_all_passes(
            'build',
            unparse_script=(UnparseScript(unparse_script)
                            if unparse_script else None),
            explicit_passes_triggers=explicit_passes_triggers
        )
        ctx.emit()
        # ... and tell about how it went
    except DiagnosticError:
        # If there is a diagnostic error, don't say anything, the diagnostics
        # are enough.
        return None
    else:
        print('Code generation was successful')
        return ctx
    finally:
        if lexer is not None:
            lexer._dfa_code = None
        langkit.reset()


def build_and_run(
    lkt_file: str,
    generate_unparser: bool = False,
    default_unparsing_config: str | None = None,
    default_unit_provider: LibraryEntity | None = None,
    symbol_canonicalizer: LibraryEntity | None = None,
    show_property_logging: bool = False,
    case_insensitive: bool = False,
    version: str | None = None,
    build_date: str | None = None,
    property_exceptions: Set[str] = set(),
    cache_collection_conf: Optional[CacheCollectionConf] = None,
    py_script: str | None = None,
    py_args: list[str] | None = None,
    gpr_mains: list[Main] | None = None,
    ocaml_main: Main | None = None,
    java_main: Main | None = None,
    ni_main: Main | None = None,
) -> None:
    """
    Compile and emit code for `ctx` and build the generated library. Then,
    execute the provided scripts/programs, if any.

    An exception is raised if any step fails (the script must return code 0).

    :param lkt_file: If provided, file from which to read the Lkt language
        spec.
    :param generate_unparser: Whether to generate unparser.
    :param default_unparsing_config: See the homonym CompileCtx constructor
        argument.
    :param default_unit_provider: Default unit provider to use for this
        context, if any.
    :param symbol_canonicalizer: Symbol canonicalizer to use for this context,
        if any.
    :param show_property_logging: If true, any property that has been marked
        with tracing activated will be traced on stdout by default, without
        need for any config file.
    :param case_insensitive: See CompileCtx's constructor.
    :param version: See CompileCtx's constructor.
    :param build_date: See CompileCtx's constructor.
    :param property_exceptions: See CompileCtx's constructor.
    :param cache_collection_conf: See CompileCtx's constructor.
    :param py_script: If not None, name of the Python script to run with the
        built library available.
    :param python_args: Arguments to pass to the Python interpreter when
        running a Python script.
    :param gpr_mains: If not None, list of name of mains (Ada and/or C) for the
        generated GPR file, to build and run with the generated library. Each
        main can be either a Main instance or a string (for the main source
        file basename, the main is run without arguments).
    :param ocaml_main: If not None, name of the OCaml source file to build and
        run with the built library available.
    :param java_main: If not None, name of the Java main sourec file to build
        and run with the Langkit Java lib through JNI.
    :param ni_main: If not None, name of the Java main sourec file to build
        and run with the Langkit Java lib through Native Image.
    """

    class Manage(ManageScript):
        def __init__(self, ctx):
            self._cached_context = ctx
            super().__init__(root_dir=os.getcwd())

        def create_context(self, args):
            return self._cached_context

    # The call to build_and_run in test.py scripts should never be considered
    # as being part of the DSL to create diagnostics.
    for frame in traceback.extract_stack():
        Diagnostics.blacklist_frame(frame)

    build_mode = 'dev'

    maven_exec = os.environ.get('MAVEN_EXECUTABLE')
    maven_repo = os.environ.get('MAVEN_LOCAL_REPO')

    ctx = prepare_context(
        grammar=None,
        lexer=None,
        lkt_file=lkt_file,
        types_from_lkt=True,
        generate_unparser=generate_unparser,
        default_unparsing_config=default_unparsing_config,
        default_unit_provider=default_unit_provider,
        symbol_canonicalizer=symbol_canonicalizer,
        show_property_logging=show_property_logging,
        case_insensitive=case_insensitive,
        version=version,
        build_date=build_date,
        property_exceptions=property_exceptions,
        cache_coll_conf=cache_collection_conf,
    )
    m = Manage(ctx)

    extensions_dir = P.abspath('extensions')
    if P.isdir(extensions_dir):
        ctx.extensions_dir = extensions_dir

    # First build the library. Forward all test.py's arguments to the libmanage
    # call so that manual testcase runs can pass "-g", for instance.
    argv = (
        ['make']
        + sys.argv[1:]
        + ['-vnone', f'-j{jobs}', "--full-error-traces"]
    )

    # If there is a Java main, enable the Java bindings building
    if java_main is not None or ni_main is not None:
        argv.append('--enable-java')
        if maven_exec:
            argv.append('--maven-executable')
            argv.append(maven_exec)
        if maven_repo:
            argv.append('--maven-local-repo')
            argv.append(maven_repo)
        if ni_main is not None and os.name == 'nt':
            argv.append('--generate-msvc-lib')

    argv.append('--build-mode={}'.format(build_mode))
    for w in WarningSet.available_warnings:
        argv.append(
            '-{}{}'.format('W' if w in ctx.warnings else 'w', w.name)
        )
    if not pretty_print:
        argv.append('--no-pretty-print')

    # No testcase uses the generated mains, so save time: never build them
    argv.append('--disable-all-mains')

    return_code = m.run_no_exit(argv)

    # Flush stdout and stderr, so that diagnostics appear deterministically
    # before the script/program output.
    sys.stdout.flush()
    sys.stderr.flush()

    if return_code != 0:
        raise DiagnosticError()

    # Write a "setenv" script to make developper investigation convenient
    with open('setenv.sh', 'w') as f:
        m.write_setenv(f)

    env = m.derived_env()

    def run(*argv, **kwargs):
        subp_env = kwargs.pop("env", env)
        valgrind = kwargs.pop('valgrind', False)
        suppressions = kwargs.pop('valgrind_suppressions', [])
        assert not kwargs

        if valgrind_enabled and valgrind:
            argv = valgrind_cmd(list(argv), suppressions)

        subprocess.check_call(argv, env=subp_env)

    if py_script is not None:
        # Run the Python script.
        #
        # Note that in order to use the generated library, we have to use the
        # special Python interpreter the testsuite provides us. See the
        # corresponding code in testsuite/drivers/python_driver.py.
        args = [os.environ['PYTHON_INTERPRETER']]
        if py_args:
            args.extend(py_args)

        # Also note that since Python 3.8, we need special PATH processing for
        # DLLs: see the path_wrapper.py script.
        args.append(P.join(python_support_dir, "path_wrapper.py"))

        args.append(py_script)
        run(*args)

    if gpr_mains:
        # Canonicalize mains to Main instances
        gpr_mains = [
            (Main(m) if isinstance(m, str) else m) for m in gpr_mains
        ]

        source_dirs = [".", c_support_dir]
        main_source_files = sorted(m.source_file for m in gpr_mains)

        # Detect languages based on the source files present in the test
        # directory.
        langs = set()
        for f in os.listdir("."):
            if any(f.endswith(ext) for ext in [".c", ".h"]):
                langs.add("C")
            if any(f.endswith(ext) for ext in [".adb", ".ads"]):
                langs.add("Ada")

        # Generate a project file to build the given mains. Do a static build
        # (the default) to improve the debugging experience.
        with open("gen.gpr", "w") as f:

            def fmt_str_list(strings: List[str]) -> str:
                return ", ".join(f'"{s}"' for s in strings)

            f.write(project_template.format(
                languages=fmt_str_list(langs),
                source_dirs=fmt_str_list(source_dirs),
                main_sources=fmt_str_list(main_source_files),
            ))
        run("gprbuild", "-Pgen", "-q", "-p")

        # Now run all mains. If there are more than one main to run, print a
        # heading before each one.
        for i, main in enumerate(gpr_mains):
            if i > 0:
                print("")
            if len(gpr_mains) > 1:
                print(f"== {main.label} ==")
            sys.stdout.flush()
            run(
                P.join("obj", main.unit_name),
                *main.args,
                valgrind=True,
                valgrind_suppressions=["gnat"],
            )

    if ocaml_main is not None:
        # Set up a Dune project
        with open('dune', 'w') as f:
            f.write("""
                (executable
                  (name {})
                  (flags (-w -9))
                  (libraries {}))
            """.format(ocaml_main.unit_name, ctx.c_api_settings.lib_name))
        with open('dune-project', 'w') as f:
            f.write('(lang dune 1.6)')

        # Build the ocaml executable
        run('dune', 'build', '--display', 'quiet', '--root', '.',
            './{}.exe'.format(ocaml_main.unit_name))

        # Run the ocaml executable
        run('./_build/default/{}.exe'.format(ocaml_main.unit_name),
            *ocaml_main.args,
            valgrind=True,
            valgrind_suppressions=['ocaml'])

    if java_main is not None:
        java_exec = P.realpath(P.join(
            env['JAVA_HOME'],
            'bin',
            'java'
        ))
        cmd = [
            java_exec,
            '-Dfile.encoding=UTF-8',
            f"-Djava.library.path={env['LD_LIBRARY_PATH']}",
        ]
        if 'graalvm' in env['JAVA_HOME']:
            cmd.append((
                '--add-opens=org.graalvm.truffle/com.oracle.truffle.api.'
                'strings=ALL-UNNAMED'
            ))
        cmd += [java_main.source_file] + java_main.args
        run(*cmd)

    if ni_main is not None:
        # Compile the Java tests
        javac_exec = P.realpath(P.join(
            env['JAVA_HOME'],
            'bin',
            'javac'
        ))
        run(javac_exec, '-encoding', 'utf8', ni_main.source_file)

        # Run native-image to compile the tests.  Building Java bindings does
        # not go through GPRbuild, so we must explicitly give access to the
        # generated C header.
        java_env = m.derived_env(direct_c_header=True)
        ni_exec = P.realpath(P.join(
            os.environ['GRAAL_HOME'],
            'bin',
            ('native-image.cmd' if os.name == 'nt' else 'native-image')
        ))
        class_path = os.path.pathsep.join([
            P.realpath('.'),
            env['CLASSPATH'],
        ])
        run(
            ni_exec,
            '-cp', class_path,
            '--no-fallback',
            '--macro:truffle',
            '-H:NumberOfThreads=1',
            '-H:+BuildOutputSilent',
            '-H:+ReportExceptionStackTraces',
            os.path.splitext(ni_main.source_file)[0],
            'main',
            env=java_env,
        )

        # Run the newly created main
        run(P.realpath('main'), *ni_main.args)


def indent(text: str, prefix: str = "  ") -> str:
    """
    Indent all lines in `text` with the given prefix.

    :param text: Text to indent.
    :param prefix: Indentation string.
    """
    return "\n".join(prefix + line for line in text.splitlines())