Bug 751795 - Part 1: mach, the new frontend to mozilla-central; r=jhammel

.gitignore

@@ -19,6 +19,7 @@ ID
 /config.cache
 /config.log
 /.clang_complete
+/mach.ini
 
 # Empty marker file that's generated when we check out NSS
 security/manager/.nss.checkout

.hgignore

@@ -18,6 +18,7 @@
 ^config\.cache$
 ^config\.log$
 ^\.clang_complete
+^mach.ini$
 
 # Empty marker file that's generated when we check out NSS
 ^security/manager/\.nss\.checkout$

mach

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import platform
+import sys
+
+# Ensure we are running Python 2.7+. We put this check here so we generate a
+# user-friendly error message rather than a cryptic stack trace on module
+# import.
+if sys.version_info[0] == 2 and sys.version_info[1] < 7:
+    print('Python 2.7 or above is required to run mach.')
+    print('You are running', platform.python_version())
+    sys.exit(1)
+
+# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
+SEARCH_PATHS = [
+    'python/mach',
+    'python/mozbuild',
+    'build',
+    'build/pymake',
+    'python/blessings',
+    'python/psutil',
+    'python/which',
+    'other-licenses/ply',
+    'xpcom/idl-parser',
+    'testing/xpcshell',
+    'testing/mozbase/mozprocess',
+    'testing/mozbase/mozinfo',
+]
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+
+try:
+    import mach.main
+except ImportError:
+    SEARCH_PATHS.reverse()
+    sys.path[0:0] = [os.path.join(our_dir, path) for path in SEARCH_PATHS]
+
+    import mach.main
+
+# All of the code is in a module because EVERYTHING IS A LIBRARY.
+mach = mach.main.Mach(our_dir)
+mach.run(sys.argv[1:])

python/mach/README.rst

@@ -0,0 +1,103 @@
+The mach Driver
+===============
+
+The *mach* driver is the command line interface (CLI) to the source tree.
+
+The *mach* driver is invoked by running the *mach* script or from
+instantiating the *Mach* class from the *mach.main* module.
+
+Implementing mach Commands
+--------------------------
+
+The *mach* driver follows the convention of popular tools like Git,
+Subversion, and Mercurial and provides a common driver for multiple
+sub-commands.
+
+Modules inside *mach* typically contain 1 or more classes which
+inherit from *mach.base.ArgumentProvider*. Modules that inherit from
+this class are hooked up to the *mach* CLI driver. So, to add a new
+sub-command/action to *mach*, one simply needs to create a new class in
+the *mach* package which inherits from *ArgumentProvider*.
+
+Currently, you also need to hook up some plumbing in
+*mach.main.Mach*. In the future, we hope to have automatic detection
+of submodules.
+
+Your command class performs the role of configuring the *mach* frontend
+argument parser as well as providing the methods invoked if a command is
+requested. These methods will take the user-supplied input, do something
+(likely by calling a backend function in a separate module), then format
+output to the terminal.
+
+The plumbing to hook up the arguments to the *mach* driver involves
+light magic. At *mach* invocation time, the driver creates a new
+*argparse* instance. For each registered class that provides commands,
+it calls the *populate_argparse* static method, passing it the parser
+instance.
+
+Your class's *populate_argparse* function should register sub-commands
+with the parser.
+
+For example, say you want to provide the *doitall* command. e.g. *mach
+doitall*. You would create the module *mach.doitall* and this
+module would contain the following class:
+
+    from mach.base import ArgumentProvider
+
+    class DoItAll(ArgumentProvider):
+        def run(self, more=False):
+            print 'I did it!'
+
+        @staticmethod
+        def populate_argparse(parser):
+            # Create the parser to handle the sub-command.
+            p = parser.add_parser('doitall', help='Do it all!')
+
+            p.add_argument('more', action='store_true', default=False,
+                help='Do more!')
+
+            # Tell driver that the handler for this sub-command is the
+            # method *run* on the class *DoItAll*.
+            p.set_defaults(cls=DoItAll, method='run')
+
+The most important line here is the call to *set_defaults*.
+Specifically, the *cls* and *method* parameters, which tell the driver
+which class to instantiate and which method to execute if this command
+is requested.
+
+The specified method will receive all arguments parsed from the command.
+It is important that you use named - not positional - arguments for your
+handler functions or things will blow up. This is because the mach driver
+is using the ``**kwargs`` notation to call the defined method.
+
+In the future, we may provide additional syntactical sugar to make all
+this easier. For example, we may provide decorators on methods to hook
+up commands and handlers.
+
+Minimizing Code in Mach
+-----------------------
+
+Mach is just a frontend. Therefore, code in this package should pertain to
+one of 3 areas:
+
+1. Obtaining user input (parsing arguments, prompting, etc)
+2. Calling into some other Python package
+3. Formatting output
+
+Mach should not contain core logic pertaining to the desired task. If you
+find yourself needing to invent some new functionality, you should implement
+it as a generic package outside of mach and then write a mach shim to call
+into it. There are many advantages to this approach, including reusability
+outside of mach (others may want to write other frontends) and easier testing
+(it is easier to test generic libraries than code that interacts with the
+command line or terminal).
+
+Keeping Frontend Modules Small
+------------------------------
+
+The frontend modules providing mach commands are currently all loaded when
+the mach CLI driver starts. Therefore, there is potential for *import bloat*.
+
+We want the CLI driver to load quickly. So, please delay load external modules
+until they are actually required. In other words, don't use a global
+*import* when you can import from inside a specific command's handler.

python/mach/mach/base.py

@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+
+class ArgumentProvider(object):
+    """Base class for classes wishing to provide CLI arguments to mach."""
+
+    @staticmethod
+    def populate_argparse(parser):
+        raise Exception("populate_argparse not implemented.")

python/mach/mach/main.py

@@ -0,0 +1,193 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import sys
+
+from mozbuild.base import BuildConfig
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+# Import sub-command modules
+# TODO Bug 794509 do this via auto-discovery. Update README once this is
+# done.
+# TODO import modules
+
+# Classes inheriting from ArgumentProvider that provide commands.
+HANDLERS = [
+]
+
+# Classes inheriting from ConfigProvider that provide settings.
+# TODO this should come from auto-discovery somehow.
+SETTINGS_PROVIDERS = [
+    BuildConfig,
+]
+
+# Settings for argument parser that don't get proxied to sub-module. i.e. these
+# are things consumed by the driver itself.
+CONSUMED_ARGUMENTS = [
+    'settings_file',
+    'verbose',
+    'logfile',
+    'log_interval',
+    'action',
+    'cls',
+    'method',
+    'func',
+]
+
+class Mach(object):
+    """Contains code for the command-line `mach` interface."""
+
+    USAGE = """%(prog)s subcommand [arguments]
+
+mach provides an interface to performing common developer tasks. You specify
+an action/sub-command and it performs it.
+
+Some common actions are:
+
+    %(prog)s help      Show full help, including the list of all commands.
+
+To see more help for a specific action, run:
+
+  %(prog)s <command> --help
+"""
+
+    def __init__(self, cwd):
+        assert os.path.isdir(cwd)
+
+        self.cwd = cwd
+        self.log_manager = LoggingManager()
+        self.logger = logging.getLogger(__name__)
+        self.settings = ConfigSettings()
+
+        self.log_manager.register_structured_logger(self.logger)
+
+    def run(self, argv):
+        """Runs mach with arguments provided from the command line."""
+        parser = self.get_argument_parser()
+
+        if not len(argv):
+            # We don't register the usage until here because if it is globally
+            # registered, argparse always prints it. This is not desired when
+            # running with --help.
+            parser.usage = Mach.USAGE
+            parser.print_usage()
+            return 0
+
+        if argv[0] == 'help':
+            parser.print_help()
+            return 0
+
+        args = parser.parse_args(argv)
+
+        # Add JSON logging to a file if requested.
+        if args.logfile:
+            self.log_manager.add_json_handler(args.logfile)
+
+        # Up the logging level if requested.
+        log_level = logging.INFO
+        if args.verbose:
+            log_level = logging.DEBUG
+
+        # Always enable terminal logging. The log manager figures out if we are
+        # actually in a TTY or are a pipe and does the right thing.
+        self.log_manager.add_terminal_logging(level=log_level,
+            write_interval=args.log_interval)
+
+        self.load_settings(args)
+        conf = BuildConfig(self.settings)
+
+        stripped = {k: getattr(args, k) for k in vars(args) if k not in
+            CONSUMED_ARGUMENTS}
+
+        # If the action is associated with a class, instantiate and run it.
+        # All classes must be Base-derived and take the expected argument list.
+        if hasattr(args, 'cls'):
+            cls = getattr(args, 'cls')
+            instance = cls(self.cwd, self.settings, self.log_manager)
+            fn = getattr(instance, getattr(args, 'method'))
+
+        # If the action is associated with a function, call it.
+        elif hasattr(args, 'func'):
+            fn = getattr(args, 'func')
+        else:
+            raise Exception('Dispatch configuration error in module.')
+
+        fn(**stripped)
+
+    def log(self, level, action, params, format_str):
+        """Helper method to record a structured log event."""
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def load_settings(self, args):
+        """Determine which settings files apply and load them.
+
+        Currently, we only support loading settings from a single file.
+        Ideally, we support loading from multiple files. This is supported by
+        the ConfigSettings API. However, that API currently doesn't track where
+        individual values come from, so if we load from multiple sources then
+        save, we effectively do a full copy. We don't want this. Until
+        ConfigSettings does the right thing, we shouldn't expose multi-file
+        loading.
+
+        We look for a settings file in the following locations. The first one
+        found wins:
+
+          1) Command line argument
+          2) Environment variable
+          3) Default path
+        """
+        for provider in SETTINGS_PROVIDERS:
+            provider.register_settings()
+            self.settings.register_provider(provider)
+
+        p = os.path.join(self.cwd, 'mach.ini')
+
+        if args.settings_file:
+            p = args.settings_file
+        elif 'MACH_SETTINGS_FILE' in os.environ:
+            p = os.environ['MACH_SETTINGS_FILE']
+
+        self.settings.load_file(p)
+
+        return os.path.exists(p)
+
+    def get_argument_parser(self):
+        """Returns an argument parser for the command-line interface."""
+
+        parser = argparse.ArgumentParser()
+
+        settings_group = parser.add_argument_group('Settings')
+        settings_group.add_argument('--settings', dest='settings_file',
+            metavar='FILENAME', help='Path to settings file.')
+
+        logging_group = parser.add_argument_group('Logging')
+        logging_group.add_argument('-v', '--verbose', dest='verbose',
+            action='store_true', default=False,
+            help='Print verbose output.')
+        logging_group.add_argument('-l', '--log-file', dest='logfile',
+            metavar='FILENAME', type=argparse.FileType('ab'),
+            help='Filename to write log data to.')
+        logging_group.add_argument('--log-interval', dest='log_interval',
+            action='store_true', default=False,
+            help='Prefix log line with interval from last message rather '
+                'than relative time. Note that this is NOT execution time '
+                'if there are parallel operations.')
+
+        subparser = parser.add_subparsers(dest='action')
+
+        # Register argument action providers with us.
+        for cls in HANDLERS:
+            cls.populate_argparse(subparser)
+
+        return parser

python/mach/mach/terminal.py

@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+from __future__ import print_function, unicode_literals
+
+import logging
+import sys
+
+
+class LoggingHandler(logging.Handler):
+    """Custom logging handler that works with terminal window dressing.
+
+    This is alternative terminal logging handler which contains smarts for
+    emitting terminal control characters properly. Currently, it has generic
+    support for "footer" elements at the bottom of the screen. Functionality
+    can be added when needed.
+    """
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+        self.fh = sys.stdout
+        self.footer = None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            self.fh.flush()
+        finally:
+            self.release()
+
+    def emit(self, record):
+        msg = self.format(record)
+
+        if self.footer:
+            self.footer.clear()
+
+        self.fh.write(msg)
+        self.fh.write('\n')
+
+        if self.footer:
+            self.footer.draw()
+
+        # If we don't flush, the footer may not get drawn.
+        self.flush()
+
+
+class TerminalFooter(object):
+    """Represents something drawn on the bottom of a terminal."""
+    def __init__(self, terminal):
+        self.t = terminal
+        self.fh = sys.stdout
+
+    def _clear_lines(self, n):
+        for i in xrange(n):
+            self.fh.write(self.t.move_x(0))
+            self.fh.write(self.t.clear_eol())
+            self.fh.write(self.t.move_up())
+
+        self.fh.write(self.t.move_down())
+        self.fh.write(self.t.move_x(0))
+
+    def clear(self):
+        raise Exception('clear() must be implemented.')
+
+    def draw(self):
+        raise Exception('draw() must be implemented.')
+

python/mach/setup.py

@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+VERSION = '0.1'
+
+setup(
+    name='mach',
+    description='CLI frontend to mozilla-central.',
+    license='MPL 2.0',
+    packages=['mach'],
+    version=VERSION
+)
+