From 53621e3d1e0622ec889ab2f25862b54f3fd988c2 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 26 Sep 2012 09:43:54 -0700 Subject: [PATCH] Bug 751795 - Part 1: mach, the new frontend to mozilla-central; r=jhammel --- .gitignore | 1 + .hgignore | 1 + mach | 48 +++++++++ python/mach/README.rst | 103 +++++++++++++++++++ python/mach/mach/__init__.py | 0 python/mach/mach/base.py | 13 +++ python/mach/mach/main.py | 193 +++++++++++++++++++++++++++++++++++ python/mach/mach/terminal.py | 75 ++++++++++++++ python/mach/setup.py | 16 +++ 9 files changed, 450 insertions(+) create mode 100755 mach create mode 100644 python/mach/README.rst create mode 100644 python/mach/mach/__init__.py create mode 100644 python/mach/mach/base.py create mode 100644 python/mach/mach/main.py create mode 100644 python/mach/mach/terminal.py create mode 100644 python/mach/setup.py diff --git a/.gitignore b/.gitignore index c8c32f8342f..b3fefef90dd 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ ID /config.cache /config.log /.clang_complete +/mach.ini # Empty marker file that's generated when we check out NSS security/manager/.nss.checkout diff --git a/.hgignore b/.hgignore index e7f8ac18a5b..d92bb2f8862 100644 --- a/.hgignore +++ b/.hgignore @@ -18,6 +18,7 @@ ^config\.cache$ ^config\.log$ ^\.clang_complete +^mach.ini$ # Empty marker file that's generated when we check out NSS ^security/manager/\.nss\.checkout$ diff --git a/mach b/mach new file mode 100755 index 00000000000..9c39e5bffb5 --- /dev/null +++ b/mach @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +import os +import platform +import sys + +# Ensure we are running Python 2.7+. We put this check here so we generate a +# user-friendly error message rather than a cryptic stack trace on module +# import. +if sys.version_info[0] == 2 and sys.version_info[1] < 7: + print('Python 2.7 or above is required to run mach.') + print('You are running', platform.python_version()) + sys.exit(1) + +# TODO Bug 794506 Integrate with the in-tree virtualenv configuration. +SEARCH_PATHS = [ + 'python/mach', + 'python/mozbuild', + 'build', + 'build/pymake', + 'python/blessings', + 'python/psutil', + 'python/which', + 'other-licenses/ply', + 'xpcom/idl-parser', + 'testing/xpcshell', + 'testing/mozbase/mozprocess', + 'testing/mozbase/mozinfo', +] + +our_dir = os.path.dirname(os.path.abspath(__file__)) + +try: + import mach.main +except ImportError: + SEARCH_PATHS.reverse() + sys.path[0:0] = [os.path.join(our_dir, path) for path in SEARCH_PATHS] + + import mach.main + +# All of the code is in a module because EVERYTHING IS A LIBRARY. +mach = mach.main.Mach(our_dir) +mach.run(sys.argv[1:]) diff --git a/python/mach/README.rst b/python/mach/README.rst new file mode 100644 index 00000000000..80a93ff8566 --- /dev/null +++ b/python/mach/README.rst @@ -0,0 +1,103 @@ +The mach Driver +=============== + +The *mach* driver is the command line interface (CLI) to the source tree. + +The *mach* driver is invoked by running the *mach* script or from +instantiating the *Mach* class from the *mach.main* module. + +Implementing mach Commands +-------------------------- + +The *mach* driver follows the convention of popular tools like Git, +Subversion, and Mercurial and provides a common driver for multiple +sub-commands. + +Modules inside *mach* typically contain 1 or more classes which +inherit from *mach.base.ArgumentProvider*. Modules that inherit from +this class are hooked up to the *mach* CLI driver. So, to add a new +sub-command/action to *mach*, one simply needs to create a new class in +the *mach* package which inherits from *ArgumentProvider*. + +Currently, you also need to hook up some plumbing in +*mach.main.Mach*. In the future, we hope to have automatic detection +of submodules. + +Your command class performs the role of configuring the *mach* frontend +argument parser as well as providing the methods invoked if a command is +requested. These methods will take the user-supplied input, do something +(likely by calling a backend function in a separate module), then format +output to the terminal. + +The plumbing to hook up the arguments to the *mach* driver involves +light magic. At *mach* invocation time, the driver creates a new +*argparse* instance. For each registered class that provides commands, +it calls the *populate_argparse* static method, passing it the parser +instance. + +Your class's *populate_argparse* function should register sub-commands +with the parser. + +For example, say you want to provide the *doitall* command. e.g. *mach +doitall*. You would create the module *mach.doitall* and this +module would contain the following class: + + from mach.base import ArgumentProvider + + class DoItAll(ArgumentProvider): + def run(self, more=False): + print 'I did it!' + + @staticmethod + def populate_argparse(parser): + # Create the parser to handle the sub-command. + p = parser.add_parser('doitall', help='Do it all!') + + p.add_argument('more', action='store_true', default=False, + help='Do more!') + + # Tell driver that the handler for this sub-command is the + # method *run* on the class *DoItAll*. + p.set_defaults(cls=DoItAll, method='run') + +The most important line here is the call to *set_defaults*. +Specifically, the *cls* and *method* parameters, which tell the driver +which class to instantiate and which method to execute if this command +is requested. + +The specified method will receive all arguments parsed from the command. +It is important that you use named - not positional - arguments for your +handler functions or things will blow up. This is because the mach driver +is using the ``**kwargs`` notation to call the defined method. + +In the future, we may provide additional syntactical sugar to make all +this easier. For example, we may provide decorators on methods to hook +up commands and handlers. + +Minimizing Code in Mach +----------------------- + +Mach is just a frontend. Therefore, code in this package should pertain to +one of 3 areas: + +1. Obtaining user input (parsing arguments, prompting, etc) +2. Calling into some other Python package +3. Formatting output + +Mach should not contain core logic pertaining to the desired task. If you +find yourself needing to invent some new functionality, you should implement +it as a generic package outside of mach and then write a mach shim to call +into it. There are many advantages to this approach, including reusability +outside of mach (others may want to write other frontends) and easier testing +(it is easier to test generic libraries than code that interacts with the +command line or terminal). + +Keeping Frontend Modules Small +------------------------------ + +The frontend modules providing mach commands are currently all loaded when +the mach CLI driver starts. Therefore, there is potential for *import bloat*. + +We want the CLI driver to load quickly. So, please delay load external modules +until they are actually required. In other words, don't use a global +*import* when you can import from inside a specific command's handler. diff --git a/python/mach/mach/__init__.py b/python/mach/mach/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/mach/mach/base.py b/python/mach/mach/base.py new file mode 100644 index 00000000000..0d743ed7246 --- /dev/null +++ b/python/mach/mach/base.py @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + + +class ArgumentProvider(object): + """Base class for classes wishing to provide CLI arguments to mach.""" + + @staticmethod + def populate_argparse(parser): + raise Exception("populate_argparse not implemented.") diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py new file mode 100644 index 00000000000..052873f0cf9 --- /dev/null +++ b/python/mach/mach/main.py @@ -0,0 +1,193 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides functionality for the command-line build tool +# (mach). It is packaged as a module because everything is a library. + +from __future__ import unicode_literals + +import argparse +import logging +import os +import sys + +from mozbuild.base import BuildConfig +from mozbuild.config import ConfigSettings +from mozbuild.logger import LoggingManager + +# Import sub-command modules +# TODO Bug 794509 do this via auto-discovery. Update README once this is +# done. +# TODO import modules + +# Classes inheriting from ArgumentProvider that provide commands. +HANDLERS = [ +] + +# Classes inheriting from ConfigProvider that provide settings. +# TODO this should come from auto-discovery somehow. +SETTINGS_PROVIDERS = [ + BuildConfig, +] + +# Settings for argument parser that don't get proxied to sub-module. i.e. these +# are things consumed by the driver itself. +CONSUMED_ARGUMENTS = [ + 'settings_file', + 'verbose', + 'logfile', + 'log_interval', + 'action', + 'cls', + 'method', + 'func', +] + +class Mach(object): + """Contains code for the command-line `mach` interface.""" + + USAGE = """%(prog)s subcommand [arguments] + +mach provides an interface to performing common developer tasks. You specify +an action/sub-command and it performs it. + +Some common actions are: + + %(prog)s help Show full help, including the list of all commands. + +To see more help for a specific action, run: + + %(prog)s --help +""" + + def __init__(self, cwd): + assert os.path.isdir(cwd) + + self.cwd = cwd + self.log_manager = LoggingManager() + self.logger = logging.getLogger(__name__) + self.settings = ConfigSettings() + + self.log_manager.register_structured_logger(self.logger) + + def run(self, argv): + """Runs mach with arguments provided from the command line.""" + parser = self.get_argument_parser() + + if not len(argv): + # We don't register the usage until here because if it is globally + # registered, argparse always prints it. This is not desired when + # running with --help. + parser.usage = Mach.USAGE + parser.print_usage() + return 0 + + if argv[0] == 'help': + parser.print_help() + return 0 + + args = parser.parse_args(argv) + + # Add JSON logging to a file if requested. + if args.logfile: + self.log_manager.add_json_handler(args.logfile) + + # Up the logging level if requested. + log_level = logging.INFO + if args.verbose: + log_level = logging.DEBUG + + # Always enable terminal logging. The log manager figures out if we are + # actually in a TTY or are a pipe and does the right thing. + self.log_manager.add_terminal_logging(level=log_level, + write_interval=args.log_interval) + + self.load_settings(args) + conf = BuildConfig(self.settings) + + stripped = {k: getattr(args, k) for k in vars(args) if k not in + CONSUMED_ARGUMENTS} + + # If the action is associated with a class, instantiate and run it. + # All classes must be Base-derived and take the expected argument list. + if hasattr(args, 'cls'): + cls = getattr(args, 'cls') + instance = cls(self.cwd, self.settings, self.log_manager) + fn = getattr(instance, getattr(args, 'method')) + + # If the action is associated with a function, call it. + elif hasattr(args, 'func'): + fn = getattr(args, 'func') + else: + raise Exception('Dispatch configuration error in module.') + + fn(**stripped) + + def log(self, level, action, params, format_str): + """Helper method to record a structured log event.""" + self.logger.log(level, format_str, + extra={'action': action, 'params': params}) + + def load_settings(self, args): + """Determine which settings files apply and load them. + + Currently, we only support loading settings from a single file. + Ideally, we support loading from multiple files. This is supported by + the ConfigSettings API. However, that API currently doesn't track where + individual values come from, so if we load from multiple sources then + save, we effectively do a full copy. We don't want this. Until + ConfigSettings does the right thing, we shouldn't expose multi-file + loading. + + We look for a settings file in the following locations. The first one + found wins: + + 1) Command line argument + 2) Environment variable + 3) Default path + """ + for provider in SETTINGS_PROVIDERS: + provider.register_settings() + self.settings.register_provider(provider) + + p = os.path.join(self.cwd, 'mach.ini') + + if args.settings_file: + p = args.settings_file + elif 'MACH_SETTINGS_FILE' in os.environ: + p = os.environ['MACH_SETTINGS_FILE'] + + self.settings.load_file(p) + + return os.path.exists(p) + + def get_argument_parser(self): + """Returns an argument parser for the command-line interface.""" + + parser = argparse.ArgumentParser() + + settings_group = parser.add_argument_group('Settings') + settings_group.add_argument('--settings', dest='settings_file', + metavar='FILENAME', help='Path to settings file.') + + logging_group = parser.add_argument_group('Logging') + logging_group.add_argument('-v', '--verbose', dest='verbose', + action='store_true', default=False, + help='Print verbose output.') + logging_group.add_argument('-l', '--log-file', dest='logfile', + metavar='FILENAME', type=argparse.FileType('ab'), + help='Filename to write log data to.') + logging_group.add_argument('--log-interval', dest='log_interval', + action='store_true', default=False, + help='Prefix log line with interval from last message rather ' + 'than relative time. Note that this is NOT execution time ' + 'if there are parallel operations.') + + subparser = parser.add_subparsers(dest='action') + + # Register argument action providers with us. + for cls in HANDLERS: + cls.populate_argparse(subparser) + + return parser diff --git a/python/mach/mach/terminal.py b/python/mach/mach/terminal.py new file mode 100644 index 00000000000..cdc3966575e --- /dev/null +++ b/python/mach/mach/terminal.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This file contains code for interacting with terminals. + +All the terminal interaction code is consolidated so the complexity can be in +one place, away from code that is commonly looked at. +""" + +from __future__ import print_function, unicode_literals + +import logging +import sys + + +class LoggingHandler(logging.Handler): + """Custom logging handler that works with terminal window dressing. + + This is alternative terminal logging handler which contains smarts for + emitting terminal control characters properly. Currently, it has generic + support for "footer" elements at the bottom of the screen. Functionality + can be added when needed. + """ + def __init__(self): + logging.Handler.__init__(self) + + self.fh = sys.stdout + self.footer = None + + def flush(self): + self.acquire() + + try: + self.fh.flush() + finally: + self.release() + + def emit(self, record): + msg = self.format(record) + + if self.footer: + self.footer.clear() + + self.fh.write(msg) + self.fh.write('\n') + + if self.footer: + self.footer.draw() + + # If we don't flush, the footer may not get drawn. + self.flush() + + +class TerminalFooter(object): + """Represents something drawn on the bottom of a terminal.""" + def __init__(self, terminal): + self.t = terminal + self.fh = sys.stdout + + def _clear_lines(self, n): + for i in xrange(n): + self.fh.write(self.t.move_x(0)) + self.fh.write(self.t.clear_eol()) + self.fh.write(self.t.move_up()) + + self.fh.write(self.t.move_down()) + self.fh.write(self.t.move_x(0)) + + def clear(self): + raise Exception('clear() must be implemented.') + + def draw(self): + raise Exception('draw() must be implemented.') + diff --git a/python/mach/setup.py b/python/mach/setup.py new file mode 100644 index 00000000000..e1fbf3d54a6 --- /dev/null +++ b/python/mach/setup.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +VERSION = '0.1' + +setup( + name='mach', + description='CLI frontend to mozilla-central.', + license='MPL 2.0', + packages=['mach'], + version=VERSION +) +