Bug 1151124 - Add a simplified version of pymake's clinetoargv to mozbuild and use it. r=gps

Pymake's clinetoargv is very specific to pymake's use case, yet has been abused
as a replacement for shlex because shlex doesn't handle things properly for our
use cases.

Using pymake's clinetoargv, however, has shortcomings, and we're better off
importing its code in mozbuild, simplifying it a little, and using that
instead.

Plus, less dependencies on pymake will help kill it for good some day.

build/mach_bootstrap.py

@@ -61,7 +61,6 @@ SEARCH_PATHS = [
     'python/requests',
     'python/slugid',
     'build',
-    'build/pymake',
     'config',
     'dom/bindings',
     'dom/bindings/parser',

python/mozbuild/mozbuild/base.py

@@ -292,13 +292,13 @@ class MozbuildObject(ProcessExecutionMixin):
     def extra_environment_variables(self):
         '''Some extra environment variables are stored in .mozconfig.mk.
         This functions extracts and returns them.'''
-        from pymake.process import ClineSplitter
+        from mozbuild import shellutil
         mozconfig_mk = os.path.join(self.topobjdir, '.mozconfig.mk')
         env = {}
         with open(mozconfig_mk) as fh:
             for line in fh:
                 if line.startswith('export '):
-                    exports = ClineSplitter(line, self.topobjdir)[1:]
+                    exports = shellutil.split(line)[1:]
                     for e in exports:
                         if '=' in e:
                             key, value = e.split('=')

python/mozbuild/mozbuild/mach_commands.py

@@ -903,13 +903,13 @@ class GTestCommands(MachCommandBase):
         # Parameters come from the CLI. We need to convert them before
         # their use.
         if debugger_args:
-            import pymake.process
-            argv, badchar = pymake.process.clinetoargv(debugger_args, os.getcwd())
-            if badchar:
+            from mozbuild import shellutil
+            try:
+                debugger_args = shellutil.split(debugger_args)
+            except shellutil.MetaCharacterException as e:
                 print("The --debugger_args you passed require a real shell to parse them.")
-                print("(We can't handle the %r character.)" % (badchar,))
+                print("(We can't handle the %r character.)" % e.char)
                 return 1
-            debugger_args = argv;
 
         # Prepend the debugger args.
         args = [debuggerInfo.path] + debuggerInfo.args + args
@@ -1104,13 +1104,13 @@ class RunProgram(MachCommandBase):
             # Parameters come from the CLI. We need to convert them before
             # their use.
             if debugparams:
-                import pymake.process
-                argv, badchar = pymake.process.clinetoargv(debugparams, os.getcwd())
-                if badchar:
+                from mozbuild import shellutil
+                try:
+                    debugparams = shellutil.split(debugparams)
+                except shellutil.MetaCharacterException as e:
                     print("The --debugparams you passed require a real shell to parse them.")
-                    print("(We can't handle the %r character.)" % (badchar,))
+                    print("(We can't handle the %r character.)" % e.char)
                     return 1
-                debugparams = argv;
 
             if not slowscript:
                 extra_env['JS_DISABLE_SLOW_SCRIPT_SIGNALS'] = '1'

python/mozbuild/mozbuild/shellutil.py

@@ -0,0 +1,172 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+
+def _tokens2re(**tokens):
+    # Create a pattern for non-escaped tokens, in the form:
+    #   (?<!\\)(?:a|b|c...)
+    # This is meant to match patterns a, b, or c, or ... if they are not
+    # preceded by a backslash.
+    # where a, b, c... are in the form
+    #   (?P<name>pattern)
+    # which matches the pattern and captures it in a named match group.
+    # The group names and patterns are given as arguments.
+    all_tokens = '|'.join('(?P<%s>%s)' % (name, value)
+                          for name, value in tokens.iteritems())
+    nonescaped = r'(?<!\\)(?:%s)' % all_tokens
+
+    # The final pattern matches either the above pattern, or an escaped
+    # backslash, captured in the "escape" match group.
+    return re.compile('(?:%s|%s)' % (nonescaped, r'(?P<escape>\\\\)'))
+
+UNQUOTED_TOKENS_RE = _tokens2re(
+  whitespace=r'[\t\r\n ]+',
+  quote=r'[\'"]',
+  comment='#',
+  special=r'[<>&|`~(){}$;\*\?]',
+  backslashed=r'\\[^\\]',
+)
+
+DOUBLY_QUOTED_TOKENS_RE = _tokens2re(
+  quote='"',
+  backslashedquote=r'\\"',
+  special='\$',
+  backslashed=r'\\[^\\"]',
+)
+
+ESCAPED_NEWLINES_RE = re.compile(r'\\\n')
+
+
+class MetaCharacterException(Exception):
+    def __init__(self, char):
+        self.char = char
+
+
+class _ClineSplitter(object):
+    '''
+    Parses a given command line string and creates a list of command
+    and arguments, with wildcard expansion.
+    '''
+    def __init__(self, cline):
+        self.arg = None
+        self.cline = cline
+        self.result = []
+        self._parse_unquoted()
+
+    def _push(self, str):
+        '''
+        Push the given string as part of the current argument
+        '''
+        if self.arg is None:
+            self.arg = ''
+        self.arg += str
+
+    def _next(self):
+        '''
+        Finalize current argument, effectively adding it to the list.
+        '''
+        if self.arg is None:
+            return
+        self.result.append(self.arg)
+        self.arg = None
+
+    def _parse_unquoted(self):
+        '''
+        Parse command line remainder in the context of an unquoted string.
+        '''
+        while self.cline:
+            # Find the next token
+            m = UNQUOTED_TOKENS_RE.search(self.cline)
+            # If we find none, the remainder of the string can be pushed to
+            # the current argument and the argument finalized
+            if not m:
+                self._push(self.cline)
+                break
+            # The beginning of the string, up to the found token, is part of
+            # the current argument
+            if m.start():
+                self._push(self.cline[:m.start()])
+            self.cline = self.cline[m.end():]
+
+            match = {name: value
+                     for name, value in m.groupdict().items() if value}
+            if 'quote' in match:
+                # " or ' start a quoted string
+                if match['quote'] == '"':
+                    self._parse_doubly_quoted()
+                else:
+                    self._parse_quoted()
+            elif 'comment' in match:
+                # Comments are ignored. The current argument can be finalized,
+                # and parsing stopped.
+                break
+            elif 'special' in match:
+                # Unquoted, non-escaped special characters need to be sent to a
+                # shell.
+                raise MetaCharacterException(match['special'])
+            elif 'whitespace' in match:
+                # Whitespaces terminate current argument.
+                self._next()
+            elif 'escape' in match:
+                # Escaped backslashes turn into a single backslash
+                self._push('\\')
+            elif 'backslashed' in match:
+                # Backslashed characters are unbackslashed
+                # e.g. echo \a -> a
+                self._push(match['backslashed'][1])
+            else:
+                raise Exception("Shouldn't reach here")
+        if self.arg:
+            self._next()
+
+    def _parse_quoted(self):
+        # Single quoted strings are preserved, except for the final quote
+        index = self.cline.find("'")
+        if index == -1:
+            raise Exception('Unterminated quoted string in command')
+        self._push(self.cline[:index])
+        self.cline = self.cline[index+1:]
+
+    def _parse_doubly_quoted(self):
+        if not self.cline:
+            raise Exception('Unterminated quoted string in command')
+        while self.cline:
+            m = DOUBLY_QUOTED_TOKENS_RE.search(self.cline)
+            if not m:
+                raise Exception('Unterminated quoted string in command')
+            self._push(self.cline[:m.start()])
+            self.cline = self.cline[m.end():]
+            match = {name: value
+                     for name, value in m.groupdict().items() if value}
+            if 'quote' in match:
+                # a double quote ends the quoted string, so go back to
+                # unquoted parsing
+                return
+            elif 'special' in match:
+                # Unquoted, non-escaped special characters in a doubly quoted
+                # string still have a special meaning and need to be sent to a
+                # shell.
+                raise MetaCharacterException(match['special'])
+            elif 'escape' in match:
+                # Escaped backslashes turn into a single backslash
+                self._push('\\')
+            elif 'backslashedquote' in match:
+                # Backslashed double quotes are un-backslashed
+                self._push('"')
+            elif 'backslashed' in match:
+                # Backslashed characters are kept backslashed
+                self._push(match['backslashed'])
+
+
+def split(cline):
+    '''
+    Split the given command line string.
+    '''
+    s = ESCAPED_NEWLINES_RE.sub('', cline)
+    return _ClineSplitter(s).result
+
+
+__all__ = ['MetaCharacterException', 'split']