Imported Upstream version 6.10.0.49

Former-commit-id: 1d6753294b2993e1fbf92de9366bb9544db4189b

external/llvm-project/clang/docs/tools/dump_ast_matchers.py

@@ -0,0 +1,384 @@
+#!/usr/bin/env python
+# A tool to parse ASTMatchers.h and update the documentation in
+# ../LibASTMatchersReference.html automatically. Run from the
+# directory in which this file is located to update the docs.
+
+import collections
+import re
+import urllib2
+
+MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h'
+
+# Each matcher is documented in one row of the form:
+#   result | name | argA
+# The subsequent row contains the documentation and is hidden by default,
+# becoming visible via javascript when the user clicks the matcher name.
+TD_TEMPLATE="""
+<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>
+<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>
+"""
+
+# We categorize the matchers into these three categories in the reference:
+node_matchers = {}
+narrowing_matchers = {}
+traversal_matchers = {}
+
+# We output multiple rows per matcher if the matcher can be used on multiple
+# node types. Thus, we need a new id per row to control the documentation
+# pop-up. ids[name] keeps track of those ids.
+ids = collections.defaultdict(int)
+
+# Cache for doxygen urls we have already verified.
+doxygen_probes = {}
+
+def esc(text):
+  """Escape any html in the given text."""
+  text = re.sub(r'&', '&amp;', text)
+  text = re.sub(r'<', '&lt;', text)
+  text = re.sub(r'>', '&gt;', text)
+  def link_if_exists(m):
+    name = m.group(1)
+    url = 'http://clang.llvm.org/doxygen/classclang_1_1%s.html' % name
+    if url not in doxygen_probes:
+      try:
+        print 'Probing %s...' % url
+        urllib2.urlopen(url)
+        doxygen_probes[url] = True
+      except:
+        doxygen_probes[url] = False
+    if doxygen_probes[url]:
+      return r'Matcher&lt;<a href="%s">%s</a>&gt;' % (url, name)
+    else:
+      return m.group(0)
+  text = re.sub(
+    r'Matcher&lt;([^\*&]+)&gt;', link_if_exists, text)
+  return text
+
+def extract_result_types(comment):
+  """Extracts a list of result types from the given comment.
+
+     We allow annotations in the comment of the matcher to specify what
+     nodes a matcher can match on. Those comments have the form:
+       Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
+
+     Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
+     Returns the empty list if no 'Usable as' specification could be
+     parsed.
+  """
+  result_types = []
+  m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S)
+  if m:
+    return ['*']
+  while True:
+    m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S)
+    if not m:
+      if re.search(r'Usable as:\s*$', comment):
+        return result_types
+      else:
+        return None
+    result_types += [m.group(2)]
+    comment = m.group(1)
+
+def strip_doxygen(comment):
+  """Returns the given comment without \-escaped words."""
+  # If there is only a doxygen keyword in the line, delete the whole line.
+  comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M)
+  
+  # If there is a doxygen \see command, change the \see prefix into "See also:".
+  # FIXME: it would be better to turn this into a link to the target instead.
+  comment = re.sub(r'\\see', r'See also:', comment)
+  
+  # Delete the doxygen command and the following whitespace.
+  comment = re.sub(r'\\[^\s]+\s+', r'', comment)
+  return comment
+
+def unify_arguments(args):
+  """Gets rid of anything the user doesn't care about in the argument list."""
+  args = re.sub(r'internal::', r'', args)
+  args = re.sub(r'const\s+(.*)&', r'\1 ', args)
+  args = re.sub(r'&', r' ', args)
+  args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args)
+  return args
+
+def add_matcher(result_type, name, args, comment, is_dyncast=False):
+  """Adds a matcher to one of our categories."""
+  if name == 'id':
+     # FIXME: Figure out whether we want to support the 'id' matcher.
+     return
+  matcher_id = '%s%d' % (name, ids[name])
+  ids[name] += 1
+  args = unify_arguments(args)
+  matcher_html = TD_TEMPLATE % {
+    'result': esc('Matcher<%s>' % result_type),
+    'name': name,
+    'args': esc(args),
+    'comment': esc(strip_doxygen(comment)),
+    'id': matcher_id,
+  }
+  if is_dyncast:
+    node_matchers[result_type + name] = matcher_html
+  # Use a heuristic to figure out whether a matcher is a narrowing or
+  # traversal matcher. By default, matchers that take other matchers as
+  # arguments (and are not node matchers) do traversal. We specifically
+  # exclude known narrowing matchers that also take other matchers as
+  # arguments.
+  elif ('Matcher<' not in args or
+        name in ['allOf', 'anyOf', 'anything', 'unless']):
+    narrowing_matchers[result_type + name + esc(args)] = matcher_html
+  else:
+    traversal_matchers[result_type + name + esc(args)] = matcher_html
+
+def act_on_decl(declaration, comment, allowed_types):
+  """Parse the matcher out of the given declaration and comment.
+
+     If 'allowed_types' is set, it contains a list of node types the matcher
+     can match on, as extracted from the static type asserts in the matcher
+     definition.
+  """
+  if declaration.strip():
+    # Node matchers are defined by writing:
+    #   VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
+    m = re.match(r""".*Variadic(?:DynCast)?AllOfMatcher\s*<
+                       \s*([^\s,]+)\s*(?:,
+                       \s*([^\s>]+)\s*)?>
+                       \s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X)
+    if m:
+      result, inner, name = m.groups()
+      if not inner:
+        inner = result
+      add_matcher(result, name, 'Matcher<%s>...' % inner,
+                  comment, is_dyncast=True)
+      return
+
+    # Parse the various matcher definition macros.
+    m = re.match(""".*AST_TYPE_MATCHER\(
+                       \s*([^\s,]+\s*),
+                       \s*([^\s,]+\s*)
+                     \)\s*;\s*$""", declaration, flags=re.X)
+    if m:
+      inner, name = m.groups()
+      add_matcher('Type', name, 'Matcher<%s>...' % inner,
+                  comment, is_dyncast=True)
+      # FIXME: re-enable once we have implemented casting on the TypeLoc
+      # hierarchy.
+      # add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner,
+      #             comment, is_dyncast=True)
+      return
+
+    m = re.match(""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER\(
+                       \s*([^\s,]+\s*),
+                       \s*(?:[^\s,]+\s*),
+                       \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
+                     \)\s*;\s*$""", declaration, flags=re.X)
+    if m:
+      loc, name, results = m.groups()[0:3]
+      result_types = [r.strip() for r in results.split(',')]
+
+      comment_result_types = extract_result_types(comment)
+      if (comment_result_types and
+          sorted(result_types) != sorted(comment_result_types)):
+        raise Exception('Inconsistent documentation for: %s' % name)
+      for result_type in result_types:
+        add_matcher(result_type, name, 'Matcher<Type>', comment)
+        if loc:
+          add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>',
+                      comment)
+      return
+
+    m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
+                          \s*([^\s,]+)\s*,
+                          \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
+                       (?:,\s*([^\s,]+)\s*
+                          ,\s*([^\s,]+)\s*)?
+                       (?:,\s*([^\s,]+)\s*
+                          ,\s*([^\s,]+)\s*)?
+                       (?:,\s*\d+\s*)?
+                      \)\s*{\s*$""", declaration, flags=re.X)
+
+    if m:
+      p, n, name, results = m.groups()[0:4]
+      args = m.groups()[4:]
+      result_types = [r.strip() for r in results.split(',')]
+      if allowed_types and allowed_types != result_types:
+        raise Exception('Inconsistent documentation for: %s' % name)
+      if n not in ['', '2']:
+        raise Exception('Cannot parse "%s"' % declaration)
+      args = ', '.join('%s %s' % (args[i], args[i+1])
+                       for i in range(0, len(args), 2) if args[i])
+      for result_type in result_types:
+        add_matcher(result_type, name, args, comment)
+      return
+
+    m = re.match(r"""^\s*AST_MATCHER_FUNCTION(_P)?(.?)(?:_OVERLOAD)?\(
+                       (?:\s*([^\s,]+)\s*,)?
+                          \s*([^\s,]+)\s*
+                       (?:,\s*([^\s,]+)\s*
+                          ,\s*([^\s,]+)\s*)?
+                       (?:,\s*([^\s,]+)\s*
+                          ,\s*([^\s,]+)\s*)?
+                       (?:,\s*\d+\s*)?
+                      \)\s*{\s*$""", declaration, flags=re.X)
+    if m:
+      p, n, result, name = m.groups()[0:4]
+      args = m.groups()[4:]
+      if n not in ['', '2']:
+        raise Exception('Cannot parse "%s"' % declaration)
+      args = ', '.join('%s %s' % (args[i], args[i+1])
+                       for i in range(0, len(args), 2) if args[i])
+      add_matcher(result, name, args, comment)
+      return
+
+    m = re.match(r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
+                       (?:\s*([^\s,]+)\s*,)?
+                          \s*([^\s,]+)\s*
+                       (?:,\s*([^,]+)\s*
+                          ,\s*([^\s,]+)\s*)?
+                       (?:,\s*([^\s,]+)\s*
+                          ,\s*([^\s,]+)\s*)?
+                       (?:,\s*\d+\s*)?
+                      \)\s*{\s*$""", declaration, flags=re.X)
+    if m:
+      p, n, result, name = m.groups()[0:4]
+      args = m.groups()[4:]
+      if not result:
+        if not allowed_types:
+          raise Exception('Did not find allowed result types for: %s' % name)
+        result_types = allowed_types
+      else:
+        result_types = [result]
+      if n not in ['', '2']:
+        raise Exception('Cannot parse "%s"' % declaration)
+      args = ', '.join('%s %s' % (args[i], args[i+1])
+                       for i in range(0, len(args), 2) if args[i])
+      for result_type in result_types:
+        add_matcher(result_type, name, args, comment)
+      return
+
+    # Parse ArgumentAdapting matchers.
+    m = re.match(
+        r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*(?:LLVM_ATTRIBUTE_UNUSED\s*)
+              ([a-zA-Z]*)\s*=\s*{};$""",
+        declaration, flags=re.X)
+    if m:
+      name = m.groups()[0]
+      add_matcher('*', name, 'Matcher<*>', comment)
+      return
+
+    # Parse Variadic functions.
+    m = re.match(
+        r"""^.*internal::VariadicFunction\s*<\s*([^,]+),\s*([^,]+),\s*[^>]+>\s*
+              ([a-zA-Z]*)\s*=\s*{.*};$""",
+        declaration, flags=re.X)
+    if m:
+      result, arg, name = m.groups()[:3]
+      add_matcher(result, name, '%s, ..., %s' % (arg, arg), comment)
+      return
+
+    # Parse Variadic operator matchers.
+    m = re.match(
+        r"""^.*VariadicOperatorMatcherFunc\s*<\s*([^,]+),\s*([^\s>]+)\s*>\s*
+              ([a-zA-Z]*)\s*=\s*{.*};$""",
+        declaration, flags=re.X)
+    if m:
+      min_args, max_args, name = m.groups()[:3]
+      if max_args == '1':
+        add_matcher('*', name, 'Matcher<*>', comment)
+        return
+      elif max_args == 'UINT_MAX':
+        add_matcher('*', name, 'Matcher<*>, ..., Matcher<*>', comment)
+        return
+
+
+    # Parse free standing matcher functions, like:
+    #   Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
+    m = re.match(r"""^\s*(.*)\s+
+                     ([^\s\(]+)\s*\(
+                     (.*)
+                     \)\s*{""", declaration, re.X)
+    if m:
+      result, name, args = m.groups()
+      args = ', '.join(p.strip() for p in args.split(','))
+      m = re.match(r'.*\s+internal::(Bindable)?Matcher<([^>]+)>$', result)
+      if m:
+        result_types = [m.group(2)]
+      else:
+        result_types = extract_result_types(comment)
+      if not result_types:
+        if not comment:
+          # Only overloads don't have their own doxygen comments; ignore those.
+          print 'Ignoring "%s"' % name
+        else:
+          print 'Cannot determine result type for "%s"' % name
+      else:
+        for result_type in result_types:
+          add_matcher(result_type, name, args, comment)
+    else:
+      print '*** Unparsable: "' + declaration + '" ***'
+
+def sort_table(matcher_type, matcher_map):
+  """Returns the sorted html table for the given row map."""
+  table = ''
+  for key in sorted(matcher_map.keys()):
+    table += matcher_map[key] + '\n'
+  return ('<!-- START_%(type)s_MATCHERS -->\n' +
+          '%(table)s' + 
+          '<!--END_%(type)s_MATCHERS -->') % {
+    'type': matcher_type,
+    'table': table,
+  }
+
+# Parse the ast matchers.
+# We alternate between two modes:
+# body = True: We parse the definition of a matcher. We need
+#   to parse the full definition before adding a matcher, as the
+#   definition might contain static asserts that specify the result
+#   type.
+# body = False: We parse the comments and declaration of the matcher.
+comment = ''
+declaration = ''
+allowed_types = []
+body = False
+for line in open(MATCHERS_FILE).read().splitlines():
+  if body:
+    if line.strip() and line[0] == '}':
+      if declaration:
+        act_on_decl(declaration, comment, allowed_types)
+        comment = ''
+        declaration = ''
+        allowed_types = []
+      body = False
+    else:
+      m = re.search(r'is_base_of<([^,]+), NodeType>', line)
+      if m and m.group(1):
+        allowed_types += [m.group(1)]
+    continue
+  if line.strip() and line.lstrip()[0] == '/':
+    comment += re.sub(r'/+\s?', '', line) + '\n'
+  else:
+    declaration += ' ' + line
+    if ((not line.strip()) or 
+        line.rstrip()[-1] == ';' or
+        (line.rstrip()[-1] == '{' and line.rstrip()[-3:] != '= {')):
+      if line.strip() and line.rstrip()[-1] == '{':
+        body = True
+      else:
+        act_on_decl(declaration, comment, allowed_types)
+        comment = ''
+        declaration = ''
+        allowed_types = []
+
+node_matcher_table = sort_table('DECL', node_matchers)
+narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers)
+traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers)
+
+reference = open('../LibASTMatchersReference.html').read()
+reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->',
+                   node_matcher_table, reference, flags=re.S)
+reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->',
+                   narrowing_matcher_table, reference, flags=re.S)
+reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->',
+                   traversal_matcher_table, reference, flags=re.S)
+
+with open('../LibASTMatchersReference.html', 'wb') as output:
+  output.write(reference)
+

external/llvm-project/clang/docs/tools/dump_format_style.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+# A tool to parse the FormatStyle struct from Format.h and update the
+# documentation in ../ClangFormatStyleOptions.rst automatically.
+# Run from the directory in which this file is located to update the docs.
+
+import collections
+import os
+import re
+import urllib2
+
+CLANG_DIR = os.path.join(os.path.dirname(__file__), '../..')
+FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, 'include/clang/Format/Format.h')
+DOC_FILE = os.path.join(CLANG_DIR, 'docs/ClangFormatStyleOptions.rst')
+
+
+def substitute(text, tag, contents):
+  replacement = '\n.. START_%s\n\n%s\n\n.. END_%s\n' % (tag, contents, tag)
+  pattern = r'\n\.\. START_%s\n.*\n\.\. END_%s\n' % (tag, tag)
+  return re.sub(pattern, '%s', text, flags=re.S) % replacement
+
+def doxygen2rst(text):
+  text = re.sub(r'<tt>\s*(.*?)\s*<\/tt>', r'``\1``', text)
+  text = re.sub(r'\\c ([^ ,;\.]+)', r'``\1``', text)
+  text = re.sub(r'\\\w+ ', '', text)
+  return text
+
+def indent(text, columns, indent_first_line=True):
+  indent = ' ' * columns
+  s = re.sub(r'\n([^\n])', '\n' + indent + '\\1', text, flags=re.S)
+  if not indent_first_line or s.startswith('\n'):
+    return s
+  return indent + s
+
+class Option:
+  def __init__(self, name, type, comment):
+    self.name = name
+    self.type = type
+    self.comment = comment.strip()
+    self.enum = None
+    self.nested_struct = None
+
+  def __str__(self):
+    s = '**%s** (``%s``)\n%s' % (self.name, self.type,
+                                 doxygen2rst(indent(self.comment, 2)))
+    if self.enum:
+      s += indent('\n\nPossible values:\n\n%s\n' % self.enum, 2)
+    if self.nested_struct:
+      s += indent('\n\nNested configuration flags:\n\n%s\n' %self.nested_struct,
+                  2)
+    return s
+
+class NestedStruct:
+  def __init__(self, name, comment):
+    self.name = name
+    self.comment = comment.strip()
+    self.values = []
+
+  def __str__(self):
+    return '\n'.join(map(str, self.values))
+
+class NestedField:
+  def __init__(self, name, comment):
+    self.name = name
+    self.comment = comment.strip()
+
+  def __str__(self):
+    return '\n* ``%s`` %s' % (
+        self.name,
+        doxygen2rst(indent(self.comment, 2, indent_first_line=False)))
+
+class Enum:
+  def __init__(self, name, comment):
+    self.name = name
+    self.comment = comment.strip()
+    self.values = []
+
+  def __str__(self):
+    return '\n'.join(map(str, self.values))
+
+class EnumValue:
+  def __init__(self, name, comment):
+    self.name = name
+    self.comment = comment
+
+  def __str__(self):
+    return '* ``%s`` (in configuration: ``%s``)\n%s' % (
+        self.name,
+        re.sub('.*_', '', self.name),
+        doxygen2rst(indent(self.comment, 2)))
+
+def clean_comment_line(line):
+  match = re.match(r'^/// \\code(\{.(\w+)\})?$', line)
+  if match:
+    lang = match.groups()[1]
+    if not lang:
+      lang = 'c++'
+    return '\n.. code-block:: %s\n\n' % lang
+  if line == '/// \\endcode':
+    return ''
+  return line[4:] + '\n'
+
+def read_options(header):
+  class State:
+    BeforeStruct, Finished, InStruct, InNestedStruct, InNestedFieldComent, \
+    InFieldComment, InEnum, InEnumMemberComment = range(8)
+  state = State.BeforeStruct
+
+  options = []
+  enums = {}
+  nested_structs = {}
+  comment = ''
+  enum = None
+  nested_struct = None
+
+  for line in header:
+    line = line.strip()
+    if state == State.BeforeStruct:
+      if line == 'struct FormatStyle {':
+        state = State.InStruct
+    elif state == State.InStruct:
+      if line.startswith('///'):
+        state = State.InFieldComment
+        comment = clean_comment_line(line)
+      elif line == '};':
+        state = State.Finished
+        break
+    elif state == State.InFieldComment:
+      if line.startswith('///'):
+        comment += clean_comment_line(line)
+      elif line.startswith('enum'):
+        state = State.InEnum
+        name = re.sub(r'enum\s+(\w+)\s*\{', '\\1', line)
+        enum = Enum(name, comment)
+      elif line.startswith('struct'):
+        state = State.InNestedStruct
+        name = re.sub(r'struct\s+(\w+)\s*\{', '\\1', line)
+        nested_struct = NestedStruct(name, comment)
+      elif line.endswith(';'):
+        state = State.InStruct
+        field_type, field_name = re.match(r'([<>:\w(,\s)]+)\s+(\w+);',
+                                          line).groups()
+        option = Option(str(field_name), str(field_type), comment)
+        options.append(option)
+      else:
+        raise Exception('Invalid format, expected comment, field or enum')
+    elif state == State.InNestedStruct:
+      if line.startswith('///'):
+        state = State.InNestedFieldComent
+        comment = clean_comment_line(line)
+      elif line == '};':
+        state = State.InStruct
+        nested_structs[nested_struct.name] = nested_struct
+    elif state == State.InNestedFieldComent:
+      if line.startswith('///'):
+        comment += clean_comment_line(line)
+      else:
+        state = State.InNestedStruct
+        nested_struct.values.append(NestedField(line.replace(';', ''), comment))
+    elif state == State.InEnum:
+      if line.startswith('///'):
+        state = State.InEnumMemberComment
+        comment = clean_comment_line(line)
+      elif line == '};':
+        state = State.InStruct
+        enums[enum.name] = enum
+      else:
+        raise Exception('Invalid format, expected enum field comment or };')
+    elif state == State.InEnumMemberComment:
+      if line.startswith('///'):
+        comment += clean_comment_line(line)
+      else:
+        state = State.InEnum
+        enum.values.append(EnumValue(line.replace(',', ''), comment))
+  if state != State.Finished:
+    raise Exception('Not finished by the end of file')
+
+  for option in options:
+    if not option.type in ['bool', 'unsigned', 'int', 'std::string',
+                           'std::vector<std::string>',
+                           'std::vector<IncludeCategory>',
+                           'std::vector<RawStringFormat>']:
+      if enums.has_key(option.type):
+        option.enum = enums[option.type]
+      elif nested_structs.has_key(option.type):
+        option.nested_struct = nested_structs[option.type]
+      else:
+        raise Exception('Unknown type: %s' % option.type)
+  return options
+
+options = read_options(open(FORMAT_STYLE_FILE))
+
+options = sorted(options, key=lambda x: x.name)
+options_text = '\n\n'.join(map(str, options))
+
+contents = open(DOC_FILE).read()
+
+contents = substitute(contents, 'FORMAT_STYLE_OPTIONS', options_text)
+
+with open(DOC_FILE, 'wb') as output:
+  output.write(contents)