scripts/codeconverter: Update to latest version

I'm not documenting every single change in the codeconverter
script because most of that code will be deleted once we finish
the QOM code conversion.  This patch updates the script to the
latest version that was used to perform changes in the QOM code.

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
Message-Id: <20200916182519.415636-2-ehabkost@redhat.com>
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>

scripts/codeconverter/codeconverter/patching.py

@@ -5,7 +5,7 @@
 #
 # This work is licensed under the terms of the GNU GPL, version 2.  See
 # the COPYING file in the top-level directory.
-from typing import IO, Match, NamedTuple, Optional, Literal, Iterable, Type, Dict, List, Any, TypeVar, NewType, Tuple
+from typing import IO, Match, NamedTuple, Optional, Literal, Iterable, Type, Dict, List, Any, TypeVar, NewType, Tuple, Union
 from pathlib import Path
 from itertools import chain
 from tempfile import NamedTemporaryFile
@@ -47,7 +47,7 @@ class FileMatch:
 
     def __init__(self, f: 'FileInfo', m: Match) -> None:
         self.file: 'FileInfo' = f
-        self.match: Match = m
+        self.match: Match[str] = m
 
     @property
     def name(self) -> str:
@@ -68,8 +68,13 @@ class FileMatch:
     def line_col(self) -> LineAndColumn:
         return self.file.line_col(self.start())
 
-    def group(self, *args):
-        return self.match.group(*args)
+    def group(self, group: Union[int, str]) -> str:
+        return self.match.group(group)
+
+    def getgroup(self, group: str) -> Optional[str]:
+        if group not in self.match.groupdict():
+            return None
+        return self.match.group(group)
 
     def log(self, level, fmt, *args) -> None:
         pos = self.line_col()
@@ -163,18 +168,51 @@ class FileMatch:
         raise NotImplementedError()
 
     @classmethod
-    def find_matches(klass, content: str) -> Iterable[Match]:
-        """Generate match objects for class
+    def finditer(klass, content: str, pos=0, endpos=-1) -> Iterable[Match]:
+        """Helper for re.finditer()"""
+        if endpos >= 0:
+            content = content[:endpos]
+        return klass.compiled_re().finditer(content, pos)
 
-        Might be reimplemented by subclasses if they
-        intend to look for matches using a different method.
-        """
-        return klass.compiled_re().finditer(content)
+    @classmethod
+    def domatch(klass, content: str, pos=0, endpos=-1) -> Optional[Match]:
+        """Helper for re.match()"""
+        if endpos >= 0:
+            content = content[:endpos]
+        return klass.compiled_re().match(content, pos)
+
+    def group_finditer(self, klass: Type['FileMatch'], group: Union[str, int]) -> Iterable['FileMatch']:
+        assert self.file.original_content
+        return (klass(self.file, m)
+                for m in klass.finditer(self.file.original_content,
+                                        self.match.start(group),
+                                        self.match.end(group)))
+
+    def try_group_match(self, klass: Type['FileMatch'], group: Union[str, int]) -> Optional['FileMatch']:
+        assert self.file.original_content
+        m = klass.domatch(self.file.original_content,
+                          self.match.start(group),
+                          self.match.end(group))
+        if not m:
+            return None
+        else:
+            return klass(self.file, m)
+
+    def group_match(self, group: Union[str, int]) -> 'FileMatch':
+        m = self.try_group_match(FullMatch, group)
+        assert m
+        return m
 
     @property
     def allfiles(self) -> 'FileList':
         return self.file.allfiles
 
+class FullMatch(FileMatch):
+    """Regexp that will match all contents of string
+    Useful when used with group_match()
+    """
+    regexp = r'(?s).*' # (?s) is re.DOTALL
+
 def all_subclasses(c: Type[FileMatch]) -> Iterable[Type[FileMatch]]:
     for sc in c.__subclasses__():
         yield sc
@@ -201,7 +239,15 @@ def apply_patches(s: str, patches: Iterable[Patch]) -> str:
     """
     r = StringIO()
     last = 0
-    for p in sorted(patches):
+    def patch_sort_key(item: Tuple[int, Patch]) -> Tuple[int, int, int]:
+        """Patches are sorted by byte position,
+        patches at the same byte position are applied in the order
+        they were generated.
+        """
+        i,p = item
+        return (p.start, p.end, i)
+
+    for i,p in sorted(enumerate(patches), key=patch_sort_key):
         DBG("Applying patch at position %d (%s) - %d (%s): %r",
             p.start, line_col(s, p.start),
             p.end, line_col(s, p.end),
@@ -220,26 +266,35 @@ class RegexpScanner:
         self.match_index: Dict[Type[Any], List[FileMatch]] = {}
         self.match_name_index: Dict[Tuple[Type[Any], str, str], Optional[FileMatch]] = {}
 
-    def _find_matches(self, klass: Type[Any]) -> Iterable[FileMatch]:
+    def _matches_of_type(self, klass: Type[Any]) -> Iterable[FileMatch]:
         raise NotImplementedError()
 
     def matches_of_type(self, t: Type[T]) -> List[T]:
         if t not in self.match_index:
-            self.match_index[t] = list(self._find_matches(t))
-        return  self.match_index[t] # type: ignore
+            self.match_index[t] = list(self._matches_of_type(t))
+        return self.match_index[t] # type: ignore
 
-    def find_match(self, t: Type[T], name: str, group: str='name') -> Optional[T]:
+    def find_matches(self, t: Type[T], name: str, group: str='name') -> List[T]:
         indexkey = (t, name, group)
         if indexkey in self.match_name_index:
             return self.match_name_index[indexkey] # type: ignore
-        r: Optional[T] = None
+        r: List[T] = []
         for m in self.matches_of_type(t):
             assert isinstance(m, FileMatch)
-            if m.group(group) == name:
-                r = m # type: ignore
+            if m.getgroup(group) == name:
+                r.append(m) # type: ignore
         self.match_name_index[indexkey] = r # type: ignore
         return r
 
+    def find_match(self, t: Type[T], name: str, group: str='name') -> Optional[T]:
+        l = self.find_matches(t, name, group)
+        if not l:
+            return None
+        if len(l) > 1:
+            logger.warn("multiple matches found for %r (%s=%r)", t, group, name)
+            return None
+        return l[0]
+
     def reset_index(self) -> None:
         self.match_index.clear()
         self.match_name_index.clear()
@@ -258,18 +313,22 @@ class FileInfo(RegexpScanner):
     def __repr__(self) -> str:
         return f'<FileInfo {repr(self.filename)}>'
 
+    def filename_matches(self, name: str) -> bool:
+        nameparts = Path(name).parts
+        return self.filename.parts[-len(nameparts):] == nameparts
+
     def line_col(self, start: int) -> LineAndColumn:
         """Return line and column for a match object inside original_content"""
         return line_col(self.original_content, start)
 
-    def _find_matches(self, klass: Type[Any]) -> List[FileMatch]:
+    def _matches_of_type(self, klass: Type[Any]) -> List[FileMatch]:
         """Build FileMatch objects for each match of regexp"""
         if not hasattr(klass, 'regexp') or klass.regexp is None:
             return []
         assert hasattr(klass, 'regexp')
         DBG("%s: scanning for %s", self.filename, klass.__name__)
         DBG("regexp: %s", klass.regexp)
-        matches = [klass(self, m) for m in klass.find_matches(self.original_content)]
+        matches = [klass(self, m) for m in klass.finditer(self.original_content)]
         DBG('%s: %d matches found for %s: %s', self.filename, len(matches),
             klass.__name__,' '.join(names(matches)))
         return matches
@@ -277,7 +336,7 @@ class FileInfo(RegexpScanner):
     def find_match(self, t: Type[T], name: str, group: str='name') -> Optional[T]:
         for m in self.matches_of_type(t):
             assert isinstance(m, FileMatch)
-            if m.group(group) == name:
+            if m.getgroup(group) == name:
                 return m # type: ignore
         return None
 
@@ -299,7 +358,16 @@ class FileInfo(RegexpScanner):
         return (m for l in lists
                   for m in l)
 
-    def scan_for_matches(self, class_names: Optional[List[str]]=None) -> None:
+    def gen_patches(self, matches: List[FileMatch]) -> None:
+        for m in matches:
+            DBG("Generating patches for %r", m)
+            for i,p in enumerate(m.gen_patches()):
+                DBG("patch %d generated by %r:", i, m)
+                DBG("replace contents at %s-%s with %r",
+                    self.line_col(p.start), self.line_col(p.end), p.replacement)
+                self.patches.append(p)
+
+    def scan_for_matches(self, class_names: Optional[List[str]]=None) -> Iterable[FileMatch]:
         DBG("class names: %r", class_names)
         class_dict = match_class_dict()
         if class_names is None:
@@ -309,40 +377,9 @@ class FileInfo(RegexpScanner):
         DBG("class_names: %r", class_names)
         for cn in class_names:
             matches = self.matches_of_type(class_dict[cn])
-            if len(matches) > 0:
-                DBG('%s: %d matches found for %s: %s', self.filename,
-                     len(matches), cn, ' '.join(names(matches)))
-
-    def gen_patches(self) -> None:
-        for m in self.all_matches:
-            for i,p in enumerate(m.gen_patches()):
-                DBG("patch %d generated by %r:", i, m)
-                DBG("replace contents at %s-%s with %r",
-                    self.line_col(p.start), self.line_col(p.end), p.replacement)
-                self.patches.append(p)
-
-    def patch_content(self, max_passes=0, class_names: Optional[List[str]]=None) -> None:
-        """Multi-pass content patching loop
-
-        We run multiple passes because there are rules that will
-        delete init functions once they become empty.
-        """
-        passes = 0
-        total_patches  = 0
-        DBG("max_passes: %r", max_passes)
-        while not max_passes or max_passes <= 0 or passes < max_passes:
-            passes += 1
-            self.scan_for_matches(class_names)
-            self.gen_patches()
-            DBG("patch content: pass %d: %d patches generated", passes, len(self.patches))
-            total_patches += len(self.patches)
-            if not self.patches:
-                break
-            try:
-                self.apply_patches()
-            except PatchingError:
-                logger.exception("%s: failed to patch file", self.filename)
-        DBG("%s: %d patches applied total in %d passes", self.filename, total_patches, passes)
+            DBG('%d matches found for %s: %s',
+                    len(matches), cn, ' '.join(names(matches)))
+            yield from matches
 
     def apply_patches(self) -> None:
         """Replace self.original_content after applying patches from self.patches"""
@@ -384,14 +421,46 @@ class FileList(RegexpScanner):
     def __iter__(self):
         return iter(self.files)
 
-    def _find_matches(self, klass: Type[Any]) -> Iterable[FileMatch]:
-        return chain(*(f._find_matches(klass) for f in self.files))
+    def _matches_of_type(self, klass: Type[Any]) -> Iterable[FileMatch]:
+        return chain(*(f._matches_of_type(klass) for f in self.files))
 
-    def find_file(self, name) -> Optional[FileInfo]:
+    def find_file(self, name: str) -> Optional[FileInfo]:
         """Get file with path ending with @name"""
-        nameparts = Path(name).parts
         for f in self.files:
-            if f.filename.parts[:len(nameparts)] == nameparts:
+            if f.filename_matches(name):
                 return f
         else:
-            return None
+            return None
+
+    def one_pass(self, class_names: List[str]) -> int:
+        total_patches = 0
+        for f in self.files:
+            INFO("Scanning file %s", f.filename)
+            matches = list(f.scan_for_matches(class_names))
+            INFO("Generating patches for file %s", f.filename)
+            f.gen_patches(matches)
+            total_patches += len(f.patches)
+        if total_patches:
+            for f in self.files:
+                try:
+                    f.apply_patches()
+                except PatchingError:
+                    logger.exception("%s: failed to patch file", f.filename)
+        return total_patches
+
+    def patch_content(self, max_passes, class_names: List[str]) -> None:
+        """Multi-pass content patching loop
+
+        We run multiple passes because there are rules that will
+        delete init functions once they become empty.
+        """
+        passes = 0
+        total_patches  = 0
+        DBG("max_passes: %r", max_passes)
+        while not max_passes or max_passes <= 0 or passes < max_passes:
+            passes += 1
+            INFO("Running pass: %d", passes)
+            count = self.one_pass(class_names)
+            DBG("patch content: pass %d: %d patches generated", passes, count)
+            total_patches += count
+        DBG("%d patches applied total in %d passes", total_patches, passes)

scripts/codeconverter/codeconverter/qom_macros.py

@@ -23,16 +23,24 @@ WARN = logger.warning
 
 RE_CONSTANT = OR(RE_STRING, RE_NUMBER)
 
-class ConstantDefine(FileMatch):
-    """Simple #define preprocessor directive for a constant"""
-    # if the macro contents are very simple, it might be included
-    # in the match group 'value'
+class DefineDirective(FileMatch):
+    """Match any #define directive"""
+    regexp = S(r'^[ \t]*#[ \t]*define', CPP_SPACE, NAMED('name', RE_IDENTIFIER), r'\b')
+
+class ExpressionDefine(FileMatch):
+    """Simple #define preprocessor directive for an expression"""
     regexp = S(r'^[ \t]*#[ \t]*define', CPP_SPACE, NAMED('name', RE_IDENTIFIER),
-               CPP_SPACE, NAMED('value', RE_CONSTANT), r'[ \t]*\n')
+               CPP_SPACE, NAMED('value', RE_EXPRESSION), r'[ \t]*\n')
 
     def provided_identifiers(self) -> Iterable[RequiredIdentifier]:
         yield RequiredIdentifier('constant', self.group('name'))
 
+class ConstantDefine(ExpressionDefine):
+    """Simple #define preprocessor directive for a number or string constant"""
+    regexp = S(r'^[ \t]*#[ \t]*define', CPP_SPACE, NAMED('name', RE_IDENTIFIER),
+               CPP_SPACE, NAMED('value', RE_CONSTANT), r'[ \t]*\n')
+
+
 class TypeIdentifiers(NamedTuple):
     """Type names found in type declarations"""
     # TYPE_MYDEVICE
@@ -236,13 +244,12 @@ class TypeCheckMacro(FileMatch):
     """OBJECT_CHECK/OBJECT_CLASS_CHECK/OBJECT_GET_CLASS macro definitions
     Will be replaced by DECLARE_*_CHECKERS macro
     """
-    #TODO: handle and convert INTERFACE_CHECK macros
     regexp = RE_CHECK_MACRO
 
     @property
     def checker(self) -> CheckerMacroName:
         """Name of checker macro being used"""
-        return self.group('checker')
+        return self.group('checker') # type: ignore
 
     @property
     def typedefname(self) -> Optional[str]:
@@ -330,6 +337,8 @@ class TypeCheckMacro(FileMatch):
                                instancetype=instancetype, uppercase=uppercase)
 
     def gen_patches(self) -> Iterable[Patch]:
+        # the implementation is a bit tricky because we need to group
+        # macros dealing with the same type into a single declaration
         if self.type_identifiers is None:
             self.warn("couldn't extract type information from macro %s", self.name)
             return
@@ -426,10 +435,61 @@ class TypeCheckMacro(FileMatch):
             yield self.prepend("/* FIXME: %s */\n" % (issue))
         yield self.append(new_decl)
 
-class DeclareInstanceChecker(FileMatch):
-    """DECLARE_INSTANCE_CHECKER use
-    Will be replaced with DECLARE_OBJ_CHECKERS if possible
+class InterfaceCheckMacro(FileMatch):
+    """Type checking macro using INTERFACE_CHECK
+    Will be replaced by DECLARE_INTERFACE_CHECKER
     """
+    regexp = S(RE_MACRO_DEFINE,
+               'INTERFACE_CHECK',
+               r'\s*\(\s*', OR(NAMED('instancetype', RE_IDENTIFIER), RE_TYPE, name='c_type'),
+               r'\s*,', CPP_SPACE,
+               OPTIONAL_PARS(RE_IDENTIFIER), r',', CPP_SPACE,
+               NAMED('qom_typename', RE_IDENTIFIER), r'\s*\)\n')
+
+    def required_identifiers(self) -> Iterable[RequiredIdentifier]:
+        yield RequiredIdentifier('include', '"qom/object.h"')
+        yield RequiredIdentifier('type', self.group('instancetype'))
+        yield RequiredIdentifier('constant', self.group('qom_typename'))
+
+    def gen_patches(self) -> Iterable[Patch]:
+        if self.file.filename_matches('qom/object.h'):
+            self.debug("skipping object.h")
+            return
+
+        typename = self.group('qom_typename')
+        uppercase = self.name
+        instancetype = self.group('instancetype')
+        c = f"DECLARE_INTERFACE_CHECKER({instancetype}, {uppercase},\n"+\
+            f"                          {typename})\n"
+        yield self.make_patch(c)
+
+
+class TypeDeclaration(FileMatch):
+    """Parent class to all type declarations"""
+    @property
+    def instancetype(self) -> Optional[str]:
+        return self.getgroup('instancetype')
+
+    @property
+    def classtype(self) -> Optional[str]:
+        return self.getgroup('classtype')
+
+    @property
+    def typename(self) -> Optional[str]:
+        return self.getgroup('typename')
+
+class TypeCheckerDeclaration(TypeDeclaration):
+    """Parent class to all type checker declarations"""
+    @property
+    def typename(self) -> str:
+        return self.group('typename')
+
+    @property
+    def uppercase(self) -> str:
+        return self.group('uppercase')
+
+class DeclareInstanceChecker(TypeCheckerDeclaration):
+    """DECLARE_INSTANCE_CHECKER use"""
     #TODO: replace lonely DECLARE_INSTANCE_CHECKER with DECLARE_OBJ_CHECKERS
     #      if all types are found.
     #      This will require looking up the correct class type in the TypeInfo
@@ -445,8 +505,45 @@ class DeclareInstanceChecker(FileMatch):
         yield RequiredIdentifier('constant', self.group('typename'))
         yield RequiredIdentifier('type', self.group('instancetype'))
 
-class DeclareClassCheckers(FileMatch):
-    """DECLARE_INSTANCE_CHECKER use"""
+class DeclareInterfaceChecker(TypeCheckerDeclaration):
+    """DECLARE_INTERFACE_CHECKER use"""
+    regexp = S(r'^[ \t]*DECLARE_INTERFACE_CHECKER\s*\(\s*',
+               NAMED('instancetype', RE_TYPE), r'\s*,\s*',
+               NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
+               OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'), SP,
+               r'\)[ \t]*;?[ \t]*\n')
+
+    def required_identifiers(self) -> Iterable[RequiredIdentifier]:
+        yield RequiredIdentifier('include', '"qom/object.h"')
+        yield RequiredIdentifier('constant', self.group('typename'))
+        yield RequiredIdentifier('type', self.group('instancetype'))
+
+class DeclareInstanceType(TypeDeclaration):
+    """DECLARE_INSTANCE_TYPE use"""
+    regexp = S(r'^[ \t]*DECLARE_INSTANCE_TYPE\s*\(\s*',
+               NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
+               NAMED('instancetype', RE_TYPE), SP,
+               r'\)[ \t]*;?[ \t]*\n')
+
+    def required_identifiers(self) -> Iterable[RequiredIdentifier]:
+        yield RequiredIdentifier('include', '"qom/object.h"')
+        yield RequiredIdentifier('type', self.group('instancetype'))
+
+class DeclareClassType(TypeDeclaration):
+    """DECLARE_CLASS_TYPE use"""
+    regexp = S(r'^[ \t]*DECLARE_CLASS_TYPE\s*\(\s*',
+               NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
+               NAMED('classtype', RE_TYPE), SP,
+               r'\)[ \t]*;?[ \t]*\n')
+
+    def required_identifiers(self) -> Iterable[RequiredIdentifier]:
+        yield RequiredIdentifier('include', '"qom/object.h"')
+        yield RequiredIdentifier('type', self.group('classtype'))
+
+
+
+class DeclareClassCheckers(TypeCheckerDeclaration):
+    """DECLARE_CLASS_CHECKER use"""
     regexp = S(r'^[ \t]*DECLARE_CLASS_CHECKERS\s*\(\s*',
                NAMED('classtype', RE_TYPE), r'\s*,\s*',
                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
@@ -458,10 +555,8 @@ class DeclareClassCheckers(FileMatch):
         yield RequiredIdentifier('constant', self.group('typename'))
         yield RequiredIdentifier('type', self.group('classtype'))
 
-class DeclareObjCheckers(FileMatch):
-    """DECLARE_OBJ_CHECKERS use
-    Will be replaced with OBJECT_DECLARE_TYPE if possible
-    """
+class DeclareObjCheckers(TypeCheckerDeclaration):
+    """DECLARE_OBJ_CHECKERS use"""
     #TODO: detect when OBJECT_DECLARE_SIMPLE_TYPE can be used
     regexp = S(r'^[ \t]*DECLARE_OBJ_CHECKERS\s*\(\s*',
                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
@@ -476,44 +571,121 @@ class DeclareObjCheckers(FileMatch):
         yield RequiredIdentifier('type', self.group('classtype'))
         yield RequiredIdentifier('type', self.group('instancetype'))
 
-    def gen_patches(self):
-        ids = TypeIdentifiers(uppercase=self.group('uppercase'),
-                              typename=self.group('typename'),
-                              classtype=self.group('classtype'),
-                              instancetype=self.group('instancetype'))
-        issues = ids.check_consistency()
-        if issues:
-            for i in issues:
-                self.warn("inconsistent identifiers: %s", i)
+class TypeDeclarationFixup(FileMatch):
+    """Common base class for code that will look at a set of type declarations"""
+    regexp = RE_FILE_BEGIN
+    def gen_patches(self) -> Iterable[Patch]:
+        if self.file.filename_matches('qom/object.h'):
+            self.debug("skipping object.h")
             return
 
-        if self.group('typename') != 'TYPE_'+self.group('uppercase'):
-            self.warn("type %s mismatch with uppercase name %s", ids.typename, ids.uppercase)
+        # group checkers by uppercase name:
+        decl_types: List[Type[TypeDeclaration]] = [DeclareInstanceChecker, DeclareInstanceType,
+                                                   DeclareClassCheckers, DeclareClassType,
+                                                   DeclareObjCheckers]
+        checker_dict: Dict[str, List[TypeDeclaration]] = {}
+        for t in decl_types:
+            for m in self.file.matches_of_type(t):
+                checker_dict.setdefault(m.group('uppercase'), []).append(m)
+        self.debug("checker_dict: %r", checker_dict)
+        for uppercase,checkers in checker_dict.items():
+            fields = ('instancetype', 'classtype', 'uppercase', 'typename')
+            fvalues = dict((field, set(getattr(m, field) for m in checkers
+                                       if getattr(m, field, None) is not None))
+                            for field in fields)
+            for field,values in fvalues.items():
+                if len(values) > 1:
+                    for c in checkers:
+                        c.warn("%s mismatch (%s)", field, ' '.join(values))
+                    return
+
+            field_dict = dict((f, v.pop() if v else None) for f,v in fvalues.items())
+            yield from self.gen_patches_for_type(uppercase, checkers, field_dict)
+
+    def find_conflicts(self, uppercase: str, checkers: List[TypeDeclaration]) -> bool:
+        """Look for conflicting declarations that would make it unsafe to add new ones"""
+        conflicting: List[FileMatch] = []
+        # conflicts in the same file:
+        conflicting.extend(chain(self.file.find_matches(DefineDirective, uppercase),
+                                 self.file.find_matches(DeclareInterfaceChecker, uppercase, 'uppercase'),
+                                 self.file.find_matches(DeclareClassType, uppercase, 'uppercase'),
+                                 self.file.find_matches(DeclareInstanceType, uppercase, 'uppercase')))
+
+        # conflicts in another file:
+        conflicting.extend(o for o in chain(self.allfiles.find_matches(DeclareInstanceChecker, uppercase, 'uppercase'),
+                                            self.allfiles.find_matches(DeclareClassCheckers, uppercase, 'uppercase'),
+                                            self.allfiles.find_matches(DeclareInterfaceChecker, uppercase, 'uppercase'),
+                                            self.allfiles.find_matches(DefineDirective, uppercase))
+                           if o is not None and o.file != self.file
+                               # if both are .c files, there's no conflict at all:
+                               and not (o.file.filename.suffix == '.c' and
+                                       self.file.filename.suffix == '.c'))
+
+        if conflicting:
+            for c in checkers:
+                c.warn("skipping due to conflicting %s macro", uppercase)
+            for o in conflicting:
+                if o is None:
+                    continue
+                o.warn("conflicting %s macro is here", uppercase)
+            return True
+
+        return False
+
+    def gen_patches_for_type(self, uppercase: str,
+                             checkers: List[TypeDeclaration],
+                             fields: Dict[str, Optional[str]]) -> Iterable[Patch]:
+        """Should be reimplemented by subclasses"""
+        return
+        yield
+
+class DeclareVoidTypes(TypeDeclarationFixup):
+    """Add DECLARE_*_TYPE(..., void) when there's no declared type"""
+    regexp = RE_FILE_BEGIN
+    def gen_patches_for_type(self, uppercase: str,
+                             checkers: List[TypeDeclaration],
+                             fields: Dict[str, Optional[str]]) -> Iterable[Patch]:
+        if self.find_conflicts(uppercase, checkers):
             return
 
-        typedefs = [(t,self.file.find_match(SimpleTypedefMatch, t))
-                    for t in (ids.instancetype, ids.classtype)]
-        for t,td in typedefs:
-            if td is None:
-                self.warn("typedef %s not found", t)
-                break
-            if td.start() > self.start():
-                self.warn("typedef %s needs to be move earlier in the file", t)
-                break
-            #HACK: check if typedef is used between its definition and the macro
-            #TODO: check if the only match is inside the "struct { ... }" declaration
-            if re.search(r'\b'+t+r'\b', self.file.original_content[td.end():self.start()]):
-                self.warn("typedef %s can't be moved, it is used before the macro", t)
-                break
-        else:
-            for t,td in typedefs:
-                yield td.make_removal_patch()
+        #_,last_checker = max((m.start(), m) for m in checkers)
+        _,first_checker = min((m.start(), m) for m in checkers)
 
-            lowercase = ids.uppercase.lower()
-            # all is OK, we can replace the macro!
-            c = (f'OBJECT_DECLARE_TYPE({ids.instancetype}, {ids.classtype},\n'
-                 f'                    {lowercase}, {ids.uppercase})\n')
-            yield self.make_patch(c)
+        if not any(m.instancetype for m in checkers):
+            yield first_checker.prepend(f'DECLARE_INSTANCE_TYPE({uppercase}, void)\n')
+        if not any(m.classtype for m in checkers):
+            yield first_checker.prepend(f'DECLARE_CLASS_TYPE({uppercase}, void)\n')
+
+        #if not all(len(v) == 1 for v in fvalues.values()):
+        #    return
+        #
+        #final_values = dict((field, values.pop())
+        #                    for field,values in fvalues.items())
+        #s = (f"DECLARE_OBJ_CHECKERS({final_values['instancetype']}, {final_values['classtype']},\n"+
+        #        f"                     {final_values['uppercase']}, {final_values['typename']})\n")
+        #for c in checkers:
+        #    yield c.make_removal_patch()
+        #yield last_checker.append(s)
+
+
+class AddDeclareTypeName(TypeDeclarationFixup):
+    """Add DECLARE_TYPE_NAME declarations if necessary"""
+    def gen_patches_for_type(self, uppercase: str,
+                             checkers: List[TypeDeclaration],
+                             fields: Dict[str, Optional[str]]) -> Iterable[Patch]:
+        typename = fields.get('typename')
+        if typename is None:
+            self.warn("typename unavailable")
+            return
+        if typename == f'TYPE_{uppercase}':
+            self.info("already using TYPE_%s as type name", uppercase)
+            return
+        if self.file.find_match(DeclareTypeName, uppercase, 'uppercase'):
+            self.info("type name for %s already declared", uppercase)
+            return
+        _,first_checker = min((m.start(), m) for m in checkers)
+        s = f'DECLARE_TYPE_NAME({uppercase}, {typename})\n'
+        yield first_checker.prepend(s)
 
 class TrivialClassStruct(FileMatch):
     """Trivial class struct"""
@@ -527,14 +699,13 @@ class DeclareTypeName(FileMatch):
                OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'),
                r'\s*\);?[ \t]*\n')
 
-class ObjectDeclareType(FileMatch):
+class ObjectDeclareType(TypeCheckerDeclaration):
     """OBJECT_DECLARE_TYPE usage
     Will be replaced with OBJECT_DECLARE_SIMPLE_TYPE if possible
     """
     regexp = S(r'^[ \t]*OBJECT_DECLARE_TYPE\s*\(',
                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
                NAMED('classtype', RE_TYPE), r'\s*,\s*',
-               NAMED('lowercase', RE_IDENTIFIER), r'\s*,\s*',
                NAMED('uppercase', RE_IDENTIFIER), SP,
                r'\)[ \t]*;?[ \t]*\n')
 
@@ -549,14 +720,42 @@ class ObjectDeclareType(FileMatch):
                  "                           %(uppercase)s, %(parent_struct)s)\n" % d)
             yield self.make_patch(c)
 
-def find_type_declaration(files: FileList, typename: str) -> Optional[FileMatch]:
-    """Find usage of DECLARE*CHECKER macro"""
-    for c in (DeclareInstanceChecker, DeclareClassCheckers, DeclareObjCheckers, DeclareTypeName):
-        d = files.find_match(c, name=typename, group='typename')
-        if d:
-            return d
+class ObjectDeclareSimpleType(TypeCheckerDeclaration):
+    """OBJECT_DECLARE_SIMPLE_TYPE usage"""
+    regexp = S(r'^[ \t]*OBJECT_DECLARE_SIMPLE_TYPE\s*\(',
+               NAMED('instancetype', RE_TYPE), r'\s*,\s*',
+               NAMED('uppercase', RE_IDENTIFIER), SP,
+               r'\)[ \t]*;?[ \t]*\n')
+
+class OldStyleObjectDeclareSimpleType(TypeCheckerDeclaration):
+    """OBJECT_DECLARE_SIMPLE_TYPE usage (old API)"""
+    regexp = S(r'^[ \t]*OBJECT_DECLARE_SIMPLE_TYPE\s*\(',
+               NAMED('instancetype', RE_TYPE), r'\s*,\s*',
+               NAMED('lowercase', RE_IDENTIFIER), r'\s*,\s*',
+               NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
+               NAMED('parent_classtype', RE_TYPE), SP,
+               r'\)[ \t]*;?[ \t]*\n')
+
+    @property
+    def classtype(self) -> Optional[str]:
+        instancetype = self.instancetype
+        assert instancetype
+        return f"{instancetype}Class"
+
+def find_typename_uppercase(files: FileList, typename: str) -> Optional[str]:
+    """Try to find what's the right MODULE_OBJ_NAME for a given type name"""
+    decl = files.find_match(DeclareTypeName, name=typename, group='typename')
+    if decl:
+        return decl.group('uppercase')
+    if typename.startswith('TYPE_'):
+        return typename[len('TYPE_'):]
     return None
 
+def find_type_checkers(files:FileList, name:str, group:str='uppercase') -> Iterable[TypeCheckerDeclaration]:
+    """Find usage of DECLARE*CHECKER macro"""
+    c: Type[TypeCheckerDeclaration]
+    for c in (DeclareInstanceChecker, DeclareClassCheckers, DeclareObjCheckers, ObjectDeclareType, ObjectDeclareSimpleType):
+        yield from files.find_matches(c, name=name, group=group)
 
 class Include(FileMatch):
     """#include directive"""
@@ -586,9 +785,13 @@ class MoveSymbols(FileMatch):
     regexp = RE_FILE_BEGIN
 
     def gen_patches(self) -> Iterator[Patch]:
+        if self.file.filename_matches('qom/object.h'):
+            self.debug("skipping object.h")
+            return
+
         index: Dict[RequiredIdentifier, SymbolUserList] = {}
         definition_classes = [SimpleTypedefMatch, FullStructTypedefMatch, ConstantDefine, Include]
-        user_classes = [TypeCheckMacro, DeclareObjCheckers, DeclareInstanceChecker, DeclareClassCheckers]
+        user_classes = [TypeCheckMacro, DeclareObjCheckers, DeclareInstanceChecker, DeclareClassCheckers, InterfaceCheckMacro]
 
         # first we scan for all symbol definitions and usage:
         for dc in definition_classes:
@@ -650,3 +853,9 @@ class MoveSymbols(FileMatch):
                 definition.warn("definition of %s %s needs to be moved earlier in the file", i.type, i.name)
                 earliest.warn("definition of %s %s is used here", i.type, i.name)
 
+
+class EmptyPreprocessorConditional(FileMatch):
+    """Delete empty preprocessor conditionals"""
+    regexp = r'^[ \t]*#(if|ifdef)[ \t].*\n+[ \t]*#endif[ \t]*\n'
+    def gen_patches(self) -> Iterable[Patch]:
+        yield self.make_removal_patch()

scripts/codeconverter/codeconverter/test_patching.py

@@ -31,7 +31,6 @@ def test_pattern_patching():
     files = FileList()
     f = FileInfo(files, of.name)
     f.load()
-    f.scan_for_matches()
     matches = f.matches_of_type(BasicPattern)
     assert len(matches) == 2
     p2 = matches[1]
@@ -40,7 +39,7 @@ def test_pattern_patching():
     f.patches.append(p2.append('XXX'))
 
     # apply all patches:
-    f.gen_patches()
+    f.gen_patches(matches)
     patched = f.get_patched_content()
     assert patched == ('one line\n'+
                        'this pattern will be patched: defBBBBBhij\n'+

scripts/codeconverter/codeconverter/test_regexps.py

@@ -9,7 +9,7 @@ from .regexps import *
 from .qom_macros import *
 from .qom_type_info import *
 
-def test_res():
+def test_res() -> None:
     def fullmatch(regexp, s):
         return re.fullmatch(regexp, s, re.MULTILINE)
 
@@ -113,10 +113,10 @@ static const TypeInfo char_file_type_info = {
              * need to set up reset or vmstate, and has no realize method.
              */''')
 
-    print(RE_TYPEINFO_DEF)
+    print(TypeInfoVar.regexp)
     test_empty = 'static const TypeInfo x86_base_cpu_type_info = {\n'+\
                  '};\n';
-    assert fullmatch(RE_TYPEINFO_DEF, test_empty)
+    assert fullmatch(TypeInfoVar.regexp, test_empty)
 
     test_simple = r'''
     static const TypeInfo x86_base_cpu_type_info = {
@@ -125,7 +125,7 @@ static const TypeInfo char_file_type_info = {
         .class_init = x86_cpu_base_class_init,
     };
     '''
-    assert re.search(RE_TYPEINFO_DEF, test_simple, re.MULTILINE)
+    assert re.search(TypeInfoVar.regexp, test_simple, re.MULTILINE)
 
     test_interfaces = r'''
     static const TypeInfo acpi_ged_info = {
@@ -141,7 +141,7 @@ static const TypeInfo char_file_type_info = {
         }
     };
     '''
-    assert re.search(RE_TYPEINFO_DEF, test_interfaces, re.MULTILINE)
+    assert re.search(TypeInfoVar.regexp, test_interfaces, re.MULTILINE)
 
     test_comments = r'''
     static const TypeInfo palm_misc_gpio_info = {
@@ -155,7 +155,7 @@ static const TypeInfo char_file_type_info = {
          */
     };
     '''
-    assert re.search(RE_TYPEINFO_DEF, test_comments, re.MULTILINE)
+    assert re.search(TypeInfoVar.regexp, test_comments, re.MULTILINE)
 
     test_comments = r'''
     static const TypeInfo tpm_crb_info = {
@@ -170,7 +170,7 @@ static const TypeInfo char_file_type_info = {
         }
     };
     '''
-    assert re.search(RE_TYPEINFO_DEF, test_comments, re.MULTILINE)
+    assert re.search(TypeInfoVar.regexp, test_comments, re.MULTILINE)
 
 def test_struct_re():
     print('---')
@@ -232,8 +232,8 @@ def test_initial_includes():
 
 /* pflash_cfi01.c */
 '''
-    print(repr(list(m.groupdict() for m in re.finditer(InitialIncludes.regexp, c, re.MULTILINE))))
-    m = re.match(InitialIncludes.regexp, c, re.MULTILINE)
+    print(repr(list(m.groupdict() for m in InitialIncludes.finditer(c))))
+    m = InitialIncludes.domatch(c)
     assert m
     print(repr(m.group(0)))
     assert m.group(0).endswith('#include "exec/hwaddr.h"\n')
@@ -247,8 +247,8 @@ def test_initial_includes():
 
 
 '''
-    print(repr(list(m.groupdict() for m in re.finditer(InitialIncludes.regexp, c, re.MULTILINE))))
-    m = re.match(InitialIncludes.regexp, c, re.MULTILINE)
+    print(repr(list(m.groupdict() for m in InitialIncludes.finditer(c))))
+    m = InitialIncludes.domatch(c)
     assert m
     print(repr(m.group(0)))
     assert m.group(0).endswith('#include "9p.h"\n')
@@ -274,8 +274,8 @@ def test_initial_includes():
 /* Missing stuff:
    SCTRL_P[12](END|ST)INC
 '''
-    print(repr(list(m.groupdict() for m in re.finditer(InitialIncludes.regexp, c, re.MULTILINE))))
-    m = re.match(InitialIncludes.regexp, c, re.MULTILINE)
+    print(repr(list(m.groupdict() for m in InitialIncludes.finditer(c))))
+    m = InitialIncludes.domatch(c)
     assert m
     print(repr(m.group(0)))
     assert m.group(0).endswith('#include "sysemu/dma.h"\n')

scripts/codeconverter/converter.py

@@ -42,7 +42,7 @@ def process_all_files(parser: argparse.ArgumentParser, args: argparse.Namespace)
             for t in f.matches_of_type(TypeInfoVar):
                 assert isinstance(t, TypeInfoVar)
                 values = [f.filename, t.name] + \
-                         [t.get_initializer_value(f).raw
+                         [t.get_raw_initializer_value(f)
                           for f in TI_FIELDS]
                 DBG('values: %r', values)
                 assert all('\t' not in v for v in values)
@@ -55,18 +55,18 @@ def process_all_files(parser: argparse.ArgumentParser, args: argparse.Namespace)
         parser.error("--pattern is required")
 
     classes = [p for arg in args.patterns
-                for p in re.split(r'[\s,]', arg)]
+               for p in re.split(r'[\s,]', arg)
+               if p.strip()]
     for c in classes:
-        if c not in match_classes:
+        if c not in match_classes \
+           or not match_classes[c].regexp:
             print("Invalid pattern name: %s" % (c), file=sys.stderr)
             print("Valid patterns:", file=sys.stderr)
             print(PATTERN_HELP, file=sys.stderr)
             sys.exit(1)
 
     DBG("classes: %r", classes)
-    for f in files:
-        DBG("patching contents of %s", f.filename)
-        f.patch_content(max_passes=args.passes, class_names=classes)
+    files.patch_content(max_passes=args.passes, class_names=classes)
 
     for f in files:
         #alltypes.extend(f.type_infos)