BaseTools: Decouple AutoGen Objects

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875

1. Separate the AutoGen.py into 3 small py files.
One is for AutoGen base class, one is for WorkspaceAutoGen class
and PlatformAutoGen class, and the one for ModuleAutoGen class.
2. Create a new class DataPipe to store the Platform scope settings.
Create a new class PlatformInfo to provide the same interface
as PlatformAutoGen. PlatformInfo class is initialized by
DataPipe instance.
Create a new class WorkspaceInfo to provide the same interface
as WorkspaceAutoGen. WorkspaceInfo class is initialized by
DataPipe instance.
3. Change ModuleAutoGen to depends on DataPipe, PlatformInfo and
WorkspaceInfo. Remove the dependency of ModuleAutoGen to PlatformAutoGen.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Acked-by: Laszlo Ersek <lersek@redhat.com>
Tested-by: Laszlo Ersek <lersek@redhat.com>
Acked-by: Liming Gao <liming.gao@intel.com>

BaseTools/Source/Python/AutoGen/DataPipe.py

@@ -0,0 +1,147 @@
+## @file
+# Create makefile for MS nmake and GNU make
+#
+# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+from __future__ import absolute_import
+from Workspace.WorkspaceDatabase import BuildDB
+from Workspace.WorkspaceCommon import GetModuleLibInstances
+import Common.GlobalData as GlobalData
+import os
+import pickle
+from pickle import HIGHEST_PROTOCOL
+
+class PCD_DATA():
+    def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInfoList,DefaultValue,
+                 MaxDatumSize,UserDefinedDefaultStoresFlag,validateranges,
+                 validlists,expressions,CustomAttribute,TokenValue):
+        self.TokenCName = TokenCName
+        self.TokenSpaceGuidCName = TokenSpaceGuidCName
+        self.Type = Type
+        self.DatumType = DatumType
+        self.SkuInfoList = SkuInfoList
+        self.DefaultValue = DefaultValue
+        self.MaxDatumSize = MaxDatumSize
+        self.UserDefinedDefaultStoresFlag = UserDefinedDefaultStoresFlag
+        self.validateranges = validateranges
+        self.validlists = validlists
+        self.expressions = expressions
+        self.CustomAttribute = CustomAttribute
+        self.TokenValue = TokenValue
+
+class DataPipe(object):
+    def __init__(self, BuildDir=None):
+        self.data_container = {}
+        self.BuildDir = BuildDir
+
+class MemoryDataPipe(DataPipe):
+
+    def Get(self,key):
+        return self.data_container.get(key)
+
+    def dump(self,file_path):
+        with open(file_path,'wb') as fd:
+            pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL)
+
+    def load(self,file_path):
+        with open(file_path,'rb') as fd:
+            self.data_container = pickle.load(fd)
+
+    @property
+    def DataContainer(self):
+        return self.data_container
+    @DataContainer.setter
+    def DataContainer(self,data):
+        self.data_container.update(data)
+
+    def FillData(self,PlatformInfo):
+        #Platform Pcds
+        self.DataContainer = {
+            "PLA_PCD" : [PCD_DATA(
+            pcd.TokenCName,pcd.TokenSpaceGuidCName,pcd.Type,
+            pcd.DatumType,pcd.SkuInfoList,pcd.DefaultValue,
+            pcd.MaxDatumSize,pcd.UserDefinedDefaultStoresFlag,pcd.validateranges,
+                 pcd.validlists,pcd.expressions,pcd.CustomAttribute,pcd.TokenValue)
+            for pcd in PlatformInfo.Platform.Pcds.values()]
+            }
+
+        #Platform Module Pcds
+        ModulePcds = {}
+        for m in PlatformInfo.Platform.Modules:
+            m_pcds =  PlatformInfo.Platform.Modules[m].Pcds
+            if m_pcds:
+                ModulePcds[(m.File,m.Root)] = [PCD_DATA(
+            pcd.TokenCName,pcd.TokenSpaceGuidCName,pcd.Type,
+            pcd.DatumType,pcd.SkuInfoList,pcd.DefaultValue,
+            pcd.MaxDatumSize,pcd.UserDefinedDefaultStoresFlag,pcd.validateranges,
+                 pcd.validlists,pcd.expressions,pcd.CustomAttribute,pcd.TokenValue)
+            for pcd in PlatformInfo.Platform.Modules[m].Pcds.values()]
+
+
+        self.DataContainer = {"MOL_PCDS":ModulePcds}
+
+        #Module's Library Instance
+        ModuleLibs = {}
+        for m in PlatformInfo.Platform.Modules:
+            module_obj = BuildDB.BuildObject[m,PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain]
+            Libs = GetModuleLibInstances(module_obj, PlatformInfo.Platform, BuildDB.BuildObject, PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain)
+            ModuleLibs[(m.File,m.Root,module_obj.Arch)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch) for l in Libs]
+        self.DataContainer = {"DEPS":ModuleLibs}
+
+        #Platform BuildOptions
+
+        platform_build_opt =  PlatformInfo.EdkIIBuildOption
+
+        ToolDefinition = PlatformInfo.ToolDefinition
+        module_build_opt = {}
+        for m in PlatformInfo.Platform.Modules:
+            ModuleTypeOptions, PlatformModuleOptions = PlatformInfo.GetGlobalBuildOptions(BuildDB.BuildObject[m,PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain])
+            if ModuleTypeOptions or PlatformModuleOptions:
+                module_build_opt.update({(m.File,m.Root): {"ModuleTypeOptions":ModuleTypeOptions, "PlatformModuleOptions":PlatformModuleOptions}})
+
+        self.DataContainer = {"PLA_BO":platform_build_opt,
+                              "TOOLDEF":ToolDefinition,
+                              "MOL_BO":module_build_opt
+                              }
+
+
+
+        #Platform Info
+        PInfo = {
+            "WorkspaceDir":PlatformInfo.Workspace.WorkspaceDir,
+            "Target":PlatformInfo.BuildTarget,
+            "ToolChain":PlatformInfo.Workspace.ToolChain,
+            "BuildRuleFile":PlatformInfo.BuildRule,
+            "Arch": PlatformInfo.Arch,
+            "ArchList":PlatformInfo.Workspace.ArchList,
+            "ActivePlatform":PlatformInfo.MetaFile
+            }
+        self.DataContainer = {'P_Info':PInfo}
+
+        self.DataContainer = {'M_Name':PlatformInfo.UniqueBaseName}
+
+        self.DataContainer = {"ToolChainFamily": PlatformInfo.ToolChainFamily}
+
+        self.DataContainer = {"BuildRuleFamily": PlatformInfo.BuildRuleFamily}
+
+        self.DataContainer = {"MixedPcd":GlobalData.MixedPcd}
+
+        self.DataContainer = {"BuildOptPcd":GlobalData.BuildOptionPcd}
+
+        self.DataContainer = {"BuildCommand": PlatformInfo.BuildCommand}
+
+        self.DataContainer = {"AsBuildModuleList": PlatformInfo._AsBuildModuleList}
+
+        self.DataContainer = {"G_defines": GlobalData.gGlobalDefines}
+
+        self.DataContainer = {"CL_defines": GlobalData.gCommandLineDefines}
+
+        self.DataContainer = {"Env_Var": {k:v for k, v in os.environ.items()}}
+
+        self.DataContainer = {"PackageList": [(dec.MetaFile,dec.Arch) for dec in PlatformInfo.PackageList]}
+
+        self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict}
+
+        self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False}
+

BaseTools/Source/Python/AutoGen/GenC.py

@@ -1629,7 +1629,7 @@ def CreatePcdCode(Info, AutoGenC, AutoGenH):
         if Pcd.Type in PCD_DYNAMIC_EX_TYPE_SET and Pcd.TokenSpaceGuidCName not in TokenSpaceList:
             TokenSpaceList.append(Pcd.TokenSpaceGuidCName)
 
-    SkuMgr = Info.Workspace.Platform.SkuIdMgr
+    SkuMgr = Info.PlatformInfo.Platform.SkuIdMgr
     AutoGenH.Append("\n// Definition of SkuId Array\n")
     AutoGenH.Append("extern UINT64 _gPcd_SkuId_Array[];\n")
     # Add extern declarations to AutoGen.h if one or more Token Space GUIDs were found

BaseTools/Source/Python/Common/Misc.py

@@ -649,7 +649,6 @@ def GuidValue(CName, PackageList, Inffile = None):
         if CName in GuidKeys:
             return P.Guids[CName]
     return None
-    return None
 
 ## A string template class
 #

BaseTools/Source/Python/PatchPcdValue/PatchPcdValue.py

@@ -11,7 +11,6 @@
 import Common.LongFilePathOs as os
 from Common.LongFilePathSupport import OpenLongFilePath as open
 import sys
-import re
 
 from optparse import OptionParser
 from optparse import make_option

BaseTools/Source/Python/Workspace/DscBuildData.py

@@ -1373,11 +1373,11 @@ class DscBuildData(PlatformBuildClassObject):
                                         self._PCD_TYPE_STRING_[MODEL_PCD_FEATURE_FLAG],
                                         self._PCD_TYPE_STRING_[MODEL_PCD_DYNAMIC],
                                         self._PCD_TYPE_STRING_[MODEL_PCD_DYNAMIC_EX]]:
-                        self.Pcds[Name, Guid] = copy.deepcopy(PcdInDec)
-                        self.Pcds[Name, Guid].DefaultValue = NoFiledValues[( Guid, Name)][0]
+                        self._Pcds[Name, Guid] = copy.deepcopy(PcdInDec)
+                        self._Pcds[Name, Guid].DefaultValue = NoFiledValues[( Guid, Name)][0]
                     if PcdInDec.Type in [self._PCD_TYPE_STRING_[MODEL_PCD_DYNAMIC],
                                         self._PCD_TYPE_STRING_[MODEL_PCD_DYNAMIC_EX]]:
-                        self.Pcds[Name, Guid].SkuInfoList = {TAB_DEFAULT:SkuInfoClass(TAB_DEFAULT, self.SkuIds[TAB_DEFAULT][0], '', '', '', '', '', NoFiledValues[( Guid, Name)][0])}
+                        self._Pcds[Name, Guid].SkuInfoList = {TAB_DEFAULT:SkuInfoClass(TAB_DEFAULT, self.SkuIds[TAB_DEFAULT][0], '', '', '', '', '', NoFiledValues[( Guid, Name)][0])}
         return AllPcds
 
     def OverrideByFdfOverAll(self,AllPcds):
@@ -1419,8 +1419,8 @@ class DscBuildData(PlatformBuildClassObject):
                     if PcdInDec.Type in [self._PCD_TYPE_STRING_[MODEL_PCD_FIXED_AT_BUILD],
                                         self._PCD_TYPE_STRING_[MODEL_PCD_PATCHABLE_IN_MODULE],
                                         self._PCD_TYPE_STRING_[MODEL_PCD_FEATURE_FLAG]]:
-                        self.Pcds[Name, Guid] = copy.deepcopy(PcdInDec)
-                        self.Pcds[Name, Guid].DefaultValue = Value
+                        self._Pcds[Name, Guid] = copy.deepcopy(PcdInDec)
+                        self._Pcds[Name, Guid].DefaultValue = Value
         return AllPcds
 
     def ParsePcdNameStruct(self,NamePart1,NamePart2):

BaseTools/Source/Python/Workspace/InfBuildData.py

@@ -154,6 +154,13 @@ class InfBuildData(ModuleBuildClassObject):
         self._PcdComments = None
         self._BuildOptions = None
         self._DependencyFileList = None
+        self.LibInstances = []
+        self.ReferenceModules = set()
+        self.Guids
+        self.Pcds
+    def SetReferenceModule(self,Module):
+        self.ReferenceModules.add(Module)
+        return self
 
     ## XXX[key] = value
     def __setitem__(self, key, value):
@@ -705,6 +712,25 @@ class InfBuildData(ModuleBuildClassObject):
         return RetVal
 
     @cached_property
+    def ModulePcdList(self):
+        RetVal = self.Pcds
+        return RetVal
+    @cached_property
+    def LibraryPcdList(self):
+        if bool(self.LibraryClass):
+            return []
+        RetVal = {}
+        Pcds = set()
+        for Library in self.LibInstances:
+            PcdsInLibrary = OrderedDict()
+            for Key in Library.Pcds:
+                if Key in self.Pcds or Key in Pcds:
+                    continue
+                Pcds.add(Key)
+                PcdsInLibrary[Key] = copy.copy(Library.Pcds[Key])
+            RetVal[Library] = PcdsInLibrary
+        return RetVal
+    @cached_property
     def PcdsName(self):
         PcdsName = set()
         for Type in (MODEL_PCD_FIXED_AT_BUILD,MODEL_PCD_PATCHABLE_IN_MODULE,MODEL_PCD_FEATURE_FLAG,MODEL_PCD_DYNAMIC,MODEL_PCD_DYNAMIC_EX):
@@ -1030,3 +1056,6 @@ class InfBuildData(ModuleBuildClassObject):
         if (self.Binaries and not self.Sources) or GlobalData.gIgnoreSource:
             return True
         return False
+def ExtendCopyDictionaryLists(CopyToDict, CopyFromDict):
+    for Key in CopyFromDict:
+        CopyToDict[Key].extend(CopyFromDict[Key])

BaseTools/Source/Python/Workspace/WorkspaceCommon.py

@@ -88,6 +88,8 @@ def GetLiabraryInstances(Module, Platform, BuildDatabase, Arch, Target, Toolchai
     return GetModuleLibInstances(Module, Platform, BuildDatabase, Arch, Target, Toolchain)
 
 def GetModuleLibInstances(Module, Platform, BuildDatabase, Arch, Target, Toolchain, FileName = '', EdkLogger = None):
+    if Module.LibInstances:
+        return Module.LibInstances
     ModuleType = Module.ModuleType
 
     # add forced library instances (specified under LibraryClasses sections)
@@ -246,4 +248,6 @@ def GetModuleLibInstances(Module, Platform, BuildDatabase, Arch, Target, Toolcha
     # The DAG Topo sort produces the destructor order, so the list of constructors must generated in the reverse order
     #
     SortedLibraryList.reverse()
+    Module.LibInstances = SortedLibraryList
+    SortedLibraryList = [lib.SetReferenceModule(Module) for lib in SortedLibraryList]
     return SortedLibraryList

BaseTools/Source/Python/Workspace/WorkspaceDatabase.py

@@ -62,6 +62,8 @@ class WorkspaceDatabase(object):
         }
 
         _CACHE_ = {}    # (FilePath, Arch)  : <object>
+        def GetCache(self):
+            return self._CACHE_
 
         # constructor
         def __init__(self, WorkspaceDb):
@@ -203,6 +205,7 @@ class WorkspaceDatabase(object):
             EdkLogger.error('build', PARSER_ERROR, "Failed to parser DSC file: %s" % Dscfile)
         return Platform
 
+BuildDB = WorkspaceDatabase()
 ##
 #
 # This acts like the main() function for the script, unless it is 'import'ed into another

BaseTools/Source/Python/build/BuildReport.py

@@ -34,7 +34,7 @@ from Common.BuildToolError import FORMAT_INVALID
 from Common.LongFilePathSupport import OpenLongFilePath as open
 from Common.MultipleWorkspace import MultipleWorkspace as mws
 import Common.GlobalData as GlobalData
-from AutoGen.AutoGen import ModuleAutoGen
+from AutoGen.ModuleAutoGen import ModuleAutoGen
 from Common.Misc import PathClass
 from Common.StringUtils import NormPath
 from Common.DataType import *
@@ -2142,7 +2142,7 @@ class PlatformReport(object):
                         INFList = GlobalData.gFdfParser.Profile.InfDict[Pa.Arch]
                         for InfName in INFList:
                             InfClass = PathClass(NormPath(InfName), Wa.WorkspaceDir, Pa.Arch)
-                            Ma = ModuleAutoGen(Wa, InfClass, Pa.BuildTarget, Pa.ToolChain, Pa.Arch, Wa.MetaFile)
+                            Ma = ModuleAutoGen(Wa, InfClass, Pa.BuildTarget, Pa.ToolChain, Pa.Arch, Wa.MetaFile,Pa.DataPile)
                             if Ma is None:
                                 continue
                             if Ma not in ModuleAutoGenList:

BaseTools/Source/Python/build/build.py

@@ -12,42 +12,45 @@
 # Import Modules
 #
 from __future__ import print_function
-import Common.LongFilePathOs as os
-import re
+from __future__ import absolute_import
+import os.path as path
 import sys
+import os
+import re
 import glob
 import time
 import platform
 import traceback
-import encodings.ascii
 import multiprocessing
-
-from struct import *
-from threading import *
+from threading import Thread,Event,BoundedSemaphore
 import threading
+from subprocess import Popen,PIPE
+from collections import OrderedDict, defaultdict
 from optparse import OptionParser
-from subprocess import *
+from AutoGen.PlatformAutoGen import PlatformAutoGen
+from AutoGen.ModuleAutoGen import ModuleAutoGen
+from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
+from AutoGen import GenMake
 from Common import Misc as Utils
 
-from Common.LongFilePathSupport import OpenLongFilePath as open
 from Common.TargetTxtClassObject import TargetTxt
 from Common.ToolDefClassObject import ToolDef
-from Common.DataType import *
-from Common.BuildVersion import gBUILD_VERSION
-from AutoGen.AutoGen import *
-from Common.BuildToolError import *
-from Workspace.WorkspaceDatabase import WorkspaceDatabase
+from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory
+from Common.StringUtils import NormPath
 from Common.MultipleWorkspace import MultipleWorkspace as mws
+from Common.BuildToolError import *
+from Common.DataType import *
+import Common.EdkLogger as EdkLogger
+from Common.BuildVersion import gBUILD_VERSION
+from Workspace.WorkspaceDatabase import BuildDB
 
 from BuildReport import BuildReport
-from GenPatchPcdTable.GenPatchPcdTable import *
-from PatchPcdValue.PatchPcdValue import *
+from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile
+from PatchPcdValue.PatchPcdValue import PatchBinaryFile
 
-import Common.EdkLogger
 import Common.GlobalData as GlobalData
 from GenFds.GenFds import GenFds, GenFdsApi
 
-from collections import OrderedDict, defaultdict
 
 # Version and Copyright
 VersionNumber = "0.60" + ' ' + gBUILD_VERSION
@@ -775,7 +778,7 @@ class Build():
         GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))
         if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):
             os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))
-        self.Db = WorkspaceDatabase()
+        self.Db = BuildDB
         self.BuildDatabase = self.Db.BuildObject
         self.Platform = None
         self.ToolChainFamily = None
@@ -1700,13 +1703,17 @@ class Build():
                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
 
                 for Arch in Wa.ArchList:
+                    PcdMaList    = []
                     GlobalData.gGlobalDefines['ARCH'] = Arch
                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
                     for Module in Pa.Platform.Modules:
                         # Get ModuleAutoGen object to generate C code file and makefile
-                        Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
+                        Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
                         if Ma is None:
                             continue
+                        if Ma.PcdIsDriver:
+                            Ma.PlatformInfo = Pa
+                            PcdMaList.append(Ma)
                         self.BuildModules.append(Ma)
                     self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict)
 
@@ -1802,7 +1809,7 @@ class Build():
                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
                     for Module in Pa.Platform.Modules:
                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
-                            Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
+                            Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
                             if Ma is None:
                                 continue
                             MaList.append(Ma)
@@ -1982,6 +1989,7 @@ class Build():
                 ExitFlag.clear()
                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
                 for Arch in Wa.ArchList:
+                    PcdMaList    = []
                     AutoGenStart = time.time()
                     GlobalData.gGlobalDefines['ARCH'] = Arch
                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
@@ -1999,10 +2007,13 @@ class Build():
                             ModuleList.append(Inf)
                     for Module in ModuleList:
                         # Get ModuleAutoGen object to generate C code file and makefile
-                        Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
+                        Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
 
                         if Ma is None:
                             continue
+                        if Ma.PcdIsDriver:
+                            Ma.PlatformInfo = Pa
+                            PcdMaList.append(Ma)
                         if Ma.CanSkipbyHash():
                             self.HashSkipModules.append(Ma)
                             if GlobalData.gBinCacheSource: