dasharo/slimbootloader
0
0
Fork 0
mirror of https://github.com/Dasharo/slimbootloader.git synced 2026-06-14 04:16:15 -07:00
Code Issues Packages Projects Releases Wiki Activity
Files
69fb535cc5082b90235bb032172513e662028dee
slimbootloader/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
T
Mike Crowe 990e3e81e6 Use LF line endings in the repository 
Convert the line endings stored for all text files in the repository to
LF. The majority previously used DOS-style CRLF line endings. Add a
.gitattributes file to enforce this and treat certain extensions as
never being text files.

Update PatchCheck.py to insist on LF line endings rather than CRLF.
However, its other checks fail on this commit due to lots of
pre-existing complaints that it only notices because the line endings
have changed.

Silicon/QemuSocPkg/FspBin/Patches/0001-Build-QEMU-FSP-2.0-binaries.patch
needs to be treated as binary since it contains a mixture of line
endings.

This change has implications depending on the client platform you are
using the repository from:

* Windows

The usual configuration for Git on Windows means that text files will
be checked out to the work tree with DOS-style CRLF line endings. If
that's not the case then you can configure Git to do so for the entire
machine with:

 git config --global core.autocrlf true

or for just the repository with:

 git config core.autocrlf true

Line endings will be normalised to LF when they are committed to the
repository. If you commit a text file with only LF line endings then it
will be converted to CRLF line endings in your work tree.

* Linux, MacOS and other Unices

The usual configuration for Git on such platforms is to check files out
of the repository with LF line endings. This is probably the right thing
for you. In the unlikely even that you are using Git on Unix but editing
or compiling on Windows for some reason then you may need to tweak your
configuration to force the use of CRLF line endings as described above.

* General

For more information see
https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings .

Fixes: https://github.com/slimbootloader/slimbootloader/issues/1400
Signed-off-by: Mike Crowe <mac@mcrowe.com>
2021-11-10 12:46:42 -08:00

330 lines
14 KiB
Python
Raw Blame History

## @file
# Create makefile for MS nmake and GNU make
#
# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
from __future__ import absolute_import
import multiprocessing as mp
import threading
from Common.Misc import PathClass
from AutoGen.ModuleAutoGen import ModuleAutoGen
from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
import Common.GlobalData as GlobalData
import Common.EdkLogger as EdkLogger
import os
from Common.MultipleWorkspace import MultipleWorkspace as mws
from AutoGen.AutoGen import AutoGen
from Workspace.WorkspaceDatabase import BuildDB
try:
from queue import Empty
except:
from Queue import Empty
import traceback
import sys
from AutoGen.DataPipe import MemoryDataPipe
import logging
import time
def clearQ(q):
try:
while True:
q.get_nowait()
except Empty:
pass
class LogAgent(threading.Thread):
def __init__(self,log_q,log_level,log_file=None):
super(LogAgent,self).__init__()
self.log_q = log_q
self.log_level = log_level
self.log_file = log_file
def InitLogger(self):
# For DEBUG level (All DEBUG_0~9 are applicable)
self._DebugLogger_agent = logging.getLogger("tool_debug_agent")
_DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")
self._DebugLogger_agent.setLevel(self.log_level)
_DebugChannel = logging.StreamHandler(sys.stdout)
_DebugChannel.setFormatter(_DebugFormatter)
self._DebugLogger_agent.addHandler(_DebugChannel)
# For VERBOSE, INFO, WARN level
self._InfoLogger_agent = logging.getLogger("tool_info_agent")
_InfoFormatter = logging.Formatter("%(message)s")
self._InfoLogger_agent.setLevel(self.log_level)
_InfoChannel = logging.StreamHandler(sys.stdout)
_InfoChannel.setFormatter(_InfoFormatter)
self._InfoLogger_agent.addHandler(_InfoChannel)
# For ERROR level
self._ErrorLogger_agent = logging.getLogger("tool_error_agent")
_ErrorFormatter = logging.Formatter("%(message)s")
self._ErrorLogger_agent.setLevel(self.log_level)
_ErrorCh = logging.StreamHandler(sys.stderr)
_ErrorCh.setFormatter(_ErrorFormatter)
self._ErrorLogger_agent.addHandler(_ErrorCh)
if self.log_file:
if os.path.exists(self.log_file):
os.remove(self.log_file)
_Ch = logging.FileHandler(self.log_file)
_Ch.setFormatter(_DebugFormatter)
self._DebugLogger_agent.addHandler(_Ch)
_Ch= logging.FileHandler(self.log_file)
_Ch.setFormatter(_InfoFormatter)
self._InfoLogger_agent.addHandler(_Ch)
_Ch = logging.FileHandler(self.log_file)
_Ch.setFormatter(_ErrorFormatter)
self._ErrorLogger_agent.addHandler(_Ch)
def run(self):
self.InitLogger()
while True:
log_message = self.log_q.get()
if log_message is None:
break
if log_message.name == "tool_error":
self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())
elif log_message.name == "tool_info":
self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
elif log_message.name == "tool_debug":
self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())
else:
self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
def kill(self):
self.log_q.put(None)
class AutoGenManager(threading.Thread):
def __init__(self,autogen_workers, feedback_q,error_event):
super(AutoGenManager,self).__init__()
self.autogen_workers = autogen_workers
self.feedback_q = feedback_q
self.Status = True
self.error_event = error_event
def run(self):
try:
fin_num = 0
while True:
badnews = self.feedback_q.get()
if badnews is None:
break
if badnews == "Done":
fin_num += 1
elif badnews == "QueueEmpty":
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
self.TerminateWorkers()
else:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
self.Status = False
self.TerminateWorkers()
if fin_num == len(self.autogen_workers):
self.clearQueue()
for w in self.autogen_workers:
w.join()
break
except Exception:
return
def clearQueue(self):
taskq = self.autogen_workers[0].module_queue
logq = self.autogen_workers[0].log_q
clearQ(taskq)
clearQ(self.feedback_q)
clearQ(logq)
# Copy the cache queue itmes to parent thread before clear
cacheq = self.autogen_workers[0].cache_q
try:
cache_num = 0
while True:
item = cacheq.get()
if item == "CacheDone":
cache_num += 1
else:
GlobalData.gModuleAllCacheStatus.add(item)
if cache_num == len(self.autogen_workers):
break
except:
print ("cache_q error")
def TerminateWorkers(self):
self.error_event.set()
def kill(self):
self.feedback_q.put(None)
class AutoGenWorkerInProcess(mp.Process):
def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):
mp.Process.__init__(self)
self.module_queue = module_queue
self.data_pipe_file_path =data_pipe_file_path
self.data_pipe = None
self.feedback_q = feedback_q
self.PlatformMetaFileSet = {}
self.file_lock = file_lock
self.cache_q = cache_q
self.log_q = log_q
self.error_event = error_event
def GetPlatformMetaFile(self,filepath,root):
try:
return self.PlatformMetaFileSet[(filepath,root)]
except:
self.PlatformMetaFileSet[(filepath,root)] = filepath
return self.PlatformMetaFileSet[(filepath,root)]
def run(self):
try:
taskname = "Init"
with self.file_lock:
try:
self.data_pipe = MemoryDataPipe()
self.data_pipe.load(self.data_pipe_file_path)
except:
self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
EdkLogger.LogClientInitialize(self.log_q)
loglevel = self.data_pipe.Get("LogLevel")
if not loglevel:
loglevel = EdkLogger.INFO
EdkLogger.SetLevel(loglevel)
target = self.data_pipe.Get("P_Info").get("Target")
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
archlist = self.data_pipe.Get("P_Info").get("ArchList")
active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
PackagesPath = os.getenv("PACKAGES_PATH")
mws.setWs(workspacedir, PackagesPath)
self.Wa = WorkSpaceInfo(
workspacedir,active_p,target,toolchain,archlist
)
self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
os.environ._data = self.data_pipe.Get("Env_Var")
GlobalData.gWorkspace = workspacedir
GlobalData.gDisableIncludePathCheck = False
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")
GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")
GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")
GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")
GlobalData.gModulePreMakeCacheStatus = dict()
GlobalData.gModuleMakeCacheStatus = dict()
GlobalData.gHashChainStatus = dict()
GlobalData.gCMakeHashFile = dict()
GlobalData.gModuleHashFile = dict()
GlobalData.gFileHashDict = dict()
GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")
GlobalData.file_lock = self.file_lock
CommandTarget = self.data_pipe.Get("CommandTarget")
pcd_from_build_option = []
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
if pcd_tuple[2].strip():
pcd_id = ".".join((pcd_id,pcd_tuple[2]))
pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
GlobalData.BuildOptionPcd = pcd_from_build_option
module_count = 0
FfsCmd = self.data_pipe.Get("FfsCommand")
if FfsCmd is None:
FfsCmd = {}
GlobalData.FfsCmd = FfsCmd
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
while True:
if self.error_event.is_set():
break
module_count += 1
try:
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
except Empty:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))
time.sleep(0.01)
continue
if module_file is None:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))
self.feedback_q.put("QueueEmpty")
time.sleep(0.01)
continue
modulefullpath = os.path.join(module_root,module_file)
taskname = " : ".join((modulefullpath,module_arch))
module_metafile = PathClass(module_file,module_root)
if module_path:
module_metafile.Path = module_path
if module_basename:
module_metafile.BaseName = module_basename
if module_originalpath:
module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
arch = module_arch
target = self.data_pipe.Get("P_Info").get("Target")
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
Ma.IsLibrary = IsLib
# SourceFileList calling sequence impact the makefile string sequence.
# Create cached SourceFileList here to unify its calling sequence for both
# CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
RetVal = Ma.SourceFileList
if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:
try:
CacheResult = Ma.CanSkipbyPreMakeCache()
except:
CacheResult = False
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))
continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))
Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
Ma.CreateAsBuiltInf()
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
try:
CacheResult = Ma.CanSkipbyMakeCache()
except:
CacheResult = False
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
except Exception as e:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e)))
self.feedback_q.put(taskname)
finally:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done"))
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
groupobj = {}
for buildobj in BuildDB.BuildObject.GetCache().values():
if str(buildobj).lower().endswith("dec"):
try:
groupobj['dec'].append(str(buildobj))
except:
groupobj['dec'] = [str(buildobj)]
if str(buildobj).lower().endswith("dsc"):
try:
groupobj['dsc'].append(str(buildobj))
except:
groupobj['dsc'] = [str(buildobj)]
if str(buildobj).lower().endswith("inf"):
try:
groupobj['inf'].append(str(buildobj))
except:
groupobj['inf'] = [str(buildobj)]
print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))
Reference in New Issue View Git Blame Copy Permalink