Add official support for Baldur's Gate 3 (#188)

__init__.py

@@ -3,10 +3,15 @@
 import glob
 import importlib
 import os
+import pathlib
 import site
 import sys
 import typing
 
+from PyQt6.QtCore import qWarning
+
+from mobase import IPlugin
+
 from .basic_game import BasicGame
 from .basic_game_ini import BasicIniGame
 
@@ -18,7 +23,7 @@ BasicGame.setup()
 
 def createPlugins():
     # List of game class from python:
-    game_plugins: typing.List[BasicGame] = []
+    game_plugins: typing.List[IPlugin] = []
 
     # We are going to list all game plugins:
     curpath = os.path.abspath(os.path.dirname(__file__))
@@ -58,5 +63,25 @@ def createPlugins():
                             "Failed to instantiate {}: {}".format(name, e),
                             file=sys.stderr,
                         )
+    for path in pathlib.Path(escaped_games_path).rglob("plugins/__init__.py"):
+        module_path = "." + os.path.relpath(path.parent, curpath).replace(os.sep, ".")
+        try:
+            module = importlib.import_module(module_path, __package__)
+            if hasattr(module, "createPlugins") and callable(module.createPlugins):
+                try:
+                    plugins: typing.Any = module.createPlugins()
+                    for item in plugins:
+                        if isinstance(item, IPlugin):
+                            game_plugins.append(item)
+                except TypeError:
+                    pass
+            if hasattr(module, "createPlugin") and callable(module.createPlugin):
+                plugin = module.createPlugin()
+                if isinstance(plugin, IPlugin):
+                    game_plugins.append(plugin)
+        except ImportError as e:
+            qWarning(f"Error importing module {module_path}: {e}")
+        except Exception as e:
+            qWarning(f"Error calling function createPlugin(s) in {module_path}: {e}")
 
     return game_plugins

games/baldursgate3/bg3_data_checker.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+
+import mobase
+
+from ...basic_features import BasicModDataChecker, GlobPatterns, utils
+from . import bg3_utils
+
+
+class BG3ModDataChecker(BasicModDataChecker):
+    def __init__(self):
+        super().__init__(
+            GlobPatterns(
+                valid=[
+                    "*.pak",
+                    str(Path("Mods") / "*.pak"),  # standard mods
+                    "bin",  # native mods / Script Extender
+                    "Script Extender",  # mods which are configured via jsons in this folder
+                    "Data",  # loose file mods
+                ]
+                + [str(Path("*") / f) for f in bg3_utils.loose_file_folders],
+                move={
+                    "Root/": "",  # root builder not needed
+                    "*.dll": "bin/",
+                    "ScriptExtenderSettings.json": "bin/",
+                }
+                | {f: "Data/" for f in bg3_utils.loose_file_folders},
+                delete=["info.json", "*.txt"],
+            )
+        )
+
+    def dataLooksValid(
+        self, filetree: mobase.IFileTree
+    ) -> mobase.ModDataChecker.CheckReturn:
+        status = mobase.ModDataChecker.INVALID
+        rp = self._regex_patterns
+        for entry in filetree:
+            name = entry.name().casefold()
+            if rp.unfold.match(name):
+                if utils.is_directory(entry):
+                    status = self.dataLooksValid(entry)
+                else:
+                    status = mobase.ModDataChecker.INVALID
+                    break
+            elif rp.valid.match(name):
+                if status is mobase.ModDataChecker.INVALID:
+                    status = mobase.ModDataChecker.VALID
+            elif isinstance(entry, mobase.IFileTree):
+                status = (
+                    mobase.ModDataChecker.VALID
+                    if all(rp.valid.match(e.pathFrom(filetree)) for e in entry)
+                    else mobase.ModDataChecker.INVALID
+                )
+            elif rp.delete.match(name) or rp.move_match(name) is not None:
+                status = mobase.ModDataChecker.FIXABLE
+            else:
+                status = mobase.ModDataChecker.INVALID
+                break
+        return status

games/baldursgate3/bg3_data_content.py

@@ -0,0 +1,54 @@
+from enum import IntEnum, auto
+
+import mobase
+
+from . import bg3_utils
+
+
+class Content(IntEnum):
+    PAK = auto()
+    WORKSPACE = auto()
+    NATIVE = auto()
+    LOOSE_FILES = auto()
+    SE_FILES = auto()
+
+
+class BG3DataContent(mobase.ModDataContent):
+    BG3_CONTENTS: list[tuple[Content, str, str, bool] | tuple[Content, str, str]] = [
+        (Content.WORKSPACE, "Mod workspace", ":/MO/gui/content/script"),
+        (Content.PAK, "Pak", ":/MO/gui/content/bsa"),
+        (Content.LOOSE_FILES, "Loose file override mod", ":/MO/gui/content/texture"),
+        (Content.SE_FILES, "Script Extender Files", ":/MO/gui/content/inifile"),
+        (Content.NATIVE, "Native DLL mod", ":/MO/gui/content/plugin"),
+    ]
+
+    def getAllContents(self) -> list[mobase.ModDataContent.Content]:
+        return [
+            mobase.ModDataContent.Content(id, name, icon, *filter_only)
+            for id, name, icon, *filter_only in self.BG3_CONTENTS
+        ]
+
+    def getContentsFor(self, filetree: mobase.IFileTree) -> list[int]:
+        contents: set[int] = set()
+        for entry in filetree:
+            if isinstance(entry, mobase.IFileTree):
+                match entry.name():
+                    case "Script Extender":
+                        contents.add(Content.SE_FILES)
+                    case "Data":
+                        contents.add(Content.LOOSE_FILES)
+                    case "Mods":
+                        for e in entry:
+                            if e.name().endswith(".pak"):
+                                contents.add(Content.PAK)
+                                break
+                    case "bin":
+                        contents.add(Content.NATIVE)
+                    case _:
+                        for e in entry:
+                            if e.name() in bg3_utils.loose_file_folders:
+                                contents.add(Content.WORKSPACE)
+                                break
+            elif entry.name().endswith(".pak"):
+                contents.add(Content.PAK)
+        return list(contents)

games/baldursgate3/bg3_file_mapper.py

@@ -0,0 +1,104 @@
+import functools
+import json
+import os
+from pathlib import Path
+from typing import Callable, Optional
+
+import yaml
+from PyQt6.QtCore import QDir, qDebug, qInfo, qWarning
+from PyQt6.QtWidgets import QApplication
+
+import mobase
+
+from . import bg3_utils
+
+
+class BG3FileMapper(mobase.IPluginFileMapper):
+    current_mappings: list[mobase.Mapping] = []
+
+    def __init__(self, utils: bg3_utils.BG3Utils, doc_dir: Callable[[], QDir]):
+        super().__init__()
+        self._utils = utils
+        self.doc_dir = doc_dir
+
+    @functools.cached_property
+    def doc_path(self):
+        return Path(self.doc_dir().path())
+
+    def mappings(self) -> list[mobase.Mapping]:
+        qInfo("creating custom bg3 mappings")
+        self.current_mappings.clear()
+        active_mods = self._utils.active_mods()
+        doc_dir = Path(self.doc_dir().path())
+        progress = self._utils.create_progress_window(
+            "Mapping files to documents folder", len(active_mods) + 1
+        )
+        docs_path_mods = doc_dir / "Mods"
+        docs_path_se = doc_dir / "Script Extender"
+        for mod in active_mods:
+            modpath = Path(mod.absolutePath())
+            self.map_files(modpath, dest=docs_path_mods, pattern="*.pak", rel=False)
+            self.map_files(modpath / "Script Extender", dest=docs_path_se)
+            progress.setValue(progress.value() + 1)
+            QApplication.processEvents()
+            if progress.wasCanceled():
+                qWarning("mapping canceled by user")
+                return self.current_mappings
+        self.map_files(self._utils.overwrite_path)
+        self.create_mapping(
+            self._utils.modsettings_path,
+            doc_dir / "PlayerProfiles" / "Public" / self._utils.modsettings_path.name,
+        )
+        progress.setValue(len(active_mods) + 1)
+        QApplication.processEvents()
+        progress.close()
+        return self.current_mappings
+
+    def map_files(
+        self,
+        path: Path,
+        dest: Optional[Path] = None,
+        pattern: str = "*",
+        rel: bool = True,
+    ):
+        dest = dest if dest else self.doc_path
+        dest_func: Callable[[Path], str] = (
+            (lambda f: os.path.relpath(f, path)) if rel else lambda f: f.name
+        )
+        found_jsons: set[Path] = set()
+        for file in list(path.rglob(pattern)):
+            if self._utils.convert_yamls_to_json and (
+                file.name.endswith(".yaml") or file.name.endswith(".yml")
+            ):
+                converted_path = file.parent / file.name.replace(
+                    ".yaml", ".json"
+                ).replace(".yml", ".json")
+                try:
+                    if not converted_path.exists() or os.path.getmtime(
+                        file
+                    ) > os.path.getmtime(converted_path):
+                        with open(file, "r") as yaml_file:
+                            with open(converted_path, "w") as json_file:
+                                json.dump(
+                                    yaml.safe_load(yaml_file), json_file, indent=2
+                                )
+                        qDebug(f"Converted {file} to JSON")
+                    found_jsons.add(converted_path)
+                except OSError as e:
+                    qWarning(f"Error accessing file {converted_path}: {e}")
+            elif file.name.endswith(".json"):
+                found_jsons.add(file)
+            else:
+                self.create_mapping(file, dest / dest_func(file))
+        for file in found_jsons:
+            self.create_mapping(file, dest / dest_func(file))
+
+    def create_mapping(self, file: Path, dest: Path):
+        self.current_mappings.append(
+            mobase.Mapping(
+                source=str(file),
+                destination=str(dest),
+                is_directory=file.is_dir(),
+                create_target=True,
+            )
+        )

games/baldursgate3/bg3_utils.py

@@ -0,0 +1,242 @@
+import functools
+import shutil
+import typing
+from pathlib import Path
+
+from PyQt6.QtCore import (
+    QCoreApplication,
+    QDir,
+    QEventLoop,
+    QRunnable,
+    Qt,
+    QThread,
+    QThreadPool,
+    qInfo,
+    qWarning,
+)
+from PyQt6.QtWidgets import QApplication, QMainWindow, QProgressDialog
+
+import mobase
+
+loose_file_folders = {
+    "Public",
+    "Mods",
+    "Generated",
+    "Localization",
+    "ScriptExtender",
+}
+
+
+class BG3Utils:
+    _mod_settings_xml_start = """<?xml version="1.0" encoding="UTF-8"?>
+    <save>
+        <version major="4" minor="8" revision="0" build="200"/>
+        <region id="ModuleSettings">
+            <node id="root">
+                <children>
+                    <node id="Mods">
+                        <children>
+                            <node id="ModuleShortDesc">
+                                <attribute id="Folder" type="LSString" value="GustavX"/>
+                                <attribute id="MD5" type="LSString" value=""/>
+                                <attribute id="Name" type="LSString" value="GustavX"/>
+                                <attribute id="PublishHandle" type="uint64" value="0"/>
+                                <attribute id="UUID" type="guid" value="cb555efe-2d9e-131f-8195-a89329d218ea"/>
+                                <attribute id="Version64" type="int64" value="36028797018963968"/>
+                            </node>"""
+    _mod_settings_xml_end = """
+                        </children>
+                    </node>
+                </children>
+            </node>
+        </region>
+    </save>"""
+
+    def __init__(self, name: str):
+        self.main_window = None
+        self._name = name
+        from . import lslib_retriever, pak_parser
+
+        self.lslib_retriever = lslib_retriever.LSLibRetriever(self)
+        self._pak_parser = pak_parser.BG3PakParser(self)
+
+    def init(self, organizer: mobase.IOrganizer):
+        self._organizer = organizer
+
+    @functools.cached_property
+    def autobuild_paks(self):
+        return bool(self.get_setting("autobuild_paks"))
+
+    @functools.cached_property
+    def extract_full_package(self):
+        return bool(self.get_setting("extract_full_package"))
+
+    @functools.cached_property
+    def remove_extracted_metadata(self):
+        return bool(self.get_setting("remove_extracted_metadata"))
+
+    @functools.cached_property
+    def force_load_dlls(self):
+        return bool(self.get_setting("force_load_dlls"))
+
+    @functools.cached_property
+    def log_diff(self):
+        return bool(self.get_setting("log_diff"))
+
+    @functools.cached_property
+    def convert_yamls_to_json(self):
+        return bool(self.get_setting("convert_yamls_to_json"))
+
+    @functools.cached_property
+    def log_dir(self):
+        return Path(self._organizer.basePath()) / "logs"
+
+    @functools.cached_property
+    def modsettings_backup(self):
+        return self.plugin_data_path / "temp" / "modsettings.lsx"
+
+    @functools.cached_property
+    def modsettings_path(self):
+        return Path(self._organizer.profilePath()) / "modsettings.lsx"
+
+    @functools.cached_property
+    def plugin_data_path(self) -> Path:
+        """Gets the path to the data folder for the current plugin."""
+        return Path(self._organizer.pluginDataPath(), self._name).absolute()
+
+    @functools.cached_property
+    def tools_dir(self):
+        return self.plugin_data_path / "tools"
+
+    @functools.cached_property
+    def overwrite_path(self):
+        return Path(self._organizer.overwritePath())
+
+    def active_mods(self) -> list[mobase.IModInterface]:
+        modlist = self._organizer.modList()
+        return [
+            modlist.getMod(mod_name)
+            for mod_name in filter(
+                lambda mod: modlist.state(mod) & mobase.ModState.ACTIVE,
+                modlist.allModsByProfilePriority(),
+            )
+        ]
+
+    def _set_setting(self, key: str, value: mobase.MoVariant):
+        self._organizer.setPluginSetting(self._name, key, value)
+
+    def get_setting(self, key: str) -> mobase.MoVariant:
+        return self._organizer.pluginSetting(self._name, key)
+
+    def tr(self, trstr: str) -> str:
+        return QCoreApplication.translate(self._name, trstr)
+
+    def create_progress_window(
+        self, title: str, max_progress: int, msg: str = "", cancelable: bool = True
+    ) -> QProgressDialog:
+        progress = QProgressDialog(
+            self.tr(msg if msg else title),
+            self.tr("Cancel") if cancelable else None,
+            0,
+            max_progress,
+            self.main_window,
+        )
+        progress.setWindowTitle(self.tr(f"BG3 Plugin: {title}"))
+        progress.setWindowModality(Qt.WindowModality.ApplicationModal)
+        progress.show()
+        return progress
+
+    def on_user_interface_initialized(self, window: QMainWindow) -> None:
+        self.main_window = window
+
+    def on_settings_changed(
+        self,
+        plugin_name: str,
+        setting: str,
+        old: mobase.MoVariant,
+        new: mobase.MoVariant,
+    ) -> None:
+        if self._name != plugin_name:
+            return
+        if setting in {
+            "extract_full_package",
+            "autobuild_paks",
+            "remove_extracted_metadata",
+            "force_load_dlls",
+            "log_diff",
+            "convert_yamls_to_json",
+        } and hasattr(self, setting):
+            delattr(self, setting)
+
+    def construct_modsettings_xml(
+        self,
+        exec_path: str = "",
+        working_dir: typing.Optional[QDir] = None,
+        args: str = "",
+        force_reparse_metadata: bool = False,
+    ) -> bool:
+        if (
+            "bin/bg3" not in exec_path
+            or not self.lslib_retriever.download_lslib_if_missing()
+        ):
+            return True
+        active_mods = self.active_mods()
+        progress = self.create_progress_window(
+            "Generating modsettings.xml", len(active_mods)
+        )
+        threadpool = QThreadPool.globalInstance()
+        if threadpool is None:
+            return False
+        metadata: dict[str, str] = {}
+
+        def retrieve_mod_metadata_in_new_thread(mod: mobase.IModInterface):
+            return lambda: metadata.update(
+                self._pak_parser.get_metadata_for_files_in_mod(
+                    mod, force_reparse_metadata
+                )
+            )
+
+        for mod in active_mods:
+            if progress.wasCanceled():
+                qWarning("processing canceled by user")
+                return False
+            threadpool.start(QRunnable.create(retrieve_mod_metadata_in_new_thread(mod)))
+        count = 0
+        num_active_mods = len(active_mods)
+        total_intervals_to_wait = (num_active_mods * 2) + 20
+        while len(metadata.keys()) < num_active_mods:
+            progress.setValue(len(metadata.keys()))
+            QApplication.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, 100)
+            count += 1
+            if count == total_intervals_to_wait or progress.wasCanceled():
+                remaining_mods = {mod.name() for mod in active_mods} - metadata.keys()
+                qWarning(f"processing did not finish in time for: {remaining_mods}")
+                progress.close()
+                break
+            QThread.msleep(100)
+        progress.setValue(num_active_mods)
+        QApplication.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, 100)
+        progress.close()
+        qInfo(f"writing mod load order to {self.modsettings_path}")
+        self.modsettings_path.parent.mkdir(parents=True, exist_ok=True)
+        self.modsettings_path.write_text(
+            (
+                self._mod_settings_xml_start
+                + "".join(
+                    metadata[mod.name()]
+                    for mod in active_mods
+                    if mod.name() in metadata
+                )
+                + self._mod_settings_xml_end
+            )
+        )
+        qInfo(
+            f"backing up generated file {self.modsettings_path} to {self.modsettings_backup}, "
+            f"check the backup after the executable runs for differences with the file used by the game if you encounter issues"
+        )
+        shutil.copy(self.modsettings_path, self.modsettings_backup)
+        return True
+
+    def on_mod_installed(self, mod: mobase.IModInterface) -> None:
+        if self.lslib_retriever.download_lslib_if_missing():
+            self._pak_parser.get_metadata_for_files_in_mod(mod, True)

games/baldursgate3/lslib_retriever.py

@@ -0,0 +1,157 @@
+import json
+import shutil
+import traceback
+import urllib.request
+import zipfile
+from functools import cached_property
+
+from PyQt6.QtCore import qDebug, qWarning
+from PyQt6.QtWidgets import QApplication, QMessageBox
+
+from . import bg3_utils
+
+
+class LSLibRetriever:
+    def __init__(self, utils: bg3_utils.BG3Utils):
+        self._utils = utils
+
+    @cached_property
+    def _needed_lslib_files(self):
+        return {
+            self._utils.tools_dir / x
+            for x in {
+                "CommandLineArgumentsParser.dll",
+                "Divine.dll",
+                "Divine.dll.config",
+                "Divine.exe",
+                "Divine.runtimeconfig.json",
+                "LSLib.dll",
+                "LSLibNative.dll",
+                "LZ4.dll",
+                "System.IO.Hashing.dll",
+                "ZstdSharp.dll",
+            }
+        }
+
+    def download_lslib_if_missing(self, force: bool = False) -> bool:
+        if not force and all(x.exists() for x in self._needed_lslib_files):
+            return True
+        try:
+            self._utils.tools_dir.mkdir(exist_ok=True, parents=True)
+            downloaded = False
+
+            def reporthook(block_num: int, block_size: int, total_size: int) -> None:
+                if total_size > 0:
+                    progress.setValue(
+                        min(int(block_num * block_size * 100 / total_size), 100)
+                    )
+                    QApplication.processEvents()
+
+            with urllib.request.urlopen(
+                "https://api.github.com/repos/Norbyte/lslib/releases/latest"
+            ) as response:
+                assets = json.loads(response.read().decode("utf-8"))["assets"][0]
+                zip_path = self._utils.tools_dir / assets["name"]
+                if not zip_path.exists():
+                    old_archives = list(self._utils.tools_dir.glob("*.zip"))
+                    msg_box = QMessageBox(self._utils.main_window)
+                    msg_box.setWindowTitle(
+                        self._utils.tr("Baldur's Gate 3 Plugin - Missing dependencies")
+                    )
+                    if old_archives:
+                        msg_box.setText(self._utils.tr("LSLib update available."))
+                    else:
+                        msg_box.setText(
+                            self._utils.tr(
+                                "LSLib tools are missing.\nThese are necessary for the plugin to create the load order file for BG3."
+                            )
+                        )
+                    msg_box.addButton(
+                        self._utils.tr("Download"),
+                        QMessageBox.ButtonRole.DestructiveRole,
+                    )
+                    exit_btn = msg_box.addButton(
+                        self._utils.tr("Exit"), QMessageBox.ButtonRole.ActionRole
+                    )
+                    msg_box.setIcon(QMessageBox.Icon.Warning)
+                    msg_box.exec()
+
+                    if msg_box.clickedButton() == exit_btn:
+                        if not old_archives:
+                            err = QMessageBox(self._utils.main_window)
+                            err.setIcon(QMessageBox.Icon.Critical)
+                            err.setText(
+                                "LSLib tools are required for the proper generation of the modsettings.xml file, file will not be generated"
+                            )
+                            return False
+                    else:
+                        progress = self._utils.create_progress_window(
+                            "Downloading LSLib", 100, cancelable=False
+                        )
+                        urllib.request.urlretrieve(
+                            assets["browser_download_url"], str(zip_path), reporthook
+                        )
+                        progress.close()
+                        downloaded = True
+                        for archive in old_archives:
+                            archive.unlink()
+                        old_archives = []
+                else:
+                    old_archives = []
+                    new_msg = QMessageBox(self._utils.main_window)
+                    new_msg.setIcon(QMessageBox.Icon.Information)
+                    new_msg.setText(
+                        self._utils.tr("Latest version of LSLib already downloaded!")
+                    )
+
+        except Exception as e:
+            qDebug(f"Download failed: {e}")
+            err = QMessageBox(self._utils.main_window)
+            err.setIcon(QMessageBox.Icon.Critical)
+            err.setText(
+                self._utils.tr(
+                    f"Failed to download LSLib tools:\n{traceback.format_exc()}"
+                )
+            )
+            err.exec()
+            return False
+        try:
+            if old_archives:
+                zip_path = sorted(old_archives)[-1]
+            if old_archives or not downloaded:
+                dialog_message = "Ensuring all necessary LSLib files have been extracted from archive..."
+                win_title = "Verifying LSLib files"
+            else:
+                dialog_message = "Extracting/Updating LSLib files..."
+                win_title = "Extracting LSLib"
+            x_progress = self._utils.create_progress_window(
+                win_title, len(self._needed_lslib_files), msg=dialog_message
+            )
+            with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                for file in self._needed_lslib_files:
+                    if downloaded or not file.exists():
+                        shutil.move(
+                            zip_ref.extract(
+                                f"Packed/Tools/{file.name}", self._utils.tools_dir
+                            ),
+                            file,
+                        )
+                    x_progress.setValue(x_progress.value() + 1)
+                    QApplication.processEvents()
+                    if x_progress.wasCanceled():
+                        qWarning("processing canceled by user")
+                        return False
+            x_progress.close()
+            shutil.rmtree(self._utils.tools_dir / "Packed", ignore_errors=True)
+        except Exception as e:
+            qDebug(f"Extraction failed: {e}")
+            err = QMessageBox(self._utils.main_window)
+            err.setIcon(QMessageBox.Icon.Critical)
+            err.setText(
+                self._utils.tr(
+                    f"Failed to extract LSLib tools:\n{traceback.format_exc()}"
+                )
+            )
+            err.exec()
+            return False
+        return True

games/baldursgate3/pak_parser.py

@@ -0,0 +1,290 @@
+import configparser
+import hashlib
+import itertools
+import os
+import re
+import shutil
+import subprocess
+import traceback
+from functools import cached_property
+from pathlib import Path
+from typing import Callable
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+from PyQt6.QtCore import (
+    qDebug,
+    qInfo,
+    qWarning,
+)
+
+import mobase
+
+from . import bg3_utils
+
+
+class BG3PakParser:
+    def __init__(self, utils: bg3_utils.BG3Utils):
+        self._utils = utils
+
+    _mod_cache: dict[Path, bool] = {}
+    _types = {
+        "Folder": "",
+        "MD5": "",
+        "Name": "",
+        "PublishHandle": "0",
+        "UUID": "",
+        "Version64": "0",
+    }
+
+    @cached_property
+    def _divine_command(self):
+        return f"{self._utils.tools_dir / 'Divine.exe'} -g bg3 -l info"
+
+    @cached_property
+    def _folder_pattern(self):
+        return re.compile("Data|Script Extender|bin|Mods")
+
+    def get_metadata_for_files_in_mod(
+        self, mod: mobase.IModInterface, force_reparse_metadata: bool
+    ):
+        return {
+            mod.name(): "".join(
+                [
+                    self._get_metadata_for_file(mod, file, force_reparse_metadata)
+                    for file in sorted(
+                        list(Path(mod.absolutePath()).rglob("*.pak"))
+                        + (
+                            [
+                                f
+                                for f in Path(mod.absolutePath()).glob("*")
+                                if f.is_dir()
+                            ]
+                            if self._utils.autobuild_paks
+                            else []
+                        )
+                    )
+                ]
+            )
+        }
+
+    def _get_metadata_for_file(
+        self,
+        mod: mobase.IModInterface,
+        file: Path,
+        force_reparse_metadata: bool,
+    ) -> str:
+        meta_ini = Path(mod.absolutePath()) / "meta.ini"
+        config = configparser.ConfigParser()
+        config.read(meta_ini, encoding="utf-8")
+        try:
+            if file.name.endswith("pak"):
+                meta_file = (
+                    self._utils.plugin_data_path
+                    / "temp"
+                    / "extracted_metadata"
+                    / f"{file.name[: int(len(file.name) / 2)]}-{hashlib.md5(str(file).encode(), usedforsecurity=False).hexdigest()[:5]}.lsx"
+                )
+                try:
+                    if (
+                        not force_reparse_metadata
+                        and config.has_section(file.name)
+                        and (
+                            "override" in config[file.name].keys()
+                            or "Folder" in config[file.name].keys()
+                        )
+                    ):
+                        return get_module_short_desc(config, file)
+                    meta_file.parent.mkdir(parents=True, exist_ok=True)
+                    meta_file.unlink(missing_ok=True)
+                    out_dir = (
+                        str(meta_file)[:-4] if self._utils.extract_full_package else ""
+                    )
+                    can_continue = True
+                    if self.run_divine(
+                        f'{"extract-package" if self._utils.extract_full_package else "extract-single-file -f meta.lsx"} -d "{meta_file if not self._utils.extract_full_package else out_dir}"',
+                        file,
+                    ).returncode:
+                        can_continue = False
+                    if can_continue and self._utils.extract_full_package:
+                        qDebug(f"archive {file} extracted to {out_dir}")
+                        if self.run_divine(
+                            f'convert-resources -d "{out_dir}" -i lsf -o lsx -x "*.lsf"',
+                            out_dir,
+                        ).returncode:
+                            qDebug(
+                                f"failed to convert lsf files in {out_dir} to readable lsx"
+                            )
+                        extracted_meta_files = list(Path(out_dir).rglob("meta.lsx"))
+                        if len(extracted_meta_files) == 0:
+                            qInfo(
+                                f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod"
+                            )
+                            can_continue = False
+                        else:
+                            shutil.copyfile(
+                                extracted_meta_files[0],
+                                meta_file,
+                            )
+                    elif can_continue and not meta_file.exists():
+                        qInfo(
+                            f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod"
+                        )
+                        can_continue = False
+                    return self.metadata_to_ini(
+                        config, file, mod, meta_ini, can_continue, lambda: meta_file
+                    )
+                finally:
+                    if self._utils.remove_extracted_metadata:
+                        meta_file.unlink(missing_ok=True)
+                        if self._utils.extract_full_package:
+                            Path(str(meta_file)[:-4]).unlink(missing_ok=True)
+            elif file.is_dir() and self._folder_pattern.search(file.name):
+                # qDebug(f"directory is not packable: {file}")
+                return ""
+            elif next(
+                itertools.chain(
+                    file.glob(f"{folder}/*") for folder in bg3_utils.loose_file_folders
+                ),
+                False,
+            ):
+                qInfo(f"packable dir: {file}")
+                if (file.parent / f"{file.name}.pak").exists() or (
+                    file.parent / "Mods" / f"{file.name}.pak"
+                ).exists():
+                    qInfo(
+                        f"pak with same name as packable dir exists in mod directory. not packing dir {file}"
+                    )
+                    return ""
+                pak_path = self._utils.overwrite_path / f"Mods/{file.name}.pak"
+                build_pak = True
+                if pak_path.exists():
+                    pak_creation_time = os.path.getmtime(pak_path)
+                    for root, _, files in os.walk(file):
+                        for f in files:
+                            file_path = os.path.join(root, f)
+                            try:
+                                if os.path.getmtime(file_path) > pak_creation_time:
+                                    break
+                            except OSError as e:
+                                qDebug(f"Error accessing file {file_path}: {e}")
+                                break
+                    else:
+                        build_pak = False
+                if build_pak:
+                    pak_path.unlink(missing_ok=True)
+                    if self.run_divine(
+                        f'create-package -d "{pak_path}"', file
+                    ).returncode:
+                        return ""
+                meta_files = list(file.glob("Mods/*/meta.lsx"))
+                return self.metadata_to_ini(
+                    config,
+                    file,
+                    mod,
+                    meta_ini,
+                    len(meta_files) > 0,
+                    lambda: meta_files[0],
+                )
+            else:
+                # qDebug(f"non packable dir, unlikely to be used by the game: {file}")
+                return ""
+        except Exception:
+            qWarning(traceback.format_exc())
+            return ""
+
+    def run_divine(
+        self, action: str, source: Path | str
+    ) -> subprocess.CompletedProcess[str]:
+        command = f'{self._divine_command} -a {action} -s "{source}"'
+        result = subprocess.run(
+            command,
+            creationflags=subprocess.CREATE_NO_WINDOW,
+            check=False,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        if result.returncode:
+            qWarning(
+                f"{command.replace(str(Path.home()), '~', 1).replace(str(Path.home()), '$HOME')}"
+                f" returned stdout: {result.stdout}, stderr: {result.stderr}, code {result.returncode}"
+            )
+        return result
+
+    def get_attr_value(self, root: Element, attr_id: str) -> str:
+        default_val = self._types.get(attr_id) or ""
+        attr = root.find(f".//attribute[@id='{attr_id}']")
+        return default_val if attr is None else attr.get("value", default_val)
+
+    def metadata_to_ini(
+        self,
+        config: configparser.ConfigParser,
+        file: Path,
+        mod: mobase.IModInterface,
+        meta_ini: Path,
+        condition: bool,
+        to_parse: Callable[[], Path],
+    ):
+        config[file.name] = {}
+        if condition:
+            root = (
+                ElementTree.parse(to_parse())
+                .getroot()
+                .find(".//node[@id='ModuleInfo']")
+            )
+            if root is None:
+                qInfo(f"No ModuleInfo node found in meta.lsx for {mod.name()} ")
+            else:
+                section = config[file.name]
+                folder_name = self.get_attr_value(root, "Folder")
+                if file.is_dir():
+                    self._mod_cache[file] = (
+                        len(list(file.glob(f"*/{folder_name}/**"))) > 1
+                        or len(
+                            list(file.glob("Public/Engine/Timeline/MaterialGroups/*"))
+                        )
+                        > 0
+                    )
+                elif file not in self._mod_cache:
+                    # a mod which has a meta.lsx and is not an override mod meets at least one of three conditions:
+                    # 1. it has files in Public/Engine/Timeline/MaterialGroups, or
+                    # 2. it has files in Mods/<folder_name>/ other than the meta.lsx file, or
+                    # 3. it has files in Public/<folder_name>
+                    result = self.run_divine(
+                        f'list-package --use-regex -x "(/{folder_name}/(?!meta\\.lsx))|(Public/Engine/Timeline/MaterialGroups)"',
+                        file,
+                    )
+                    self._mod_cache[file] = (
+                        result.returncode == 0 and result.stdout.strip() != ""
+                    )
+                if self._mod_cache[file]:
+                    for key in self._types:
+                        section[key] = self.get_attr_value(root, key)
+                else:
+                    qInfo(f"pak {file.name} determined to be an override mod")
+                    section["override"] = "True"
+                    section["Folder"] = folder_name
+        else:
+            config[file.name]["override"] = "True"
+        with open(meta_ini, "w+", encoding="utf-8") as f:
+            config.write(f)
+        return get_module_short_desc(config, file)
+
+
+def get_module_short_desc(config: configparser.ConfigParser, file: Path) -> str:
+    return (
+        ""
+        if not config.has_section(file.name)
+        or "override" in config[file.name].keys()
+        or "Name" not in config[file.name].keys()
+        else f"""
+                <node id="ModuleShortDesc">
+                    <attribute id="Folder" type="LSString" value="{config[file.name]["Folder"]}"/>
+                    <attribute id="MD5" type="LSString" value="{config[file.name]["MD5"]}"/>
+                    <attribute id="Name" type="LSString" value="{config[file.name]["Name"]}"/>
+                    <attribute id="PublishHandle" type="uint64" value="{config[file.name]["PublishHandle"]}"/>
+                    <attribute id="UUID" type="guid" value="{config[file.name]["UUID"]}"/>
+                    <attribute id="Version64" type="int64" value="{config[file.name]["Version64"]}"/>
+                </node>"""
+    )

games/baldursgate3/plugins/__init__.py

@@ -0,0 +1,13 @@
+import mobase
+
+from .check_for_lslib_updates_plugin import BG3ToolCheckForLsLibUpdates
+from .convert_jsons_to_yaml_plugin import BG3ToolConvertJsonsToYaml
+from .reparse_pak_metadata_plugin import BG3ToolReparsePakMetadata
+
+
+def createPlugins() -> list[mobase.IPluginTool]:
+    return [
+        BG3ToolCheckForLsLibUpdates(),
+        BG3ToolReparsePakMetadata(),
+        BG3ToolConvertJsonsToYaml(),
+    ]

games/baldursgate3/plugins/bg3_tool_plugin.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+from PyQt6.QtCore import QCoreApplication
+from PyQt6.QtGui import QIcon
+
+import mobase
+
+
+class BG3ToolPlugin(mobase.IPluginTool, mobase.IPlugin):
+    icon_file = desc = sub_name = ""
+
+    def __init__(self):
+        mobase.IPluginTool.__init__(self)
+        mobase.IPlugin.__init__(self)
+        self._pluginName = self._displayName = "BG3 Tools"
+        self._pluginVersion = mobase.VersionInfo(1, 0, 0)
+
+    def init(self, organizer: mobase.IOrganizer) -> bool:
+        self._organizer = organizer
+        return True
+
+    def version(self):
+        return self._pluginVersion
+
+    def author(self):
+        return "daescha"
+
+    def name(self):
+        return f"{self._pluginName}: {self.sub_name}"
+
+    def displayName(self):
+        return f"{self._displayName}/{self.sub_name}"
+
+    def tooltip(self):
+        return self.description()
+
+    def enabledByDefault(self):
+        return self._organizer.managedGame().name() == "Baldur's Gate 3 Plugin"
+
+    def settings(self) -> list[mobase.PluginSetting]:
+        return []
+
+    def icon(self) -> QIcon:
+        return QIcon(str(Path(__file__).parent / "icons" / self.icon_file))
+
+    def description(self) -> str:
+        return QCoreApplication.translate(self._pluginName, self.desc)

games/baldursgate3/plugins/check_for_lslib_updates_plugin.py

@@ -0,0 +1,14 @@
+from .bg3_tool_plugin import BG3ToolPlugin
+
+
+class BG3ToolCheckForLsLibUpdates(BG3ToolPlugin):
+    icon_file = "ui-update.ico"
+    sub_name = "Check For LsLib Updates"
+    desc = "Check to see if there has been a new release of LSLib and create download dialog if so."
+
+    def display(self):
+        from ...game_baldursgate3 import BG3Game
+
+        game_plugin = self._organizer.managedGame()
+        if isinstance(game_plugin, BG3Game):
+            game_plugin.utils.lslib_retriever.download_lslib_if_missing(True)

games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py

@@ -0,0 +1,57 @@
+import json
+import os
+from pathlib import Path
+
+from PyQt6.QtCore import qInfo, qWarning
+from PyQt6.QtWidgets import QApplication
+
+from .bg3_tool_plugin import BG3ToolPlugin
+
+
+class BG3ToolConvertJsonsToYaml(BG3ToolPlugin):
+    icon_file = "ui-next.ico"
+    sub_name = "Convert JSONS to YAML"
+    desc = "Convert all jsons in active mods to yaml immediately."
+
+    def display(self):
+        from ...game_baldursgate3 import BG3Game
+
+        game_plugin = self._organizer.managedGame()
+        if not isinstance(game_plugin, BG3Game):
+            return
+        utils = game_plugin.utils
+        qInfo("converting all json files to yaml")
+        active_mods = utils.active_mods()
+        progress = utils.create_progress_window(
+            "Converting all json files to yaml", len(active_mods) + 1
+        )
+        for mod in active_mods:
+            _convert_jsons_in_dir_to_yaml(Path(mod.absolutePath()))
+            progress.setValue(progress.value() + 1)
+            QApplication.processEvents()
+            if progress.wasCanceled():
+                qWarning("conversion canceled by user")
+                return
+        _convert_jsons_in_dir_to_yaml(utils.overwrite_path)
+        progress.setValue(len(active_mods) + 1)
+        QApplication.processEvents()
+        progress.close()
+
+
+def _convert_jsons_in_dir_to_yaml(path: Path):
+    import yaml
+
+    for file in list(path.rglob("*.json")):
+        converted_path = file.parent / file.name.replace(".json", ".yaml")
+        try:
+            if not converted_path.exists() or os.path.getmtime(file) > os.path.getmtime(
+                converted_path
+            ):
+                with open(file, "r") as json_file:
+                    with open(converted_path, "w") as yaml_file:
+                        yaml.dump(
+                            json.load(json_file), yaml_file, indent=2, sort_keys=False
+                        )
+                qInfo(f"Converted {file} to YAML")
+        except OSError as e:
+            qWarning(f"Error accessing file {converted_path}: {e}")

games/baldursgate3/plugins/reparse_pak_metadata_plugin.py

@@ -0,0 +1,16 @@
+from .bg3_tool_plugin import BG3ToolPlugin
+
+
+class BG3ToolReparsePakMetadata(BG3ToolPlugin):
+    icon_file = "ui-refresh.ico"
+    sub_name = "Reparse Pak Metadata"
+    desc = "Force reparsing mod metadata immediately."
+
+    def display(self):
+        from ...game_baldursgate3 import BG3Game
+
+        game_plugin = self._organizer.managedGame()
+        if isinstance(game_plugin, BG3Game):
+            game_plugin.utils.construct_modsettings_xml(
+                exec_path="bin/bg3", force_reparse_metadata=True
+            )

games/game_baldursgate3.py

@@ -0,0 +1,214 @@
+import datetime
+import difflib
+import os
+import shutil
+from functools import cached_property
+from pathlib import Path
+from typing import Any
+
+from PyQt6.QtCore import (
+    qDebug,
+    qInfo,
+)
+
+import mobase
+
+from ..basic_features import (
+    BasicGameSaveGameInfo,
+    BasicLocalSavegames,
+)
+from ..basic_game import BasicGame
+from .baldursgate3 import bg3_file_mapper
+
+
+class BG3Game(BasicGame, bg3_file_mapper.BG3FileMapper):
+    Name = "Baldur's Gate 3 Plugin"
+    Author = "daescha"
+    Version = "0.1.0"
+    GameName = "Baldur's Gate 3"
+    GameShortName = "baldursgate3"
+    GameNexusName = "baldursgate3"
+    GameValidShortNames = ["bg3"]
+    GameLauncher = "Launcher/LariLauncher.exe"
+    GameBinary = "bin/bg3.exe"
+    GameDataPath = ""
+    GameDocumentsDirectory = (
+        "%USERPROFILE%/AppData/Local/Larian Studios/Baldur's Gate 3"
+    )
+    GameSavesDirectory = "%GAME_DOCUMENTS%/PlayerProfiles/Public/Savegames/Story"
+    GameSaveExtension = "lsv"
+    GameNexusId = 3474
+    GameSteamId = 1086940
+    GameGogId = 1456460669
+
+    def __init__(self):
+        BasicGame.__init__(self)
+        from .baldursgate3 import bg3_utils
+
+        self.utils = bg3_utils.BG3Utils(self.name())
+        bg3_file_mapper.BG3FileMapper.__init__(
+            self, self.utils, self.documentsDirectory
+        )
+
+    def init(self, organizer: mobase.IOrganizer) -> bool:
+        super().init(organizer)
+        self.utils.init(organizer)
+        from .baldursgate3 import (
+            bg3_data_checker,
+            bg3_data_content,
+        )
+
+        self._register_feature(bg3_data_checker.BG3ModDataChecker())
+        self._register_feature(bg3_data_content.BG3DataContent())
+        self._register_feature(BasicGameSaveGameInfo(lambda s: s.with_suffix(".webp")))
+        self._register_feature(BasicLocalSavegames(self.savesDirectory()))
+        organizer.onAboutToRun(self.utils.construct_modsettings_xml)
+        organizer.onFinishedRun(self._on_finished_run)
+        organizer.onUserInterfaceInitialized(self.utils.on_user_interface_initialized)
+        organizer.modList().onModInstalled(self.utils.on_mod_installed)
+        organizer.onPluginSettingChanged(self.utils.on_settings_changed)
+        return True
+
+    def settings(self):
+        base_settings = super().settings()
+        custom_settings = [
+            mobase.PluginSetting(
+                "force_load_dlls",
+                "Force load all dlls detected in active mods. Removes the need for 'Native Mod Loader' and similar mods.",
+                True,
+            ),
+            mobase.PluginSetting(
+                "log_diff",
+                "Log a diff of the modsettings.xml file before and after the game runs to check for differences.",
+                False,
+            ),
+            mobase.PluginSetting(
+                "delete_levelcache_folders_older_than_x_days",
+                "Maximum number of days a file in overwrite/LevelCache is allowed to exist before being deleted "
+                "after the executable finishes. Set to negative to disable.",
+                3,
+            ),
+            mobase.PluginSetting(
+                "autobuild_paks",
+                "Autobuild folders likely to be PAK folders with every run of an executable.",
+                True,
+            ),
+            mobase.PluginSetting(
+                "remove_extracted_metadata",
+                "Remove extracted meta.lsx files when they are no longer needed.",
+                True,
+            ),
+            mobase.PluginSetting(
+                "extract_full_package",
+                "Extract the full pak when parsing metadata, instead of just meta.lsx.",
+                False,
+            ),
+            mobase.PluginSetting(
+                "convert_yamls_to_json",
+                "Convert YAMLs to JSONs when executable runs. Allows one to configure ScriptExtender and related mods with YAML files.",
+                False,
+            ),
+        ]
+        for setting in custom_settings:
+            setting.description = self.utils.tr(setting.description)
+            base_settings.append(setting)
+        return base_settings
+
+    def executables(self) -> list[mobase.ExecutableInfo]:
+        return [
+            mobase.ExecutableInfo(
+                f"{self.gameName()}: DX11",
+                self.gameDirectory().absoluteFilePath("bin/bg3_dx11.exe"),
+            ),
+            mobase.ExecutableInfo(
+                f"{self.gameName()}: Vulkan",
+                self.gameDirectory().absoluteFilePath(self.binaryName()),
+            ),
+            mobase.ExecutableInfo(
+                "Larian Launcher",
+                self.gameDirectory().absoluteFilePath(self.getLauncherName()),
+            ),
+        ]
+
+    def executableForcedLoads(self) -> list[mobase.ExecutableForcedLoadSetting]:
+        try:
+            efls = super().executableForcedLoads()
+        except AttributeError:
+            efls = []
+        if self.utils.force_load_dlls:
+            qInfo("detecting dlls in enabled mods")
+            libs: set[str] = set()
+            tree: mobase.IFileTree | mobase.FileTreeEntry | None = (
+                self._organizer.virtualFileTree().find("bin")
+            )
+            if type(tree) is not mobase.IFileTree:
+                return efls
+
+            def find_dlls(
+                _: Any, entry: mobase.FileTreeEntry
+            ) -> mobase.IFileTree.WalkReturn:
+                relpath = entry.pathFrom(tree)
+                if (
+                    relpath
+                    and entry.hasSuffix("dll")
+                    and relpath not in self._base_dlls
+                ):
+                    libs.add(relpath)
+                return mobase.IFileTree.WalkReturn.CONTINUE
+
+            tree.walk(find_dlls)
+            exes = self.executables()
+            qDebug(f"dlls to force load: {libs}")
+            efls = efls + [
+                mobase.ExecutableForcedLoadSetting(
+                    exe.binary().fileName(), lib
+                ).withEnabled(True)
+                for lib in libs
+                for exe in exes
+            ]
+        return efls
+
+    def loadOrderMechanism(self) -> mobase.LoadOrderMechanism:
+        return mobase.LoadOrderMechanism.PLUGINS_TXT
+
+    @cached_property
+    def _base_dlls(self) -> set[str]:
+        base_bin = Path(self.gameDirectory().absoluteFilePath("bin"))
+        return {str(f.relative_to(base_bin)) for f in base_bin.glob("*.dll")}
+
+    def _on_finished_run(self, exec_path: str, exit_code: int):
+        if "bin/bg3" not in exec_path:
+            return
+        if self.utils.log_diff:
+            for x in difflib.unified_diff(
+                open(self.utils.modsettings_backup).readlines(),
+                open(self.utils.modsettings_path).readlines(),
+                fromfile=str(self.utils.modsettings_backup),
+                tofile=str(self.utils.modsettings_path),
+                lineterm="",
+            ):
+                qDebug(x)
+        for path in self.utils.overwrite_path.rglob("*.log"):
+            try:
+                qDebug(f"moving {path} to {self.utils.log_dir}")
+                shutil.move(path, self.utils.log_dir / path.name)
+            except PermissionError as e:
+                qDebug(str(e))
+        days = self.utils.get_setting("delete_levelcache_folders_older_than_x_days")
+        if type(days) is int and days >= 0:
+            cutoff_time = datetime.datetime.now() - datetime.timedelta(days=days)
+            qDebug(f"cleaning folders in overwrite/LevelCache older than {cutoff_time}")
+            for path in self.utils.overwrite_path.glob("LevelCache/*"):
+                if (
+                    datetime.datetime.fromtimestamp(os.path.getmtime(path))
+                    < cutoff_time
+                ):
+                    shutil.rmtree(path, ignore_errors=True)
+        qDebug("cleaning empty dirs from overwrite directory")
+        for folder in sorted(
+            list(os.walk(self.utils.overwrite_path))[1:], reverse=True
+        ):
+            try:
+                os.rmdir(folder[0])
+            except OSError:
+                pass

plugin-requirements.txt

@@ -1,3 +1,4 @@
 psutil==5.8.0
 vdf==3.4
 lzokay==1.1.5
+pyyaml==6.0.2

poetry.lock

@@ -210,7 +210,7 @@ version = "6.0.2"
 description = "YAML parser and emitter for Python"
 optional = false
 python-versions = ">=3.8"
-groups = ["dev"]
+groups = ["main", "dev"]
 files = [
     {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
     {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -334,4 +334,4 @@ files = [
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.12,<4.0"
-content-hash = "10b882dbbaae72534c631994926861048e8183e18ebbb85c2c46376bdbc70446"
+content-hash = "b3a65da37845a56d4b259c4d73913b8957a61382499eeda8bdce2343a48e8c66"

pyproject.toml

@@ -18,6 +18,7 @@ psutil = "^5.8.0"
 vdf = "3.4"
 lzokay = "1.1.5"
 pyqt6 = "6.7.0"
+pyyaml = "^6.0.2"
 
 [tool.poetry.group.dev.dependencies]
 mobase-stubs = "^2.5.2"