From ecd882beccde510b3c2784df9f7045b52e8d0cd1 Mon Sep 17 00:00:00 2001 From: Zash Date: Wed, 30 Mar 2022 00:47:37 +0200 Subject: [PATCH 1/8] fix: Valheim sync overwrite also run for other games. --- games/game_valheim.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/games/game_valheim.py b/games/game_valheim.py index e118495..91dca7e 100644 --- a/games/game_valheim.py +++ b/games/game_valheim.py @@ -548,6 +548,8 @@ class ValheimGame(BasicGame): self._sync_overwrite() def _sync_overwrite(self) -> None: + if self._organizer.managedGame() is not self: + return if self._organizer.pluginSetting(self.name(), "sync_overwrite") is not False: self._overwrite_sync.search_file_contents = ( self._organizer.pluginSetting( From a127433da5082a258abbd4d504b777a290830cc5 Mon Sep 17 00:00:00 2001 From: Zash Date: Wed, 30 Mar 2022 21:04:23 +0200 Subject: [PATCH 2/8] + basic_feature: BasicModDataChecker - ModDataChecker with simple file patterns (glob). --- basic_features/__init__.py | 1 + basic_features/basic_mod_data_checker.py | 192 +++++++++++++++++++ games/game_valheim.py | 234 +++++------------------ 3 files changed, 246 insertions(+), 181 deletions(-) create mode 100644 basic_features/basic_mod_data_checker.py diff --git a/basic_features/__init__.py b/basic_features/__init__.py index cc2d236..918f9dd 100644 --- a/basic_features/__init__.py +++ b/basic_features/__init__.py @@ -1,3 +1,4 @@ # -*- encoding: utf-8 -*- from .basic_save_game_info import BasicGameSaveGameInfo # noqa +from .basic_mod_data_checker import BasicModDataChecker # noqa diff --git a/basic_features/basic_mod_data_checker.py b/basic_features/basic_mod_data_checker.py new file mode 100644 index 0000000..3738d4d --- /dev/null +++ b/basic_features/basic_mod_data_checker.py @@ -0,0 +1,192 @@ +# -*- encoding: utf-8 -*- +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field, fields +from typing import Optional, Protocol + + +import mobase + + +def convert_entry_to_tree(entry: mobase.FileTreeEntry) -> Optional[mobase.IFileTree]: + if not entry.isDir(): + return None + if isinstance(entry, mobase.IFileTree): + return entry + if (parent := entry.parent()) is None: + return None + converted_entry = parent.find( + entry.name(), mobase.FileTreeEntry.FileTypes.DIRECTORY + ) + if isinstance(converted_entry, mobase.IFileTree): + return converted_entry + return None + + +class HasGlobPatterns(Protocol): + set_as_root: Optional[Iterable[str]] + valid: Optional[Iterable[str]] + delete: Optional[Iterable[str]] + move: Optional[Iterable[str]] + + +@dataclass +class FileRegexPatterns: + """Regex patterns for validation in `BasicModDataChecker`.""" + + set_as_root: Optional[re.Pattern] = None + valid: Optional[re.Pattern] = None + delete: Optional[re.Pattern] = None + move: Optional[re.Pattern] = None + + @classmethod + def from_glob_patterns(cls, glob_patterns: HasGlobPatterns) -> FileRegexPatterns: + """Returns an instance of `FileRegexPatterns`, + with the glob pattern fields in `glob_patterns` translated to regex. + """ + return cls( + **{ + f.name: ( + cls.regex_from_glob_list(value) + if (value := getattr(glob_patterns, f.name)) + else None + ) + for f in fields(FileRegexPatterns) + } + ) + + @staticmethod + def regex_from_glob_list(glob_list: Iterable[str]) -> re.Pattern: + """Returns a regex pattern form a list of glob patterns. + + Every pattern has a capturing group, + so that `match.lastindex - 1` will give the `file_list` index. + """ + return re.compile( + f'(?:{"|".join(f"({fnmatch.translate(f)})" for f in glob_list)})', re.I + ) + + @staticmethod + def get_match_index(search_str: str, pattern: re.Pattern) -> Optional[int]: + """Get the index of the matched group if `search_str` matches `pattern`. + + Args: + search_str: the string to search + pattern: a pattern from `FileRegexPatterns`. + + Returns: + The 0-based index of the matched group or None for no match. + """ + if not (match := pattern.match(search_str)) or match.lastindex is None: + return None + return match.lastindex - 1 + + +@dataclass +class BasicModDataChecker(mobase.ModDataChecker): + """Game feature that is used to check and fix the content of a data tree + via simple file definitions. + + The file definitions support glob pattern (without subfolders) and are checked in + order. + """ + + set_as_root: Optional[Iterable[str]] = None + """If a folder from this set is found, it will be set as new root dir (unfolded) and + its contents are checked again. + """ + + valid: Optional[Iterable[str]] = None + """Files and folders in the right path. + Check result: `mobase.ModDataChecker.VALID`. + """ + + delete: Optional[Iterable[str]] = None + """Files/folders to delete. Check result: `mobase.ModDataChecker.FIXABLE`.""" + + move: Optional[dict[str, str]] = None + """Files/folders to move and their target. + + If the path ends with `/` or `\\`, the entry will be inserted + in the corresponding directory instead of replacing it. + + Check result: `mobase.ModDataChecker.FIXABLE`. + + Example:: + + {"*.ext": "path/to/target_folder/"} + + See Also: + `mobase.IFileTree.move` + """ + + _regex: FileRegexPatterns = field(init=False, repr=False) + """The regex patterns derived from the file (glob) patterns.""" + + _move_targets: Sequence[str] = field(init=False, repr=False) + + def __post_init__(self): + self.update_patterns() + + def update_patterns(self): + """Update patterns after init field changes.""" + self._regex = FileRegexPatterns.from_glob_patterns(self) + if self.move: + self._move_targets = list(self.move.values()) + + def dataLooksValid( + self, filetree: mobase.IFileTree + ) -> mobase.ModDataChecker.CheckReturn: + status = mobase.ModDataChecker.INVALID + for entry in filetree: + name = entry.name().casefold() + regex = self._regex + if regex.set_as_root and regex.set_as_root.match(name): + return mobase.ModDataChecker.FIXABLE + elif regex.valid and regex.valid.match(name): + if status is not mobase.ModDataChecker.FIXABLE: + status = mobase.ModDataChecker.VALID + elif (regex.move and regex.move.match(name)) or ( + regex.delete and regex.delete.match(name) + ): + status = mobase.ModDataChecker.FIXABLE + else: + return mobase.ModDataChecker.INVALID + return status + + def fix(self, filetree: mobase.IFileTree) -> Optional[mobase.IFileTree]: + for entry in list(filetree): + name = entry.name().casefold() + regex = self._regex + if regex.set_as_root and regex.set_as_root.match(name): + new_root = convert_entry_to_tree(entry) + return self.fix(new_root) if new_root else None + elif regex.valid and regex.valid.match(name): + continue + elif regex.delete and regex.delete.match(name): + entry.detach() + elif regex.move: + move_target = self._find_move_target(name) + if move_target is not None: + filetree.move(entry, move_target) + return filetree + + def _find_move_target(self, filename: str) -> Optional[str]: + """Find a matching file pattern (key) from `.move` and return the target (value). + + Args: + filename: the file name to match. + + Returns: + The move target or None for no match or no `.move` pattern. + """ + if ( + self._regex.move + and (i := self._regex.get_match_index(filename, self._regex.move)) + is not None + ): + return self._move_targets[i] + return None diff --git a/games/game_valheim.py b/games/game_valheim.py index 91dca7e..9179a3f 100644 --- a/games/game_valheim.py +++ b/games/game_valheim.py @@ -2,12 +2,11 @@ from __future__ import annotations -import fnmatch import itertools import re import shutil from collections.abc import Collection, Container, Iterable, Mapping, Sequence -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field from pathlib import Path from typing import Optional, TextIO, Union @@ -15,25 +14,11 @@ from PyQt5.QtCore import QDir import mobase +from ..basic_features import BasicModDataChecker from ..basic_features.basic_save_game_info import BasicGameSaveGame from ..basic_game import BasicGame -def convert_entry_to_tree(entry: mobase.FileTreeEntry) -> Optional[mobase.IFileTree]: - if not entry.isDir(): - return None - if isinstance(entry, mobase.IFileTree): - return entry - if (parent := entry.parent()) is None: - return None - converted_entry = parent.find( - entry.name(), mobase.FileTreeEntry.FileTypes.DIRECTORY - ) - if isinstance(converted_entry, mobase.IFileTree): - return converted_entry - return None - - def move_file(source: Path, target: Path): """Move `source` to `target`. Creates missing (parent) directories and overwrites existing `target`.""" @@ -130,7 +115,8 @@ class DebugTable: if self._table: for line in self._table: print("|", " | ".join(line.values()), "|", file=output_file) - output_file and output_file.flush() + if output_file: + output_file.flush() self._table = [] @@ -277,165 +263,6 @@ class OverwriteSync: ) -@dataclass -class RegexFilesDefinition: - """Regex pattern for the file lists in `FilesDefinition` - for globbing support. - Fields should match `RegexFilesDefinition`. - """ - - set_as_root: Optional[re.Pattern] - valid: Optional[re.Pattern] - delete: Optional[re.Pattern] - move: Optional[re.Pattern] - - @classmethod - def from_filesmap(cls, filesdef: FilesDefinition) -> RegexFilesDefinition: - """Returns an instance of `RegexFilesDefinition`, - with the file list fields from `FilesDefinition` as regex patterns. - """ - return cls( - **{ - f.name: ( - cls.file_list_regex(value) - if (value := getattr(filesdef, f.name)) - else None - ) - for f in fields(cls) - } - ) - - @staticmethod - def file_list_regex(file_list: Iterable[str]) -> re.Pattern: - """Returns a regex pattern for a file list with glob patterns. - - Every pattern has a capturing group, - so that match.lastindex - 1 will give the file_list index. - """ - return re.compile( - f'(?:{"|".join(f"({fnmatch.translate(f)})" for f in file_list)})', re.I - ) - - -@dataclass -class FilesDefinition: - """File (pattern) definitions for the `mobase.ModDataChecker`. - Fields should match `RegexFilesDefinition`. - """ - - set_as_root: Optional[set[str]] - """If a folder from this set is found, it will be set as new root dir (unfolded).""" - valid: Optional[set[str]] - """Files and folders in the right path.""" - delete: Optional[set[str]] - """Files/folders to delete.""" - move: Optional[dict[str, str]] - """Files/folders to move, like `{"*.ext": "path/to/folder/"}`. - If the path ends with / or \\, the entry will be inserted - in the corresponding directory instead of replacing it. - """ - - regex: RegexFilesDefinition = field(init=False) - _move_targets: Sequence[str] = field(init=False, repr=False) - - def __post_init__(self): - self.regex = RegexFilesDefinition.from_filesmap(self) - if self.move: - self._move_targets = list(self.move.values()) - - def get_move_target(self, index: int) -> str: - return self._move_targets[index] - - -class ValheimGameModDataChecker(mobase.ModDataChecker): - files_map = FilesDefinition( - set_as_root={ - "BepInExPack_Valheim", - }, - valid={ - "meta.ini", # Included in installed mod folder. - "BepInEx", - "doorstop_libs", - "unstripped_corlib", - "doorstop_config.ini", - "start_game_bepinex.sh", - "start_server_bepinex.sh", - "winhttp.dll", - # - "InSlimVML", - "valheim_Data", - "inslimvml.ini", - # - "unstripped_managed", - # - "AdvancedBuilder", - }, - delete={ - "*.txt", - "*.md", - "icon.png", - "license", - "manifest.json", - }, - move={ - "*_VML.dll": "InSlimVML/Mods/", - # - "plugins": "BepInEx/", - "*.dll": "BepInEx/plugins/", - "config": "BepInEx/", - "*.cfg": "BepInEx/config/", - # - "CustomTextures": "BepInEx/plugins/", - "*.png": "BepInEx/plugins/CustomTextures/", - # - "Builds": "AdvancedBuilder/", - "*.vbuild": "AdvancedBuilder/Builds/", - # - "*.assets": "valheim_Data/", - }, - ) - - def dataLooksValid( - self, filetree: mobase.IFileTree - ) -> mobase.ModDataChecker.CheckReturn: - status = mobase.ModDataChecker.INVALID - for entry in filetree: - name = entry.name().casefold() - regex = self.files_map.regex - if regex.set_as_root and regex.set_as_root.match(name): - return mobase.ModDataChecker.FIXABLE - elif regex.valid and regex.valid.match(name): - if status is not mobase.ModDataChecker.FIXABLE: - status = mobase.ModDataChecker.VALID - elif (regex.move and regex.move.match(name)) or ( - regex.delete and regex.delete.match(name) - ): - status = mobase.ModDataChecker.FIXABLE - else: - return mobase.ModDataChecker.INVALID - return status - - def fix(self, filetree: mobase.IFileTree) -> Optional[mobase.IFileTree]: - for entry in list(filetree): - name = entry.name().casefold() - regex = self.files_map.regex - if regex.set_as_root and regex.set_as_root.match(name): - new_root = convert_entry_to_tree(entry) - return self.fix(new_root) if new_root else None - elif regex.valid and regex.valid.match(name): - continue - elif regex.delete and regex.delete.match(name): - entry.detach() - elif regex.move and (match := regex.move.match(name)): - if match.lastindex is None: - return None - else: - # Get index of matched group - map_index = match.lastindex - 1 - # Get the move target corresponding to the matched group - filetree.move(entry, self.files_map.get_move_target(map_index)) - return filetree - - class ValheimSaveGame(BasicGameSaveGame): def getName(self) -> str: return f"[{self.getSaveGroupIdentifier().rstrip('s')}] {self._filepath.stem}" @@ -485,7 +312,7 @@ class ValheimGame(BasicGame): Name = "Valheim Support Plugin" Author = "Zash" - Version = "1.1.1" + Version = "1.2" GameName = "Valheim" GameShortName = "valheim" @@ -495,11 +322,56 @@ class ValheimGame(BasicGame): GameDataPath = "" GameSavesDirectory = r"%USERPROFILE%/AppData/LocalLow/IronGate/Valheim" - forced_libraries = ["winhttp.dll"] + _forced_libraries = ["winhttp.dll"] def init(self, organizer: mobase.IOrganizer) -> bool: super().init(organizer) - self._featureMap[mobase.ModDataChecker] = ValheimGameModDataChecker() + self._featureMap[mobase.ModDataChecker] = BasicModDataChecker( + set_as_root=[ + "BepInExPack_Valheim", + ], + valid=[ + "meta.ini", # Included in installed mod folder. + "BepInEx", + "doorstop_libs", + "unstripped_corlib", + "doorstop_config.ini", + "start_game_bepinex.sh", + "start_server_bepinex.sh", + "winhttp.dll", + # + "InSlimVML", + "valheim_Data", + "inslimvml.ini", + # + "unstripped_managed", + # + "AdvancedBuilder", + ], + delete=[ + "*.txt", + "*.md", + "icon.png", + "license", + "manifest.json", + ], + move={ + "*_VML.dll": "InSlimVML/Mods/", + # + "plugins": "BepInEx/", + "*.dll": "BepInEx/plugins/", + "config": "BepInEx/", + "*.cfg": "BepInEx/config/", + # + "CustomTextures": "BepInEx/plugins/", + "*.png": "BepInEx/plugins/CustomTextures/", + # + "Builds": "AdvancedBuilder/", + "*.vbuild": "AdvancedBuilder/Builds/", + # + "*.assets": "valheim_Data/", + }, + ) self._featureMap[mobase.LocalSavegames] = ValheimLocalSavegames( self.savesDirectory() ) @@ -510,7 +382,7 @@ class ValheimGame(BasicGame): def executableForcedLoads(self) -> list[mobase.ExecutableForcedLoadSetting]: return [ mobase.ExecutableForcedLoadSetting(self.binaryName(), lib).withEnabled(True) - for lib in self.forced_libraries + for lib in self._forced_libraries ] def listSaves(self, folder: QDir) -> list[mobase.ISaveGame]: From 0567e4ff316d6c30e69c3223c5d6760b41c1e4cc Mon Sep 17 00:00:00 2001 From: Zash Date: Sat, 2 Apr 2022 00:07:28 +0200 Subject: [PATCH 3/8] BasicModDataChecker: check / fix keys in definition order. --- basic_features/basic_mod_data_checker.py | 280 ++++++++++++++--------- games/game_valheim.py | 2 +- 2 files changed, 172 insertions(+), 110 deletions(-) diff --git a/basic_features/basic_mod_data_checker.py b/basic_features/basic_mod_data_checker.py index 3738d4d..ab9a41f 100644 --- a/basic_features/basic_mod_data_checker.py +++ b/basic_features/basic_mod_data_checker.py @@ -3,10 +3,9 @@ from __future__ import annotations import fnmatch import re +import sys from collections.abc import Iterable, Sequence -from dataclasses import dataclass, field, fields -from typing import Optional, Protocol - +from typing import MutableMapping, Optional, TypedDict, overload import mobase @@ -26,56 +25,39 @@ def convert_entry_to_tree(entry: mobase.FileTreeEntry) -> Optional[mobase.IFileT return None -class HasGlobPatterns(Protocol): - set_as_root: Optional[Iterable[str]] - valid: Optional[Iterable[str]] - delete: Optional[Iterable[str]] - move: Optional[Iterable[str]] - - -@dataclass -class FileRegexPatterns: +class RegexPatternDict(dict, MutableMapping[str, re.Pattern]): """Regex patterns for validation in `BasicModDataChecker`.""" - set_as_root: Optional[re.Pattern] = None - valid: Optional[re.Pattern] = None - delete: Optional[re.Pattern] = None - move: Optional[re.Pattern] = None - @classmethod - def from_glob_patterns(cls, glob_patterns: HasGlobPatterns) -> FileRegexPatterns: - """Returns an instance of `FileRegexPatterns`, - with the glob pattern fields in `glob_patterns` translated to regex. + def from_glob_patterns(cls, glob_patterns: GlobPatternDict) -> RegexPatternDict: + """Returns an instance of `RegexPatternDict`, with the `glob_patterns` + translated to regex. """ return cls( - **{ - f.name: ( - cls.regex_from_glob_list(value) - if (value := getattr(glob_patterns, f.name)) - else None - ) - for f in fields(FileRegexPatterns) - } + ( + key, + cls.regex_from_glob_list(value) + if isinstance(value, Iterable) + else None, + ) + for key, value in glob_patterns.items() ) - @staticmethod - def regex_from_glob_list(glob_list: Iterable[str]) -> re.Pattern: - """Returns a regex pattern form a list of glob patterns. + def get_match_index(self, key: str, search_str: str) -> Optional[int]: + """Get the index of the matched group if the `self[key]` matches `search_str`. - Every pattern has a capturing group, - so that `match.lastindex - 1` will give the `file_list` index. + Returns: + The 0-based index of the matched group or None for no match. """ - return re.compile( - f'(?:{"|".join(f"({fnmatch.translate(f)})" for f in glob_list)})', re.I - ) + if (pattern := self.get(key)) is None: + return None + else: + return self.match_index_of_pattern(search_str, pattern) @staticmethod - def get_match_index(search_str: str, pattern: re.Pattern) -> Optional[int]: - """Get the index of the matched group if `search_str` matches `pattern`. - - Args: - search_str: the string to search - pattern: a pattern from `FileRegexPatterns`. + def match_index_of_pattern(search_str: str, pattern: re.Pattern) -> Optional[int]: + """Get the index of the matched group if `search_str` matches `pattern` + (from this dict). Returns: The 0-based index of the matched group or None for no match. @@ -84,58 +66,141 @@ class FileRegexPatterns: return None return match.lastindex - 1 + @staticmethod + def regex_from_glob_list(glob_list: Iterable[str]) -> re.Pattern: + """Returns a regex pattern form a list of glob patterns. + + Every pattern has a capturing group, so that `match.lastindex - 1` will + give the `glob_list` index. + """ + return re.compile( + f'(?:{"|".join(f"({fnmatch.translate(f)})" for f in glob_list)})', re.I + ) + + +class GlobPatternDict(TypedDict, total=False): + """See: `BasicModDataChecker`""" + + unfold: Iterable[str] | None + valid: Iterable[str] | None + delete: Iterable[str] | None + move: MutableMapping[str, str] | None + -@dataclass class BasicModDataChecker(mobase.ModDataChecker): """Game feature that is used to check and fix the content of a data tree via simple file definitions. The file definitions support glob pattern (without subfolders) and are checked in - order. - """ + definition order (passed either as dict or kwargs). - set_as_root: Optional[Iterable[str]] = None - """If a folder from this set is found, it will be set as new root dir (unfolded) and - its contents are checked again. - """ + Args: + file_patterns (optional): An dict with the keys below or as kwargs: - valid: Optional[Iterable[str]] = None - """Files and folders in the right path. - Check result: `mobase.ModDataChecker.VALID`. - """ + unfold (optional): Folders to unfold (remove and move contents to parent), + after being checked and fixed recursively. - delete: Optional[Iterable[str]] = None - """Files/folders to delete. Check result: `mobase.ModDataChecker.FIXABLE`.""" + valid (optional): Files and folders in the right path. + Check result: `mobase.ModDataChecker.VALID`. - move: Optional[dict[str, str]] = None - """Files/folders to move and their target. + delete (optional): Files/folders to delete. + Check result: `mobase.ModDataChecker.FIXABLE`. - If the path ends with `/` or `\\`, the entry will be inserted - in the corresponding directory instead of replacing it. + move (optional): Files/folders to move and their target. + If the path ends with `/` or `\\`, the entry will be inserted + in the corresponding directory instead of replacing it. - Check result: `mobase.ModDataChecker.FIXABLE`. + Check result: `mobase.ModDataChecker.FIXABLE`. - Example:: + Example:: - {"*.ext": "path/to/target_folder/"} + BasicModDataChecker( + valid=["valid_folder", "*.ext1"] + move={"*.ext2": "path/to/target_folder/"} + ) + + BasicModDataChecker( + { + "valid": ["valid_folder", "*.ext1"], + "move": {"*.ext2": "path/to/target_folder/"} + } + ) See Also: `mobase.IFileTree.move` """ - _regex: FileRegexPatterns = field(init=False, repr=False) + file_patterns = None # type: GlobPatternDict + """Use `update_patterns` for modifications.""" + + _regex: RegexPatternDict """The regex patterns derived from the file (glob) patterns.""" - _move_targets: Sequence[str] = field(init=False, repr=False) + _move_targets: Sequence[str] - def __post_init__(self): - self.update_patterns() + # Overloads as workaround for **kwargs: Unpack[GlobPatternDict] + @overload + def __init__( + self, + *, + unfold: Iterable[str] | None = None, + valid: Iterable[str] | None = None, + delete: Iterable[str] | None = None, + move: dict[str, str] | None = None, + ): + ... - def update_patterns(self): - """Update patterns after init field changes.""" - self._regex = FileRegexPatterns.from_glob_patterns(self) - if self.move: - self._move_targets = list(self.move.values()) + @overload + def __init__( + self, + file_patterns: Optional[GlobPatternDict] = None, + *, + unfold: Iterable[str] | None = None, + valid: Iterable[str] | None = None, + delete: Iterable[str] | None = None, + move: dict[str, str] | None = None, + ): + ... + + def __init__( + self, file_patterns: Optional[GlobPatternDict] = None, **kwargs + ): # Unpack[GlobPatternDict] + super().__init__() + # Init with copy from class var + self.file_patterns = fp.copy() if (fp := self.file_patterns) else {} + self.update_patterns(file_patterns, **kwargs) + + @overload + def update_patterns( + self, + *, + unfold: Iterable[str] | None = None, + valid: Iterable[str] | None = None, + delete: Iterable[str] | None = None, + move: dict[str, str] | None = None, + ): + ... + + @overload + def update_patterns( + self, + file_patterns: Optional[GlobPatternDict] = None, + *, + unfold: Iterable[str] | None = None, + valid: Iterable[str] | None = None, + delete: Iterable[str] | None = None, + move: dict[str, str] | None = None, + ): + ... + + def update_patterns(self, file_patterns=None, **kwargs): # Unpack[GlobPatternDict] + """Update file patterns.""" + if file_patterns: + self.file_patterns.update(file_patterns) + self.file_patterns.update(kwargs) + self._regex = RegexPatternDict.from_glob_patterns(self.file_patterns) + if move_map := self.file_patterns.get("move"): + self._move_targets = list(move_map.values()) def dataLooksValid( self, filetree: mobase.IFileTree @@ -143,16 +208,17 @@ class BasicModDataChecker(mobase.ModDataChecker): status = mobase.ModDataChecker.INVALID for entry in filetree: name = entry.name().casefold() - regex = self._regex - if regex.set_as_root and regex.set_as_root.match(name): - return mobase.ModDataChecker.FIXABLE - elif regex.valid and regex.valid.match(name): - if status is not mobase.ModDataChecker.FIXABLE: - status = mobase.ModDataChecker.VALID - elif (regex.move and regex.move.match(name)) or ( - regex.delete and regex.delete.match(name) - ): - status = mobase.ModDataChecker.FIXABLE + for key, regex in self._regex.items(): + if not regex or not regex.match(name): + continue + if key == "unfold": + return mobase.ModDataChecker.FIXABLE + elif key == "valid": + if status is not mobase.ModDataChecker.FIXABLE: + status = mobase.ModDataChecker.VALID + elif key in ("move", "delete"): + status = mobase.ModDataChecker.FIXABLE + break else: return mobase.ModDataChecker.INVALID return status @@ -160,33 +226,29 @@ class BasicModDataChecker(mobase.ModDataChecker): def fix(self, filetree: mobase.IFileTree) -> Optional[mobase.IFileTree]: for entry in list(filetree): name = entry.name().casefold() - regex = self._regex - if regex.set_as_root and regex.set_as_root.match(name): - new_root = convert_entry_to_tree(entry) - return self.fix(new_root) if new_root else None - elif regex.valid and regex.valid.match(name): - continue - elif regex.delete and regex.delete.match(name): - entry.detach() - elif regex.move: - move_target = self._find_move_target(name) - if move_target is not None: - filetree.move(entry, move_target) + # Fix entries in pattern definition order. + for key, regex in self._regex.items(): + if not regex or not regex.match(name) or key == "valid": + continue + if key == "unfold": + if (folder_tree := convert_entry_to_tree(entry)) is not None: + if folder_tree: # Not empty + # Recursively fix subtree and unfold. + if not (fixed_folder_tree := self.fix(folder_tree)): + return None + filetree.merge(fixed_folder_tree) + folder_tree.detach() + else: + print(f"Cannot unfold {name}!", file=sys.stderr) + elif key == "delete": + entry.detach() + elif key == "move": + if (move_target := self._get_move_target(name)) is not None: + filetree.move(entry, move_target) + break return filetree - def _find_move_target(self, filename: str) -> Optional[str]: - """Find a matching file pattern (key) from `.move` and return the target (value). - - Args: - filename: the file name to match. - - Returns: - The move target or None for no match or no `.move` pattern. - """ - if ( - self._regex.move - and (i := self._regex.get_match_index(filename, self._regex.move)) - is not None - ): - return self._move_targets[i] - return None + def _get_move_target(self, filename: str) -> Optional[str]: + if (i := self._regex.get_match_index("move", filename)) is None: + return None + return self._move_targets[i] diff --git a/games/game_valheim.py b/games/game_valheim.py index 9179a3f..ed8c96b 100644 --- a/games/game_valheim.py +++ b/games/game_valheim.py @@ -327,7 +327,7 @@ class ValheimGame(BasicGame): def init(self, organizer: mobase.IOrganizer) -> bool: super().init(organizer) self._featureMap[mobase.ModDataChecker] = BasicModDataChecker( - set_as_root=[ + unfold=[ "BepInExPack_Valheim", ], valid=[ From 87157e72836dc94230f4cc28aa31481e761dcd7d Mon Sep 17 00:00:00 2001 From: Zash Date: Sun, 3 Apr 2022 14:06:54 +0200 Subject: [PATCH 4/8] README update: BasicModDataChecker --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 961ed8d..435af87 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ You can rename `modorganizer-basic_games-xxx` to whatever you want (e.g., `basic | S.T.A.L.K.E.R. Anomaly — [MOD](https://www.stalker-anomaly.com/) | [Qudix](https://github.com/Qudix) | [game_stalkeranomaly.py](games/game_stalkeranomaly.py) |
  • mod data checker
| | Stardew Valley — [GOG](https://www.gog.com/game/stardew_valley) / [STEAM](https://store.steampowered.com/app/413150/Stardew_Valley/) | [Syer10](https://github.com/Syer10), [Holt59](https://github.com/holt59/) | [game_stardewvalley.py](games/game_stardewvalley.py) |
  • mod data checker
| | STAR WARS™ Empire at War: Gold Pack - [GOG](https://www.gog.com/game/star_wars_empire_at_war_gold_pack) / [STEAM](https://store.steampowered.com/app/32470/) | [erri120](https://github.com/erri120) |
  • Empire at War: [game_starwars-empire-at-war.py](games/game_starwars-empire-at-war.py)
  • Force of Corruption: [game_starwars-empire-at-war-foc.py](games/game_starwars-empire-at-war-foc.py)
| | -| Valheim — [STEAM](https://store.steampowered.com/app/892970/Valheim/) | [Zash](https://github.com/ZashIn) | [game_valheim.py](games/game_valheim.py) |
  • mod data checker
  • overwrite config sync
  • save game support (no preview)
+| Valheim — [STEAM](https://store.steampowered.com/app/892970/Valheim/) | [Zash](https://github.com/ZashIn) | [game_valheim.py](games/game_valheim.py) |
  • mod data checker
  • overwrite config sync
  • save game support (no preview)
| | The Witcher: Enhanced Edition - [GOG](https://www.gog.com/game/the_witcher) / [STEAM](https://store.steampowered.com/app/20900/The_Witcher_Enhanced_Edition_Directors_Cut/) | [erri120](https://github.com/erri120) | [game_witcher1.py](games/game_witcher1.py) |
  • save game parsing (no preview)
| | The Witcher 3: Wild Hunt — [GOG](https://www.gog.com/game/the_witcher_3_wild_hunt) / [STEAM](https://store.steampowered.com/app/292030/The_Witcher_3_Wild_Hunt/) | [Holt59](https://github.com/holt59/) | [game_witcher3.py](games/game_witcher3.py) |
  • save game preview
| | Yu-Gi-Oh! Master Duel — [STEAM](https://store.steampowered.com/app/1449850/) | [The Conceptionist](https://github.com/the-conceptionist) & [uwx](https://github.com/uwx) | [game_masterduel.py](games/game_masterduel.py) | | @@ -172,6 +172,10 @@ The meta-plugin provides some useful extra feature: 2. **Basic save game preview:** If you use the Python version, and if you can easily obtain a picture (file) for any saves, you can provide basic save-game preview by using the `BasicGameSaveGameInfo`. See [games/game_witcher3.py](games/game_witcher3.py) for more details. +3. **Basic mod data checker** (Python): + Check and fix different mod archive layouts for an automatic installation with the proper + file structure, using simple (glob) patterns via `BasicModDataChecker`. + See [games/game_valheim.py](games/game_valheim.py) for an example. Game IDs can be found here: From 3e9217b3c3b1bd69ee25418c451bf060ae66c70c Mon Sep 17 00:00:00 2001 From: Zash Date: Tue, 5 Apr 2022 23:31:33 +0200 Subject: [PATCH 5/8] fix: valid files not accepted. --- basic_features/basic_mod_data_checker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/basic_features/basic_mod_data_checker.py b/basic_features/basic_mod_data_checker.py index ab9a41f..2b96aa4 100644 --- a/basic_features/basic_mod_data_checker.py +++ b/basic_features/basic_mod_data_checker.py @@ -224,13 +224,17 @@ class BasicModDataChecker(mobase.ModDataChecker): return status def fix(self, filetree: mobase.IFileTree) -> Optional[mobase.IFileTree]: + print(f"{self.file_patterns}") + print(self._regex) for entry in list(filetree): - name = entry.name().casefold() + name = entry.name() # Fix entries in pattern definition order. for key, regex in self._regex.items(): - if not regex or not regex.match(name) or key == "valid": + if not regex or not regex.match(name): continue - if key == "unfold": + if key == "valid": + break + elif key == "unfold": if (folder_tree := convert_entry_to_tree(entry)) is not None: if folder_tree: # Not empty # Recursively fix subtree and unfold. From f3977a9235067d9709321d7c200cce9b0925b345 Mon Sep 17 00:00:00 2001 From: Zash Date: Thu, 7 Apr 2022 16:01:53 +0200 Subject: [PATCH 6/8] Refactor: clear distinction between setting and updating file patterns, removed kwargs + overloads. --- basic_features/basic_mod_data_checker.py | 135 ++++++++--------------- games/game_valheim.py | 90 +++++++-------- 2 files changed, 91 insertions(+), 134 deletions(-) diff --git a/basic_features/basic_mod_data_checker.py b/basic_features/basic_mod_data_checker.py index 2b96aa4..5fcd83e 100644 --- a/basic_features/basic_mod_data_checker.py +++ b/basic_features/basic_mod_data_checker.py @@ -5,7 +5,7 @@ import fnmatch import re import sys from collections.abc import Iterable, Sequence -from typing import MutableMapping, Optional, TypedDict, overload +from typing import ClassVar, MutableMapping, Optional, TypedDict import mobase @@ -49,10 +49,9 @@ class RegexPatternDict(dict, MutableMapping[str, re.Pattern]): Returns: The 0-based index of the matched group or None for no match. """ - if (pattern := self.get(key)) is None: - return None - else: + if pattern := self.get(key): return self.match_index_of_pattern(search_str, pattern) + return None @staticmethod def match_index_of_pattern(search_str: str, pattern: re.Pattern) -> Optional[int]: @@ -62,9 +61,9 @@ class RegexPatternDict(dict, MutableMapping[str, re.Pattern]): Returns: The 0-based index of the matched group or None for no match. """ - if not (match := pattern.match(search_str)) or match.lastindex is None: - return None - return match.lastindex - 1 + if (match := pattern.match(search_str)) and match.lastindex: + return match.lastindex - 1 + return None @staticmethod def regex_from_glob_list(glob_list: Iterable[str]) -> re.Pattern: @@ -91,115 +90,73 @@ class BasicModDataChecker(mobase.ModDataChecker): """Game feature that is used to check and fix the content of a data tree via simple file definitions. - The file definitions support glob pattern (without subfolders) and are checked in - definition order (passed either as dict or kwargs). + The file definitions support glob pattern (without subfolders) and are + checked and fixed in definition order of the `file_patterns` dict. Args: - file_patterns (optional): An dict with the keys below or as kwargs: + file_patterns (optional): A dict (GlobPatternDict) with the following keys:: - unfold (optional): Folders to unfold (remove and move contents to parent), - after being checked and fixed recursively. + { + "unfold": [ "list of folders to unfold" ], + # (remove and move contents to parent), after being checked and + # fixed recursively. + # Check result: `mobase.ModDataChecker.VALID`. - valid (optional): Files and folders in the right path. - Check result: `mobase.ModDataChecker.VALID`. + "valid": [ "list of files and folders in the right path." ], + # Check result: `mobase.ModDataChecker.VALID`. - delete (optional): Files/folders to delete. - Check result: `mobase.ModDataChecker.FIXABLE`. + "delete": [ "list of files/folders to delete." ], + # Check result: `mobase.ModDataChecker.FIXABLE`. - move (optional): Files/folders to move and their target. - If the path ends with `/` or `\\`, the entry will be inserted - in the corresponding directory instead of replacing it. - - Check result: `mobase.ModDataChecker.FIXABLE`. + "move": {"Files/folders to move": "target path"} + # If the path ends with `/` or `\\`, the entry will be inserted + # in the corresponding directory instead of replacing it. + # Check result: `mobase.ModDataChecker.FIXABLE`. + } Example:: - BasicModDataChecker( - valid=["valid_folder", "*.ext1"] - move={"*.ext2": "path/to/target_folder/"} - ) - BasicModDataChecker( { - "valid": ["valid_folder", "*.ext1"], + "valid": ["valid_folder", "*.ext1"] "move": {"*.ext2": "path/to/target_folder/"} } ) See Also: - `mobase.IFileTree.move` + `mobase.IFileTree.move` for the `"move"` target path specs. """ - file_patterns = None # type: GlobPatternDict - """Use `update_patterns` for modifications.""" + default_file_patterns: ClassVar[GlobPatternDict] = {} + """Default for `file_patterns` - for subclasses.""" + + _file_patterns: GlobPatternDict + """Private `file_patterns`, updated together with `._regex` and `._move_targets`.""" _regex: RegexPatternDict """The regex patterns derived from the file (glob) patterns.""" _move_targets: Sequence[str] + """Target paths from `file_patterns["move"]`.""" - # Overloads as workaround for **kwargs: Unpack[GlobPatternDict] - @overload - def __init__( - self, - *, - unfold: Iterable[str] | None = None, - valid: Iterable[str] | None = None, - delete: Iterable[str] | None = None, - move: dict[str, str] | None = None, - ): - ... - - @overload - def __init__( - self, - file_patterns: Optional[GlobPatternDict] = None, - *, - unfold: Iterable[str] | None = None, - valid: Iterable[str] | None = None, - delete: Iterable[str] | None = None, - move: dict[str, str] | None = None, - ): - ... - - def __init__( - self, file_patterns: Optional[GlobPatternDict] = None, **kwargs - ): # Unpack[GlobPatternDict] + def __init__(self, file_patterns: Optional[GlobPatternDict]): super().__init__() - # Init with copy from class var - self.file_patterns = fp.copy() if (fp := self.file_patterns) else {} - self.update_patterns(file_patterns, **kwargs) + # Init with copy from class var by default (for unique instance var). + self.set_patterns(file_patterns or (self.default_file_patterns.copy())) - @overload - def update_patterns( - self, - *, - unfold: Iterable[str] | None = None, - valid: Iterable[str] | None = None, - delete: Iterable[str] | None = None, - move: dict[str, str] | None = None, - ): - ... + def set_patterns(self, file_patterns: GlobPatternDict): + """Sets the file patterns, replacing previous/default values and order.""" + self._file_patterns = file_patterns + self.update_patterns() - @overload - def update_patterns( - self, - file_patterns: Optional[GlobPatternDict] = None, - *, - unfold: Iterable[str] | None = None, - valid: Iterable[str] | None = None, - delete: Iterable[str] | None = None, - move: dict[str, str] | None = None, - ): - ... - - def update_patterns(self, file_patterns=None, **kwargs): # Unpack[GlobPatternDict] - """Update file patterns.""" + def update_patterns(self, file_patterns: Optional[GlobPatternDict] = None): + """Update file patterns. Preserves previous/default definition + (check/fix) order. + """ if file_patterns: - self.file_patterns.update(file_patterns) - self.file_patterns.update(kwargs) - self._regex = RegexPatternDict.from_glob_patterns(self.file_patterns) - if move_map := self.file_patterns.get("move"): + self._file_patterns.update(file_patterns) + self._regex = RegexPatternDict.from_glob_patterns(self._file_patterns) + if move_map := self._file_patterns.get("move"): self._move_targets = list(move_map.values()) def dataLooksValid( @@ -224,8 +181,6 @@ class BasicModDataChecker(mobase.ModDataChecker): return status def fix(self, filetree: mobase.IFileTree) -> Optional[mobase.IFileTree]: - print(f"{self.file_patterns}") - print(self._regex) for entry in list(filetree): name = entry.name() # Fix entries in pattern definition order. diff --git a/games/game_valheim.py b/games/game_valheim.py index ed8c96b..40977b6 100644 --- a/games/game_valheim.py +++ b/games/game_valheim.py @@ -327,50 +327,52 @@ class ValheimGame(BasicGame): def init(self, organizer: mobase.IOrganizer) -> bool: super().init(organizer) self._featureMap[mobase.ModDataChecker] = BasicModDataChecker( - unfold=[ - "BepInExPack_Valheim", - ], - valid=[ - "meta.ini", # Included in installed mod folder. - "BepInEx", - "doorstop_libs", - "unstripped_corlib", - "doorstop_config.ini", - "start_game_bepinex.sh", - "start_server_bepinex.sh", - "winhttp.dll", - # - "InSlimVML", - "valheim_Data", - "inslimvml.ini", - # - "unstripped_managed", - # - "AdvancedBuilder", - ], - delete=[ - "*.txt", - "*.md", - "icon.png", - "license", - "manifest.json", - ], - move={ - "*_VML.dll": "InSlimVML/Mods/", - # - "plugins": "BepInEx/", - "*.dll": "BepInEx/plugins/", - "config": "BepInEx/", - "*.cfg": "BepInEx/config/", - # - "CustomTextures": "BepInEx/plugins/", - "*.png": "BepInEx/plugins/CustomTextures/", - # - "Builds": "AdvancedBuilder/", - "*.vbuild": "AdvancedBuilder/Builds/", - # - "*.assets": "valheim_Data/", - }, + { + "unfold": [ + "BepInExPack_Valheim", + ], + "valid": [ + "meta.ini", # Included in installed mod folder. + "BepInEx", + "doorstop_libs", + "unstripped_corlib", + "doorstop_config.ini", + "start_game_bepinex.sh", + "start_server_bepinex.sh", + "winhttp.dll", + # + "InSlimVML", + "valheim_Data", + "inslimvml.ini", + # + "unstripped_managed", + # + "AdvancedBuilder", + ], + "delete": [ + "*.txt", + "*.md", + "icon.png", + "license", + "manifest.json", + ], + "move": { + "*_VML.dll": "InSlimVML/Mods/", + # + "plugins": "BepInEx/", + "*.dll": "BepInEx/plugins/", + "config": "BepInEx/", + "*.cfg": "BepInEx/config/", + # + "CustomTextures": "BepInEx/plugins/", + "*.png": "BepInEx/plugins/CustomTextures/", + # + "Builds": "AdvancedBuilder/", + "*.vbuild": "AdvancedBuilder/Builds/", + # + "*.assets": "valheim_Data/", + }, + } ) self._featureMap[mobase.LocalSavegames] = ValheimLocalSavegames( self.savesDirectory() From c568c0b943878cd98111ec7fe77bcdb4cc0f2f55 Mon Sep 17 00:00:00 2001 From: Zash Date: Thu, 7 Apr 2022 16:02:56 +0200 Subject: [PATCH 7/8] fix: failed unfold should result in unfixed tree (not valid). --- basic_features/basic_mod_data_checker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/basic_features/basic_mod_data_checker.py b/basic_features/basic_mod_data_checker.py index 5fcd83e..673e0ed 100644 --- a/basic_features/basic_mod_data_checker.py +++ b/basic_features/basic_mod_data_checker.py @@ -199,6 +199,7 @@ class BasicModDataChecker(mobase.ModDataChecker): folder_tree.detach() else: print(f"Cannot unfold {name}!", file=sys.stderr) + return None elif key == "delete": entry.detach() elif key == "move": From 7eb501027dea27c409bcbf368110ead60ebd88f2 Mon Sep 17 00:00:00 2001 From: Zash Date: Thu, 7 Apr 2022 22:35:59 +0200 Subject: [PATCH 8/8] fix: = None missing in optional custructor arg --- basic_features/basic_mod_data_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basic_features/basic_mod_data_checker.py b/basic_features/basic_mod_data_checker.py index 673e0ed..d849484 100644 --- a/basic_features/basic_mod_data_checker.py +++ b/basic_features/basic_mod_data_checker.py @@ -139,7 +139,7 @@ class BasicModDataChecker(mobase.ModDataChecker): _move_targets: Sequence[str] """Target paths from `file_patterns["move"]`.""" - def __init__(self, file_patterns: Optional[GlobPatternDict]): + def __init__(self, file_patterns: Optional[GlobPatternDict] = None): super().__init__() # Init with copy from class var by default (for unique instance var). self.set_patterns(file_patterns or (self.default_file_patterns.copy()))