diff --git a/src/plugin_python_en.ts b/src/plugin_python_en.ts index 9ef7836..158fcc7 100644 --- a/src/plugin_python_en.ts +++ b/src/plugin_python_en.ts @@ -4,78 +4,35 @@ ProxyPython - - Python Initialization failed + + The path to Mod Organizer (%1) contains a semicolon. <br>While this is legal on NTFS drives, many softwares do not handle it correctly.<br>Unfortunately MO depends on libraries that seem to fall into that group.<br>As a result the python plugin cannot be loaded, and the only solution we canoffer is to remove the semicolon or move MO to a path without a semicolon. - - On a previous start the Python Plugin failed to initialize. -Do you want to try initializing python again (at the risk of another crash)? - Suggestion: Select "no", and click the warning sign for further help.Afterwards you have to re-enable the python plugin. - - - - - Python Proxy - - - - - Proxy Plugin to allow plugins written in python to be loaded - - - - - ModOrganizer path contains a semicolon - - - - - Python DLL not found - - - - - Invalid Python DLL - - - - - Initializing Python failed - - - - - - invalid problem key %1 - - - - - The path to Mod Organizer (%1) contains a semicolon.<br>While this is legal on NTFS drives, many applications do not handle it correctly.<br>Unfortunately MO depends on libraries that seem to fall into that group.<br>As a result the python plugin cannot be loaded, and the only solution we can offer is to remove the semicolon or move MO to a path without a semicolon. - - - - + The Python plugin DLL was not found, maybe your antivirus deleted it. Re-installing MO2 might fix the problem. - + The Python plugin DLL is invalid, maybe your antivirus is blocking it. Re-installing MO2 and adding exclusions for it to your AV might fix the problem. - + The initialization of the Python plugin DLL failed, unfortunately without any details. + + + no failure + + QObject - + An unknown exception was thrown in python code. diff --git a/src/proxy/CMakeLists.txt b/src/proxy/CMakeLists.txt index c37b219..3ec965b 100644 --- a/src/proxy/CMakeLists.txt +++ b/src/proxy/CMakeLists.txt @@ -2,28 +2,33 @@ cmake_minimum_required(VERSION 3.16) find_package(mo2-uibase CONFIG REQUIRED) -set(PLUGIN_NAME "plugin_python") +set(PROXY_NAME "python") add_library(proxy SHARED proxypython.cpp proxypython.h) mo2_configure_plugin(proxy NO_SOURCES WARNINGS 4 EXTERNAL_WARNINGS 4 - TRANSLATIONS OFF + TRANSLATIONS ON EXTRA_TRANSLATIONS ${CMAKE_CURRENT_SOURCE_DIR}/../runner ${CMAKE_CURRENT_SOURCE_DIR}/../mobase ${CMAKE_CURRENT_SOURCE_DIR}/../pybind11-qt) mo2_default_source_group() target_link_libraries(proxy PRIVATE runner mo2::uibase) -set_target_properties(proxy PROPERTIES OUTPUT_NAME ${PLUGIN_NAME}) -mo2_install_plugin(proxy FOLDER) +set_target_properties(proxy PROPERTIES OUTPUT_NAME "python_proxy") -set(PLUGIN_PYTHON_DIR bin/plugins/${PLUGIN_NAME}) +set(PROXY_PYTHON_DIR ${MO2_INSTALL_BIN}/proxies/python) -# install runner +# install runner and proxy +install(FILES $ DESTINATION ${PROXY_PYTHON_DIR}/python) +install(FILES $ DESTINATION ${PROXY_PYTHON_DIR}/python/dlls) + +# install PDB +install(FILES $ DESTINATION pdb) + +# delay loading since the dll is not in the standard folder target_link_options(proxy PRIVATE "/DELAYLOAD:runner.dll") -install(FILES $ DESTINATION ${PLUGIN_PYTHON_DIR}/dlls) # translations (custom location) mo2_add_translations(proxy @@ -35,14 +40,14 @@ mo2_add_translations(proxy ${CMAKE_CURRENT_SOURCE_DIR}/../pybind11-qt) # install DLLs files needed -set(DLL_DIRS ${PLUGIN_PYTHON_DIR}/dlls) +set(DLL_DIRS ${PROXY_PYTHON_DIR}/dlls) file(GLOB dlls_to_install # ${PYTHON_BUILD_PATH}/libffi*.dll ${Python_HOME}/python${Python_VERSION_MAJOR}*.dll) install(FILES ${dlls_to_install} DESTINATION ${DLL_DIRS}) # install Python .pyd files -set(PYLIB_DIR ${PLUGIN_PYTHON_DIR}/libs) +set(PYLIB_DIR ${PROXY_PYTHON_DIR}/libs) file(GLOB libs_to_install ${Python_DLL_DIR}/*.pyd) install(FILES ${libs_to_install} DESTINATION ${PYLIB_DIR}) diff --git a/src/proxy/proxypython.cpp b/src/proxy/proxypython.cpp index 0ecbfc0..8cfc3e3 100644 --- a/src/proxy/proxypython.cpp +++ b/src/proxy/proxypython.cpp @@ -53,59 +53,26 @@ fs::path getPluginFolder() return fs::path(path).parent_path(); } -ProxyPython::ProxyPython() - : m_MOInfo{nullptr}, m_RunnerLib{nullptr}, m_Runner{nullptr}, - m_LoadFailure(FailureType::NONE) -{ -} +ProxyPython::ProxyPython() : m_RunnerLib{nullptr}, m_Runner{nullptr} {} -bool ProxyPython::init(IOrganizer* moInfo) +bool ProxyPython::initialize(QString& errorMessage) { - m_MOInfo = moInfo; - - if (m_MOInfo && !m_MOInfo->isPluginEnabled(this)) { - return false; - } + errorMessage = ""; if (QCoreApplication::applicationDirPath().contains(';')) { - m_LoadFailure = FailureType::SEMICOLON; + errorMessage = failureMessage(FailureType::SEMICOLON); return true; } const auto pluginFolder = getPluginFolder(); - if (pluginFolder.empty()) { - DWORD error = ::GetLastError(); - m_LoadFailure = FailureType::DLL_NOT_FOUND; + DWORD error = ::GetLastError(); + errorMessage = failureMessage(FailureType::DLL_NOT_FOUND); log::error("failed to resolve Python proxy directory ({}): {}", error, qUtf8Printable(windowsErrorString(::GetLastError()))); return false; } - if (m_MOInfo && m_MOInfo->persistent(name(), "tryInit", false).toBool()) { - m_LoadFailure = FailureType::INITIALIZATION; - if (QMessageBox::question( - parentWidget(), tr("Python Initialization failed"), - tr("On a previous start the Python Plugin failed to initialize.\n" - "Do you want to try initializing python again (at the risk of " - "another crash)?\n " - "Suggestion: Select \"no\", and click the warning sign for further " - "help.Afterwards you have to re-enable the python plugin."), - QMessageBox::Yes | QMessageBox::No, - QMessageBox::No) == QMessageBox::No) { - // we force enabled here (note: this is a persistent settings since MO2 2.4 - // or something), plugin - // usually should not handle enabled/disabled themselves but this is a base - // plugin so... - m_MOInfo->setPersistent(name(), "enabled", false, true); - return true; - } - } - - if (m_MOInfo) { - m_MOInfo->setPersistent(name(), "tryInit", true); - } - // load the pythonrunner library, this is done in multiple steps: // // 1. we set the dlls/ subfolder (from the plugin) as the DLL directory so Windows @@ -114,8 +81,8 @@ bool ProxyPython::init(IOrganizer* moInfo) // const auto dllPaths = pluginFolder / "dlls"; if (SetDllDirectoryW(dllPaths.c_str()) == 0) { - DWORD error = ::GetLastError(); - m_LoadFailure = FailureType::DLL_NOT_FOUND; + DWORD error = ::GetLastError(); + errorMessage = failureMessage(FailureType::DLL_NOT_FOUND); log::error("failed to add python DLL directory ({}): {}", error, qUtf8Printable(windowsErrorString(::GetLastError()))); return false; @@ -129,21 +96,16 @@ bool ProxyPython::init(IOrganizer* moInfo) if (m_Runner) { const auto libpath = pluginFolder / "libs"; - const std::vector paths{ - libpath / "pythoncore.zip", libpath, - std::filesystem::path{IOrganizer::getPluginDataPath().toStdWString()}}; + const std::vector paths{libpath / "pythoncore.zip", libpath}; m_Runner->initialize(paths); } - if (m_MOInfo) { - m_MOInfo->setPersistent(name(), "tryInit", false); - } - // reset DLL directory SetDllDirectoryW(NULL); if (!m_Runner || !m_Runner->isInitialized()) { - m_LoadFailure = FailureType::INITIALIZATION; + errorMessage = failureMessage(FailureType::INITIALIZATION); + return false; } else { m_Runner->addDllSearchPath(pluginFolder / "dlls"); @@ -152,110 +114,57 @@ bool ProxyPython::init(IOrganizer* moInfo) return true; } -QString ProxyPython::name() const -{ - return "Python Proxy"; -} - -QString ProxyPython::localizedName() const -{ - return tr("Python Proxy"); -} - -QString ProxyPython::author() const -{ - return "AnyOldName3, Holt59, Silarn, Tannin"; -} - -QString ProxyPython::description() const -{ - return tr("Proxy Plugin to allow plugins written in python to be loaded"); -} - -VersionInfo ProxyPython::version() const -{ - return VersionInfo(3, 0, 0, VersionInfo::RELEASE_FINAL); -} - -QList ProxyPython::settings() const -{ - return {}; -} - -QStringList ProxyPython::pluginList(const QDir& pluginPath) const -{ - QDir dir(pluginPath); - dir.setFilter(dir.filter() | QDir::NoDotAndDotDot); - QDirIterator iter(dir); - - // Note: We put python script (.py) and directory names, not the __init__.py - // files in those since it is easier for the runner to import them. - QStringList result; - while (iter.hasNext()) { - QString name = iter.next(); - QFileInfo info = iter.fileInfo(); - - if (info.isFile() && name.endsWith(".py")) { - result.append(name); - } - else if (info.isDir() && QDir(info.absoluteFilePath()).exists("__init__.py")) { - result.append(name); - } - } - - return result; -} - -QList ProxyPython::load(const QString& identifier) +QList> ProxyPython::load(const PluginExtension& extension) { if (!m_Runner) { return {}; } - return m_Runner->load(identifier); + + // currently, only handle __init__.py directly in the folder + const auto pyIniFile = extension.directory() / "__init__.py"; + if (!exists(pyIniFile)) { + return {}; + } + + m_ExtensionModules[&extension] = {pyIniFile}; + + QList> plugins; + for (auto&& pythonModule : m_ExtensionModules[&extension]) { + plugins.append(m_Runner->load(pythonModule)); + } + + return plugins; } -void ProxyPython::unload(const QString& identifier) +void ProxyPython::unload(const PluginExtension& extension) +{ + if (!m_Runner) { + return; + } + + if (auto it = m_ExtensionModules.find(&extension); it != m_ExtensionModules.end()) { + for (auto&& pythonModule : it->second) { + m_Runner->unload(pythonModule); + } + m_ExtensionModules.erase(it); + } +} + +void ProxyPython::unloadAll() { if (m_Runner) { - return m_Runner->unload(identifier); + for (auto& [ext, modules] : m_ExtensionModules) { + for (auto& pythonModule : modules) { + m_Runner->unload(pythonModule); + } + } } + m_ExtensionModules.clear(); } -std::vector ProxyPython::activeProblems() const +QString ProxyPython::failureMessage(FailureType key) { - auto failure = m_LoadFailure; - - // don't know how this could happen but wth - if (m_Runner && !m_Runner->isInitialized()) { - failure = FailureType::INITIALIZATION; - } - - if (failure != FailureType::NONE) { - return {static_cast>(failure)}; - } - - return {}; -} - -QString ProxyPython::shortDescription(unsigned int key) const -{ - switch (static_cast(key)) { - case FailureType::SEMICOLON: - return tr("ModOrganizer path contains a semicolon"); - case FailureType::DLL_NOT_FOUND: - return tr("Python DLL not found"); - case FailureType::INVALID_DLL: - return tr("Invalid Python DLL"); - case FailureType::INITIALIZATION: - return tr("Initializing Python failed"); - default: - return tr("invalid problem key %1").arg(key); - } -} - -QString ProxyPython::fullDescription(unsigned int key) const -{ - switch (static_cast(key)) { + switch (key) { case FailureType::SEMICOLON: return tr("The path to Mod Organizer (%1) contains a semicolon.
" "While this is legal on NTFS drives, many applications do not " @@ -278,13 +187,6 @@ QString ProxyPython::fullDescription(unsigned int key) const return tr("The initialization of the Python plugin DLL failed, unfortunately " "without any details."); default: - return tr("invalid problem key %1").arg(key); + return tr("no failure"); } } - -bool ProxyPython::hasGuidedFix(unsigned int) const -{ - return false; -} - -void ProxyPython::startGuidedFix(unsigned int) const {} diff --git a/src/proxy/proxypython.h b/src/proxy/proxypython.h index 3d52da5..a6f239a 100644 --- a/src/proxy/proxypython.h +++ b/src/proxy/proxypython.h @@ -23,45 +23,27 @@ along with python proxy plugin. If not, see . #include #include +#include #include -#include + +#include #include -class ProxyPython : public QObject, - public MOBase::IPluginProxy, - public MOBase::IPluginDiagnose { +class ProxyPython : public MOBase::IPluginLoader { Q_OBJECT - Q_INTERFACES(MOBase::IPlugin MOBase::IPluginProxy MOBase::IPluginDiagnose) + Q_INTERFACES(MOBase::IPluginLoader) Q_PLUGIN_METADATA(IID "org.mo2.ProxyPython") public: ProxyPython(); - virtual bool init(MOBase::IOrganizer* moInfo); - virtual QString name() const override; - virtual QString localizedName() const override; - virtual QString author() const override; - virtual QString description() const override; - virtual MOBase::VersionInfo version() const override; - virtual QList settings() const override; - - QStringList pluginList(const QDir& pluginPath) const override; - QList load(const QString& identifier) override; - void unload(const QString& identifier) override; - -public: // IPluginDiagnose - virtual std::vector activeProblems() const override; - virtual QString shortDescription(unsigned int key) const override; - virtual QString fullDescription(unsigned int key) const override; - virtual bool hasGuidedFix(unsigned int key) const override; - virtual void startGuidedFix(unsigned int key) const override; + bool initialize(QString& errorMessage) override; + QList> load(const MOBase::PluginExtension& extension) override; + void unload(const MOBase::PluginExtension& extension) override; + void unloadAll() override; private: - MOBase::IOrganizer* m_MOInfo; - HMODULE m_RunnerLib; - std::unique_ptr m_Runner; - enum class FailureType : unsigned int { NONE = 0, SEMICOLON = 1, @@ -70,7 +52,14 @@ private: INITIALIZATION = 4 }; - FailureType m_LoadFailure; + static QString failureMessage(FailureType failureType); + +private: + HMODULE m_RunnerLib; + std::unique_ptr m_Runner; + std::unordered_map> + m_ExtensionModules; }; #endif // PROXYPYTHON_H diff --git a/src/runner/CMakeLists.txt b/src/runner/CMakeLists.txt index faee5c6..cd16884 100644 --- a/src/runner/CMakeLists.txt +++ b/src/runner/CMakeLists.txt @@ -21,7 +21,8 @@ target_link_libraries(runner PUBLIC mo2::uibase PRIVATE pybind11::embed pybind11 target_include_directories(runner PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_definitions(runner PRIVATE RUNNER_BUILD) -# proxy will install runner +# proxy will install runner but we install the PDB +install(FILES $ DESTINATION pdb) # force runner to build mobase add_dependencies(runner mobase) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index 3538bb7..3fb87ae 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -36,24 +36,12 @@ namespace mo2::python { PythonRunner() = default; ~PythonRunner() = default; - QList load(const QString& identifier) override; - void unload(const QString& identifier) override; + QList> load(std::filesystem::path const& pythonModule) override; + void unload(std::filesystem::path const& pythonModule) override; bool initialize(std::vector const& pythonPaths) override; void addDllSearchPath(std::filesystem::path const& dllPath) override; bool isInitialized() const override; - - private: - /** - * @brief Ensure that the given folder is in sys.path. - */ - void ensureFolderInPath(QString folder); - - private: - // for each "identifier" (python file or python module folder), contains the - // list of python objects - this does not keep the objects alive, it simply used - // to unload plugins - std::unordered_map> m_PythonObjects; }; std::unique_ptr createPythonRunner() @@ -162,72 +150,41 @@ namespace mo2::python { py::module_::import("os").attr("add_dll_directory")(absolute(dllPath)); } - void PythonRunner::ensureFolderInPath(QString folder) - { - py::module_ sys = py::module_::import("sys"); - py::list sysPath = sys.attr("path"); - - // Converting to QStringList for Qt::CaseInsensitive and because .index() - // raise an exception: - const QStringList currentPath = sysPath.cast(); - if (!currentPath.contains(folder, Qt::CaseInsensitive)) { - sysPath.insert(0, folder); - } - } - - QList PythonRunner::load(const QString& identifier) + QList> PythonRunner::load(const std::filesystem::path& pythonModule) { py::gil_scoped_acquire lock; - // `pluginName` can either be a python file (single-file plugin or a folder - // (whole module). - // - // For whole module, we simply add the parent folder to path, then we load - // the module with a simple py::import, and we retrieve the associated - // __dict__ from which we extract either createPlugin or createPlugins. - // - // For single file, we need to use py::eval_file, and we will use the - // context (global variables) from __main__ (already contains mobase, and - // other required module). Since the context is shared between called of - // `instantiate`, we need to make sure to remove createPlugin(s) from - // previous call. try { - // dictionary that will contain createPlugin() or createPlugins(). - py::dict moduleDict; + // some needed import + auto sys = py::module_::import("sys"); + auto importlib_util = py::module_::import("importlib.util"); - if (identifier.endsWith(".py")) { - py::object mainModule = py::module_::import("__main__"); + // check the file type + const auto moduleName = + pythonModule.filename() == "__init__.py" + ? pythonModule.parent_path().filename().u8string() + : pythonModule.filename().u8string(); - // make a copy, otherwise we might end up calling the createPlugin() or - // createPlugins() function multiple time - py::dict moduleNamespace = mainModule.attr("__dict__").attr("copy")(); - - std::string temp = ToString(identifier); - py::eval_file(temp, moduleNamespace).is_none(); - moduleDict = moduleNamespace; + // check if the module is already loaded + py::dict modules = sys.attr("modules"); + py::module_ pymodule; + if (modules.contains(moduleName)) { + pymodule = modules[py::str(moduleName)]; + pymodule.reload(); } else { - // Retrieve the module name: - QStringList parts = identifier.split("/"); - std::string moduleName = ToString(parts.takeLast()); - ensureFolderInPath(parts.join("/")); - - // check if the module is already loaded - py::dict modules = py::module_::import("sys").attr("modules"); - if (modules.contains(moduleName)) { - py::module_ prev = modules[py::str(moduleName)]; - py::module_(prev).reload(); - moduleDict = prev.attr("__dict__"); - } - else { - moduleDict = - py::module_::import(moduleName.c_str()).attr("__dict__"); - } + // load the module + auto spec = importlib_util.attr("find_spec")(moduleName, pythonModule); + pymodule = importlib_util.attr("module_from_spec")(spec); + sys.attr("modules")[py::str(moduleName)] = pymodule; + spec.attr("loader").attr("exec_module")(pymodule); } + py::dict moduleDict = pymodule.attr("__dict__"); + if (py::len(moduleDict) == 0) { - MOBase::log::error("No plugins found in {}.", identifier); + MOBase::log::error("no plugins found in {}", pythonModule); return {}; } @@ -240,8 +197,8 @@ namespace mo2::python { else if (moduleDict.contains("createPlugins")) { py::object pyPlugins = moduleDict["createPlugins"](); if (!py::isinstance(pyPlugins)) { - MOBase::log::error( - "Plugin {}: createPlugins must return a sequence.", identifier); + MOBase::log::error("{}: createPlugins must return a sequence", + pythonModule); } else { py::sequence pyList(pyPlugins); @@ -252,30 +209,26 @@ namespace mo2::python { } } else { - MOBase::log::error("Plugin {}: missing a createPlugin(s) function.", - identifier); + MOBase::log::error("{}: missing createPlugin(s) function", + pythonModule); } - // If we have no plugins, there was an issue, and we already logged the - // problem: + // if we have no plugins, there was an issue, and we already logged the + // problem if (plugins.empty()) { - return QList(); + return {}; } - QList allInterfaceList; + QList> allInterfaceList; for (py::object pluginObj : plugins) { - - // save to be able to unload it - m_PythonObjects[identifier].push_back(pluginObj); - QList interfaceList = py::module_::import("mobase.private") .attr("extract_plugins")(pluginObj) .cast>(); if (interfaceList.isEmpty()) { - MOBase::log::error("Plugin {}: no plugin interface implemented.", - identifier); + MOBase::log::error("{}: no plugin interface implemented.", + pythonModule); } // Append the plugins to the main list: @@ -285,57 +238,37 @@ namespace mo2::python { return allInterfaceList; } catch (const py::error_already_set& ex) { - MOBase::log::error("Failed to import plugin from {}.", identifier); + MOBase::log::error("failed to import plugin from {}", pythonModule); throw pyexcept::PythonError(ex); } } - void PythonRunner::unload(const QString& identifier) + void PythonRunner::unload(const std::filesystem::path& pythonModule) { - auto it = m_PythonObjects.find(identifier); - if (it != m_PythonObjects.end()) { + py::gil_scoped_acquire lock; - py::gil_scoped_acquire lock; + // At this point, the identifier is the full path to the module. + QDir folder(pythonModule); - if (!identifier.endsWith(".py")) { + // we want to "unload" (remove from sys.modules) modules that come + // from this plugin (whose __path__ points under this module, + // including the module of the plugin itself) + py::object sys = py::module_::import("sys"); + py::dict modules = sys.attr("modules"); + py::list keys = modules.attr("keys")(); + for (std::size_t i = 0; i < py::len(keys); ++i) { + py::object mod = modules[keys[i]]; + if (PyObject_HasAttrString(mod.ptr(), "__path__")) { + QString mpath = mod.attr("__path__")[py::int_(0)].cast(); - // At this point, the identifier is the full path to the module. - QDir folder(identifier); + if (!folder.relativeFilePath(mpath).startsWith("..")) { + // if the path is under identifier, we need to unload it + log::debug("unloading module {} from {}", + keys[i].cast(), mpath); - // We want to "unload" (remove from sys.modules) modules that come - // from this plugin (whose __path__ points under this module, - // including the module of the plugin itself). - py::object sys = py::module_::import("sys"); - py::dict modules = sys.attr("modules"); - py::list keys = modules.attr("keys")(); - for (std::size_t i = 0; i < py::len(keys); ++i) { - py::object mod = modules[keys[i]]; - if (PyObject_HasAttrString(mod.ptr(), "__path__")) { - QString mpath = - mod.attr("__path__")[py::int_(0)].cast(); - - if (!folder.relativeFilePath(mpath).startsWith("..")) { - // If the path is under identifier, we need to unload - // it. - log::debug("Unloading module {} from {} for {}.", - keys[i].cast(), mpath, identifier); - - PyDict_DelItem(modules.ptr(), keys[i].ptr()); - } - } + PyDict_DelItem(modules.ptr(), keys[i].ptr()); } } - - // Boost.Python does not handle cyclic garbace collection, so we need to - // release everything hold by the objects before deleting the objects - // themselves (done when erasing from m_PythonObjects). - for (auto& obj : it->second) { - obj.attr("__dict__").attr("clear")(); - } - - log::debug("Deleting {} python objects for {}.", it->second.size(), - identifier); - m_PythonObjects.erase(it); } } diff --git a/src/runner/pythonrunner.h b/src/runner/pythonrunner.h index 5f9751b..0664bef 100644 --- a/src/runner/pythonrunner.h +++ b/src/runner/pythonrunner.h @@ -9,6 +9,8 @@ #include #include +#include + #ifdef RUNNER_BUILD #define RUNNER_DLL_EXPORT Q_DECL_EXPORT #else @@ -21,8 +23,9 @@ namespace mo2::python { // class IPythonRunner { public: - virtual QList load(const QString& identifier) = 0; - virtual void unload(const QString& identifier) = 0; + virtual QList> + load(std::filesystem::path const& pythonModule) = 0; + virtual void unload(std::filesystem::path const& pythonModule) = 0; // initialize Python //