From e6c5923bb1f4047a6c764b28912dab0fe5ea02b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Fri, 6 May 2022 17:40:11 +0200 Subject: [PATCH] Make runner a static lib. Update translation paths. Clean CMake stuff. --- CMakeLists.txt | 24 +- src/CMakeLists.txt | 3 - src/mobase/CMakeLists.txt | 3 - src/{proxy => }/plugin_python_en.ts | 30 +- src/proxy/CMakeLists.txt | 45 ++- src/proxy/proxypython.cpp | 34 +- src/proxy/proxypython.h | 7 +- src/pybind11-qt/CMakeLists.txt | 3 +- src/pybind11-utils/CMakeLists.txt | 3 +- src/runner/CMakeLists.txt | 38 +-- src/runner/pythonrunner.cpp | 494 ++++++++++++++-------------- src/runner/pythonrunner.h | 57 ++-- tests/runner/CMakeLists.txt | 23 +- tests/runner/test_diagnose.cpp | 2 +- tests/runner/test_filemapper.cpp | 2 +- tests/runner/test_game.cpp | 2 +- tests/runner/test_installer.cpp | 2 +- tests/runner/test_iplugin.cpp | 2 +- tests/runner/test_lifetime.cpp | 2 +- 19 files changed, 380 insertions(+), 396 deletions(-) rename src/{proxy => }/plugin_python_en.ts (75%) diff --git a/CMakeLists.txt b/CMakeLists.txt index e2f247a..a65e6d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,22 +6,28 @@ else() include(${CMAKE_CURRENT_LIST_DIR}/../cmake_common/mo2.cmake) endif() +project(plugin_python CXX) + set(PYTHON_BUILD_PATH ${PYTHON_ROOT}/PCBuild/amd64) -set(PYVERSION 310) -set(PYBIND11_FINDPYTHON true) -set(PYTHON_EXECUTABLE ${PYTHON_BUILD_PATH}/python.exe) -set(PYTHON_INCLUDE_DIRS ${PYTHON_ROOT}/Include) -set(PYTHON_LIBRARIES ${MO2_INSTALL_LIBS_PATH}/python${PYVERSION}.lib) +# find Python - lots of "Hints" since we have a weird setup +set(Python_USE_STATIC_LIBS False) +set(Python_INCLUDE_DIR ${PYTHON_ROOT}/Include) +set(Python_EXECUTABLE ${PYTHON_BUILD_PATH}/python.exe) +file(GLOB Python_LIBRARY ${PYTHON_BUILD_PATH}/python[0-9]+.lib) +find_package(Python COMPONENTS Interpreter Development REQUIRED) +# useful for naming DLL, zip, etc. (3.10 -> 310) +set(Python_VERSION_SHORT ${Python_VERSION_MAJOR}${Python_VERSION_MINOR}) + +# pybind11 add_subdirectory(${MO2_BUILD_PATH}/pybind11 ${CMAKE_CURRENT_BINARY_DIR}/pybind11) -project(plugin_python) - +# projects add_subdirectory(src) +set_property(DIRECTORY ${PROJECT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT proxy) -set_property(DIRECTORY ${PROJECT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT plugin_python) - +# tests (if requested) set(PLUGIN_PYTHON_TESTS ${PLUGIN_PYTHON_TESTS} CACHE BOOL "build tests for plugin_python") if (PLUGIN_PYTHON_TESTS) enable_testing() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 161cb0a..533a5ab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,3 @@ add_subdirectory(pybind11-utils) add_subdirectory(mobase) add_subdirectory(runner) add_subdirectory(proxy) - -# force plugin_python to build mobase -add_dependencies(plugin_python mobase) diff --git a/src/mobase/CMakeLists.txt b/src/mobase/CMakeLists.txt index e6b35f7..87fd525 100644 --- a/src/mobase/CMakeLists.txt +++ b/src/mobase/CMakeLists.txt @@ -10,6 +10,3 @@ mo2_configure_library(mobase PRIVATE_DEPENDS uibase Qt::Core ) target_link_libraries(mobase PRIVATE pybind11::qt pybind11::utils) -target_include_directories(mobase PRIVATE ${PYTHON_ROOT}/Include) - -install(TARGETS mobase DESTINATION bin/plugins/plugin_python/libs) diff --git a/src/proxy/plugin_python_en.ts b/src/plugin_python_en.ts similarity index 75% rename from src/proxy/plugin_python_en.ts rename to src/plugin_python_en.ts index b8eb0a7..1aa9ef7 100644 --- a/src/proxy/plugin_python_en.ts +++ b/src/plugin_python_en.ts @@ -4,70 +4,70 @@ ProxyPython - + Python Initialization failed - + On a previous start the Python Plugin failed to initialize. Do you want to try initializing python again (at the risk of another crash)? Suggestion: Select "no", and click the warning sign for further help.Afterwards you have to re-enable the python plugin. - + Python Proxy - + Proxy Plugin to allow plugins written in python to be loaded - + ModOrganizer path contains a semicolon - + Python DLL not found - + Invalid Python DLL - + Initializing Python failed - - + + invalid problem key %1 - + The path to Mod Organizer (%1) contains a semicolon. <br>While this is legal on NTFS drives, many softwares do not handle it correctly.<br>Unfortunately MO depends on libraries that seem to fall into that group.<br>As a result the python plugin cannot be loaded, and the only solution we canoffer is to remove the semicolon or move MO to a path without a semicolon. - + The Python plugin DLL was not found, maybe your antivirus deleted it. Re-installing MO2 might fix the problem. - + The Python plugin DLL is invalid, maybe your antivirus is blocking it. Re-installing MO2 and adding exclusions for it to your AV might fix the problem. - + The initialization of the Python plugin DLL failed, unfortunately without any details. @@ -75,7 +75,7 @@ Do you want to try initializing python again (at the risk of another crash)? QObject - + An unknown exception was thrown in python code. diff --git a/src/proxy/CMakeLists.txt b/src/proxy/CMakeLists.txt index 2188bc9..5cff16d 100644 --- a/src/proxy/CMakeLists.txt +++ b/src/proxy/CMakeLists.txt @@ -1,12 +1,49 @@ cmake_minimum_required(VERSION 3.16) -add_library(plugin_python SHARED) -mo2_configure_plugin(plugin_python +set(PLUGIN_NAME "plugin_python") + +add_library(proxy SHARED) +mo2_configure_plugin(proxy WARNINGS 4 EXTERNAL_WARNINGS 4 + TRANSLATIONS OFF EXTRA_TRANSLATIONS ${CMAKE_CURRENT_SOURCE_DIR}/../runner ${CMAKE_CURRENT_SOURCE_DIR}/../mobase ${CMAKE_CURRENT_SOURCE_DIR}/../pybind11-qt) -target_link_libraries(plugin_python PRIVATE pythonrunner) -mo2_install_target(plugin_python FOLDER) +target_link_libraries(proxy PRIVATE runner) +set_target_properties(proxy PROPERTIES OUTPUT_NAME ${PLUGIN_NAME}) +mo2_install_target(proxy FOLDER) + +# translations (custom location) +mo2_add_translations(proxy + TS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/../${PLUGIN_NAME}_en.ts + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../runner + ${CMAKE_CURRENT_SOURCE_DIR}/../mobase + ${CMAKE_CURRENT_SOURCE_DIR}/../pybind11-qt) + +# install DLLs files needed +set(DLL_DIRS ${MO2_INSTALL_PATH}/bin/plugins/${PLUGIN_NAME}/dlls) +file(GLOB dlls_to_install + ${PYTHON_BUILD_PATH}/libffi*.dll + ${PYTHON_BUILD_PATH}/python${Python_VERSION_SHORT}.dll) +install(FILES ${dlls_to_install} DESTINATION ${DLL_DIRS}) + +# install Python files +set(PYLIB_DIR ${MO2_INSTALL_PATH}/bin/plugins/${PLUGIN_NAME}/libs) +file(GLOB libs_to_install ${PYTHON_BUILD_PATH}/pythoncore/*.pyd) +install(FILES ${libs_to_install} DESTINATION ${PYLIB_DIR}) +install(FILES ${PYTHON_BUILD_PATH}/pythoncore/python${Python_VERSION_SHORT}.zip + DESTINATION ${PYLIB_DIR} RENAME pythoncore.zip) + +# install mobase +install(TARGETS mobase DESTINATION ${PYLIB_DIR}) + +# install PyQt6 +file(GLOB PYQT_DIR ${MO2_BUILD_PATH}/PyQt${QT_MAJOR_VERSION}*) +set(PYQT_LIB_DIR ${PYQT_DIR}/Lib/site-packages/PyQt${QT_MAJOR_VERSION}) +set(PYQT_TARGET_DIR ${PYLIB_DIR}/PyQt${QT_MAJOR_VERSION}) +file(GLOB pyqt_files ${PYQT_LIB_DIR}/*.py ${PYQT_LIB_DIR}/*.pyd ${PYQT_LIB_DIR}/*.pyi) +install(FILES ${pyqt_files} DESTINATION ${PYQT_TARGET_DIR}) diff --git a/src/proxy/proxypython.cpp b/src/proxy/proxypython.cpp index 1485799..f050735 100644 --- a/src/proxy/proxypython.cpp +++ b/src/proxy/proxypython.cpp @@ -20,13 +20,15 @@ along with python proxy plugin. If not, see . #include +#include + #include #include #include #include #include -#include "log.h" +#include #include #include @@ -52,15 +54,12 @@ fs::path getPluginFolder() } ProxyPython::ProxyPython() - : m_MOInfo{nullptr}, m_RunnerLib{nullptr}, m_Runner{nullptr}, - m_LoadFailure(FailureType::NONE) + : m_MOInfo{nullptr}, m_Runner{nullptr}, m_LoadFailure(FailureType::NONE) { } bool ProxyPython::init(IOrganizer* moInfo) { - using CreatePythonRunner_func = IPythonRunner* (*)(); - m_MOInfo = moInfo; if (m_MOInfo && !m_MOInfo->isPluginEnabled(this)) { @@ -92,29 +91,6 @@ bool ProxyPython::init(IOrganizer* moInfo) return false; } - const auto runnerPath = pluginFolder / "pythonrunner.dll"; - m_RunnerLib = ::LoadLibraryW(runnerPath.c_str()); - - if (!m_RunnerLib) { - DWORD error = ::GetLastError(); - log::error("failed to load python runner ({}): {}", error, - qUtf8Printable(windowsErrorString(error))); - if (error == ERROR_MOD_NOT_FOUND) { - m_LoadFailure = FailureType::DLL_NOT_FOUND; - } - else { - m_LoadFailure = FailureType::INVALID_DLL; - } - return true; - } - - const CreatePythonRunner_func createPythonRunner = - (CreatePythonRunner_func)::GetProcAddress(m_RunnerLib, "CreatePythonRunner"); - if (!createPythonRunner) { - m_LoadFailure = FailureType::INVALID_DLL; - return true; - } - if (m_MOInfo && m_MOInfo->persistent(name(), "tryInit", false).toBool()) { m_LoadFailure = FailureType::INITIALIZATION; if (QMessageBox::question( @@ -139,7 +115,7 @@ bool ProxyPython::init(IOrganizer* moInfo) m_MOInfo->setPersistent(name(), "tryInit", true); } - m_Runner = std::unique_ptr{createPythonRunner()}; + m_Runner = mo2::python::createPythonRunner(); if (m_Runner) { const auto libpath = pluginFolder / "libs"; diff --git a/src/proxy/proxypython.h b/src/proxy/proxypython.h index 9da1b9d..34ac7f2 100644 --- a/src/proxy/proxypython.h +++ b/src/proxy/proxypython.h @@ -23,8 +23,6 @@ along with python proxy plugin. If not, see . #include #include -#include - #include #include @@ -35,9 +33,7 @@ class ProxyPython : public QObject, public MOBase::IPluginDiagnose { Q_OBJECT Q_INTERFACES(MOBase::IPlugin MOBase::IPluginProxy MOBase::IPluginDiagnose) -#if QT_VERSION >= QT_VERSION_CHECK(5, 0, 0) Q_PLUGIN_METADATA(IID "org.mo2.ProxyPython") -#endif public: ProxyPython(); @@ -63,8 +59,7 @@ public: // IPluginDiagnose private: MOBase::IOrganizer* m_MOInfo; - HMODULE m_RunnerLib; - std::unique_ptr m_Runner; + std::unique_ptr m_Runner; enum class FailureType : unsigned int { NONE = 0, diff --git a/src/pybind11-qt/CMakeLists.txt b/src/pybind11-qt/CMakeLists.txt index 0553b58..b75fbc7 100644 --- a/src/pybind11-qt/CMakeLists.txt +++ b/src/pybind11-qt/CMakeLists.txt @@ -10,8 +10,7 @@ mo2_configure_library(pybind11-qt PRIVATE_DEPENDS Qt::Core Qt::Widgets ) target_link_libraries(pybind11-qt PUBLIC pybind11::pybind11) -target_include_directories(pybind11-qt - PUBLIC ${PYTHON_ROOT}/Include ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_include_directories(pybind11-qt PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) # this is kind of broken but it only works with this... target_compile_definitions(pybind11-qt PUBLIC QT_NO_KEYWORDS) diff --git a/src/pybind11-utils/CMakeLists.txt b/src/pybind11-utils/CMakeLists.txt index 32a13d7..3d05b2e 100644 --- a/src/pybind11-utils/CMakeLists.txt +++ b/src/pybind11-utils/CMakeLists.txt @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.16) add_library(pybind11-utils INTERFACE) target_link_libraries(pybind11-utils INTERFACE pybind11::pybind11) -target_include_directories(pybind11-utils - INTERFACE ${PYTHON_ROOT}/Include ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_include_directories(pybind11-utils INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) add_library(pybind11::utils ALIAS pybind11-utils) diff --git a/src/runner/CMakeLists.txt b/src/runner/CMakeLists.txt index 33c0d9e..dde6288 100644 --- a/src/runner/CMakeLists.txt +++ b/src/runner/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.16) -add_library(pythonrunner SHARED) -mo2_configure_library(pythonrunner +add_library(runner STATIC) +mo2_configure_library(runner SOURCE_TREE WARNINGS 4 EXTERNAL_WARNINGS 4 @@ -9,33 +9,9 @@ mo2_configure_library(pythonrunner TRANSLATIONS OFF PUBLIC_DEPENDS uibase Qt::Core ) -target_link_libraries(pythonrunner PRIVATE pybind11::embed pybind11::qt) -target_include_directories(pythonrunner - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${PYTHON_ROOT}/Include) +target_link_libraries(runner PRIVATE pybind11::embed pybind11::qt) +target_include_directories(runner PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +mo2_install_target(runner) -# this is kind of broken but it only works with this... -target_compile_definitions(pythonrunner - PRIVATE QT_NO_KEYWORDS PYTHONRUNNER_LIBRARY) -mo2_install_target(pythonrunner INSTALLDIR bin/plugins/plugin_python) - -# install DLLs files needed -set(DLL_DIRS ${MO2_INSTALL_PATH}/bin/plugins/plugin_python/dlls) -file(GLOB dlls_to_install - ${PYTHON_BUILD_PATH}/libffi*.dll - ${PYTHON_BUILD_PATH}/python${PYVERSION}.dll) -install(FILES ${dlls_to_install} DESTINATION ${DLL_DIRS}) - -# install Python files -set(PYLIB_DIR ${MO2_INSTALL_PATH}/bin/plugins/plugin_python/libs) -file(GLOB libs_to_install ${PYTHON_BUILD_PATH}/pythoncore/*.pyd) -install(FILES ${libs_to_install} DESTINATION ${PYLIB_DIR}) -install(FILES ${PYTHON_BUILD_PATH}/pythoncore/python${PYVERSION}.zip - DESTINATION ${PYLIB_DIR} RENAME pythoncore.zip) - -# install PyQt6 -file(GLOB PYQT_DIR ${MO2_BUILD_PATH}/PyQt${QT_MAJOR_VERSION}*) -set(PYQT_LIB_DIR ${PYQT_DIR}/Lib/site-packages/PyQt${QT_MAJOR_VERSION}) -set(PYQT_TARGET_DIR ${PYLIB_DIR}/PyQt${QT_MAJOR_VERSION}) -file(GLOB pyqt_files ${PYQT_LIB_DIR}/*.py ${PYQT_LIB_DIR}/*.pyd ${PYQT_LIB_DIR}/*.pyi) -install(FILES ${pyqt_files} DESTINATION ${PYQT_TARGET_DIR}) +# force runner to build mobase +add_dependencies(runner mobase) diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp index a2e132a..eac1e6b 100644 --- a/src/runner/pythonrunner.cpp +++ b/src/runner/pythonrunner.cpp @@ -22,279 +22,287 @@ using namespace MOBase; namespace py = pybind11; -/** - * - */ -class PythonRunner : public IPythonRunner { +namespace mo2::python { -public: - PythonRunner() = default; - ~PythonRunner() = default; - - QList load(const QString& identifier) override; - void unload(const QString& identifier) override; - - bool initialize(QStringList const& paths) override; - bool isInitialized() const override; - -private: /** - * @brief Ensure that the given folder is in sys.path. + * */ - void ensureFolderInPath(QString folder); + class PythonRunner : public IPythonRunner { -private: - // for each "identifier" (python file or python module folder), contains the - // list of python objects - this does not keep the objects alive, it simply used to - // unload plugins - std::unordered_map> m_PythonObjects; -}; + public: + PythonRunner() = default; + ~PythonRunner() = default; -IPythonRunner* CreatePythonRunner() -{ - return new PythonRunner(); -} + QList load(const QString& identifier) override; + void unload(const QString& identifier) override; -bool PythonRunner::initialize(QStringList const& paths) -{ - // we only initialize Python once for the whole lifetime of the program, even if MO2 - // is restarted and the proxy or PythonRunner objects are deleted and recreated, - // Python is not re-initialized - // - // in an ideal world, we would initialize Python here (or in the constructor) and - // then finalize it in the destructor - // - // unfortunately, many library, including PyQt6, do not handle properly - // re-initializing the Python interpreter, so we cannot do that and we keep the - // interpreter alive - // + bool initialize(QStringList const& paths) override; + bool isInitialized() const override; - if (Py_IsInitialized()) { - return true; + private: + /** + * @brief Ensure that the given folder is in sys.path. + */ + void ensureFolderInPath(QString folder); + + private: + // for each "identifier" (python file or python module folder), contains the + // list of python objects - this does not keep the objects alive, it simply used + // to unload plugins + std::unordered_map> m_PythonObjects; + }; + + std::unique_ptr createPythonRunner() + { + return std::make_unique(); } - try { - static const char* argv0 = "ModOrganizer.exe"; - - // initialize the core Path of Python, this must be done before initialization + bool PythonRunner::initialize(QStringList const& paths) + { + // we only initialize Python once for the whole lifetime of the program, even if + // MO2 is restarted and the proxy or PythonRunner objects are deleted and + // recreated, Python is not re-initialized // - if (!paths.isEmpty()) { - Py_SetPath(paths.join(';').toStdWString().c_str()); + // in an ideal world, we would initialize Python here (or in the constructor) + // and then finalize it in the destructor + // + // unfortunately, many library, including PyQt6, do not handle properly + // re-initializing the Python interpreter, so we cannot do that and we keep the + // interpreter alive + // + + if (Py_IsInitialized()) { + return true; } - Py_OptimizeFlag = 2; - Py_NoSiteFlag = 1; + try { + static const char* argv0 = "ModOrganizer.exe"; - py::initialize_interpreter(false, 1, &argv0); - - if (!Py_IsInitialized()) { - MOBase::log::error( - "failed to init python: failed to initialize interpreter."); - - if (PyGILState_Check()) { - PyEval_SaveThread(); + // initialize the core Path of Python, this must be done before + // initialization + // + if (!paths.isEmpty()) { + Py_SetPath(paths.join(';').toStdWString().c_str()); } + Py_OptimizeFlag = 2; + Py_NoSiteFlag = 1; + + py::initialize_interpreter(false, 1, &argv0); + + if (!Py_IsInitialized()) { + MOBase::log::error( + "failed to init python: failed to initialize interpreter."); + + if (PyGILState_Check()) { + PyEval_SaveThread(); + } + + return false; + } + + py::module_ mainModule = py::module_::import("__main__"); + py::object mainNamespace = mainModule.attr("__dict__"); + mainNamespace["sys"] = py::module_::import("sys"); + mainNamespace["mobase"] = py::module_::import("mobase"); + + mo2::python::configure_python_stream(); + mo2::python::configure_python_logging(mainNamespace["mobase"]); + + // we need to release the GIL here - which is what this does + // + // when Python is initialized, the GIl is acquired, and if it is not + // release, trying to acquire it on a different thread will deadlock + PyEval_SaveThread(); + + return true; + } + catch (const py::error_already_set& ex) { + MOBase::log::error("failed to init python: {}", ex.what()); return false; } - - py::module_ mainModule = py::module_::import("__main__"); - py::object mainNamespace = mainModule.attr("__dict__"); - mainNamespace["sys"] = py::module_::import("sys"); - mainNamespace["mobase"] = py::module_::import("mobase"); - - mo2::python::configure_python_stream(); - mo2::python::configure_python_logging(mainNamespace["mobase"]); - - // we need to release the GIL here - which is what this does - // - // when Python is initialized, the GIl is acquired, and if it is not release, - // trying to acquire it on a different thread will deadlock - PyEval_SaveThread(); - - return true; } - catch (const py::error_already_set& ex) { - MOBase::log::error("failed to init python: {}", ex.what()); - return false; + + void PythonRunner::ensureFolderInPath(QString folder) + { + py::module_ sys = py::module_::import("sys"); + py::list sysPath = sys.attr("path"); + + // Converting to QStringList for Qt::CaseInsensitive and because .index() + // raise an exception: + const QStringList currentPath = sysPath.cast(); + if (!currentPath.contains(folder, Qt::CaseInsensitive)) { + sysPath.insert(0, folder); + } } -} - -void PythonRunner::ensureFolderInPath(QString folder) -{ - py::module_ sys = py::module_::import("sys"); - py::list sysPath = sys.attr("path"); - - // Converting to QStringList for Qt::CaseInsensitive and because .index() - // raise an exception: - const QStringList currentPath = sysPath.cast(); - if (!currentPath.contains(folder, Qt::CaseInsensitive)) { - sysPath.insert(0, folder); - } -} - -QList PythonRunner::load(const QString& identifier) -{ - py::gil_scoped_acquire lock; - - // `pluginName` can either be a python file (single-file plugin or a folder - // (whole module). - // - // For whole module, we simply add the parent folder to path, then we load - // the module with a simple py::import, and we retrieve the associated - // __dict__ from which we extract either createPlugin or createPlugins. - // - // For single file, we need to use py::eval_file, and we will use the - // context (global variables) from __main__ (already contains mobase, and - // other required module). Since the context is shared between called of - // `instantiate`, we need to make sure to remove createPlugin(s) from - // previous call. - try { - - // dictionary that will contain createPlugin() or createPlugins(). - py::dict moduleDict; - - if (identifier.endsWith(".py")) { - py::object mainModule = py::module_::import("__main__"); - - // make a copy, otherwise we might end up calling the createPlugin() or - // createPlugins() function multiple time - py::dict moduleNamespace = mainModule.attr("__dict__").attr("copy")(); - - std::string temp = ToString(identifier); - py::eval_file(temp, moduleNamespace).is_none(); - moduleDict = moduleNamespace; - } - else { - // Retrieve the module name: - QStringList parts = identifier.split("/"); - std::string moduleName = ToString(parts.takeLast()); - ensureFolderInPath(parts.join("/")); - - // check if the module is already loaded - py::dict modules = py::module_::import("sys").attr("modules"); - if (modules.contains(moduleName)) { - py::module_ prev = modules[py::str(moduleName)]; - py::module_(prev).reload(); - moduleDict = prev.attr("__dict__"); - } - else { - moduleDict = py::module_::import(moduleName.c_str()).attr("__dict__"); - } - } - - if (py::len(moduleDict) == 0) { - MOBase::log::error("No plugins found in {}.", identifier); - return {}; - } - - // Create the plugins: - std::vector plugins; - - if (moduleDict.contains("createPlugin")) { - plugins.push_back(moduleDict["createPlugin"]()); - } - else if (moduleDict.contains("createPlugins")) { - py::object pyPlugins = moduleDict["createPlugins"](); - if (!py::isinstance(pyPlugins)) { - MOBase::log::error("Plugin {}: createPlugins must return a sequence.", - identifier); - } - else { - py::sequence pyList(pyPlugins); - size_t nPlugins = pyList.size(); - for (size_t i = 0; i < nPlugins; ++i) { - plugins.push_back(pyList[i]); - } - } - } - else { - MOBase::log::error("Plugin {}: missing a createPlugin(s) function.", - identifier); - } - - // If we have no plugins, there was an issue, and we already logged the - // problem: - if (plugins.empty()) { - return QList(); - } - - QList allInterfaceList; - - for (py::object pluginObj : plugins) { - - // save to be able to unload it - m_PythonObjects[identifier].push_back(pluginObj); - - QList interfaceList = py::module_::import("mobase.private") - .attr("extract_plugins")(pluginObj) - .cast>(); - - if (interfaceList.isEmpty()) { - MOBase::log::error("Plugin {}: no plugin interface implemented.", - identifier); - } - - // Append the plugins to the main list: - allInterfaceList.append(interfaceList); - } - - return allInterfaceList; - } - catch (const py::error_already_set& ex) { - MOBase::log::error("Failed to import plugin from {}.", identifier); - throw pyexcept::PythonError(ex); - } -} - -void PythonRunner::unload(const QString& identifier) -{ - auto it = m_PythonObjects.find(identifier); - if (it != m_PythonObjects.end()) { + QList PythonRunner::load(const QString& identifier) + { py::gil_scoped_acquire lock; - if (!identifier.endsWith(".py")) { + // `pluginName` can either be a python file (single-file plugin or a folder + // (whole module). + // + // For whole module, we simply add the parent folder to path, then we load + // the module with a simple py::import, and we retrieve the associated + // __dict__ from which we extract either createPlugin or createPlugins. + // + // For single file, we need to use py::eval_file, and we will use the + // context (global variables) from __main__ (already contains mobase, and + // other required module). Since the context is shared between called of + // `instantiate`, we need to make sure to remove createPlugin(s) from + // previous call. + try { - // At this point, the identifier is the full path to the module. - QDir folder(identifier); + // dictionary that will contain createPlugin() or createPlugins(). + py::dict moduleDict; - // We want to "unload" (remove from sys.modules) modules that come - // from this plugin (whose __path__ points under this module, - // including the module of the plugin itself). - py::object sys = py::module_::import("sys"); - py::dict modules = sys.attr("modules"); - py::list keys = modules.attr("keys")(); - for (std::size_t i = 0; i < py::len(keys); ++i) { - py::object mod = modules[keys[i]]; - if (PyObject_HasAttrString(mod.ptr(), "__path__")) { - QString mpath = mod.attr("__path__")[py::int_(0)].cast(); + if (identifier.endsWith(".py")) { + py::object mainModule = py::module_::import("__main__"); - if (!folder.relativeFilePath(mpath).startsWith("..")) { - // If the path is under identifier, we need to unload - // it. - log::debug("Unloading module {} from {} for {}.", - keys[i].cast(), mpath, identifier); + // make a copy, otherwise we might end up calling the createPlugin() or + // createPlugins() function multiple time + py::dict moduleNamespace = mainModule.attr("__dict__").attr("copy")(); - PyDict_DelItem(modules.ptr(), keys[i].ptr()); + std::string temp = ToString(identifier); + py::eval_file(temp, moduleNamespace).is_none(); + moduleDict = moduleNamespace; + } + else { + // Retrieve the module name: + QStringList parts = identifier.split("/"); + std::string moduleName = ToString(parts.takeLast()); + ensureFolderInPath(parts.join("/")); + + // check if the module is already loaded + py::dict modules = py::module_::import("sys").attr("modules"); + if (modules.contains(moduleName)) { + py::module_ prev = modules[py::str(moduleName)]; + py::module_(prev).reload(); + moduleDict = prev.attr("__dict__"); + } + else { + moduleDict = + py::module_::import(moduleName.c_str()).attr("__dict__"); + } + } + + if (py::len(moduleDict) == 0) { + MOBase::log::error("No plugins found in {}.", identifier); + return {}; + } + + // Create the plugins: + std::vector plugins; + + if (moduleDict.contains("createPlugin")) { + plugins.push_back(moduleDict["createPlugin"]()); + } + else if (moduleDict.contains("createPlugins")) { + py::object pyPlugins = moduleDict["createPlugins"](); + if (!py::isinstance(pyPlugins)) { + MOBase::log::error( + "Plugin {}: createPlugins must return a sequence.", identifier); + } + else { + py::sequence pyList(pyPlugins); + size_t nPlugins = pyList.size(); + for (size_t i = 0; i < nPlugins; ++i) { + plugins.push_back(pyList[i]); } } } - } + else { + MOBase::log::error("Plugin {}: missing a createPlugin(s) function.", + identifier); + } - // Boost.Python does not handle cyclic garbace collection, so we need to - // release everything hold by the objects before deleting the objects - // themselves (done when erasing from m_PythonObjects). - for (auto& obj : it->second) { - obj.attr("__dict__").attr("clear")(); - } + // If we have no plugins, there was an issue, and we already logged the + // problem: + if (plugins.empty()) { + return QList(); + } - log::debug("Deleting {} python objects for {}.", it->second.size(), identifier); - m_PythonObjects.erase(it); + QList allInterfaceList; + + for (py::object pluginObj : plugins) { + + // save to be able to unload it + m_PythonObjects[identifier].push_back(pluginObj); + + QList interfaceList = py::module_::import("mobase.private") + .attr("extract_plugins")(pluginObj) + .cast>(); + + if (interfaceList.isEmpty()) { + MOBase::log::error("Plugin {}: no plugin interface implemented.", + identifier); + } + + // Append the plugins to the main list: + allInterfaceList.append(interfaceList); + } + + return allInterfaceList; + } + catch (const py::error_already_set& ex) { + MOBase::log::error("Failed to import plugin from {}.", identifier); + throw pyexcept::PythonError(ex); + } } -} -bool PythonRunner::isInitialized() const -{ - return Py_IsInitialized() != 0; -} + void PythonRunner::unload(const QString& identifier) + { + auto it = m_PythonObjects.find(identifier); + if (it != m_PythonObjects.end()) { + + py::gil_scoped_acquire lock; + + if (!identifier.endsWith(".py")) { + + // At this point, the identifier is the full path to the module. + QDir folder(identifier); + + // We want to "unload" (remove from sys.modules) modules that come + // from this plugin (whose __path__ points under this module, + // including the module of the plugin itself). + py::object sys = py::module_::import("sys"); + py::dict modules = sys.attr("modules"); + py::list keys = modules.attr("keys")(); + for (std::size_t i = 0; i < py::len(keys); ++i) { + py::object mod = modules[keys[i]]; + if (PyObject_HasAttrString(mod.ptr(), "__path__")) { + QString mpath = + mod.attr("__path__")[py::int_(0)].cast(); + + if (!folder.relativeFilePath(mpath).startsWith("..")) { + // If the path is under identifier, we need to unload + // it. + log::debug("Unloading module {} from {} for {}.", + keys[i].cast(), mpath, identifier); + + PyDict_DelItem(modules.ptr(), keys[i].ptr()); + } + } + } + } + + // Boost.Python does not handle cyclic garbace collection, so we need to + // release everything hold by the objects before deleting the objects + // themselves (done when erasing from m_PythonObjects). + for (auto& obj : it->second) { + obj.attr("__dict__").attr("clear")(); + } + + log::debug("Deleting {} python objects for {}.", it->second.size(), + identifier); + m_PythonObjects.erase(it); + } + } + + bool PythonRunner::isInitialized() const + { + return Py_IsInitialized() != 0; + } + +} // namespace mo2::python diff --git a/src/runner/pythonrunner.h b/src/runner/pythonrunner.h index 5e96fb8..a7f5d58 100644 --- a/src/runner/pythonrunner.h +++ b/src/runner/pythonrunner.h @@ -1,44 +1,41 @@ #ifndef PYTHONRUNNER_H #define PYTHONRUNNER_H -#include -#include +#include +#include #include #include #include -#include -#include +namespace mo2::python { -class IPythonRunner { -public: - virtual QList load(const QString& identifier) = 0; - virtual void unload(const QString& identifier) = 0; - - // initialize Python + // python runner interface // - // paths contains the list of built-in paths for the Python library (pythonxxx.zip, - // etc.), an empty list uses the default Python paths (e.g., the PYTHONPATH - // environment variable) + class IPythonRunner { + public: + virtual QList load(const QString& identifier) = 0; + virtual void unload(const QString& identifier) = 0; + + // initialize Python + // + // paths contains the list of built-in paths for the Python library + // (pythonxxx.zip, etc.), an empty list uses the default Python paths (e.g., the + // PYTHONPATH environment variable) + // + virtual bool initialize(QStringList const& paths = {}) = 0; + + // check if the runner has been initialized, i.e., initialize() has been called + // and succeeded + virtual bool isInitialized() const = 0; + + virtual ~IPythonRunner() {} + }; + + // create the Python runner // - virtual bool initialize(QStringList const& paths = {}) = 0; + std::unique_ptr createPythonRunner(); - // check if the runner has been initialized, i.e., initialize() has been called and - // succeeded - virtual bool isInitialized() const = 0; - - virtual ~IPythonRunner() {} -}; - -#ifdef PYTHONRUNNER_LIBRARY -#define PYDLLEXPORT Q_DECL_EXPORT -#else // PYTHONRUNNER_LIBRARY -#define PYDLLEXPORT Q_DECL_IMPORT -#endif // PYTHONRUNNER_LIBRARY - -// create the Python runner -// -extern "C" PYDLLEXPORT IPythonRunner* CreatePythonRunner(); +} // namespace mo2::python #endif // PYTHONRUNNER_H diff --git a/tests/runner/CMakeLists.txt b/tests/runner/CMakeLists.txt index 6747c83..2122404 100644 --- a/tests/runner/CMakeLists.txt +++ b/tests/runner/CMakeLists.txt @@ -4,37 +4,34 @@ cmake_minimum_required(VERSION 3.22) # things # first we configure the tests as with other tests -add_executable(pythonrunner-tests EXCLUDE_FROM_ALL) -mo2_configure_tests(pythonrunner-tests WARNINGS OFF) +add_executable(runner-tests EXCLUDE_FROM_ALL) +mo2_configure_tests(runner-tests WARNINGS OFF) # add mocks -target_include_directories(pythonrunner-tests +target_include_directories(runner-tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../mocks) -# link to pythonrunner - we set PYTHONRUNNER_LIBRARY to not export the symbol but -# loads it directly, we are going to add the DLL path below -target_compile_definitions(pythonrunner-tests PUBLIC PYTHONRUNNER_LIBRARY) -target_link_libraries(pythonrunner-tests PUBLIC pythonrunner) +# link to runner +target_link_libraries(runner-tests PUBLIC runner) set(PYLIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/pylibs) -mo2_python_pip_install(pythonrunner-tests +mo2_python_pip_install(runner-tests DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/pylibs PACKAGES pytest PyQt6==6.3.0) -add_dependencies(pythonrunner-tests mobase) +add_dependencies(runner-tests mobase) # we set multiple properties, including: -# - updated PATH for DLLs (Qt, etc.), Python and pythonrunner +# - updated PATH for DLLs (Qt, etc.), Python and runner # - PYTHONPATH set(pythoncore "${PYTHON_ROOT}/PCbuild/amd64/pythoncore") -file(GLOB pythoncorezip "${pythoncore}/python*.zip") +set(pythoncorezip "${pythoncore}/python${Python_VERSION_SHORT}.zip") set(PYTHONPATH "${PYLIB_DIR}\\;$\\;${pythoncore}\\;${pythoncorezip}") set(extra_paths "${MO2_INSTALL_PATH}/bin/dlls") string(APPEND extra_paths "\\;${PYTHON_ROOT}/PCbuild/amd64") -string(APPEND extra_paths "\\;$") -set_tests_properties(${pythonrunner-tests_gtests} +set_tests_properties(${runner-tests_gtests} PROPERTIES WORKING_DIRECTORY "${MO2_INSTALL_PATH}/bin" ENVIRONMENT "PLUGIN_DIR=${CMAKE_CURRENT_SOURCE_DIR}/plugins" diff --git a/tests/runner/test_diagnose.cpp b/tests/runner/test_diagnose.cpp index 3639a2f..b0a6152 100644 --- a/tests/runner/test_diagnose.cpp +++ b/tests/runner/test_diagnose.cpp @@ -17,7 +17,7 @@ TEST(IPluginDiagnose, Simple) { const auto plugins_folder = QString(std::getenv("PLUGIN_DIR")); - std::unique_ptr runner(CreatePythonRunner()); + auto runner = mo2::python::createPythonRunner(); runner->initialize(); // load objects diff --git a/tests/runner/test_filemapper.cpp b/tests/runner/test_filemapper.cpp index 1bd7b67..7591177 100644 --- a/tests/runner/test_filemapper.cpp +++ b/tests/runner/test_filemapper.cpp @@ -15,7 +15,7 @@ TEST(IPluginFileMapper, Simple) { const auto plugins_folder = QString(std::getenv("PLUGIN_DIR")); - std::unique_ptr runner(CreatePythonRunner()); + auto runner = mo2::python::createPythonRunner(); runner->initialize(); // load objects diff --git a/tests/runner/test_game.cpp b/tests/runner/test_game.cpp index 6bdaf9c..cba4ea3 100644 --- a/tests/runner/test_game.cpp +++ b/tests/runner/test_game.cpp @@ -18,7 +18,7 @@ TEST(IPluginGame, Simple) { const auto plugins_folder = QString(std::getenv("PLUGIN_DIR")); - std::unique_ptr runner(CreatePythonRunner()); + auto runner = mo2::python::createPythonRunner(); runner->initialize(); // load objects diff --git a/tests/runner/test_installer.cpp b/tests/runner/test_installer.cpp index d2f2389..a96ddf6 100644 --- a/tests/runner/test_installer.cpp +++ b/tests/runner/test_installer.cpp @@ -15,7 +15,7 @@ TEST(IPluginInstaller, Simple) { const auto plugins_folder = QString(std::getenv("PLUGIN_DIR")); - std::unique_ptr runner(CreatePythonRunner()); + auto runner = mo2::python::createPythonRunner(); runner->initialize(); // load objects diff --git a/tests/runner/test_iplugin.cpp b/tests/runner/test_iplugin.cpp index 94e29a5..cadfa11 100644 --- a/tests/runner/test_iplugin.cpp +++ b/tests/runner/test_iplugin.cpp @@ -13,7 +13,7 @@ TEST(IPlugin, Basic) { const auto plugins_folder = QString(std::getenv("PLUGIN_DIR")); - std::unique_ptr runner(CreatePythonRunner()); + auto runner = mo2::python::createPythonRunner(); runner->initialize(); // load objects diff --git a/tests/runner/test_lifetime.cpp b/tests/runner/test_lifetime.cpp index 1eb92db..bf057d3 100644 --- a/tests/runner/test_lifetime.cpp +++ b/tests/runner/test_lifetime.cpp @@ -9,7 +9,7 @@ TEST(Lifetime, Plugins) { const auto plugins_folder = QString(std::getenv("PLUGIN_DIR")); - std::unique_ptr runner(CreatePythonRunner()); + auto runner = mo2::python::createPythonRunner(); runner->initialize(); {