From 68060b0e6b338be477d2afaeea690c5609ffcb3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Thu, 28 Apr 2022 22:44:15 +0200 Subject: [PATCH] Global re-organization to split mobase from runner. --- .gitmodules | 3 - CMakeLists.txt | 9 +- pybind11 | 1 - src/mobase/CMakeLists.txt | 14 + src/mobase/deprecation.cpp | 54 ++ .../pythonutils.h => mobase/deprecation.h} | 13 - src/mobase/mobase.cpp | 269 +++++++++ .../pybind11_all.h | 0 .../pybind11_utils/functional.h | 0 .../pybind11_utils/shared_cpp_owner.h | 0 .../wrappers/basic_classes.cpp | 2 +- .../wrappers/game_features.cpp | 0 .../wrappers/pyfiletree.cpp | 7 +- .../wrappers/pyfiletree.h | 0 .../wrappers/pyplugins.cpp | 0 .../wrappers/pyplugins.h | 0 .../wrappers/widgets.cpp | 0 .../wrappers/wrappers.cpp | 0 .../wrappers/wrappers.h | 0 src/proxy/plugin_python_en.ts | 31 +- src/proxy/proxypython.cpp | 12 +- src/pybind11-qt/CMakeLists.txt | 18 + .../pybind11_qt/details/pybind11_qt_enum.h | 0 .../pybind11_qt/details/pybind11_qt_qlist.h | 0 .../pybind11_qt/details/pybind11_qt_qmap.h | 0 .../pybind11_qt/details/pybind11_qt_sip.h | 0 .../pybind11_qt/details/pybind11_qt_utils.h | 0 .../include}/pybind11_qt/pybind11_qt.h | 0 .../include}/pybind11_qt/pybind11_qt_basic.h | 0 .../pybind11_qt/pybind11_qt_containers.h | 0 .../include}/pybind11_qt/pybind11_qt_enums.h | 0 .../include}/pybind11_qt/pybind11_qt_holder.h | 0 .../pybind11_qt/pybind11_qt_objects.h | 0 .../include}/pybind11_qt/pybind11_qt_qflags.h | 0 .../pybind11_qt_basic.cpp | 4 +- .../pybind11_qt_sip.cpp | 2 +- .../pybind11_qt_utils.cpp | 2 +- src/runner-pybind11/pythonrunner.cpp | 565 ------------------ .../CMakeLists.txt | 9 +- src/{runner-pybind11 => runner}/error.h | 0 src/runner/pythonrunner.cpp | 308 ++++++++++ .../pythonrunner.h | 0 .../pythonutils.cpp | 40 -- src/runner/pythonutils.h | 25 + 44 files changed, 724 insertions(+), 664 deletions(-) delete mode 100644 .gitmodules delete mode 160000 pybind11 create mode 100644 src/mobase/CMakeLists.txt create mode 100644 src/mobase/deprecation.cpp rename src/{runner-pybind11/pythonutils.h => mobase/deprecation.h} (72%) create mode 100644 src/mobase/mobase.cpp rename src/{runner-pybind11 => mobase}/pybind11_all.h (100%) rename src/{runner-pybind11 => mobase}/pybind11_utils/functional.h (100%) rename src/{runner-pybind11 => mobase}/pybind11_utils/shared_cpp_owner.h (100%) rename src/{runner-pybind11 => mobase}/wrappers/basic_classes.cpp (99%) rename src/{runner-pybind11 => mobase}/wrappers/game_features.cpp (100%) rename src/{runner-pybind11 => mobase}/wrappers/pyfiletree.cpp (98%) rename src/{runner-pybind11 => mobase}/wrappers/pyfiletree.h (100%) rename src/{runner-pybind11 => mobase}/wrappers/pyplugins.cpp (100%) rename src/{runner-pybind11 => mobase}/wrappers/pyplugins.h (100%) rename src/{runner-pybind11 => mobase}/wrappers/widgets.cpp (100%) rename src/{runner-pybind11 => mobase}/wrappers/wrappers.cpp (100%) rename src/{runner-pybind11 => mobase}/wrappers/wrappers.h (100%) create mode 100644 src/pybind11-qt/CMakeLists.txt rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/details/pybind11_qt_enum.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/details/pybind11_qt_qlist.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/details/pybind11_qt_qmap.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/details/pybind11_qt_sip.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/details/pybind11_qt_utils.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt_basic.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt_containers.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt_enums.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt_holder.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt_objects.h (100%) rename src/{runner-pybind11 => pybind11-qt/include}/pybind11_qt/pybind11_qt_qflags.h (100%) rename src/{runner-pybind11/pybind11_qt => pybind11-qt}/pybind11_qt_basic.cpp (98%) rename src/{runner-pybind11/pybind11_qt/details => pybind11-qt}/pybind11_qt_sip.cpp (98%) rename src/{runner-pybind11/pybind11_qt/details => pybind11-qt}/pybind11_qt_utils.cpp (85%) delete mode 100644 src/runner-pybind11/pythonrunner.cpp rename src/{runner-pybind11 => runner}/CMakeLists.txt (87%) rename src/{runner-pybind11 => runner}/error.h (100%) create mode 100644 src/runner/pythonrunner.cpp rename src/{runner-pybind11 => runner}/pythonrunner.h (100%) rename src/{runner-pybind11 => runner}/pythonutils.cpp (75%) create mode 100644 src/runner/pythonutils.h diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2726187..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "pybind11"] - path = pybind11 - url = https://github.com/pybind/pybind11 diff --git a/CMakeLists.txt b/CMakeLists.txt index 27c9638..5fd23a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,14 +14,19 @@ set(PYTHON_EXECUTABLE ${PYTHON_BUILD_PATH}/python.exe) set(PYTHON_INCLUDE_DIRS ${PYTHON_ROOT}/Include) set(PYTHON_LIBRARIES ${MO2_INSTALL_LIBS_PATH}/python${PYVERSION}.lib) -add_subdirectory(pybind11) +add_subdirectory(${MO2_BUILD_PATH}/pybind11 ${CMAKE_CURRENT_BINARY_DIR}/pybind11) project(plugin_python) # order matters! -add_subdirectory(src/runner-pybind11) +add_subdirectory(src/pybind11-qt) +add_subdirectory(src/mobase) +add_subdirectory(src/runner) add_subdirectory(src/proxy) +# force plugin_python to build mobase +add_dependencies(plugin_python mobase) + # add_subdirectory(tests) set_property(DIRECTORY ${PROJECT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT plugin_python) diff --git a/pybind11 b/pybind11 deleted file mode 160000 index 9bc2704..0000000 --- a/pybind11 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9bc270443068c90298c0220cd866fbf9ea1bb589 diff --git a/src/mobase/CMakeLists.txt b/src/mobase/CMakeLists.txt new file mode 100644 index 0000000..fb596e2 --- /dev/null +++ b/src/mobase/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.16) + +pybind11_add_module(mobase MODULE) +mo2_configure_library(mobase + SOURCE_TREE + WARNINGS OFF + AUTOMOC ON + TRANSLATIONS OFF + PRIVATE_DEPENDS uibase Qt::Core +) +target_link_libraries(mobase PRIVATE pybind11::qt) +target_include_directories(mobase PRIVATE ${PYTHON_ROOT}/Include) + +install(TARGETS mobase DESTINATION bin/plugins/plugin_python/libs) diff --git a/src/mobase/deprecation.cpp b/src/mobase/deprecation.cpp new file mode 100644 index 0000000..860d071 --- /dev/null +++ b/src/mobase/deprecation.cpp @@ -0,0 +1,54 @@ +#include "deprecation.h" + +#include +#include + +#include + +#include + +#include "log.h" + +namespace py = pybind11; + +namespace mo2::python { + + void show_deprecation_warning(std::string_view name, std::string_view message, + bool show_once) + { + + // Contains the list of filename / line number for which a deprecation + // warning has already been shown. + static std::set> DeprecatedLines; + + // Find the caller: + auto inspect = py::module_::import("inspect"); + auto current_frame = inspect.attr("currentframe")(); + py::sequence callable_frame = inspect.attr("getouterframes")(current_frame, 2); + auto filename = callable_frame[-1].attr("filename").cast(); + auto function = callable_frame[-1].attr("function").cast(); + auto lineno = callable_frame[-1].attr("lineno").cast(); + + // Only show once if requested: + if (show_once && DeprecatedLines.contains({filename, lineno})) { + return; + } + + // Register the deprecation: + DeprecatedLines.emplace(filename, lineno); + + auto path = relative(std::filesystem::path(filename), + QCoreApplication::applicationDirPath().toStdWString()); + + // Show the message: + if (message.empty()) { + MOBase::log::warn("[deprecated] {} in {} [{}:{}].", name, function, + path.native(), lineno); + } + else { + MOBase::log::warn("[deprecated] {} in {} [{}:{}]: {}", name, function, + path.native(), lineno, message); + } + } + +} // namespace mo2::python diff --git a/src/runner-pybind11/pythonutils.h b/src/mobase/deprecation.h similarity index 72% rename from src/runner-pybind11/pythonutils.h rename to src/mobase/deprecation.h index 8e6cb76..d9a445c 100644 --- a/src/runner-pybind11/pythonutils.h +++ b/src/mobase/deprecation.h @@ -7,19 +7,6 @@ namespace mo2::python { - /** - * @brief Configure Python stdout and stderr to log to MO2. - * - */ - void configure_python_stream(); - - /** - * @brief Configure logging for MO2 python plugin. - * - * @param mobase The mobase module. - */ - void configure_python_logging(pybind11::module_ mobase); - /** * @brief Show a deprecation warning. * diff --git a/src/mobase/mobase.cpp b/src/mobase/mobase.cpp new file mode 100644 index 0000000..cf9d252 --- /dev/null +++ b/src/mobase/mobase.cpp @@ -0,0 +1,269 @@ +#pragma warning(disable : 4100) +#pragma warning(disable : 4996) + +#include +#include + +#include "pybind11_all.h" + +#include "wrappers/pyfiletree.h" +#include "wrappers/wrappers.h" + +// TODO: remove these include (only for testing) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace MOBase; +namespace py = pybind11; + +PYBIND11_MODULE(mobase, m) +{ + using namespace mo2::python; + + py::module_::import("PyQt6.QtCore"); + py::module_::import("PyQt6.QtWidgets"); + + // bindings + // + mo2::python::add_basic_bindings(m); + mo2::python::add_wrapper_bindings(m); + + // game features must be added before plugins + mo2::python::add_game_feature_bindings(m); + + mo2::python::add_plugins_bindings(m); + + // widgets + // + py::module_ widgets( + py::reinterpret_borrow(PyImport_AddModule("mobase.widgets"))); + m.attr("widgets") = widgets; + mo2::python::add_widget_bindings(widgets); + + // functions + // + m.def("getFileVersion", &MOBase::getFileVersion, py::arg("filepath")); + m.def("getProductVersion", &MOBase::getProductVersion, py::arg("executable")); + m.def("getIconForExecutable", &MOBase::iconForExecutable, py::arg("executable")); + + // expose MoVariant - MoVariant is a fake object whose only purpose is to be + // used as a type-hint on the python side (e.g., def foo(x: + // mobase.MoVariant)) + // + // the real MoVariant is defined in the generated stubs, since it's only + // relevant when doing type-checking, but this needs to be defined, + // otherwise MoVariant is not found when actually running plugins through + // MO2, making them crash + m.attr("MoVariant") = py::none(); + + // private stuff for debugging/test + py::module_ moprivate( + py::reinterpret_borrow(PyImport_AddModule("mobase.private"))); + m.attr("private") = moprivate; + + // expose a function to create a particular tree, only for debugging + // purpose, not in mobase. + mo2::python::add_make_tree_function(moprivate); + moprivate.def("extract_plugins", &mo2::python::extract_plugins); + + // == BEGIN TESTS == + + m.def("testPlugin", [](py::object pyobj) { + py::scoped_ostream_redirect s{std::cout}; + std::cout << "type: " << pyobj.get_type().attr("__name__").cast() + << "\n"; + auto qobjects = mo2::python::extract_plugins(pyobj); + std::cout << " found " << qobjects.size() << " plugins\n"; + + // cast as IPlugin + for (int i = 0; i < qobjects.size(); ++i) { + IPlugin* plugin = qobject_cast(qobjects[i]); + std::cout << fmt::format(" plugin {}: {} -> {}\n", i, (void*)qobjects[i], + (void*)plugin); + std::cout << fmt::format(" name: {}\n", plugin->name().toStdString()); + // std::cout << fmt::format( + // " installer?: {}\n", + // (void*)qobject_cast(qobjects[i])); + // std::cout << fmt::format( + // " installer simple?: {}\n", + // (void*)qobject_cast(qobjects[i])); + + if (IPluginGame* game = dynamic_cast(plugin)) { + auto saves = game->listSaves(QDir()); + std::cout << " saves: " << saves.size() << "\n"; + for (auto& save : saves) { + std::cout << " save: " << (void*)save.get() << ", " + << py::reinterpret_borrow(py::cast(save)) + << ", " << save->getFilepath().toStdString() << "\n"; + } + } + } + }); + + py::detail::type_caster t1; + py::detail::type_caster t2; + + m.def("testQStringList", [](QStringList const& list) { + QStringList res = list; + for (QString& value : res) { + value = value + "_CPP"; + } + return res; + }); + + m.def("testGuessedString", [](GuessedValue const& value) { + return std::make_tuple(value.operator const QString&(), value.variants()); + }); + + m.def("testQStringList", [](QStringList const& list) { + QStringList res = list; + for (QString& value : res) { + value = value + "_CPP"; + } + return res; + }); + + m.def("testQMap1", [](QMap const& map) { + QMap res; + for (auto it = map.begin(); it != map.end(); ++it) { + res[it.key()] = it.value().size(); + } + return res; + }); + + m.def("testQMap2", [](QMap const& map) { + QMap res; + for (auto it = map.begin(); it != map.end(); ++it) { + res[it.key()] = QString::number(it.value()); + } + return res; + }); + + m.def("testDateTime1", []() { + return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); + }); + + m.def("testDateTime2", [](QDateTime const& datetime) { + return datetime.toString(); + }); + + m.def("testEnum0", []() { + return Qt::GlobalColor::darkRed; + }); + + m.def("testEnum", [](Qt::GlobalColor color) { + return py::make_tuple(color, QMessageBox::Icon::Information); + }); + + m.def("testPixmap", [](QPixmap const& pixmap) { + return pixmap.size(); + }); + + m.def("createSaveGame", []() -> ISaveGame* { + class SaveGame : public ISaveGame { + QString getFilepath() const override { return "filepath"; } + QDateTime getCreationTime() const override + { + return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); + } + QString getName() const override { return "name"; } + QString getSaveGroupIdentifier() const override { return "group"; } + QStringList allFiles() const override { return {"file1", "file2"}; } + }; + return new SaveGame(); + }); + + m.def("testSaveGameWidget", [](ISaveGameInfoWidget* widget) { + class SaveGame : public ISaveGame { + QString getFilepath() const override { return "filepath-c++"; } + QDateTime getCreationTime() const override + { + return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); + } + QString getName() const override { return "name"; } + QString getSaveGroupIdentifier() const override { return "group"; } + QStringList allFiles() const override { return {"file1", "file2"}; } + }; + static SaveGame s; + widget->setSave(s); + }); + + m.def("testSaveGameRef", [](const ISaveGame& game) { + std::cout << "getFilepath(): " << game.getFilepath().toStdString() << "\n"; + std::cout << "getCreationTime(): " + << game.getCreationTime().toString().toStdString() << "\n"; + std::cout << "getName(): " << game.getName().toStdString() << "\n"; + std::cout << "getSaveGroupIdentifier(): " + << game.getSaveGroupIdentifier().toStdString() << "\n"; + std::cout << "allFiles(): [ " << game.allFiles().join(" ").toStdString() + << " ]\n"; + }); + + m.def("testSaveGamePtr", [](const ISaveGame* game) { + std::cout << "getFilepath(): " << game->getFilepath().toStdString() << "\n"; + std::cout << "getCreationTime(): " + << game->getCreationTime().toString().toStdString() << "\n"; + std::cout << "getName(): " << game->getName().toStdString() << "\n"; + std::cout << "getSaveGroupIdentifier(): " + << game->getSaveGroupIdentifier().toStdString() << "\n"; + std::cout << "allFiles(): [ " << game->allFiles().join(" ").toStdString() + << " ]\n"; + }); + + m.def("testWidget1", [](QWidget* w) { + if (w) { + std::cout << "background role: " << w->backgroundRole() << "\n"; + } + else { + std::cout << "null widget\n"; + } + }); + + m.def("testWidget2", []() { + QWidget* w = new QWidget(); + w->setBackgroundRole(QPalette::ColorRole::HighlightedText); + return w; + }); + + m.def("testFlags1", [](IPluginList::PluginStates states) { + std::vector res; + if (states.testFlag(IPluginList::STATE_MISSING)) { + res.push_back("missing"); + } + if (states.testFlag(IPluginList::STATE_INACTIVE)) { + res.push_back("inactive"); + } + if (states.testFlag(IPluginList::STATE_ACTIVE)) { + res.push_back("active"); + } + return res; + }); + + m.def("testFlags2", [](QStringList const& states) { + IPluginList::PluginStates res; + if (states.contains("missing")) { + res |= IPluginList::STATE_MISSING; + } + if (states.contains("inactive")) { + res |= IPluginList::STATE_INACTIVE; + } + if (states.contains("active")) { + res |= IPluginList::STATE_ACTIVE; + } + return res; + }); +} diff --git a/src/runner-pybind11/pybind11_all.h b/src/mobase/pybind11_all.h similarity index 100% rename from src/runner-pybind11/pybind11_all.h rename to src/mobase/pybind11_all.h diff --git a/src/runner-pybind11/pybind11_utils/functional.h b/src/mobase/pybind11_utils/functional.h similarity index 100% rename from src/runner-pybind11/pybind11_utils/functional.h rename to src/mobase/pybind11_utils/functional.h diff --git a/src/runner-pybind11/pybind11_utils/shared_cpp_owner.h b/src/mobase/pybind11_utils/shared_cpp_owner.h similarity index 100% rename from src/runner-pybind11/pybind11_utils/shared_cpp_owner.h rename to src/mobase/pybind11_utils/shared_cpp_owner.h diff --git a/src/runner-pybind11/wrappers/basic_classes.cpp b/src/mobase/wrappers/basic_classes.cpp similarity index 99% rename from src/runner-pybind11/wrappers/basic_classes.cpp rename to src/mobase/wrappers/basic_classes.cpp index bdb0974..6016988 100644 --- a/src/runner-pybind11/wrappers/basic_classes.cpp +++ b/src/mobase/wrappers/basic_classes.cpp @@ -20,7 +20,7 @@ #include #include -#include "../pythonutils.h" +#include "../deprecation.h" #include "pyfiletree.h" using namespace MOBase; diff --git a/src/runner-pybind11/wrappers/game_features.cpp b/src/mobase/wrappers/game_features.cpp similarity index 100% rename from src/runner-pybind11/wrappers/game_features.cpp rename to src/mobase/wrappers/game_features.cpp diff --git a/src/runner-pybind11/wrappers/pyfiletree.cpp b/src/mobase/wrappers/pyfiletree.cpp similarity index 98% rename from src/runner-pybind11/wrappers/pyfiletree.cpp rename to src/mobase/wrappers/pyfiletree.cpp index c77a00b..8a08039 100644 --- a/src/runner-pybind11/wrappers/pyfiletree.cpp +++ b/src/mobase/wrappers/pyfiletree.cpp @@ -3,12 +3,7 @@ #include #include -#include -#include -#include -#include - -#include "../pybind11_qt/pybind11_qt.h" +#include "../pybind11_all.h" #include #include diff --git a/src/runner-pybind11/wrappers/pyfiletree.h b/src/mobase/wrappers/pyfiletree.h similarity index 100% rename from src/runner-pybind11/wrappers/pyfiletree.h rename to src/mobase/wrappers/pyfiletree.h diff --git a/src/runner-pybind11/wrappers/pyplugins.cpp b/src/mobase/wrappers/pyplugins.cpp similarity index 100% rename from src/runner-pybind11/wrappers/pyplugins.cpp rename to src/mobase/wrappers/pyplugins.cpp diff --git a/src/runner-pybind11/wrappers/pyplugins.h b/src/mobase/wrappers/pyplugins.h similarity index 100% rename from src/runner-pybind11/wrappers/pyplugins.h rename to src/mobase/wrappers/pyplugins.h diff --git a/src/runner-pybind11/wrappers/widgets.cpp b/src/mobase/wrappers/widgets.cpp similarity index 100% rename from src/runner-pybind11/wrappers/widgets.cpp rename to src/mobase/wrappers/widgets.cpp diff --git a/src/runner-pybind11/wrappers/wrappers.cpp b/src/mobase/wrappers/wrappers.cpp similarity index 100% rename from src/runner-pybind11/wrappers/wrappers.cpp rename to src/mobase/wrappers/wrappers.cpp diff --git a/src/runner-pybind11/wrappers/wrappers.h b/src/mobase/wrappers/wrappers.h similarity index 100% rename from src/runner-pybind11/wrappers/wrappers.h rename to src/mobase/wrappers/wrappers.h diff --git a/src/proxy/plugin_python_en.ts b/src/proxy/plugin_python_en.ts index f47d583..142f2b9 100644 --- a/src/proxy/plugin_python_en.ts +++ b/src/proxy/plugin_python_en.ts @@ -4,58 +4,58 @@ ProxyPython - + Python Proxy - + Proxy Plugin to allow plugins written in python to be loaded - + ModOrganizer path contains a semicolon - + Python DLL not found - + Invalid Python DLL - + Initializing Python failed - - + + invalid problem key %1 - + The path to Mod Organizer (%1) contains a semicolon. <br>While this is legal on NTFS drives, many softwares do not handle it correctly.<br>Unfortunately MO depends on libraries that seem to fall into that group.<br>As a result the python plugin cannot be loaded, and the only solution we canoffer is to remove the semicolon or move MO to a path without a semicolon. - + The Python plugin DLL was not found, maybe your antivirus deleted it. Re-installing MO2 might fix the problem. - + The Python plugin DLL is invalid, maybe your antivirus is blocking it. Re-installing MO2 and adding exclusions for it to your AV might fix the problem. - + The initialization of the Python plugin DLL failed, unfortunately without any details. @@ -63,12 +63,7 @@ QObject - - An unexpected C++ exception was thrown in python code. - - - - + An unknown exception was thrown in python code. diff --git a/src/proxy/proxypython.cpp b/src/proxy/proxypython.cpp index 354b9d8..afb6b38 100644 --- a/src/proxy/proxypython.cpp +++ b/src/proxy/proxypython.cpp @@ -134,18 +134,22 @@ bool ProxyPython::init(IOrganizer* moInfo) if (m_MOInfo) { m_MOInfo->setPersistent(name(), "tryInit", true); - - m_Runner = std::unique_ptr{createPythonRunner()}; } - if (!m_Runner->initialize(pluginFolder / "libs")) { - m_LoadFailure = FailureType::INITIALIZATION; + m_Runner = std::unique_ptr{createPythonRunner()}; + + if (m_Runner) { + m_Runner->initialize(pluginFolder / "libs"); } if (m_MOInfo) { m_MOInfo->setPersistent(name(), "tryInit", false); } + if (!m_Runner || !m_Runner->isInitialized()) { + m_LoadFailure = FailureType::INITIALIZATION; + } + // reset DLL directory SetDllDirectoryW(NULL); diff --git a/src/pybind11-qt/CMakeLists.txt b/src/pybind11-qt/CMakeLists.txt new file mode 100644 index 0000000..6b39049 --- /dev/null +++ b/src/pybind11-qt/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.16) + +add_library(pybind11-qt STATIC) +mo2_configure_library(pybind11-qt + SOURCE_TREE + WARNINGS OFF + AUTOMOC OFF + TRANSLATIONS OFF + PRIVATE_DEPENDS Qt::Core Qt::Widgets +) +target_link_libraries(pybind11-qt PUBLIC pybind11::pybind11) +target_include_directories(pybind11-qt + PUBLIC ${PYTHON_ROOT}/Include ${CMAKE_CURRENT_SOURCE_DIR}/include) + +# this is kind of broken but it only works with this... +target_compile_definitions(pybind11-qt PUBLIC QT_NO_KEYWORDS) + +add_library(pybind11::qt ALIAS pybind11-qt) diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_enum.h b/src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_enum.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_enum.h rename to src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_enum.h diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_qlist.h b/src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_qlist.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_qlist.h rename to src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_qlist.h diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_qmap.h b/src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_qmap.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_qmap.h rename to src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_qmap.h diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_sip.h b/src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_sip.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_sip.h rename to src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_sip.h diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_utils.h b/src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_utils.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_utils.h rename to src/pybind11-qt/include/pybind11_qt/details/pybind11_qt_utils.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_basic.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt_basic.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_basic.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt_basic.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_containers.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt_containers.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_containers.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt_containers.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_enums.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt_enums.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_enums.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt_enums.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_holder.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt_holder.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_holder.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt_holder.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_objects.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt_objects.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_objects.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt_objects.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_qflags.h b/src/pybind11-qt/include/pybind11_qt/pybind11_qt_qflags.h similarity index 100% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_qflags.h rename to src/pybind11-qt/include/pybind11_qt/pybind11_qt_qflags.h diff --git a/src/runner-pybind11/pybind11_qt/pybind11_qt_basic.cpp b/src/pybind11-qt/pybind11_qt_basic.cpp similarity index 98% rename from src/runner-pybind11/pybind11_qt/pybind11_qt_basic.cpp rename to src/pybind11-qt/pybind11_qt_basic.cpp index 7d12d25..488f1c2 100644 --- a/src/runner-pybind11/pybind11_qt/pybind11_qt_basic.cpp +++ b/src/pybind11-qt/pybind11_qt_basic.cpp @@ -1,10 +1,10 @@ -#include "pybind11_qt_basic.h" +#include "pybind11_qt/pybind11_qt_basic.h" #include #include -#include "details/pybind11_qt_utils.h" +#include "pybind11_qt/details/pybind11_qt_utils.h" namespace pybind11::detail { diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_sip.cpp b/src/pybind11-qt/pybind11_qt_sip.cpp similarity index 98% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_sip.cpp rename to src/pybind11-qt/pybind11_qt_sip.cpp index edc48db..4ef8fb3 100644 --- a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_sip.cpp +++ b/src/pybind11-qt/pybind11_qt_sip.cpp @@ -1,4 +1,4 @@ -#include "pybind11_qt_sip.h" +#include "pybind11_qt/details/pybind11_qt_sip.h" #include diff --git a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_utils.cpp b/src/pybind11-qt/pybind11_qt_utils.cpp similarity index 85% rename from src/runner-pybind11/pybind11_qt/details/pybind11_qt_utils.cpp rename to src/pybind11-qt/pybind11_qt_utils.cpp index 4cd98d0..249bed5 100644 --- a/src/runner-pybind11/pybind11_qt/details/pybind11_qt_utils.cpp +++ b/src/pybind11-qt/pybind11_qt_utils.cpp @@ -1,4 +1,4 @@ -#include "pybind11_qt_utils.h" +#include "pybind11_qt/details/pybind11_qt_utils.h" namespace pybind11::detail::qt { diff --git a/src/runner-pybind11/pythonrunner.cpp b/src/runner-pybind11/pythonrunner.cpp deleted file mode 100644 index 95b0786..0000000 --- a/src/runner-pybind11/pythonrunner.cpp +++ /dev/null @@ -1,565 +0,0 @@ -#include "pythonrunner.h" - -#pragma warning(disable : 4100) -#pragma warning(disable : 4996) - -#include -#include - -#include - -#include -#include -#include -#include - -#include "pybind11_all.h" -#include - -#include -#include - -#include "error.h" -#include "pythonutils.h" -#include "wrappers/pyfiletree.h" -#include "wrappers/wrappers.h" - -// TODO: remove these include (only for testing) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace MOBase; -namespace py = pybind11; - -PYBIND11_MODULE(mobase, m) -{ - using namespace mo2::python; - - py::module_::import("PyQt6.QtCore"); - py::module_::import("PyQt6.QtWidgets"); - - // bindings - // - mo2::python::add_basic_bindings(m); - mo2::python::add_wrapper_bindings(m); - - // game features must be added before plugins - mo2::python::add_game_feature_bindings(m); - - mo2::python::add_plugins_bindings(m); - - // widgets - // - py::module_ widgets( - py::reinterpret_borrow(PyImport_AddModule("mobase.widgets"))); - m.attr("widgets") = widgets; - mo2::python::add_widget_bindings(widgets); - - // functions - // - m.def("getFileVersion", &MOBase::getFileVersion, py::arg("filepath")); - m.def("getProductVersion", &MOBase::getProductVersion, py::arg("executable")); - m.def("getIconForExecutable", &MOBase::iconForExecutable, py::arg("executable")); - - // expose MoVariant - MoVariant is a fake object whose only purpose is to be - // used as a type-hint on the python side (e.g., def foo(x: - // mobase.MoVariant)) - // - // the real MoVariant is defined in the generated stubs, since it's only - // relevant when doing type-checking, but this needs to be defined, - // otherwise MoVariant is not found when actually running plugins through - // MO2, making them crash - m.attr("MoVariant") = py::none(); - - // == BEGIN TESTS == - - m.def("testPlugin", [](py::object pyobj) { - py::scoped_ostream_redirect s{std::cout}; - std::cout << "type: " << pyobj.get_type().attr("__name__").cast() - << "\n"; - auto qobjects = mo2::python::extract_plugins(pyobj); - std::cout << " found " << qobjects.size() << " plugins\n"; - - // cast as IPlugin - for (int i = 0; i < qobjects.size(); ++i) { - IPlugin* plugin = qobject_cast(qobjects[i]); - std::cout << fmt::format(" plugin {}: {} -> {}\n", i, (void*)qobjects[i], - (void*)plugin); - std::cout << fmt::format(" name: {}\n", plugin->name().toStdString()); - // std::cout << fmt::format( - // " installer?: {}\n", - // (void*)qobject_cast(qobjects[i])); - // std::cout << fmt::format( - // " installer simple?: {}\n", - // (void*)qobject_cast(qobjects[i])); - - if (IPluginGame* game = dynamic_cast(plugin)) { - auto saves = game->listSaves(QDir()); - std::cout << " saves: " << saves.size() << "\n"; - for (auto& save : saves) { - std::cout << " save: " << (void*)save.get() << ", " - << py::reinterpret_borrow(py::cast(save)) - << ", " << save->getFilepath().toStdString() << "\n"; - } - } - } - }); - - py::detail::type_caster t1; - py::detail::type_caster t2; - - m.def("testQStringList", [](QStringList const& list) { - QStringList res = list; - for (QString& value : res) { - value = value + "_CPP"; - } - return res; - }); - - m.def("testGuessedString", [](GuessedValue const& value) { - return std::make_tuple(value.operator const QString&(), value.variants()); - }); - - m.def("testQStringList", [](QStringList const& list) { - QStringList res = list; - for (QString& value : res) { - value = value + "_CPP"; - } - return res; - }); - - m.def("testQMap1", [](QMap const& map) { - QMap res; - for (auto it = map.begin(); it != map.end(); ++it) { - res[it.key()] = it.value().size(); - } - return res; - }); - - m.def("testQMap2", [](QMap const& map) { - QMap res; - for (auto it = map.begin(); it != map.end(); ++it) { - res[it.key()] = QString::number(it.value()); - } - return res; - }); - - m.def("testDateTime1", []() { - return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); - }); - - m.def("testDateTime2", [](QDateTime const& datetime) { - return datetime.toString(); - }); - - m.def("testEnum0", []() { - return Qt::GlobalColor::darkRed; - }); - - m.def("testEnum", [](Qt::GlobalColor color) { - MOBase::log::error("In C++: {}", color); - return py::make_tuple(color, QMessageBox::Icon::Information); - }); - - m.def("testPixmap", [](QPixmap const& pixmap) { - return pixmap.size(); - }); - - m.def("createSaveGame", []() -> ISaveGame* { - class SaveGame : public ISaveGame { - QString getFilepath() const override { return "filepath"; } - QDateTime getCreationTime() const override - { - return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); - } - QString getName() const override { return "name"; } - QString getSaveGroupIdentifier() const override { return "group"; } - QStringList allFiles() const override { return {"file1", "file2"}; } - }; - return new SaveGame(); - }); - - m.def("testSaveGameWidget", [](ISaveGameInfoWidget* widget) { - class SaveGame : public ISaveGame { - QString getFilepath() const override { return "filepath-c++"; } - QDateTime getCreationTime() const override - { - return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); - } - QString getName() const override { return "name"; } - QString getSaveGroupIdentifier() const override { return "group"; } - QStringList allFiles() const override { return {"file1", "file2"}; } - }; - static SaveGame s; - widget->setSave(s); - }); - - m.def("testSaveGameRef", [](const ISaveGame& game) { - std::cout << "getFilepath(): " << game.getFilepath().toStdString() << "\n"; - std::cout << "getCreationTime(): " - << game.getCreationTime().toString().toStdString() << "\n"; - std::cout << "getName(): " << game.getName().toStdString() << "\n"; - std::cout << "getSaveGroupIdentifier(): " - << game.getSaveGroupIdentifier().toStdString() << "\n"; - std::cout << "allFiles(): [ " << game.allFiles().join(" ").toStdString() - << " ]\n"; - }); - - m.def("testSaveGamePtr", [](const ISaveGame* game) { - std::cout << "getFilepath(): " << game->getFilepath().toStdString() << "\n"; - std::cout << "getCreationTime(): " - << game->getCreationTime().toString().toStdString() << "\n"; - std::cout << "getName(): " << game->getName().toStdString() << "\n"; - std::cout << "getSaveGroupIdentifier(): " - << game->getSaveGroupIdentifier().toStdString() << "\n"; - std::cout << "allFiles(): [ " << game->allFiles().join(" ").toStdString() - << " ]\n"; - }); - - m.def("testWidget1", [](QWidget* w) { - if (w) { - std::cout << "background role: " << w->backgroundRole() << "\n"; - } - else { - std::cout << "null widget\n"; - } - }); - - m.def("testWidget2", []() { - QWidget* w = new QWidget(); - w->setBackgroundRole(QPalette::ColorRole::HighlightedText); - return w; - }); - - m.def("testFlags1", [](IPluginList::PluginStates states) { - std::vector res; - if (states.testFlag(IPluginList::STATE_MISSING)) { - res.push_back("missing"); - } - if (states.testFlag(IPluginList::STATE_INACTIVE)) { - res.push_back("inactive"); - } - if (states.testFlag(IPluginList::STATE_ACTIVE)) { - res.push_back("active"); - } - return res; - }); - - m.def("testFlags2", [](QStringList const& states) { - IPluginList::PluginStates res; - if (states.contains("missing")) { - res |= IPluginList::STATE_MISSING; - } - if (states.contains("inactive")) { - res |= IPluginList::STATE_INACTIVE; - } - if (states.contains("active")) { - res |= IPluginList::STATE_ACTIVE; - } - return res; - }); -} - -/** - * - */ -class PythonRunner : public IPythonRunner { - -public: - PythonRunner() = default; - ~PythonRunner() = default; - - QList load(const QString& identifier) override; - void unload(const QString& identifier) override; - - bool initialize(std::filesystem::path const& libpath) override; - bool isInitialized() const override; - -private: - /** - * @brief Ensure that the given folder is in sys.path. - */ - void ensureFolderInPath(QString folder); - -private: - // for each "identifier" (python file or python module folder), contains the - // list of python objects - this does not keep the objects alive, it simply used to - // unload plugins - std::unordered_map> m_PythonObjects; -}; - -IPythonRunner* CreatePythonRunner() -{ - return new PythonRunner(); -} - -PYBIND11_MODULE(moprivate, m) -{ - // expose a function to create a particular tree, only for debugging - // purpose, not in mobase. - mo2::python::add_make_tree_function(m); -} - -bool PythonRunner::initialize(std::filesystem::path const& libpath) -{ - // we only initialize Python once for the whole lifetime of the program, even if MO2 - // is restarted and the proxy or PythonRunner objects are deleted and recreated, - // Python is not re-initialized - // - // in an ideal world, we would initialize Python here (or in the constructor) and - // then finalize it in the destructor - // - // unfortunately, many library, including PyQt6, do not handle properly - // re-initializing the Python interpreter, so we cannot do that and we keep the - // interpreter alive - // - - if (Py_IsInitialized()) { - return true; - } - - try { - static const char* argv0 = "ModOrganizer.exe"; - - // initialize the core Path of Python, this must be done before initialization - // - const QStringList paths{ - QFileInfo(libpath / "pythoncore.zip").absoluteFilePath(), - QFileInfo(libpath).absoluteFilePath(), IOrganizer::getPluginDataPath()}; - Py_SetPath(paths.join(';').toStdWString().c_str()); - - if (PyImport_AppendInittab("mobase", &PyInit_mobase) == -1) { - MOBase::log::error("failed to init python: failed to append mobase."); - return false; - } - - if (PyImport_AppendInittab("moprivate", &PyInit_moprivate) == -1) { - MOBase::log::error("failed to init python: failed to append moprivate."); - return false; - } - - Py_OptimizeFlag = 2; - Py_NoSiteFlag = 1; - - py::initialize_interpreter(false, 1, &argv0); - - if (!Py_IsInitialized()) { - MOBase::log::error( - "failed to init python: failed to initialize interpreter."); - - if (PyGILState_Check()) { - PyEval_SaveThread(); - } - - return false; - } - - py::module_ mainModule = py::module_::import("__main__"); - py::object mainNamespace = mainModule.attr("__dict__"); - mainNamespace["sys"] = py::module_::import("sys"); - mainNamespace["moprivate"] = py::module_::import("moprivate"); - mainNamespace["mobase"] = py::module_::import("mobase"); - - mo2::python::configure_python_stream(); - mo2::python::configure_python_logging(mainNamespace["mobase"]); - - // we need to release the GIL here - which is what this does - // - // when Python is initialized, the GIl is acquired, and if it is not release, - // trying to acquire it on a different thread will deadlock - PyEval_SaveThread(); - - return true; - } - catch (const py::error_already_set& ex) { - MOBase::log::error("failed to init python: {}", ex.what()); - return false; - } -} - -void PythonRunner::ensureFolderInPath(QString folder) -{ - py::module_ sys = py::module_::import("sys"); - py::list sysPath = sys.attr("path"); - - // Converting to QStringList for Qt::CaseInsensitive and because .index() - // raise an exception: - const QStringList currentPath = sysPath.cast(); - if (!currentPath.contains(folder, Qt::CaseInsensitive)) { - sysPath.insert(0, folder); - } -} - -QList PythonRunner::load(const QString& identifier) -{ - py::gil_scoped_acquire lock; - - // `pluginName` can either be a python file (single-file plugin or a folder - // (whole module). - // - // For whole module, we simply add the parent folder to path, then we load - // the module with a simple py::import, and we retrieve the associated - // __dict__ from which we extract either createPlugin or createPlugins. - // - // For single file, we need to use py::eval_file, and we will use the - // context (global variables) from __main__ (already contains mobase, and - // other required module). Since the context is shared between called of - // `instantiate`, we need to make sure to remove createPlugin(s) from - // previous call. - try { - - // dictionary that will contain createPlugin() or createPlugins(). - py::dict moduleDict; - - if (identifier.endsWith(".py")) { - py::object mainModule = py::module_::import("__main__"); - - // make a copy, otherwise we might end up calling the createPlugin() or - // createPlugins() function multiple time - py::dict moduleNamespace = mainModule.attr("__dict__").attr("copy")(); - - std::string temp = ToString(identifier); - py::eval_file(temp, moduleNamespace).is_none(); - moduleDict = moduleNamespace; - } - else { - // Retrieve the module name: - QStringList parts = identifier.split("/"); - std::string moduleName = ToString(parts.takeLast()); - ensureFolderInPath(parts.join("/")); - - // check if the module is already loaded - py::dict modules = py::module_::import("sys").attr("modules"); - if (modules.contains(moduleName)) { - py::module_ prev = modules[py::str(moduleName)]; - py::module_(prev).reload(); - moduleDict = prev.attr("__dict__"); - } - else { - moduleDict = py::module_::import(moduleName.c_str()).attr("__dict__"); - } - } - - if (py::len(moduleDict) == 0) { - MOBase::log::error("No plugins found in {}.", identifier); - return {}; - } - - // Create the plugins: - std::vector plugins; - - if (moduleDict.contains("createPlugin")) { - plugins.push_back(moduleDict["createPlugin"]()); - } - else if (moduleDict.contains("createPlugins")) { - py::object pyPlugins = moduleDict["createPlugins"](); - if (!py::isinstance(pyPlugins)) { - MOBase::log::error("Plugin {}: createPlugins must return a sequence.", - identifier); - } - else { - py::sequence pyList(pyPlugins); - size_t nPlugins = pyList.size(); - for (size_t i = 0; i < nPlugins; ++i) { - plugins.push_back(pyList[i]); - } - } - } - else { - MOBase::log::error("Plugin {}: missing a createPlugin(s) function.", - identifier); - } - - // If we have no plugins, there was an issue, and we already logged the - // problem: - if (plugins.empty()) { - return QList(); - } - - QList allInterfaceList; - - for (py::object pluginObj : plugins) { - - // save to be able to unload it - m_PythonObjects[identifier].push_back(pluginObj); - - QList interfaceList = mo2::python::extract_plugins(pluginObj); - - if (interfaceList.isEmpty()) { - MOBase::log::error("Plugin {}: no plugin interface implemented.", - identifier); - } - - // Append the plugins to the main list: - allInterfaceList.append(interfaceList); - } - - return allInterfaceList; - } - catch (const py::error_already_set& ex) { - MOBase::log::error("Failed to import plugin from {}.", identifier); - throw pyexcept::PythonError(ex); - } -} - -void PythonRunner::unload(const QString& identifier) -{ - auto it = m_PythonObjects.find(identifier); - if (it != m_PythonObjects.end()) { - - py::gil_scoped_acquire lock; - - if (!identifier.endsWith(".py")) { - - // At this point, the identifier is the full path to the module. - QDir folder(identifier); - - // We want to "unload" (remove from sys.modules) modules that come - // from this plugin (whose __path__ points under this module, - // including the module of the plugin itself). - py::object sys = py::module_::import("sys"); - py::dict modules = sys.attr("modules"); - py::list keys = modules.attr("keys")(); - for (std::size_t i = 0; i < py::len(keys); ++i) { - py::object mod = modules[keys[i]]; - if (PyObject_HasAttrString(mod.ptr(), "__path__")) { - QString mpath = mod.attr("__path__")[0].cast(); - - if (!folder.relativeFilePath(mpath).startsWith("..")) { - // If the path is under identifier, we need to unload - // it. - log::debug("Unloading module {} from {} for {}.", - keys[i].cast(), mpath, identifier); - - PyDict_DelItem(modules.ptr(), keys[i].ptr()); - } - } - } - } - - // Boost.Python does not handle cyclic garbace collection, so we need to - // release everything hold by the objects before deleting the objects - // themselves (done when erasing from m_PythonObjects). - for (auto& obj : it->second) { - obj.attr("__dict__").attr("clear")(); - } - - log::debug("Deleting {} python objects for {}.", it->second.size(), identifier); - m_PythonObjects.erase(it); - } -} - -bool PythonRunner::isInitialized() const -{ - return Py_IsInitialized() != 0; -} diff --git a/src/runner-pybind11/CMakeLists.txt b/src/runner/CMakeLists.txt similarity index 87% rename from src/runner-pybind11/CMakeLists.txt rename to src/runner/CMakeLists.txt index 3800ff1..52429a8 100644 --- a/src/runner-pybind11/CMakeLists.txt +++ b/src/runner/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.16) -pybind11_add_module(pythonrunner SHARED) +add_library(pythonrunner SHARED) mo2_configure_library(pythonrunner SOURCE_TREE WARNINGS OFF @@ -8,12 +8,7 @@ mo2_configure_library(pythonrunner TRANSLATIONS OFF PRIVATE_DEPENDS uibase Qt::Core ) -set_target_properties(pythonrunner - PROPERTIES - OUTPUT_NAME "pythonrunner" - PREFIX "" - SUFFIX ".dll") -target_link_libraries(pythonrunner PRIVATE pybind11::embed) +target_link_libraries(pythonrunner PRIVATE pybind11::embed pybind11::qt) target_include_directories(pythonrunner PRIVATE ${PYTHON_ROOT}/Include) # this is kind of broken but it only works with this... diff --git a/src/runner-pybind11/error.h b/src/runner/error.h similarity index 100% rename from src/runner-pybind11/error.h rename to src/runner/error.h diff --git a/src/runner/pythonrunner.cpp b/src/runner/pythonrunner.cpp new file mode 100644 index 0000000..74e3cde --- /dev/null +++ b/src/runner/pythonrunner.cpp @@ -0,0 +1,308 @@ +#include "pythonrunner.h" + +#pragma warning(disable : 4100) +#pragma warning(disable : 4996) + +#include + +#include +#include +#include + +#include "pybind11_qt/pybind11_qt.h" +#include +#include + +#include +#include + +#include "error.h" +#include "pythonutils.h" + +using namespace MOBase; +namespace py = pybind11; + +/** + * + */ +class PythonRunner : public IPythonRunner { + +public: + PythonRunner() = default; + ~PythonRunner() = default; + + QList load(const QString& identifier) override; + void unload(const QString& identifier) override; + + bool initialize(std::filesystem::path const& libpath) override; + bool isInitialized() const override; + +private: + /** + * @brief Ensure that the given folder is in sys.path. + */ + void ensureFolderInPath(QString folder); + +private: + // for each "identifier" (python file or python module folder), contains the + // list of python objects - this does not keep the objects alive, it simply used to + // unload plugins + std::unordered_map> m_PythonObjects; + + // handle to extract_plugins() from mobase.private + std::function(py::object)> m_extract_plugins; +}; + +IPythonRunner* CreatePythonRunner() +{ + return new PythonRunner(); +} + +bool PythonRunner::initialize(std::filesystem::path const& libpath) +{ + // we only initialize Python once for the whole lifetime of the program, even if MO2 + // is restarted and the proxy or PythonRunner objects are deleted and recreated, + // Python is not re-initialized + // + // in an ideal world, we would initialize Python here (or in the constructor) and + // then finalize it in the destructor + // + // unfortunately, many library, including PyQt6, do not handle properly + // re-initializing the Python interpreter, so we cannot do that and we keep the + // interpreter alive + // + + if (Py_IsInitialized()) { + return true; + } + + try { + static const char* argv0 = "ModOrganizer.exe"; + + // initialize the core Path of Python, this must be done before initialization + // + const QStringList paths{ + QFileInfo(libpath / "pythoncore.zip").absoluteFilePath(), + QFileInfo(libpath).absoluteFilePath(), IOrganizer::getPluginDataPath()}; + Py_SetPath(paths.join(';').toStdWString().c_str()); + + Py_OptimizeFlag = 2; + Py_NoSiteFlag = 1; + + py::initialize_interpreter(false, 1, &argv0); + + if (!Py_IsInitialized()) { + MOBase::log::error( + "failed to init python: failed to initialize interpreter."); + + if (PyGILState_Check()) { + PyEval_SaveThread(); + } + + return false; + } + + py::module_ mainModule = py::module_::import("__main__"); + py::object mainNamespace = mainModule.attr("__dict__"); + mainNamespace["sys"] = py::module_::import("sys"); + mainNamespace["mobase"] = py::module_::import("mobase"); + + mo2::python::configure_python_stream(); + mo2::python::configure_python_logging(mainNamespace["mobase"]); + + m_extract_plugins = py::module_::import("mobase.private") + .attr("extract_plugins") + .cast(); + + // we need to release the GIL here - which is what this does + // + // when Python is initialized, the GIl is acquired, and if it is not release, + // trying to acquire it on a different thread will deadlock + PyEval_SaveThread(); + + return true; + } + catch (const py::error_already_set& ex) { + MOBase::log::error("failed to init python: {}", ex.what()); + return false; + } +} + +void PythonRunner::ensureFolderInPath(QString folder) +{ + py::module_ sys = py::module_::import("sys"); + py::list sysPath = sys.attr("path"); + + // Converting to QStringList for Qt::CaseInsensitive and because .index() + // raise an exception: + const QStringList currentPath = sysPath.cast(); + if (!currentPath.contains(folder, Qt::CaseInsensitive)) { + sysPath.insert(0, folder); + } +} + +QList PythonRunner::load(const QString& identifier) +{ + py::gil_scoped_acquire lock; + + // `pluginName` can either be a python file (single-file plugin or a folder + // (whole module). + // + // For whole module, we simply add the parent folder to path, then we load + // the module with a simple py::import, and we retrieve the associated + // __dict__ from which we extract either createPlugin or createPlugins. + // + // For single file, we need to use py::eval_file, and we will use the + // context (global variables) from __main__ (already contains mobase, and + // other required module). Since the context is shared between called of + // `instantiate`, we need to make sure to remove createPlugin(s) from + // previous call. + try { + + // dictionary that will contain createPlugin() or createPlugins(). + py::dict moduleDict; + + if (identifier.endsWith(".py")) { + py::object mainModule = py::module_::import("__main__"); + + // make a copy, otherwise we might end up calling the createPlugin() or + // createPlugins() function multiple time + py::dict moduleNamespace = mainModule.attr("__dict__").attr("copy")(); + + std::string temp = ToString(identifier); + py::eval_file(temp, moduleNamespace).is_none(); + moduleDict = moduleNamespace; + } + else { + // Retrieve the module name: + QStringList parts = identifier.split("/"); + std::string moduleName = ToString(parts.takeLast()); + ensureFolderInPath(parts.join("/")); + + // check if the module is already loaded + py::dict modules = py::module_::import("sys").attr("modules"); + if (modules.contains(moduleName)) { + py::module_ prev = modules[py::str(moduleName)]; + py::module_(prev).reload(); + moduleDict = prev.attr("__dict__"); + } + else { + moduleDict = py::module_::import(moduleName.c_str()).attr("__dict__"); + } + } + + if (py::len(moduleDict) == 0) { + MOBase::log::error("No plugins found in {}.", identifier); + return {}; + } + + // Create the plugins: + std::vector plugins; + + if (moduleDict.contains("createPlugin")) { + plugins.push_back(moduleDict["createPlugin"]()); + } + else if (moduleDict.contains("createPlugins")) { + py::object pyPlugins = moduleDict["createPlugins"](); + if (!py::isinstance(pyPlugins)) { + MOBase::log::error("Plugin {}: createPlugins must return a sequence.", + identifier); + } + else { + py::sequence pyList(pyPlugins); + size_t nPlugins = pyList.size(); + for (size_t i = 0; i < nPlugins; ++i) { + plugins.push_back(pyList[i]); + } + } + } + else { + MOBase::log::error("Plugin {}: missing a createPlugin(s) function.", + identifier); + } + + // If we have no plugins, there was an issue, and we already logged the + // problem: + if (plugins.empty()) { + return QList(); + } + + QList allInterfaceList; + + for (py::object pluginObj : plugins) { + + // save to be able to unload it + m_PythonObjects[identifier].push_back(pluginObj); + + QList interfaceList = py::module_::import("mobase.private") + .attr("extract_plugins")(pluginObj) + .cast>(); + + if (interfaceList.isEmpty()) { + MOBase::log::error("Plugin {}: no plugin interface implemented.", + identifier); + } + + // Append the plugins to the main list: + allInterfaceList.append(interfaceList); + } + + return allInterfaceList; + } + catch (const py::error_already_set& ex) { + MOBase::log::error("Failed to import plugin from {}.", identifier); + throw pyexcept::PythonError(ex); + } +} + +void PythonRunner::unload(const QString& identifier) +{ + auto it = m_PythonObjects.find(identifier); + if (it != m_PythonObjects.end()) { + + py::gil_scoped_acquire lock; + + if (!identifier.endsWith(".py")) { + + // At this point, the identifier is the full path to the module. + QDir folder(identifier); + + // We want to "unload" (remove from sys.modules) modules that come + // from this plugin (whose __path__ points under this module, + // including the module of the plugin itself). + py::object sys = py::module_::import("sys"); + py::dict modules = sys.attr("modules"); + py::list keys = modules.attr("keys")(); + for (std::size_t i = 0; i < py::len(keys); ++i) { + py::object mod = modules[keys[i]]; + if (PyObject_HasAttrString(mod.ptr(), "__path__")) { + QString mpath = mod.attr("__path__")[0].cast(); + + if (!folder.relativeFilePath(mpath).startsWith("..")) { + // If the path is under identifier, we need to unload + // it. + log::debug("Unloading module {} from {} for {}.", + keys[i].cast(), mpath, identifier); + + PyDict_DelItem(modules.ptr(), keys[i].ptr()); + } + } + } + } + + // Boost.Python does not handle cyclic garbace collection, so we need to + // release everything hold by the objects before deleting the objects + // themselves (done when erasing from m_PythonObjects). + for (auto& obj : it->second) { + obj.attr("__dict__").attr("clear")(); + } + + log::debug("Deleting {} python objects for {}.", it->second.size(), identifier); + m_PythonObjects.erase(it); + } +} + +bool PythonRunner::isInitialized() const +{ + return Py_IsInitialized() != 0; +} diff --git a/src/runner-pybind11/pythonrunner.h b/src/runner/pythonrunner.h similarity index 100% rename from src/runner-pybind11/pythonrunner.h rename to src/runner/pythonrunner.h diff --git a/src/runner-pybind11/pythonutils.cpp b/src/runner/pythonutils.cpp similarity index 75% rename from src/runner-pybind11/pythonutils.cpp rename to src/runner/pythonutils.cpp index 9998c10..5e7e900 100644 --- a/src/runner-pybind11/pythonutils.cpp +++ b/src/runner/pythonutils.cpp @@ -6,8 +6,6 @@ #include -#include - #include "log.h" namespace py = pybind11; @@ -141,42 +139,4 @@ namespace mo2::python { mobase.attr("logger") = logger; } - void show_deprecation_warning(std::string_view name, std::string_view message, - bool show_once) - { - - // Contains the list of filename / line number for which a deprecation - // warning has already been shown. - static std::set> DeprecatedLines; - - // Find the caller: - auto inspect = py::module_::import("inspect"); - auto current_frame = inspect.attr("currentframe")(); - py::sequence callable_frame = inspect.attr("getouterframes")(current_frame, 2); - auto filename = callable_frame[-1].attr("filename").cast(); - auto function = callable_frame[-1].attr("function").cast(); - auto lineno = callable_frame[-1].attr("lineno").cast(); - - // Only show once if requested: - if (show_once && DeprecatedLines.contains({filename, lineno})) { - return; - } - - // Register the deprecation: - DeprecatedLines.emplace(filename, lineno); - - auto path = relative(std::filesystem::path(filename), - QCoreApplication::applicationDirPath().toStdWString()); - - // Show the message: - if (message.empty()) { - MOBase::log::warn("[deprecated] {} in {} [{}:{}].", name, function, - path.native(), lineno); - } - else { - MOBase::log::warn("[deprecated] {} in {} [{}:{}]: {}", name, function, - path.native(), lineno, message); - } - } - } // namespace mo2::python diff --git a/src/runner/pythonutils.h b/src/runner/pythonutils.h new file mode 100644 index 0000000..019e782 --- /dev/null +++ b/src/runner/pythonutils.h @@ -0,0 +1,25 @@ +#ifndef PYTHONRUNNER_UTILS_H +#define PYTHONRUNNER_UTILS_H + +#include + +#include + +namespace mo2::python { + + /** + * @brief Configure Python stdout and stderr to log to MO2. + * + */ + void configure_python_stream(); + + /** + * @brief Configure logging for MO2 python plugin. + * + * @param mobase The mobase module. + */ + void configure_python_logging(pybind11::module_ mobase); + +} // namespace mo2::python + +#endif