#include "pythonrunner.h" #pragma warning(disable : 4100) #pragma warning(disable : 4996) #include #include #include #include #include #include #include #include #include #include #include #include "pybind11_qt/pybind11_qt.h" #include #include #include "error.h" #include "pythonutils.h" #include "wrappers/pyfiletree.h" #include "wrappers/wrappers.h" // TODO: remove these include (only for testing) #include #include #include #include #include #include #include #include #include #include #include #include using namespace MOBase; namespace py = pybind11; PYBIND11_MODULE(mobase, m) { using namespace mo2::python; py::module_::import("PyQt6.QtCore"); py::module_::import("PyQt6.QtWidgets"); // bindings // mo2::python::add_basic_bindings(m); mo2::python::add_wrapper_bindings(m); // game features must be added before plugins mo2::python::add_game_feature_bindings(m); mo2::python::add_plugins_bindings(m); // widgets // py::module_ widgets( py::reinterpret_borrow(PyImport_AddModule("mobase.widgets"))); m.attr("widgets") = widgets; mo2::python::add_widget_bindings(widgets); // functions // m.def("getFileVersion", &MOBase::getFileVersion, py::arg("filepath")); m.def("getProductVersion", &MOBase::getProductVersion, py::arg("executable")); m.def("getIconForExecutable", &MOBase::iconForExecutable, py::arg("executable")); // expose MoVariant - MoVariant is a fake object whose only purpose is to be // used as a type-hint on the python side (e.g., def foo(x: // mobase.MoVariant)) // // the real MoVariant is defined in the generated stubs, since it's only // relevant when doing type-checking, but this needs to be defined, // otherwise MoVariant is not found when actually running plugins through // MO2, making them crash m.attr("MoVariant") = py::none(); // == BEGIN TESTS == m.def("testPlugin", [](py::object pyobj) { py::scoped_ostream_redirect s{std::cout}; std::cout << "type: " << pyobj.get_type().attr("__name__").cast() << "\n"; auto qobjects = mo2::python::extract_plugins(pyobj); std::cout << " found " << qobjects.size() << " plugins\n"; // cast as IPlugin for (int i = 0; i < qobjects.size(); ++i) { IPlugin* plugin = qobject_cast(qobjects[i]); std::cout << fmt::format(" plugin {}: {} -> {}\n", i, (void*)qobjects[i], (void*)plugin); std::cout << fmt::format(" name: {}\n", plugin->name().toStdString()); std::cout << fmt::format( " installer?: {}\n", (void*)qobject_cast(qobjects[i])); std::cout << fmt::format( " installer simple?: {}\n", (void*)qobject_cast(qobjects[i])); } }); py::detail::type_caster t1; py::detail::type_caster t2; m.def("testQStringList", [](QStringList const& list) { QStringList res = list; for (QString& value : res) { value = value + "_CPP"; } return res; }); m.def("testGuessedString", [](GuessedValue const& value) { return std::make_tuple(value.operator const QString&(), value.variants()); }); m.def("testQStringList", [](QStringList const& list) { QStringList res = list; for (QString& value : res) { value = value + "_CPP"; } return res; }); m.def("testQMap1", [](QMap const& map) { QMap res; for (auto it = map.begin(); it != map.end(); ++it) { res[it.key()] = it.value().size(); } return res; }); m.def("testQMap2", [](QMap const& map) { QMap res; for (auto it = map.begin(); it != map.end(); ++it) { res[it.key()] = QString::number(it.value()); } return res; }); m.def("testDateTime1", []() { return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); }); m.def("testDateTime2", [](QDateTime const& datetime) { return datetime.toString(); }); m.def("testEnum0", []() { return Qt::GlobalColor::darkRed; }); m.def("testEnum", [](Qt::GlobalColor color) { MOBase::log::error("In C++: {}", color); return py::make_tuple(color, QMessageBox::Icon::Information); }); m.def("testPixmap", [](QPixmap const& pixmap) { return pixmap.size(); }); m.def("createSaveGame", []() -> ISaveGame* { class SaveGame : public ISaveGame { QString getFilepath() const override { return "filepath"; } QDateTime getCreationTime() const override { return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); } QString getName() const override { return "name"; } QString getSaveGroupIdentifier() const override { return "group"; } QStringList allFiles() const override { return {"file1", "file2"}; } }; return new SaveGame(); }); m.def("testSaveGameWidget", [](ISaveGameInfoWidget* widget) { class SaveGame : public ISaveGame { QString getFilepath() const override { return "filepath-c++"; } QDateTime getCreationTime() const override { return QDateTime::fromString("2022-02-15T12:33:45", Qt::ISODate); } QString getName() const override { return "name"; } QString getSaveGroupIdentifier() const override { return "group"; } QStringList allFiles() const override { return {"file1", "file2"}; } }; static SaveGame s; widget->setSave(s); }); m.def("testSaveGameRef", [](const ISaveGame& game) { std::cout << "getFilepath(): " << game.getFilepath().toStdString() << "\n"; std::cout << "getCreationTime(): " << game.getCreationTime().toString().toStdString() << "\n"; std::cout << "getName(): " << game.getName().toStdString() << "\n"; std::cout << "getSaveGroupIdentifier(): " << game.getSaveGroupIdentifier().toStdString() << "\n"; std::cout << "allFiles(): [ " << game.allFiles().join(" ").toStdString() << " ]\n"; }); m.def("testSaveGamePtr", [](const ISaveGame* game) { std::cout << "getFilepath(): " << game->getFilepath().toStdString() << "\n"; std::cout << "getCreationTime(): " << game->getCreationTime().toString().toStdString() << "\n"; std::cout << "getName(): " << game->getName().toStdString() << "\n"; std::cout << "getSaveGroupIdentifier(): " << game->getSaveGroupIdentifier().toStdString() << "\n"; std::cout << "allFiles(): [ " << game->allFiles().join(" ").toStdString() << " ]\n"; }); m.def("testWidget1", [](QWidget* w) { if (w) { std::cout << "background role: " << w->backgroundRole() << "\n"; } else { std::cout << "null widget\n"; } }); m.def("testWidget2", []() { QWidget* w = new QWidget(); w->setBackgroundRole(QPalette::ColorRole::HighlightedText); return w; }); m.def("testFlags1", [](IPluginList::PluginStates states) { std::vector res; if (states.testFlag(IPluginList::STATE_MISSING)) { res.push_back("missing"); } if (states.testFlag(IPluginList::STATE_INACTIVE)) { res.push_back("inactive"); } if (states.testFlag(IPluginList::STATE_ACTIVE)) { res.push_back("active"); } return res; }); m.def("testFlags2", [](QStringList const& states) { IPluginList::PluginStates res; if (states.contains("missing")) { res |= IPluginList::STATE_MISSING; } if (states.contains("inactive")) { res |= IPluginList::STATE_INACTIVE; } if (states.contains("active")) { res |= IPluginList::STATE_ACTIVE; } return res; }); } /** * */ class PythonRunner : public IPythonRunner { public: PythonRunner(); ~PythonRunner(); bool initPython(); QList load(const QString& identifier); void unload(const QString& identifier); bool isPythonInitialized() const; bool isPythonVersionSupported() const; private: void initPath(); /** * @brief Ensure that the given folder is in sys.path. */ void ensureFolderInPath(QString folder); private: // For each "identifier" (python file or python module folder), contains the // list of python objects to keep "alive" during the execution. std::unordered_map> m_PythonObjects; }; IPythonRunner* CreatePythonRunner() { std::unique_ptr result = std::make_unique(); if (result->initPython()) { return result.release(); } else { return nullptr; } } PythonRunner::PythonRunner() {} PythonRunner::~PythonRunner() { // Boost.Python does not handle cyclic garbace collection, so we need to release // everything hold by the objects before deleting the objects themselves: // for (auto& [name, objects] : m_PythonObjects) { // for (auto& obj : objects) { // obj.attr("__dict__").attr("clear")(); // } // } // we need to clear this here otherwise there is a crash in // finalize_interpreter() m_PythonObjects.clear(); py::finalize_interpreter(); } // ErrWrapper is in error.h PYBIND11_MODULE(moprivate, m) { // expose a function to create a particular tree, only for debugging // purpose, not in mobase. mo2::python::add_make_tree_function(m); } bool PythonRunner::initPython() { if (Py_IsInitialized()) return true; try { static const char* argv0 = "ModOrganizer.exe"; PyImport_AppendInittab("mobase", &PyInit_mobase); PyImport_AppendInittab("moprivate", &PyInit_moprivate); Py_OptimizeFlag = 2; Py_NoSiteFlag = 1; initPath(); py::initialize_interpreter(false, 1, &argv0); if (!Py_IsInitialized()) { return false; } py::module_ mainModule = py::module_::import("__main__"); py::object mainNamespace = mainModule.attr("__dict__"); mainNamespace["sys"] = py::module_::import("sys"); mainNamespace["moprivate"] = py::module_::import("moprivate"); mainNamespace["mobase"] = py::module_::import("mobase"); mo2::python::configure_python_stream(); mo2::python::configure_python_logging(mainNamespace["mobase"]); return true; } catch (const py::error_already_set& ex) { MOBase::log::error("failed to init python: {}", ex.what()); return false; } } void PythonRunner::initPath() { static QStringList paths = {QCoreApplication::applicationDirPath() + "/pythoncore.zip", QCoreApplication::applicationDirPath() + "/pythoncore", IOrganizer::getPluginDataPath()}; Py_SetPath(paths.join(';').toStdWString().c_str()); } void PythonRunner::ensureFolderInPath(QString folder) { py::module_ sys = py::module_::import("sys"); py::list sysPath = sys.attr("path"); // Converting to QStringList for Qt::CaseInsensitive and because .index() // raise an exception: const QStringList currentPath = sysPath.cast(); if (!currentPath.contains(folder, Qt::CaseInsensitive)) { sysPath.insert(0, folder); } } QList PythonRunner::load(const QString& identifier) { py::gil_scoped_acquire lock; // `pluginName` can either be a python file (single-file plugin or a folder // (whole module). // // For whole module, we simply add the parent folder to path, then we load // the module with a simple py::import, and we retrieve the associated // __dict__ from which we extract either createPlugin or createPlugins. // // For single file, we need to use py::eval_file, and we will use the // context (global variables) from __main__ (already contains mobase, and // other required module). Since the context is shared between called of // `instantiate`, we need to make sure to remove createPlugin(s) from // previous call. try { // Dictionary that will contain createPlugin() or createPlugins(). py::dict moduleDict; if (identifier.endsWith(".py")) { py::object mainModule = py::module_::import("__main__"); py::dict moduleNamespace = mainModule.attr("__dict__"); std::string temp = ToString(identifier); py::eval_file(temp.c_str(), moduleNamespace).is_none(); moduleDict = moduleNamespace; } else { // Retrieve the module name: QStringList parts = identifier.split("/"); std::string moduleName = ToString(parts.takeLast()); ensureFolderInPath(parts.join("/")); moduleDict = py::module_::import(moduleName.c_str()).attr("__dict__"); } if (py::len(moduleDict) == 0) { MOBase::log::error("No plugins found in {}.", identifier); return {}; } // Create the plugins: std::vector plugins; if (moduleDict.contains("createPlugin")) { plugins.push_back(moduleDict["createPlugin"]()); // Clear for future call PyDict_DelItemString(moduleDict.ptr(), "createPlugin"); } else if (moduleDict.contains("createPlugins")) { py::object pyPlugins = moduleDict["createPlugins"](); if (!PySequence_Check(pyPlugins.ptr())) { MOBase::log::error("Plugin {}: createPlugins must return a list.", identifier); } else { py::list pyList(pyPlugins); int nPlugins = py::len(pyList); for (int i = 0; i < nPlugins; ++i) { plugins.push_back(pyList[i]); } } // Clear for future call PyDict_DelItemString(moduleDict.ptr(), "createPlugins"); } else { MOBase::log::error("Plugin {}: missing a createPlugin(s) function.", identifier); } // If we have no plugins, there was an issue, and we already logged the // problem: if (plugins.empty()) { return QList(); } QList allInterfaceList; for (py::object pluginObj : plugins) { // Add the plugin to keep it alive: m_PythonObjects[identifier].push_back(pluginObj); QList interfaceList = mo2::python::extract_plugins(pluginObj); if (interfaceList.isEmpty()) { MOBase::log::error("Plugin {}: no plugin interface implemented.", identifier); } // Append the plugins to the main list: allInterfaceList.append(interfaceList); } return allInterfaceList; } catch (const py::error_already_set& ex) { MOBase::log::error("Failed to import plugin from {}.", identifier); throw pyexcept::PythonError(ex); } } void PythonRunner::unload(const QString& identifier) { auto it = m_PythonObjects.find(identifier); if (it != m_PythonObjects.end()) { py::gil_scoped_acquire lock; if (!identifier.endsWith(".py")) { // At this point, the identifier is the full path to the module. QDir folder(identifier); // We want to "unload" (remove from sys.modules) modules that come // from this plugin (whose __path__ points under this module, // including the module of the plugin itself). py::object sys = py::module_::import("sys"); py::dict modules = sys.attr("modules"); py::list keys = modules.attr("keys")(); for (std::size_t i = 0; i < py::len(keys); ++i) { py::object mod = modules[keys[i]]; if (PyObject_HasAttrString(mod.ptr(), "__path__")) { QString mpath = mod.attr("__path__")[0].cast(); if (!folder.relativeFilePath(mpath).startsWith("..")) { // If the path is under identifier, we need to unload // it. log::debug("Unloading module {} from {} for {}.", keys[i].cast(), mpath, identifier); PyDict_DelItem(modules.ptr(), keys[i].ptr()); } } } } // Boost.Python does not handle cyclic garbace collection, so we need to // release everything hold by the objects before deleting the objects // themselves (done when erasing from m_PythonObjects). for (auto& obj : it->second) { obj.attr("__dict__").attr("clear")(); } log::debug("Deleting {} python objects for {}.", it->second.size(), identifier); m_PythonObjects.erase(it); } } bool PythonRunner::isPythonInitialized() const { return Py_IsInitialized() != 0; }