Merge pull request #26 from ModOrganizer2/Develop

Stage for release 2.2.1

CMakeLists.txt

@@ -1,6 +1,6 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12)
 
-ADD_COMPILE_OPTIONS($<$<CXX_COMPILER_ID:MSVC>:/MP>)
+ADD_COMPILE_OPTIONS($<$<CXX_COMPILER_ID:MSVC>:/MP> $<$<CXX_COMPILER_ID:MSVC>:$<$<CONFIG:RELEASE>:/O2>> $<$<CXX_COMPILER_ID:MSVC>:$<$<CONFIG:RELWITHDEBINFO>:/O2>>)
 
 SET(PROJ_NAME plugin_python)
 

appveyor.yml

@@ -1,9 +1,11 @@
 version: 1.0.{build}
 skip_branch_with_pr: true
-image: Visual Studio 2017
+image: Visual Studio 2019 Preview
 environment:
   WEBHOOK_URL:
     secure: gOKbXaZM9ImtMD5XrYITvdyZUW/az082G9OIN1EC1Vbg57wBaeLhi49uGjxPw5GVujHku6kxN6ab89zhbS5GVeluR76GM83IbKV4Sh7udXzoYZZdg6YudtYHzdhCgUeiedpswbuczTq9ceIkkfSEWZuh/lMAAVVwvcGsJAnoPFw=
+build:
+  parallel: true
 build_script:
 - cmd: >-
     git clone --depth=1 --branch=%APPVEYOR_REPO_BRANCH% https://github.com/ModOrganizer2/modorganizer-umbrella.git c:\projects\modorganizer-umbrella 2> $null
@@ -14,17 +16,17 @@ build_script:
 
     C:\Python37-x64\python.exe unimake.py -d c:\projects\modorganizer-build -s Appveyor_Build=True %APPVEYOR_PROJECT_NAME%
 artifacts:
-- path: vsbuild\src\proxy\RelWithDebInfo\plugin_python.dll
+- path: build\src\proxy\plugin_python.dll
   name: plugin_python_dll
-- path: vsbuild\src\proxy\RelWithDebInfo\plugin_python.pdb
+- path: build\src\proxy\plugin_python.pdb
   name: plugin_python_pdb
-- path: vsbuild\src\proxy\RelWithDebInfo\plugin_python.lib
+- path: build\src\proxy\plugin_python.lib
   name: plugin_python_lib
-- path: vsbuild\src\runner\RelWithDebInfo\pythonrunner.dll
+- path: build\src\runner\pythonrunner.dll
   name: pythonrunner_dll
-- path: vsbuild\src\runner\RelWithDebInfo\pythonrunner.pdb
+- path: build\src\runner\pythonrunner.pdb
   name: pythonrunner_pdb
-- path: vsbuild\src\runner\RelWithDebInfo\pythonrunner.lib
+- path: build\src\runner\pythonrunner.lib
   name: pythonrunner_lib
 on_success:
   - ps: Set-Location -Path $env:APPVEYOR_BUILD_FOLDER
@@ -32,5 +34,7 @@ on_success:
   - ps: ./send.ps1 success $env:WEBHOOK_URL
 on_failure:
   - ps: Set-Location -Path $env:APPVEYOR_BUILD_FOLDER
+  - ps: Push-AppveyorArtifact ${env:APPVEYOR_BUILD_FOLDER}\stdout.log
+  - ps: Push-AppveyorArtifact ${env:APPVEYOR_BUILD_FOLDER}\stderr.log
   - ps: Invoke-RestMethod https://raw.githubusercontent.com/DiscordHooks/appveyor-discord-webhook/master/send.ps1 -o send.ps1
   - ps: ./send.ps1 failure $env:WEBHOOK_URL

src/runner/error.cpp

@@ -1,8 +1,9 @@
 #ifndef Q_MOC_RUN
 #include <boost/python.hpp>
 #endif
-#include <QString>
 #include <utility.h>
+#include <QDebug>
+#include "error.h"
 
 using namespace MOBase;
 namespace bpy = boost::python;
@@ -10,17 +11,55 @@ namespace bpy = boost::python;
 void reportPythonError()
 {
   if (PyErr_Occurred()) {
-    // prints to s_ErrIO buffer
+    ErrWrapper &errWrapper = ErrWrapper::instance();
+
+    errWrapper.startRecordingExceptionMessage();
     PyErr_Print();
-    // extract data from python buffer
-    bpy::object mainModule = bpy::import("__main__");
-    bpy::object mainNamespace = mainModule.attr("__dict__");
-    bpy::object errMsgObj = bpy::eval("s_ErrIO.getvalue()", mainNamespace);
-    QString errMsg = bpy::extract<QString>(errMsgObj.ptr());
-    bpy::eval("s_ErrIO.truncate(0)", mainNamespace);
+    errWrapper.stopRecordingExceptionMessage();
+
+    QString errMsg = errWrapper.getLastExceptionMessage();
 
     throw MyException(errMsg);
   } else {
     throw MyException("An unexpected C++ exception was thrown in python code");
   }
 }
+
+ErrWrapper & ErrWrapper::instance()
+{
+  static ErrWrapper err;
+  return err;
+}
+
+void ErrWrapper::write(const char * message)
+{
+  buffer << message;
+  if (buffer.tellp() != 0 && buffer.str().back() == '\n')
+  {
+    // actually put the string in a variable so it doesn't get destroyed as soon as we get a pointer to its data
+    std::string string = buffer.str().substr(0, buffer.str().length() - 1);
+    qCritical().nospace().noquote() << string.c_str();
+    buffer = std::stringstream();
+  }
+  
+  if (recordingExceptionMessage)
+  {
+    lastException << message;
+  }
+}
+
+void ErrWrapper::startRecordingExceptionMessage()
+{
+  recordingExceptionMessage = true;
+  lastException = std::stringstream();
+}
+
+void ErrWrapper::stopRecordingExceptionMessage()
+{
+  recordingExceptionMessage = false;
+}
+
+QString ErrWrapper::getLastExceptionMessage()
+{
+  return QString::fromStdString(lastException.str());
+}

src/runner/error.h

@@ -1,7 +1,26 @@
 #ifndef ERROR_H
 #define ERROR_H
+#include <QString>
+#include <sstream>
 
 // turn an error from the python interpreter into an exception
 void reportPythonError();
 
+struct ErrWrapper
+{
+  static ErrWrapper & instance();
+  
+  void write(const char * message);
+
+  void startRecordingExceptionMessage();
+
+  void stopRecordingExceptionMessage();
+
+  QString getLastExceptionMessage();
+
+  std::stringstream buffer;
+  bool recordingExceptionMessage;
+  std::stringstream lastException;
+};
+
 #endif // ERROR_H

src/runner/proxypluginwrappers.cpp

@@ -381,12 +381,13 @@ QWidget *IPluginPreviewWrapper::genFilePreview(const QString &fileName, const QS
 {
   // This is complicated, so we can't use the basic implementation
   try {
+    GILock lock;
     boost::python::override implementation = this->get_override("genFilePreview");
     if (!implementation)
       throw MissingImplementation(this->className, "genFilePreview");
     boost::python::object pyVersion = implementation(fileName, maxSize);
     // We need responsibility for deleting the QWidget to be transferred to C++
-    sipAPI()->api_transfer_to(pyVersion.ptr(), 0);
+    sipAPIAccess::sipAPI()->api_transfer_to(pyVersion.ptr(), Py_None);
     return boost::python::extract<QWidget *>(pyVersion)();
   } PYCATCH;
 }

src/runner/pythonrunner.cpp

@@ -530,14 +530,14 @@ PyObject *toPyQt(T *objPtr)
     qDebug("no input object");
     return bpy::incref(Py_None);
   }
-  const sipTypeDef *type = sipAPI()->api_find_type(MetaData<T>::className());
+  const sipTypeDef *type = sipAPIAccess::sipAPI()->api_find_type(MetaData<T>::className());
 
   if (type == nullptr) {
     qDebug("failed to determine type: %s", MetaData<T>::className());
     return bpy::incref(Py_None);
   }
 
-  PyObject *sipObj = sipAPI()->api_convert_from_type(objPtr, type, 0);
+  PyObject *sipObj = sipAPIAccess::sipAPI()->api_convert_from_type(objPtr, type, 0);
   if (sipObj == nullptr) {
     qDebug("failed to convert");
     return bpy::incref(Py_None);
@@ -564,19 +564,19 @@ struct QClass_converters
     }
 
     static PyObject *convert(const T &object) {
-      const sipTypeDef *type = sipAPI()->api_find_type(MetaData<T>::className());
+      const sipTypeDef *type = sipAPIAccess::sipAPI()->api_find_type(MetaData<T>::className());
       if (type == nullptr) {
         return bpy::incref(Py_None);
       }
 
-      PyObject *sipObj = sipAPI()->api_convert_from_type((void*)getSafeCopy<T>((T*)&object), type, 0);
+      PyObject *sipObj = sipAPIAccess::sipAPI()->api_convert_from_type((void*)getSafeCopy<T>((T*)&object), type, 0);
       if (sipObj == nullptr) {
         return bpy::incref(Py_None);
       }
 
       if (std::is_copy_constructible_v<T>)
         // Ensure Python deletes the C++ component
-        sipAPI()->api_transfer_back(sipObj);
+        sipAPIAccess::sipAPI()->api_transfer_back(sipObj);
 
       return bpy::incref(sipObj);
     }
@@ -586,19 +586,19 @@ struct QClass_converters
         return bpy::incref(Py_None);
       }
 
-      const sipTypeDef *type = sipAPI()->api_find_type(MetaData<T>::className());
+      const sipTypeDef *type = sipAPIAccess::sipAPI()->api_find_type(MetaData<T>::className());
       if (type == nullptr) {
         return bpy::incref(Py_None);
       }
 
-      PyObject *sipObj = sipAPI()->api_convert_from_type(getSafeCopy<T>(object), type, 0);
+      PyObject *sipObj = sipAPIAccess::sipAPI()->api_convert_from_type(getSafeCopy<T>(object), type, 0);
       if (sipObj == nullptr) {
         return bpy::incref(Py_None);
       }
 
       if (std::is_copy_constructible_v<T>)
         // Ensure Python deletes the C++ component
-        sipAPI()->api_transfer_back(sipObj);
+        sipAPIAccess::sipAPI()->api_transfer_back(sipObj);
 
       return bpy::incref(sipObj);
     }
@@ -612,12 +612,12 @@ struct QClass_converters
   {
     // This would transfer responsibility for deconstructing the object to C++, but Boost assumes l-value converters (such as this) don't do that
     // Instead, this should be called within the wrappers for functions which return deletable pointers.
-    //sipAPI()->api_transfer_to(objPtr, 0);
-    if (PyObject_TypeCheck(objPtr, sipAPI()->api_simplewrapper_type)) {
+    //sipAPI()->api_transfer_to(objPtr, Py_None);
+    if (PyObject_TypeCheck(objPtr, sipAPIAccess::sipAPI()->api_simplewrapper_type)) {
       sipSimpleWrapper *wrapper;
       wrapper = reinterpret_cast<sipSimpleWrapper*>(objPtr);
       return wrapper->data;
-    } else if (PyObject_TypeCheck(objPtr, sipAPI()->api_wrapper_type)) {
+    } else if (PyObject_TypeCheck(objPtr, sipAPIAccess::sipAPI()->api_wrapper_type)) {
       sipWrapper *wrapper;
       wrapper = reinterpret_cast<sipWrapper*>(objPtr);
       return wrapper->super.data;
@@ -651,12 +651,12 @@ struct QInterface_converters
   struct QInterface_to_PyQt
   {
     static PyObject *convert(const T &object) {
-      const sipTypeDef *type = sipAPI()->api_find_type(MetaData<T>::className());
+      const sipTypeDef *type = sipAPIAccess::sipAPI()->api_find_type(MetaData<T>::className());
       if (type == nullptr) {
         return bpy::incref(Py_None);
       }
 
-      PyObject *sipObj = sipAPI()->api_convert_from_type((void*)(&object), type, 0);
+      PyObject *sipObj = sipAPIAccess::sipAPI()->api_convert_from_type((void*)(&object), type, 0);
       if (sipObj == nullptr) {
         return bpy::incref(Py_None);
       }
@@ -669,12 +669,12 @@ struct QInterface_converters
         return bpy::incref(Py_None);
       }
 
-      const sipTypeDef *type = sipAPI()->api_find_type(MetaData<T>::className());
+      const sipTypeDef *type = sipAPIAccess::sipAPI()->api_find_type(MetaData<T>::className());
       if (type == nullptr) {
         return bpy::incref(Py_None);
       }
 
-      PyObject *sipObj = sipAPI()->api_convert_from_type(object, type, 0);
+      PyObject *sipObj = sipAPIAccess::sipAPI()->api_convert_from_type(object, type, 0);
       if (sipObj == nullptr) {
         return bpy::incref(Py_None);
       }
@@ -689,13 +689,13 @@ struct QInterface_converters
 
   static void *QInterface_from_PyQt(PyObject *objPtr)
   {
-    if (!PyObject_TypeCheck(objPtr, sipAPI()->api_wrapper_type)) {
+    if (!PyObject_TypeCheck(objPtr, sipAPIAccess::sipAPI()->api_wrapper_type)) {
       bpy::throw_error_already_set();
     }
 
     // This would transfer responsibility for deconstructing the object to C++, but Boost assumes l-value converters (such as this) don't do that
     // Instead, this should be called within the wrappers for functions which return deletable pointers.
-    //sipAPI()->api_transfer_to(objPtr, 0);
+    //sipAPI()->api_transfer_to(objPtr, Py_None);
 
     sipSimpleWrapper *wrapper = reinterpret_cast<sipSimpleWrapper*>(objPtr);
     return wrapper->data;
@@ -1299,6 +1299,36 @@ PythonRunner::PythonRunner(const MOBase::IOrganizer *moInfo)
 
 static const char *argv0 = "ModOrganizer.exe";
 
+struct PrintWrapper
+{
+  void write(const char * message)
+  {
+    buffer << message;
+    if (buffer.tellp() != 0 && buffer.str().back() == '\n')
+    {
+      // actually put the string in a variable so it doesn't get destroyed as soon as we get a pointer to its data
+      std::string string = buffer.str().substr(0, buffer.str().length() - 1);
+      qDebug().nospace().noquote() << string.c_str();
+      buffer = std::stringstream();
+    }
+  }
+
+  std::stringstream buffer;
+};
+
+// ErrWrapper is in error.h
+
+BOOST_PYTHON_MODULE(moprivate)
+{
+  bpy::class_<PrintWrapper, boost::noncopyable>("PrintWrapper", bpy::init<>())
+    .def("write", &PrintWrapper::write);
+  bpy::class_<ErrWrapper, boost::noncopyable>("ErrWrapper", bpy::init<>())
+    .def("instance", &ErrWrapper::instance, bpy::return_value_policy<bpy::reference_existing_object>()).staticmethod("instance")
+    .def("write", &ErrWrapper::write)
+    .def("startRecordingExceptionMessage", &ErrWrapper::startRecordingExceptionMessage)
+    .def("stopRecordingExceptionMessage", &ErrWrapper::stopRecordingExceptionMessage)
+    .def("getLastExceptionMessage", &ErrWrapper::getLastExceptionMessage);
+}
 
 bool PythonRunner::initPython(const QString &pythonPath)
 {
@@ -1317,6 +1347,7 @@ bool PythonRunner::initPython(const QString &pythonPath)
 
     Py_SetProgramName(argBuffer);
     PyImport_AppendInittab("mobase", &PyInit_mobase);
+    PyImport_AppendInittab("moprivate", &PyInit_moprivate);
     Py_OptimizeFlag = 2;
     Py_NoSiteFlag = 1;
     initPath();
@@ -1331,11 +1362,13 @@ bool PythonRunner::initPython(const QString &pythonPath)
     bpy::object mainModule = bpy::import("__main__");
     bpy::object mainNamespace = mainModule.attr("__dict__");
     mainNamespace["sys"] = bpy::import("sys");
+    mainNamespace["moprivate"] = bpy::import("moprivate");
     bpy::import("site");
-    mainNamespace["io"] = bpy::import("io");
-    bpy::exec("s_ErrIO = io.StringIO()\n"
-                        "sys.stderr = s_ErrIO",
+    bpy::exec("sys.stdout = moprivate.PrintWrapper()\n"
+              "sys.stderr = moprivate.ErrWrapper.instance()\n"
+              "sys.excepthook = lambda x, y, z: sys.__excepthook__(x, y, z)\n",
                         mainNamespace);
+
     return true;
   } catch (const bpy::error_already_set&) {
     qDebug("failed to init python");

src/runner/sipapiaccess.cpp

@@ -0,0 +1,86 @@
+#include "sipapiaccess.h"
+#include <boost/python.hpp>
+#include <QString>
+#include <utility.h>
+
+const sipAPIDef* sipAPIAccess::sipAPI()
+{
+    QString exception;
+    static const sipAPIDef* sipApi = nullptr;
+    if (sipApi == nullptr) {
+    #if defined(SIP_USE_PYCAPSULE)
+        PyImport_ImportModule("PyQt5.sip");
+
+        auto errorObj = PyErr_Occurred();
+        if (errorObj != NULL) {
+            PyObject* type, * value, * traceback;
+            PyErr_Fetch(&type, &value, &traceback);
+            PyErr_NormalizeException(&type, &value, &traceback);
+            if (traceback != NULL) {
+                boost::python::handle<> h_type(type);
+                boost::python::handle<> h_val(value);
+                boost::python::handle<> h_tb(traceback);
+                boost::python::object tb(boost::python::import("traceback"));
+                boost::python::object fmt_exp(tb.attr("format_exception"));
+                boost::python::object exp_list(fmt_exp(h_type, h_val, h_tb));
+                boost::python::object exp_str(boost::python::str("\n").join(exp_list));
+                boost::python::extract<std::string> returned(exp_str);
+                exception = QString::fromStdString(returned());
+            }
+            PyErr_Restore(type, value, traceback);
+            throw MOBase::MyException(QString("Failed to load PyQt5: %1").arg(exception));
+        }
+
+        sipApi = (const sipAPIDef*)PyCapsule_Import("PyQt5.sip._C_API", 0);
+        if (sipApi == NULL) {
+            auto errorObj = PyErr_Occurred();
+            if (errorObj != NULL) {
+                PyObject* type, * value, * traceback;
+                PyErr_Fetch(&type, &value, &traceback);
+                PyErr_NormalizeException(&type, &value, &traceback);
+                if (traceback != NULL) {
+                    boost::python::handle<> h_type(type);
+                    boost::python::handle<> h_val(value);
+                    boost::python::handle<> h_tb(traceback);
+                    boost::python::object tb(boost::python::import("traceback"));
+                    boost::python::object fmt_exp(tb.attr("format_exception"));
+                    boost::python::object exp_list(fmt_exp(h_type, h_val, h_tb));
+                    boost::python::object exp_str(boost::python::str("\n").join(exp_list));
+                    boost::python::extract<std::string> returned(exp_str);
+                    exception = QString::fromStdString(returned());
+                }
+                PyErr_Restore(type, value, traceback);
+            }
+            throw MOBase::MyException(QString("Failed to load SIP API: %1").arg(exception));
+        }
+    #else
+        PyObject* sip_module;
+        PyObject* sip_module_dict;
+        PyObject* c_api;
+
+        /* Import the SIP module. */
+        sip_module = PyImport_ImportModule("PyQt5.sip");
+
+        if (sip_module == NULL)
+            return NULL;
+
+        /* Get the module's dictionary. */
+        sip_module_dict = PyModule_GetDict(sip_module);
+
+        /* Get the "_C_API" attribute. */
+        c_api = PyDict_GetItemString(sip_module_dict, "_C_API");
+
+        if (c_api == NULL)
+            return NULL;
+
+        /* Sanity check that it is the right type. */
+        if (!PyCObject_Check(c_api))
+            return NULL;
+
+        /* Get the actual pointer from the object. */
+        sipApi = (const sipAPIDef*)PyCObject_AsVoidPtr(c_api);
+    #endif
+    }
+
+    return sipApi;
+}

src/runner/sipapiaccess.h

@@ -3,43 +3,10 @@
 
 #include <sip.h>
 
-static const sipAPIDef *sipAPI()
+class sipAPIAccess
 {
-  static const sipAPIDef *sipApi = nullptr;
-  if (sipApi == nullptr) {
-#if defined(SIP_USE_PYCAPSULE)
-    PyImport_ImportModule("PyQt5.sip");
-    sipApi = (const sipAPIDef *)PyCapsule_Import("PyQt5.sip._C_API", 0);
-#else
-    PyObject *sip_module;
-    PyObject *sip_module_dict;
-    PyObject *c_api;
-
-    /* Import the SIP module. */
-    sip_module = PyImport_ImportModule("PyQt5.sip");
-
-    if (sip_module == NULL)
-      return NULL;
-
-    /* Get the module's dictionary. */
-    sip_module_dict = PyModule_GetDict(sip_module);
-
-    /* Get the "_C_API" attribute. */
-    c_api = PyDict_GetItemString(sip_module_dict, "_C_API");
-
-    if (c_api == NULL)
-      return NULL;
-
-    /* Sanity check that it is the right type. */
-    if (!PyCObject_Check(c_api))
-      return NULL;
-
-    /* Get the actual pointer from the object. */
-    sipApi = (const sipAPIDef *)PyCObject_AsVoidPtr(c_api);
-#endif
-  }
-
-  return sipApi;
-}
+public:
+    static const sipAPIDef* sipAPI();
+};
 
 #endif // SIPAPIACCESS_H