Better support for dev PyQt builds

- Add alternate index support to pip
- Utilize dev pypi index for PyQt
- Use Qt 6.7.1 for testing

mob.ini

@@ -88,7 +88,7 @@ pull      = true
 boost   = true
 lz4     = true
 openssl = true
-pyqt    = true
+pyqt    = false
 python  = true
 
 [versions]
@@ -101,22 +101,23 @@ boost          = 1.85.0
 boost_vs       = 14.3
 fmt            = 10.2.1
 gtest          = main
-libloot        = 0.22.4
+directxtex     = main
+libloot        = 0.23.0
 lz4            = v1.9.4
 nmm            = 0.71.2
 openssl        = 3.3.0
-pyqt           = 6.7.0
+pyqt           = 6.7.1.dev2407071716
-pyqt_builder   = 1.16.2
+pyqt_builder   = 1.16.4.dev2+g2802761
-pyqt_sip       = 13.7.0
+pyqt_sip       = 13.8.0
 python         = v3.12.3
 pybind11       = v2.12.0
 bzip2          = 1.0.8
-sip            = 6.8.3
+sip            = 6.8.6.dev4+gf624736c
 spdlog         = v1.14.1
-qt             = 6.7.0
+qt             = 6.7.1
 qt_vs          = 2019
 zlib           = v1.3.1
-libbsarch      = 0.0.9
+libbsarch      = 0.0.12
 usvfs          = master
 explorerpp     = 1.4.0
 ss_paper_lad_6788      = 7.2

scripts/make-boost-prebuilt.ps1

@@ -1,9 +0,0 @@
-param(
-    [Parameter(Mandatory = $true)][String]$Path,
-    [Parameter(Mandatory = $true)][String]$Output
-)
-
-7z a $Output (Join-Path -Path $Path -ChildPath "boost")
-7z a $Output (Join-Path -Path $Path -ChildPath "lib32-msvc-14.3")
-7z a $Output (Join-Path -Path $Path -ChildPath "lib64-msvc-14.3")
-7z a $Output (Join-Path -Path $Path -ChildPath "LICENSE_1_0.txt")

scripts/make-pyqt-prebuilt.ps1

@@ -1,13 +0,0 @@
-param(
-    [Parameter(Mandatory = $true)][String]$PyQtPath,
-    [Parameter(Mandatory = $true)][String]$PythonPath,
-    [Parameter(Mandatory = $true)][String]$Output
-)
-
-$Output = [IO.Path]::GetFullPath($Output)
-
-7z a $Output (Join-Path -Path $PyQtPath -ChildPath "LICENSE")
-
-Push-Location $PythonPath
-7z a $Output "Lib/site-packages/PyQt6" "-xr!__pycache__"
-Pop-Location

scripts/make-python-prebuilt.ps1

@@ -1,10 +0,0 @@
-param(
-    [Parameter(Mandatory = $true)][String]$Path,
-    [Parameter(Mandatory = $true)][String]$Output
-)
-
-7z a $Output (Join-Path -Path $Path -ChildPath "PCbuild/amd64") "-xr!pythoncore_temp"
-7z rn $Output "amd64" "PCbuild/amd64"
-7z a $Output (Join-Path -Path $Path -ChildPath "Include") "-xr!internal" "-xr!sip.h"
-7z a $Output (Join-Path -Path $Path -ChildPath "Lib") "-xr!__pycache__" "-xr!site-packages"
-7z a $Output (Join-Path -Path $Path -ChildPath "LICENSE")

src/main.cpp

@@ -33,7 +33,8 @@ namespace mob {
             .add_task<libbsarch>()
             .add_task<libloot>()
             .add_task<openssl>()
-            .add_task<bzip2>();
+            .add_task<bzip2>()
+            .add_task<directxtex>();
 
         add_task<parallel_tasks>()
             .add_task<tasks::python>()

src/tasks/directxtex.cpp

@@ -0,0 +1,80 @@
+#include "pch.h"
+#include "tasks.h"
+
+namespace mob::tasks {
+
+    namespace {
+
+        msbuild create_msbuild_tool(arch a, config config,
+                                    msbuild::ops o = msbuild::build)
+        {
+            return std::move(msbuild(o).architecture(a).configuration(config).solution(
+                directxtex::source_path() / "DirectXTex" /
+                "DirectXTex_Desktop_2022.vcxproj"));
+        }
+
+    }  // namespace
+
+    directxtex::directxtex() : basic_task("directxtex") {}
+
+    std::string directxtex::version()
+    {
+        return conf().version().get("directxtex");
+    }
+
+    bool directxtex::prebuilt()
+    {
+        return false;
+    }
+
+    fs::path directxtex::source_path()
+    {
+        return conf().path().build() / "DirectXTex";
+    }
+
+    void directxtex::do_clean(clean c)
+    {
+        if (is_set(c, clean::reclone)) {
+            git_wrap::delete_directory(cx(), source_path());
+            return;
+        }
+
+        if (is_set(c, clean::rebuild)) {
+            run_tool(create_msbuild_tool(arch::x64, config::release, msbuild::clean));
+            run_tool(create_msbuild_tool(arch::x64, config::debug, msbuild::clean));
+        }
+    }
+
+    void directxtex::do_fetch()
+    {
+        run_tool(make_git()
+                     .url(make_git_url("microsoft", "DirectXTex"))
+                     .branch(version())
+                     .root(source_path()));
+    }
+
+    void directxtex::do_build_and_install()
+    {
+        op::create_directories(cx(), directxtex::source_path() / "Include");
+        op::create_directories(cx(), directxtex::source_path() / "Lib" / "Debug");
+        op::create_directories(cx(), directxtex::source_path() / "Lib" / "Release");
+
+        // DO NOT run these in parallel because both generate files that are shared
+        // between release and debug
+        run_tool(create_msbuild_tool(arch::x64, config::release));
+        run_tool(create_msbuild_tool(arch::x64, config::debug));
+
+        const auto binary_path =
+            source_path() / "DirectXTex" / "Bin" / "Desktop_2022" / "x64";
+
+        for (const auto& header : {"DDS.h", "DirectXTex.h", "DirectXTex.inl "}) {
+            op::copy_file_to_dir_if_better(cx(), source_path() / "DirectXTex" / header,
+                                           source_path() / "Include");
+        }
+        op::copy_file_to_dir_if_better(cx(), binary_path / "Debug" / "DirectXTex.lib",
+                                       source_path() / "Lib" / "Debug");
+        op::copy_file_to_dir_if_better(cx(), binary_path / "Release" / "DirectXTex.lib",
+                                       source_path() / "Lib" / "Release");
+    }
+
+}  // namespace mob::tasks

src/tasks/modorganizer.cpp

@@ -226,6 +226,7 @@ namespace mob::tasks {
                      gtest::build_path(arch::x64, c == config::debug ? config::debug
                                                                      : config::release))
                 .def("OPENSSL_ROOT_DIR", openssl::source_path())
+                .def("DIRECTXTEX_ROOT", directxtex::source_path())
                 .root(root));
     }
 

src/tasks/pyqt.cpp

@@ -173,7 +173,8 @@ namespace mob::tasks {
     void pyqt::build_and_install_from_source()
     {
         // use pip to install the pyqt builder
-        if (python::build_type() == config::debug) {
+        if (python::build_type() == config::debug ||
+            builder_version().find("dev") != std::string::npos) {
             // PyQt-builder has sip as a dependency, so installing it directly will
             // replace the sip we have installed manually, but the installed sip will
             // not work (see comment in sip::build() for details)
@@ -184,6 +185,7 @@ namespace mob::tasks {
             run_tool(pip(pip::install).package("packaging"));
             run_tool(pip(pip::install)
                          .package("PyQt-builder")
+                         .index("https://www.riverbankcomputing.com/pypi/simple")
                          .no_dependencies()
                          .version(builder_version()));
         }

src/tasks/sip.cpp

@@ -116,7 +116,10 @@ namespace mob::tasks {
         }
         else {
             // download
-            run_tool(pip(pip::download).package("sip").version(version()));
+            run_tool(pip(pip::download)
+                         .package("sip")
+                         .index("https://www.riverbankcomputing.com/pypi/simple")
+                         .version(version()));
         }
 
         // extract

src/tasks/tasks.h

@@ -77,6 +77,20 @@ namespace mob::tasks {
         void do_fetch() override;
     };
 
+    class directxtex : public basic_task<directxtex> {
+    public:
+        directxtex();
+
+        static std::string version();
+        static bool prebuilt();
+        static fs::path source_path();
+
+    protected:
+        void do_clean(clean c) override;
+        void do_fetch() override;
+        void do_build_and_install() override;
+    };
+
     class explorerpp : public basic_task<explorerpp> {
     public:
         explorerpp();

src/tools/python.cpp

@@ -69,6 +69,12 @@ namespace mob {
         return *this;
     }
 
+    pip& pip::index(const std::string& s)
+    {
+        index_ = s;
+        return *this;
+    }
+
     pip& pip::no_dependencies()
     {
         no_deps_ = true;
@@ -147,6 +153,10 @@ namespace mob {
                      .arg("--no-warn-script-location")
                      .arg("--disable-pip-version-check");
 
+        if (!index_.empty()) {
+            p.arg("-i", index_).arg("--extra-index-url", "https://pypi.org/simple")
+        }
+
         if (!package_.empty()) {
             if (version_.empty()) {
                 p.arg(package_);
@@ -169,19 +179,26 @@ namespace mob {
 
     void pip::do_download()
     {
-        execute_and_join(process()
+        auto p = process()
-                             .binary(tasks::python::python_exe())
+                     .binary(tasks::python::python_exe())
-                             .chcp(65001)
+                     .chcp(65001)
-                             .stdout_encoding(encodings::utf8)
+                     .stdout_encoding(encodings::utf8)
-                             .stderr_encoding(encodings::utf8)
+                     .stderr_encoding(encodings::utf8)
-                             .arg("-X", "utf8")
+                     .arg("-X", "utf8")
-                             .arg("-m", "pip")
+                     .arg("-m", "pip")
-                             .arg("download")
+                     .arg("download");
-                             .arg("--no-binary=:all:")
+
-                             .arg("--no-deps")
+        if (!index_.empty()) {
-                             .arg("-d", conf().path().cache())
+            p.arg("-i", index_).arg("--extra-index-url", "https://pypi.org/simple")
-                             .arg(package_ + "==" + version_)
+        }
-                             .env(this_env::get().set("PYTHONUTF8", "1")));
+
+        p.arg("--no-binary=:all:")
+            .arg("--no-deps")
+            .arg("-d", conf().path().cache())
+            .arg(package_ + "==" + version_)
+            .env(this_env::get().set("PYTHONUTF8", "1"));
+
+        execute_and_join(p);
     }
 
 }  // namespace mob

src/tools/tools.h

@@ -702,6 +702,7 @@ namespace mob {
         pip& package(const std::string& s);
         pip& version(const std::string& s);
         pip& file(const fs::path& p);
+        pip& index(const std::string& s);
 
         // do not install dependencies for the package
         //
@@ -720,6 +721,7 @@ namespace mob {
         std::string package_;
         std::string version_;
         fs::path file_;
+        std::string index_;
 
         // no depndencies
         bool no_deps_;