Merge pull request #1178 from isanae/2_3_0

Merge 2.3.1 changes

.github/ISSUE_TEMPLATE/issue-report.md

@@ -28,10 +28,11 @@ If necessary, describe the problem you have been experiencing with more details
 
 
 ## Link to Mod Organizer logs:
-To access the log files you can press the Open folder icon over the modlist and select "Open MO2 Logs folder". You should close MO2 before sending or examining the logs as they are finished writing only after the program is closed.
+Right click on the log window at the bottom of MO2, set log level to debug, restart Mo2 and reproduce the problem.
+To access the log files you can press the Open folder icon over the modlist and select "Open MO2 Logs folder".
 
 ### USVFS:
-Create a [GIST](https://gist.github.com) which is a paste of your USVFS logs, and link them here.
+Create a [GIST](https://gist.github.com) which is a paste of your latest USVFS log, and link them here.
 Do _NOT_ paste your full USVFS logs here, as it will make this issue very long and hard to read!
 If you are reporting a bug, _always_ include the USVFS logs!
 

.gitignore

@@ -13,3 +13,4 @@ edit
 /*std*.log
 /*build
 
+/src/version.aps

CMakeLists.txt

@@ -1,39 +1,20 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.16)
 
-ADD_COMPILE_OPTIONS(
-	$<$<CXX_COMPILER_ID:MSVC>:/MP>
-	$<$<CXX_COMPILER_ID:MSVC>:/D_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING>
-	$<$<CXX_COMPILER_ID:MSVC>:$<$<CONFIG:RELEASE>:/O2>>
-	$<$<CXX_COMPILER_ID:MSVC>:$<$<CONFIG:RELWITHDEBINFO>:/O2>>)
+project(organizer)
+set(project_type exe)
+set(executable_name ModOrganizer)
+set(enable_warnings OFF)
 
-PROJECT(organizer)
-
-# sets ModOrganizer as StartUp Project in Visual Studio
-set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY VS_STARTUP_PROJECT "ModOrganizer")
-
-set(CMAKE_VS_INCLUDE_INSTALL_TO_DEFAULT_BUILD 1)
-set(CMAKE_INSTALL_MESSAGE NEVER)
-
-SET(DEPENDENCIES_DIR CACHE PATH "")
-
-# hint to find qt in dependencies path
-LIST(APPEND CMAKE_PREFIX_PATH ${QT_ROOT}/lib/cmake)
-LIST(APPEND CMAKE_PREFIX_PATH ${LZ4_ROOT}/dll)
-LIST(APPEND CMAKE_PREFIX_PATH ${FMT_ROOT}/build)
-
-ADD_SUBDIRECTORY(src)
-
-set(vcxproj_user_file "${CMAKE_CURRENT_BINARY_DIR}/INSTALL.vcxproj.user")
-if(NOT EXISTS ${vcxproj_user_file})
-	FILE(WRITE ${vcxproj_user_file}
-		"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
-			"<Project ToolsVersion=\"Current\" xmlns=\"http://schemas.microsoft.com/developer/msbuild/2003\">\n"
-			  "<PropertyGroup>\n"
-				"<LocalDebuggerWorkingDirectory>${CMAKE_INSTALL_PREFIX}/bin</LocalDebuggerWorkingDirectory>\n"
-				"<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>"
-				"<LocalDebuggerCommand>${CMAKE_INSTALL_PREFIX}/bin/ModOrganizer.exe</LocalDebuggerCommand>\n"
-			  "</PropertyGroup>\n"
-			"</Project>\n")
+# appveyor does not build modorganizer in its standard location, so use
+# DEPENDENCIES_DIR to find cmake_common
+if(DEFINED DEPENDENCIES_DIR)
+	include(${DEPENDENCIES_DIR}/modorganizer_super/cmake_common/project.cmake)
+else()
+	include(../cmake_common/project.cmake)
 endif()
 
-INSTALL(FILES dump_running_process.bat DESTINATION bin)
+set(additional_translations ${uibase_path}/src)
+
+add_subdirectory(src)
+
+install(FILES dump_running_process.bat DESTINATION bin)

appveyor.yml

@@ -1,32 +1,69 @@
-version: 1.0.{build}
+version: dev-appveyor{build}
 skip_branch_with_pr: true
 image: Visual Studio 2019
+init:
+- ps: >-
+    # define build version depending on nightly or normal build
+
+    if($env:APPVEYOR_SCHEDULED_BUILD -eq 'True'){
+        $timestamp= Get-Date -Format "ddMMyyyy-HHmm"
+        Update-AppveyorBuild -Version "$($env:MO_VERSION)$($env:VER_STUB_NIGHTLY)$timestamp"
+    } else {
+        Update-AppveyorBuild -Version "$($env:MO_VERSION)$($env:VER_STUB_NORMAL)$($env:APPVEYOR_BUILD_NUMBER)"
+    }
+
+    Write-Host Build version set to: $env:APPVEYOR_BUILD_VERSION
 environment:
   WEBHOOK_URL:
-    secure: gOKbXaZM9ImtMD5XrYITvdyZUW/az082G9OIN1EC1VYoNNsKROB4s6VsQiL6qbLNLc4XQhc8Y9DhdU9Hfh49vhypcjUEQmMsENxnp+1hYMJctTqIYDunVSd767eez5wz6rKHqk+XOHmg5gyVNCV9u4DVcCZJSVe4UTKn1lc47H0=
-build:
-  parallel: true
+    secure: gOKbXaZM9ImtMD5XrYITvdyZUW/az082G9OIN1EC1VaKiI9iefpxhBavJ6Al6CzIZvQ+3pxnLqjmNgA7cDc22wcj2kB4hSG5qhbTI8wGa8jLQ5L65nuRZ3vrIqghBz9G3GLglgZkg6eqH9r3Kqc6UzcpCGzxxPOqm550nRcIiUU=
+  MO_VERSION: 2.3.0
+  VER_STUB_NORMAL: dev-appveyor
+  VER_STUB_NIGHTLY: dev-nightly-
 build_script:
 - cmd: >-
-    git clone --depth=1 --branch=%APPVEYOR_REPO_BRANCH% https://github.com/ModOrganizer2/modorganizer-umbrella.git c:\projects\modorganizer-umbrella 2> $null
+    REM Maintenance comments:
+
+    REM APPVEYOR_BUILD_FOLDER= "c:\projects\modorganizer-slug"
+
+    REM -Need to update py3 version used to invoke unimake.py once in a while.
+
+    REM -Need update MO_VERSION env variable after each release.
+
+    REM -Always clones umbrella master
+
+    REM -Will checkout all the branches matching the one that triggered the build on the main repo.
+
+    REM End comments.
+
+
+    git clone --depth=1 https://github.com/ModOrganizer2/modorganizer-umbrella.git c:\projects\modorganizer-umbrella 2> $null
+
+
+    mkdir %APPVEYOR_BUILD_FOLDER%\modorganizer-build -type directory
 
-    mkdir c:\projects\modorganizer-build -type directory
 
     cd c:\projects\modorganizer-umbrella
 
-    C:\Python37-x64\python.exe unimake.py -d c:\projects\modorganizer-build -s Appveyor_Build=True %APPVEYOR_PROJECT_NAME%
+
+    C:\Python38-x64\python.exe unimake.py -d %APPVEYOR_BUILD_FOLDER%\modorganizer-build -s Appveyor_Build=True -s Feature_Branch=%APPVEYOR_REPO_BRANCH% -s override_build_version=%APPVEYOR_BUILD_VERSION%
+test: off
 artifacts:
-- path: build\src\ModOrganizer.exe
-  name: modorganizer_exe
-- path: build\src\ModOrganizer.pdb
-  name: modorganizer_pdb
+- path: '\modorganizer-build\install\bin'
+  name: Mod.Organizer-$(APPVEYOR_BUILD_VERSION)
+- path: '\modorganizer-build\install\pdb'
+  name: PDBs-Mod.Organizer-$(APPVEYOR_BUILD_VERSION)
+deploy: off
 on_success:
-  - ps: Set-Location -Path $env:APPVEYOR_BUILD_FOLDER
-  - ps: Invoke-RestMethod https://raw.githubusercontent.com/DiscordHooks/appveyor-discord-webhook/master/send.ps1 -o send.ps1
-  - ps: ./send.ps1 success $env:WEBHOOK_URL
+- ps: >-
+    Set-Location -Path $env:APPVEYOR_BUILD_FOLDER
+
+    Invoke-RestMethod https://raw.githubusercontent.com/DiscordHooks/appveyor-discord-webhook/master/send.ps1 -o send.ps1
+
+    ./send.ps1 success $env:WEBHOOK_URL
 on_failure:
-  - ps: Set-Location -Path $env:APPVEYOR_BUILD_FOLDER
-  - ps: Push-AppveyorArtifact ${env:APPVEYOR_BUILD_FOLDER}\stdout.log
-  - ps: Push-AppveyorArtifact ${env:APPVEYOR_BUILD_FOLDER}\stderr.log
-  - ps: Invoke-RestMethod https://raw.githubusercontent.com/DiscordHooks/appveyor-discord-webhook/master/send.ps1 -o send.ps1
-  - ps: ./send.ps1 failure $env:WEBHOOK_URL
+- ps: >-
+    Set-Location -Path $env:APPVEYOR_BUILD_FOLDER
+
+    Invoke-RestMethod https://raw.githubusercontent.com/DiscordHooks/appveyor-discord-webhook/master/send.ps1 -o send.ps1
+
+    ./send.ps1 failure $env:WEBHOOK_URL

modorganizer.natvis

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
+<Type Name="FileTreeItem"><DisplayString>{{
+file={m_wsFile}
+ dir={m_isDirectory}
+ origin={m_originID}
+ flags={m_flags.i}
+ loaded={m_loaded}
+ expanded={m_expanded}
+}}</DisplayString></Type>
+
+</AutoVisualizer>

readme.md

@@ -1,4 +1,4 @@
-[![Build status](https://ci.appveyor.com/api/projects/status/23qmd9rufv9vpmg7?svg=true)](https://ci.appveyor.com/project/Modorganizer2/modorganizer)
+[![Build status](https://ci.appveyor.com/api/projects/status/hxenwxmpaob5xung?svg=true)](https://ci.appveyor.com/project/ModOrganizer2/modorganizer-736bd)
 
 # Mod Organizer
 
@@ -43,6 +43,7 @@ MO2 consists of multiple repositories on github. The Umbrella project will downl
 Here is a complete list:
 
 * https://github.com/LePresidente/cpython-1
+* https://github.com/ModOrganizer2/cmake_common
 * https://github.com/ModOrganizer2/githubpp
 * https://github.com/ModOrganizer2/modorganizer
 * https://github.com/ModOrganizer2/modorganizer-archive

src/aboutdialog.cpp

@@ -20,7 +20,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 
 #include "aboutdialog.h"
 #include "ui_aboutdialog.h"
-#include "util.h"
+#include "shared/util.h"
 #include <utility.h>
 
 #include <QApplication>

src/aboutdialog.ui

@@ -217,7 +217,7 @@
               </item>
               <item>
                <property name="text">
-                <string notr="true">isanae</string>
+                <string notr="true">isa</string>
                </property>
               </item>
              </widget>
@@ -256,6 +256,11 @@
                 <string notr="true">przester</string>
                </property>
               </item>
+              <item>
+               <property name="text">
+                <string notr="true">Holt59</string>
+               </property>
+              </item>
              </widget>
             </item>
            </layout>
@@ -390,6 +395,11 @@
                 <string>Nubbie (Swedish)</string>
                </property>
               </item>
+              <item>
+               <property name="text">
+                <string notr="true">Hakan &quot;Nyks45&quot; Albayrak (Turkish)</string>
+               </property>
+              </item>
               <item>
                <property name="text">
                 <string>...and all other contributors!</string>
@@ -429,6 +439,11 @@
                 <string notr="true">DoubleYou</string>
                </property>
               </item>
+              <item>
+               <property name="text">
+                <string notr="true">Drew Warwick</string>
+               </property>
+              </item>
               <item>
                <property name="text">
                 <string notr="true">deathneko11</string>
@@ -474,6 +489,11 @@
                 <string notr="true">outdatedtv</string>
                </property>
               </item>
+              <item>
+               <property name="text">
+                <string notr="true">PurpleFez</string>
+               </property>
+              </item>
               <item>
                <property name="text">
                 <string notr="true">reedts</string>

src/activatemodsdialog.h

@@ -53,7 +53,7 @@ public:
 
   /**
    * @brief get a list of mods that the user chose to activate
-   * 
+   *
    * @note This can of ocurse only be called after the dialog has been displayed
    *
    * @return set< QString > the mods to activate

src/archivefiletree.cpp

@@ -0,0 +1,260 @@
+/*
+Copyright (C) MO2 Team. All rights reserved.
+
+This file is part of Mod Organizer.
+
+Mod Organizer is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Mod Organizer is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// For QObject::tr:
+#include <QObject>
+
+#include "archivefiletree.h"
+
+#include "log.h"
+
+using namespace MOBase;
+
+/**
+ * We use custom file entries to store the index.
+ */
+class ArchiveFileEntry : public virtual FileTreeEntry {
+public:
+
+  /**
+   * @brief Create a new entry.
+   *
+   * @param parent The tree containing this entry.
+   * @param name The name of this entry.
+   * @param index The index of the entry in the archive.
+   */
+  ArchiveFileEntry(std::shared_ptr<const IFileTree> parent, QString name, int index) :
+    FileTreeEntry(parent, name), m_Index(index) {
+  }
+
+  virtual std::shared_ptr<FileTreeEntry> clone() const override {
+    return std::make_shared<ArchiveFileEntry>(nullptr, name(), m_Index);
+  }
+
+  // No private since we are in an implementation file:
+  const int m_Index;
+};
+
+
+/**
+ *
+ */
+class ArchiveFileTreeImpl: public virtual ArchiveFileTree, public virtual ArchiveFileEntry {
+public:
+
+  using File = std::tuple<QStringList, bool, int>;
+
+public: // Public for make_shared (but not accessible by other since not exposed in .h):
+
+  ArchiveFileTreeImpl(std::shared_ptr<const IFileTree> parent, QString name, int index, std::vector<File> files)
+    : FileTreeEntry(parent, name), ArchiveFileEntry(parent, name, index), IFileTree(), m_Files(std::move(files)) { }
+
+public: // Override to avoid VS warnings:
+
+  virtual std::shared_ptr<IFileTree> astree() override {
+    return IFileTree::astree();
+  }
+
+  virtual std::shared_ptr<const IFileTree> astree() const override {
+    return IFileTree::astree();
+  }
+
+protected:
+
+  virtual std::shared_ptr<FileTreeEntry> clone() const override {
+    return IFileTree::clone();
+  }
+
+public: // Overrides:
+
+  /**
+   *
+   */
+  static void mapToArchive(IFileTree const& tree, QString path, std::vector<FileData*> const& data)
+  {
+    if (path.length() > 0) {
+      // when using a long windows path (starting with \\?\) we apparently can have redundant
+      // . components in the path. This wasn't a problem with "regular" path names.
+      if (path == ".") {
+        path.clear();
+      }
+      else {
+        path.append("\\");
+      }
+    }
+
+    for (auto const& entry : tree) {
+      if (entry->isDir()) {
+        const ArchiveFileTreeImpl& archiveEntry = dynamic_cast<const ArchiveFileTreeImpl&>(*entry);
+        QString tmp = path + archiveEntry.name();
+        if (archiveEntry.m_Index != -1) {
+          data[archiveEntry.m_Index]->addOutputFilePath(tmp.toStdWString());
+        }
+        mapToArchive(*archiveEntry.astree(), tmp, data);
+      }
+      else {
+        const ArchiveFileEntry& archiveFileEntry = dynamic_cast<const ArchiveFileEntry&>(*entry);
+        if (archiveFileEntry.m_Index != -1) {
+          data[archiveFileEntry.m_Index]->addOutputFilePath((path + archiveFileEntry.name()).toStdWString());
+        }
+      }
+    }
+  }
+
+  /**
+   *
+   */
+  void mapToArchive(Archive &archive) const override {
+    mapToArchive(*this, "", archive.getFileList());
+  }
+
+protected:
+
+  /**
+   * Overriding makeDirectory and makeFile to create file tree or file entry with index -1.
+   *
+   */
+  virtual std::shared_ptr<IFileTree> makeDirectory(
+    std::shared_ptr<const IFileTree> parent, QString name) const override {
+    return std::make_shared<ArchiveFileTreeImpl>(parent, name, -1, std::vector<File>{});
+  }
+
+  virtual std::shared_ptr<FileTreeEntry> makeFile(
+    std::shared_ptr<const IFileTree> parent, QString name) const override {
+    return std::make_shared<ArchiveFileEntry>(parent, name, -1);
+  }
+
+  virtual bool doPopulate(std::shared_ptr<const IFileTree> parent, std::vector<std::shared_ptr<FileTreeEntry>>& entries) const override {
+
+    // Sort by name:
+    std::sort(std::begin(m_Files), std::end(m_Files),
+      [](const auto& a, const auto& b) {
+        return std::get<0>(a)[0].compare(std::get<0>(b)[0], Qt::CaseInsensitive) < 0; });
+
+    // We know that the files are sorted:
+    QString currentName = "";
+    int currentIndex = -1;
+    std::vector<File> currentFiles;
+    for (auto& p : m_Files) {
+
+      // At the start or if we have reset, just retrieve the current name:
+      if (currentName == "") {
+        currentName = std::get<0>(p)[0];
+      }
+
+      // If the name is different, we need to create a directory from what we have 
+      // accumulated:
+      if (currentName != std::get<0>(p)[0]) {
+
+        // We may or may not have an index here, it depends on the type of archive (some archives list
+        // intermediate non-empty folders, some don't):
+        entries.push_back(std::make_shared<ArchiveFileTreeImpl>(parent, currentName, currentIndex, std::move(currentFiles)));
+        
+        currentFiles.clear(); // Back to a valid state.
+
+        // Reset the index:
+        currentIndex = -1;
+      }
+
+      // We can always override the current name:
+      currentName = std::get<0>(p)[0];
+
+      // If the current path contains only one components:
+      if (std::get<0>(p).size() == 1) {
+
+        // If it is not a directory, then it is a file in directly under this tree:
+        if (!std::get<1>(p)) {
+          entries.push_back(
+            std::make_shared<ArchiveFileEntry>(parent, currentName, std::get<2>(p)));
+          currentName = "";
+        }
+        else {
+          // Otherwize, it is the actual "file" corresponding to the directory we are listing, so we can retrieve
+          // the index here:
+          currentIndex = std::get<2>(p);
+        }
+      }
+      else {
+        currentFiles.push_back({
+          QStringList(std::get<0>(p).begin() + 1, std::get<0>(p).end()), std::get<1>(p), std::get<2>(p)
+        });
+      }
+    }
+
+    if (currentName != "") {
+      entries.push_back(std::make_shared<ArchiveFileTreeImpl>(parent, currentName, currentIndex, std::move(currentFiles)));
+    }
+    
+    // Let the parent class sort the entries:
+    return false;
+  }
+
+  virtual std::shared_ptr<IFileTree> doClone() const override {
+    return std::make_shared<ArchiveFileTreeImpl>(nullptr, name(), m_Index, m_Files);
+  }
+
+private:
+
+  mutable std::vector<File> m_Files;
+};
+
+std::shared_ptr<ArchiveFileTree> ArchiveFileTree::makeTree(Archive const& archive) 
+{
+  auto const& data = archive.getFileList();
+
+  std::vector<ArchiveFileTreeImpl::File> files;
+  files.reserve(data.size());
+
+  for (size_t i = 0; i < data.size(); ++i) {
+    files.push_back(std::make_tuple(
+      QString::fromStdWString(data[i]->getArchiveFilePath()).replace("\\", "/").split("/", Qt::SkipEmptyParts), 
+      data[i]->isDirectory(), 
+      (int) i));
+  }
+
+  auto tree = std::make_shared<ArchiveFileTreeImpl>(nullptr, "", -1, std::move(files));
+  return tree;
+}
+
+/**
+ * @brief Recursive function for the ArchiveFileTree::mapToArchive method. Need a template
+ * here because iterators from a vector of entries are not exactly the same as the iterators
+ * returned by a IFileTree.
+ *
+ */
+template <class It>
+void mapToArchive(std::vector<FileData*> const& data, It begin, It end) {
+  for (auto it = begin; it != end; ++it) {
+    auto entry = *it;
+    auto* aentry = dynamic_cast<const ArchiveFileEntry*>(entry.get());
+
+    if (aentry->m_Index != -1) {
+      data[aentry->m_Index]->addOutputFilePath(aentry->path().toStdWString());
+    }
+
+    if (entry->isDir()) {
+      auto tree = entry->astree();
+      mapToArchive(data, tree->begin(), tree->end());
+    }
+  }
+}
+
+void ArchiveFileTree::mapToArchive(Archive &archive, std::vector<std::shared_ptr<const FileTreeEntry>> const& entries) {
+  ::mapToArchive(archive.getFileList(), entries.cbegin(), entries.cend());
+}

src/archivefiletree.h

@@ -0,0 +1,72 @@
+/*
+Copyright (C) MO2 Team. All rights reserved.
+
+This file is part of Mod Organizer.
+
+Mod Organizer is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Mod Organizer is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef ARCHIVEFILENETRY_H
+#define ARCHIVEFILENTRY_H
+
+#include "archive.h"
+#include "ifiletree.h"
+
+
+/**
+ *
+ */
+class ArchiveFileTree: public virtual MOBase::IFileTree {
+public:
+
+  /**
+   * @brief Create a new file tree representing the given archive.
+   *
+   * @param archive Archive to represent by the file tree.
+   *
+   * @return a file tree representing the given archive.
+   */
+  static std::shared_ptr<ArchiveFileTree> makeTree(Archive const& archive);
+
+  /**
+   * @brief Update the given archive to reflect change in this tree.
+   *
+   * This method disables files that have been removed from the file
+   * tree and move the ones that have been moved.
+   *
+   * @param archive The archive to update. Must be the one used to
+   *     create the tree.
+   */
+  virtual void mapToArchive(Archive &archive) const = 0;
+
+  /**
+   * @brief Update the given archive to prepare for the extraction
+   *     of the given entries.
+   *
+   * This method "enables" files that correspond to the given entry.
+   *
+   * @param archive The archive to update. Must be the one used to
+   *     create the tree.
+   * @param entries List of entries to mark for extraction. All the entries must
+   *     come from a tree created with the given archive.
+   */
+  static void mapToArchive(Archive &archive, std::vector<std::shared_ptr<const FileTreeEntry>> const& entries);
+
+protected:
+
+  using IFileTree::IFileTree;
+
+};
+
+#endif

src/bbcode.cpp

@@ -246,7 +246,14 @@ QString convertToHTML(const QString &inputParam)
     if ((pos < (input.size() - 1)) && (input.at(pos + 1) == '/')) {
        // skip invalid end tag
       int tagEnd = input.indexOf(']',  pos) + 1;
-      pos = tagEnd;
+      if (tagEnd == 0) {
+        //no closing tag found
+        //move the pos up one so that the opening bracket is ignored next iteration
+        pos++;
+      }
+      else {
+        pos = tagEnd;
+      }
     } else {
       // convert the tag and content if necessary
       int length = -1;

src/categories.cpp

@@ -113,7 +113,7 @@ void CategoryFactory::reset()
   // 43 = Savegames (makes no sense to install them through MO)
   // 45 = Videos and trailers
   // 87 = Miscelanous
-  addCategory(0, "None", MakeVector<int>(4, 28, 43, 45, 87), 0);
+  addCategory(0, "None", { 4, 28, 43, 45, 87 }, 0);
 }
 
 
@@ -206,63 +206,63 @@ void CategoryFactory::loadDefaultCategories()
 {
   // the order here is relevant as it defines the order in which the
   // mods appear in the combo box
-  addCategory(1, "Animations", MakeVector<int>(2, 4, 51), 0);
-  addCategory(52, "Poses", MakeVector<int>(1, 29), 1);
-  addCategory(2, "Armour", MakeVector<int>(2, 5, 54), 0);
-  addCategory(53, "Power Armor", MakeVector<int>(1, 53), 2);
-  addCategory(3, "Audio", MakeVector<int>(3, 33, 35, 106), 0);
-  addCategory(38, "Music", MakeVector<int>(2, 34, 61), 0);
-  addCategory(39, "Voice", MakeVector<int>(2, 36, 107), 0);
-  addCategory(5, "Clothing", MakeVector<int>(2, 9, 60), 0);
-  addCategory(41, "Jewelry", MakeVector<int>(1, 102), 5);
-  addCategory(42, "Backpacks", MakeVector<int>(1, 49), 5);
-  addCategory(6, "Collectables", MakeVector<int>(2, 10, 92), 0);
-  addCategory(28, "Companions", MakeVector<int>(3, 11, 66, 96), 0);
-  addCategory(7, "Creatures, Mounts, & Vehicles", MakeVector<int>(4, 12, 65, 83, 101), 0);
-  addCategory(8, "Factions", MakeVector<int>(2, 16, 25), 0);
-  addCategory(9, "Gameplay", MakeVector<int>(2, 15, 24), 0);
-  addCategory(27, "Combat", MakeVector<int>(1, 77), 9);
-  addCategory(43, "Crafting", MakeVector<int>(2, 50, 100), 9);
-  addCategory(48, "Overhauls", MakeVector<int>(2, 24, 79), 9);
-  addCategory(49, "Perks", MakeVector<int>(1, 27), 9);
-  addCategory(54, "Radio", MakeVector<int>(1, 31), 9);
-  addCategory(55, "Shouts", MakeVector<int>(1, 104), 9);
-  addCategory(22, "Skills & Levelling", MakeVector<int>(2, 46, 73), 9);
-  addCategory(58, "Weather & Lighting", MakeVector<int>(1, 56), 9);
-  addCategory(44, "Equipment", MakeVector<int>(1, 44), 43);
-  addCategory(45, "Home/Settlement", MakeVector<int>(1, 45), 43);
-  addCategory(10, "Body, Face, & Hair", MakeVector<int>(2, 17, 26), 0);
-  addCategory(39, "Tattoos", MakeVector<int>(1, 57), 10);
-  addCategory(40, "Character Presets", MakeVector<int>(1, 58), 0);
-  addCategory(11, "Items", MakeVector<int>(2, 27, 85), 0);
-  addCategory(32, "Mercantile", MakeVector<int>(2, 23, 69), 0);
-  addCategory(37, "Ammo", MakeVector<int>(1, 3), 11);
-  addCategory(19, "Weapons", MakeVector<int>(2, 41, 55), 11);
-  addCategory(36, "Weapon & Armour Sets", MakeVector<int>(1, 42), 11);
-  addCategory(23, "Player Homes", MakeVector<int>(2, 28, 67), 0);
-  addCategory(25, "Castles & Mansions", MakeVector<int>(1, 68), 23);
-  addCategory(51, "Settlements", MakeVector<int>(1, 48), 23);
-  addCategory(12, "Locations", MakeVector<int>(10, 20, 21, 22, 30, 47, 70, 88, 89, 90, 91), 0);
-  addCategory(4, "Cities", MakeVector<int>(1, 53), 12);
-  addCategory(31, "Landscape Changes", MakeVector<int>(1, 58), 0);
-  addCategory(29, "Environment", MakeVector<int>(2, 14, 74), 0);
-  addCategory(30, "Immersion", MakeVector<int>(2, 51, 78), 0);
-  addCategory(20, "Magic", MakeVector<int>(3, 75, 93, 94), 0);
-  addCategory(21, "Models & Textures", MakeVector<int>(2, 19, 29), 0);
-  addCategory(33, "Modders resources", MakeVector<int>(2, 18, 82), 0);
-  addCategory(13, "NPCs", MakeVector<int>(3, 22, 33, 99), 0);
-  addCategory(24, "Bugfixes", MakeVector<int>(2, 6, 95), 0);
-  addCategory(14, "Patches", MakeVector<int>(2, 25, 84), 24);
-  addCategory(35, "Utilities", MakeVector<int>(2, 38, 39), 0);
-  addCategory(26, "Cheats", MakeVector<int>(1, 8), 0);
-  addCategory(15, "Quests", MakeVector<int>(2, 30, 35), 0);
-  addCategory(16, "Races & Classes", MakeVector<int>(1, 34), 0);
-  addCategory(34, "Stealth", MakeVector<int>(1, 76), 0);
-  addCategory(17, "UI", MakeVector<int>(2, 37, 42), 0);
-  addCategory(18, "Visuals", MakeVector<int>(2, 40, 62), 0);
-  addCategory(50, "Pip-Boy", MakeVector<int>(1, 52), 18);
-  addCategory(46, "Shader Presets", MakeVector<int>(3, 13, 97, 105), 0);
-  addCategory(47, "Miscellaneous", MakeVector<int>(2, 2, 28), 0);
+  addCategory(1, "Animations", { 2, 4, 51 }, 0);
+  addCategory(52, "Poses", { 1, 29 }, 1);
+  addCategory(2, "Armour", { 2, 5, 54 }, 0);
+  addCategory(53, "Power Armor", { 1, 53 }, 2);
+  addCategory(3, "Audio", { 3, 33, 35, 106 }, 0);
+  addCategory(38, "Music", { 2, 34, 61 }, 0);
+  addCategory(39, "Voice", { 2, 36, 107 }, 0);
+  addCategory(5, "Clothing", { 2, 9, 60 }, 0);
+  addCategory(41, "Jewelry", { 1, 102 }, 5);
+  addCategory(42, "Backpacks", { 1, 49 }, 5);
+  addCategory(6, "Collectables", { 2, 10, 92 }, 0);
+  addCategory(28, "Companions", { 3, 11, 66, 96 }, 0);
+  addCategory(7, "Creatures, Mounts, & Vehicles", { 4, 12, 65, 83, 101 }, 0);
+  addCategory(8, "Factions", { 2, 16, 25 }, 0);
+  addCategory(9, "Gameplay", { 2, 15, 24 }, 0);
+  addCategory(27, "Combat", { 1, 77 }, 9);
+  addCategory(43, "Crafting", { 2, 50, 100 }, 9);
+  addCategory(48, "Overhauls", { 2, 24, 79 }, 9);
+  addCategory(49, "Perks", { 1, 27 }, 9);
+  addCategory(54, "Radio", { 1, 31 }, 9);
+  addCategory(55, "Shouts", { 1, 104 }, 9);
+  addCategory(22, "Skills & Levelling", { 2, 46, 73 }, 9);
+  addCategory(58, "Weather & Lighting", { 1, 56 }, 9);
+  addCategory(44, "Equipment", { 1, 44 }, 43);
+  addCategory(45, "Home/Settlement", { 1, 45 }, 43);
+  addCategory(10, "Body, Face, & Hair", { 2, 17, 26 }, 0);
+  addCategory(39, "Tattoos", { 1, 57 }, 10);
+  addCategory(40, "Character Presets", { 1, 58 }, 0);
+  addCategory(11, "Items", { 2, 27, 85 }, 0);
+  addCategory(32, "Mercantile", { 2, 23, 69 }, 0);
+  addCategory(37, "Ammo", { 1, 3 }, 11);
+  addCategory(19, "Weapons", { 2, 41, 55 }, 11);
+  addCategory(36, "Weapon & Armour Sets", { 1, 42 }, 11);
+  addCategory(23, "Player Homes", { 2, 28, 67 }, 0);
+  addCategory(25, "Castles & Mansions", { 1, 68 }, 23);
+  addCategory(51, "Settlements", { 1, 48 }, 23);
+  addCategory(12, "Locations", { 10, 20, 21, 22, 30, 47, 70, 88, 89, 90, 91 }, 0);
+  addCategory(4, "Cities", { 1, 53 }, 12);
+  addCategory(31, "Landscape Changes", { 1, 58 }, 0);
+  addCategory(29, "Environment", { 2, 14, 74 }, 0);
+  addCategory(30, "Immersion", { 2, 51, 78 }, 0);
+  addCategory(20, "Magic", { 3, 75, 93, 94 }, 0);
+  addCategory(21, "Models & Textures", { 2, 19, 29 }, 0);
+  addCategory(33, "Modders resources", { 2, 18, 82 }, 0);
+  addCategory(13, "NPCs", { 3, 22, 33, 99 }, 0);
+  addCategory(24, "Bugfixes", { 2, 6, 95 }, 0);
+  addCategory(14, "Patches", { 2, 25, 84 }, 24);
+  addCategory(35, "Utilities", { 2, 38, 39 }, 0);
+  addCategory(26, "Cheats", { 1, 8 }, 0);
+  addCategory(15, "Quests", { 2, 30, 35 }, 0);
+  addCategory(16, "Races & Classes", { 1, 34 }, 0);
+  addCategory(34, "Stealth", { 1, 76 }, 0);
+  addCategory(17, "UI", { 2, 37, 42 }, 0);
+  addCategory(18, "Visuals", { 2, 40, 62 }, 0);
+  addCategory(50, "Pip-Boy", { 1, 52 }, 18);
+  addCategory(46, "Shader Presets", { 3, 13, 97, 105 }, 0);
+  addCategory(47, "Miscellaneous", { 2, 2, 28 }, 0);
 }
 
 
@@ -329,6 +329,7 @@ QString CategoryFactory::getSpecialCategoryName(SpecialCategories type) const
     case UpdateAvailable: label = QObject::tr("Update available"); break;
     case HasCategory:     label = QObject::tr("Has category"); break;
     case Conflict:        label = QObject::tr("Conflicted"); break;
+    case HasHiddenFiles:  label = QObject::tr("Has hidden files"); break;
     case Endorsed:        label = QObject::tr("Endorsed"); break;
     case Backup:          label = QObject::tr("Has backup"); break;
     case Managed:         label = QObject::tr("Managed"); break;

src/categories.h

@@ -43,6 +43,7 @@ public:
     UpdateAvailable,
     HasCategory,
     Conflict,
+    HasHiddenFiles,
     Endorsed,
     Backup,
     Managed,

src/datatab.cpp

@@ -0,0 +1,153 @@
+#include "datatab.h"
+#include "ui_mainwindow.h"
+#include "settings.h"
+#include "organizercore.h"
+#include "messagedialog.h"
+#include "filetree.h"
+#include "filetreemodel.h"
+#include <log.h>
+#include <report.h>
+
+using namespace MOShared;
+using namespace MOBase;
+
+// in mainwindow.cpp
+QString UnmanagedModName();
+
+
+DataTab::DataTab(
+  OrganizerCore& core, PluginContainer& pc,
+  QWidget* parent, Ui::MainWindow* mwui) :
+    m_core(core), m_pluginContainer(pc), m_parent(parent),
+    ui{
+      mwui->dataTabRefresh, mwui->dataTree,
+      mwui->dataTabShowOnlyConflicts, mwui->dataTabShowFromArchives},
+    m_firstActivation(true)
+{
+  m_filetree.reset(new FileTree(core, m_pluginContainer, ui.tree));
+  m_filter.setUseSourceSort(true);
+  m_filter.setFilterColumn(FileTreeModel::FileName);
+  m_filter.setEdit(mwui->dataTabFilter);
+  m_filter.setList(mwui->dataTree);
+  m_filter.setUpdateDelay(true);
+
+  if (auto* m=m_filter.proxyModel()) {
+    m->setDynamicSortFilter(false);
+  }
+
+  connect(
+    &m_filter, &FilterWidget::aboutToChange,
+    [&]{ ensureFullyLoaded(); });
+
+  connect(
+    ui.refresh, &QPushButton::clicked,
+    [&]{ onRefresh(); });
+
+  connect(
+    ui.conflicts, &QCheckBox::toggled,
+    [&]{ onConflicts(); });
+
+  connect(
+    ui.archives, &QCheckBox::toggled,
+    [&]{ onArchives(); });
+
+  connect(
+    m_filetree.get(), &FileTree::executablesChanged,
+    this, &DataTab::executablesChanged);
+
+  connect(
+    m_filetree.get(), &FileTree::originModified,
+    this, &DataTab::originModified);
+
+  connect(
+    m_filetree.get(), &FileTree::displayModInformation,
+    this, &DataTab::displayModInformation);
+}
+
+void DataTab::saveState(Settings& s) const
+{
+  s.geometry().saveState(ui.tree->header());
+  s.widgets().saveChecked(ui.conflicts);
+  s.widgets().saveChecked(ui.archives);
+}
+
+void DataTab::restoreState(const Settings& s)
+{
+  s.geometry().restoreState(ui.tree->header());
+
+  // prior to 2.3, the list was not sortable, and this remembered in the
+  // widget state, for whatever reason
+  ui.tree->setSortingEnabled(true);
+
+  s.widgets().restoreChecked(ui.conflicts);
+  s.widgets().restoreChecked(ui.archives);
+}
+
+void DataTab::activated()
+{
+  if (m_firstActivation) {
+    m_firstActivation = false;
+    updateTree();
+  }
+}
+
+void DataTab::onRefresh()
+{
+  if (QGuiApplication::keyboardModifiers() & Qt::ShiftModifier) {
+    m_filetree->model()->setEnabled(false);
+    m_filetree->clear();
+  }
+
+  m_core.refreshDirectoryStructure();
+}
+
+void DataTab::updateTree()
+{
+  m_filetree->model()->setEnabled(true);
+  m_filetree->refresh();
+
+  if (!m_filter.empty()) {
+    ensureFullyLoaded();
+
+    if (auto* m=m_filter.proxyModel()) {
+      m->invalidate();
+    }
+  }
+}
+
+void DataTab::ensureFullyLoaded()
+{
+  if (!m_filetree->fullyLoaded()) {
+    m_filter.setFilteringEnabled(false);
+    m_filetree->ensureFullyLoaded();
+    m_filter.setFilteringEnabled(true);
+  }
+}
+
+void DataTab::onConflicts()
+{
+  updateOptions();
+}
+
+void DataTab::onArchives()
+{
+  updateOptions();
+}
+
+void DataTab::updateOptions()
+{
+  using M = FileTreeModel;
+
+  M::Flags flags = M::NoFlags;
+
+  if (ui.conflicts->isChecked()) {
+    flags |= M::ConflictsOnly | M::PruneDirectories;
+  }
+
+  if (ui.archives->isChecked()) {
+    flags |= M::Archives;
+  }
+
+  m_filetree->model()->setFlags(flags);
+  updateTree();
+}

src/datatab.h

@@ -0,0 +1,60 @@
+#include "modinfodialogfwd.h"
+#include "modinfo.h"
+#include <filterwidget.h>
+#include <QPushButton>
+#include <QTreeWidget>
+#include <QCheckBox>
+
+namespace Ui { class MainWindow; }
+class OrganizerCore;
+class Settings;
+class PluginContainer;
+class FileTree;
+
+namespace MOShared { class DirectoryEntry; }
+
+class DataTab : public QObject
+{
+  Q_OBJECT;
+
+public:
+  DataTab(
+    OrganizerCore& core, PluginContainer& pc,
+    QWidget* parent, Ui::MainWindow* ui);
+
+  void saveState(Settings& s) const;
+  void restoreState(const Settings& s);
+  void activated();
+
+  void updateTree();
+
+signals:
+  void executablesChanged();
+  void originModified(int originID);
+  void displayModInformation(ModInfo::Ptr m, unsigned int i, ModInfoTabIDs tab);
+
+private:
+  struct DataTabUi
+  {
+    QPushButton* refresh;
+    QTreeView* tree;
+    QCheckBox* conflicts;
+    QCheckBox* archives;
+  };
+
+  OrganizerCore& m_core;
+  PluginContainer& m_pluginContainer;
+  QWidget* m_parent;
+  DataTabUi ui;
+  std::unique_ptr<FileTree> m_filetree;
+  std::vector<QTreeWidgetItem*> m_removeLater;
+  MOBase::FilterWidget m_filter;
+  bool m_firstActivation;
+
+  void onRefresh();
+  void onItemExpanded(QTreeWidgetItem* item);
+  void onConflicts();
+  void onArchives();
+  void updateOptions();
+  void ensureFullyLoaded();
+};

src/directoryrefresher.h

@@ -20,15 +20,14 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef DIRECTORYREFRESHER_H
 #define DIRECTORYREFRESHER_H
 
-#include <directoryentry.h>
+#include "shared/fileregisterfwd.h"
+#include "profile.h"
 #include <QObject>
 #include <QMutex>
 #include <QStringList>
 #include <vector>
 #include <set>
 #include <tuple>
-#include "profile.h"
-
 
 /**
  * @brief used to asynchronously generate the virtual view of the combined data directory
@@ -39,24 +38,37 @@ class DirectoryRefresher : public QObject
   Q_OBJECT
 
 public:
+  struct EntryInfo
+  {
+    EntryInfo(const QString &modName, const QString &absolutePath,
+      const QStringList &stealFiles, const QStringList &archives, int priority)
+      : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles)
+      , archives(archives), priority(priority)
+    {
+    }
 
-  /**
-   * @brief constructor
-   *
-   **/
-  DirectoryRefresher();
+    QString modName;
+    QString absolutePath;
+    QStringList stealFiles;
+    QStringList archives;
+    int priority;
+  };
 
-  ~DirectoryRefresher();
+  DirectoryRefresher(std::size_t threadCount);
+
+  // noncopyable
+  DirectoryRefresher(const DirectoryRefresher&) = delete;
+  DirectoryRefresher& operator=(const DirectoryRefresher&) = delete;
 
   /**
    * @brief retrieve the updated directory structure
    *
    * returns a pointer to the updated directory structure. DirectoryRefresher
    * deletes its own pointer and the caller takes custody of the pointer
-   * 
+   *
    * @return updated directory structure
    **/
-  MOShared::DirectoryEntry *getDirectoryStructure();
+  MOShared::DirectoryEntry* stealDirectoryStructure();
 
   /**
    * @brief sets up the mods to be included in the directory structure
@@ -107,7 +119,16 @@ public:
    * @param directory
    * @param stealFiles
    */
-  void addModFilesToStructure(MOShared::DirectoryEntry *directoryStructure, const QString &modName, int priority, const QString &directory, const QStringList &stealFiles);
+  void addModFilesToStructure(
+    MOShared::DirectoryEntry *directoryStructure, const QString &modName,
+    int priority, const QString &directory, const QStringList &stealFiles);
+
+  void addMultipleModsFilesToStructure(
+    MOShared::DirectoryEntry *directoryStructure,
+    const std::vector<EntryInfo>& entries,
+    DirectoryRefreshProgress* progress=nullptr);
+
+  void updateProgress(const DirectoryRefreshProgress* p);
 
 public slots:
 
@@ -118,31 +139,76 @@ public slots:
 
 signals:
 
-  void progress(int progress);
+  void progress(const DirectoryRefreshProgress* p);
   void error(const QString &error);
   void refreshed();
 
 private:
-
-  struct EntryInfo {
-    EntryInfo(const QString &modName, const QString &absolutePath,
-              const QStringList &stealFiles, const QStringList &archives, int priority)
-      : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles)
-      , archives(archives), priority(priority) {}
-    QString modName;
-    QString absolutePath;
-    QStringList stealFiles;
-    QStringList archives;
-    int priority;
-  };
-
-private:
-
   std::vector<EntryInfo> m_Mods;
   std::set<QString> m_EnabledArchives;
-  MOShared::DirectoryEntry *m_DirectoryStructure;
+  std::unique_ptr<MOShared::DirectoryEntry> m_Root;
   QMutex m_RefreshLock;
+  std::size_t m_threadCount;
+  std::size_t m_lastFileCount;
 
+  void stealModFilesIntoStructure(
+    MOShared::DirectoryEntry *directoryStructure, const QString &modName,
+    int priority, const QString &directory, const QStringList &stealFiles);
+};
+
+
+class DirectoryRefreshProgress : QObject
+{
+  Q_OBJECT;
+
+public:
+  DirectoryRefreshProgress(DirectoryRefresher* r) :
+    QObject(r), m_refresher(r), m_modCount(0), m_modDone(0), m_finished(false)
+  {
+  }
+
+  void start(std::size_t modCount)
+  {
+    m_modCount = modCount;
+    m_modDone = 0;
+    m_finished = false;
+  }
+
+
+  bool finished() const
+  {
+    return m_finished;
+  }
+
+  int percentDone() const
+  {
+    int percent = 100;
+
+    if (m_modCount > 0) {
+      const double d = static_cast<double>(m_modDone) / m_modCount;
+      percent = static_cast<int>(d * 100);
+    }
+
+    return percent;
+  }
+
+
+  void finish()
+  {
+    m_finished = true;
+  }
+
+  void addDone()
+  {
+    ++m_modDone;
+    m_refresher->updateProgress(this);
+  }
+
+private:
+  DirectoryRefresher* m_refresher;
+  std::size_t m_modCount;
+  std::atomic<std::size_t> m_modDone;
+  bool m_finished;
 };
 
 #endif // DIRECTORYREFRESHER_H

src/dlls.manifest.qt5

@@ -14,11 +14,13 @@
   <file name="Qt5Positioning.dll"/>
   <file name="Qt5PrintSupport.dll"/>
   <file name="Qt5Qml.dll"/>
+  <file name="Qt5QmlModels.dll"/>
+  <file name="Qt5QmlWorkerScript.dll"/>
   <file name="Qt5Quick.dll"/>
   <file name="Qt5QuickWidgets.dll"/>
-  <file name="Qt5WebChannel.dll"/>
   <file name="Qt5SerialPort.dll"/>
   <file name="Qt5Svg.dll"/>
+  <file name="Qt5WebChannel.dll"/>
   <file name="Qt5WebEngineCore.dll"/>
   <file name="Qt5WebEngineWidgets.dll"/>
   <file name="Qt5WebSockets.dll"/>