Replace libespm with esplugin

esplugin is a Rust rewrite of libespm that's safer, more language-portable, better tested and more performant. However, it exposes a C API to C++, so it's not a straight swap.
2026-07-27 14:16:01 -07:00 · 2017-10-08 15:27:35 +01:00
parent 8b6da099d5
commit 02a2b8bd4c
5 changed files with 125 additions and 58 deletions
@@ -1,6 +1,5 @@
 sudo: false
-language: cpp
-compiler: gcc
+language: rust

 addons:
  apt:
@@ -16,9 +15,10 @@ cache:
  directories:
    - $HOME/boost_1_64_0/boost
    - $HOME/boost_1_64_0/stage/64/lib
+  cargo: true

 install:
-  - if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi
+  - export CXX="g++-6" CC="gcc-6"
  # Build Boost.
  - wget https://raw.githubusercontent.com/WrinklyNinja/ci-scripts/1.5.0/install_boost.py
  - python install_boost.py --directory ~ --boost-version 1.64.0 -a 64 -t gcc-6 atomic chrono date_time filesystem iostreams locale log regex system thread
@@ -51,6 +51,16 @@ IF (NOT Boost_USE_STATIC_LIBS)
    add_definitions(-DBOOST_LOG_DYN_LINK)
 ENDIF ()

+IF (CMAKE_SYSTEM_NAME MATCHES "Windows")
+    IF (NOT "${CMAKE_GENERATOR}" MATCHES "(Win64|IA64)")
+        set(RUST_TARGET i686-pc-windows-msvc)
+    ELSE ()
+        set(RUST_TARGET x86_64-pc-windows-msvc)
+    ENDIF ()
+ELSE ()
+    set(RUST_TARGET x86_64-unknown-linux-gnu)
+ENDIF ()
+
 find_package(Boost REQUIRED COMPONENTS atomic log log_setup regex locale thread date_time chrono filesystem system iostreams)

 ExternalProject_Add(GTest
@@ -62,14 +72,21 @@ ExternalProject_Get_Property(GTest SOURCE_DIR BINARY_DIR)
 set (GTEST_INCLUDE_DIRS "${SOURCE_DIR}/googletest/include")
 set (GTEST_LIBRARIES "${BINARY_DIR}/googlemock/gtest/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")

-ExternalProject_Add(libespm
+ExternalProject_Add(esplugin
                    PREFIX "external"
-                    URL "https://github.com/WrinklyNinja/libespm/archive/2.5.5.tar.gz"
+                    URL "https://github.com/WrinklyNinja/esplugin/archive/1.0.3.tar.gz"
                    CONFIGURE_COMMAND ""
-                    BUILD_COMMAND ""
+                    BUILD_IN_SOURCE 1
+                    BUILD_COMMAND cargo build --release --all --all-features --target ${RUST_TARGET}
                    INSTALL_COMMAND "")
-ExternalProject_Get_Property(libespm SOURCE_DIR)
-set (LIBESPM_INCLUDE_DIRS "${SOURCE_DIR}/include")
+ExternalProject_Get_Property(esplugin SOURCE_DIR)
+set (ESPLUGIN_INCLUDE_DIRS "${SOURCE_DIR}/ffi/include")
+set (ESPLUGIN_LIBRARIES "${SOURCE_DIR}/target/${RUST_TARGET}/release/${CMAKE_STATIC_LIBRARY_PREFIX}esplugin_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}")
+IF (CMAKE_SYSTEM_NAME MATCHES "Windows")
+    set (ESPLUGIN_LIBRARIES ${ESPLUGIN_LIBRARIES} Userenv)
+ELSE ()
+    set (ESPLUGIN_LIBRARIES ${ESPLUGIN_LIBRARIES} dl)
+ENDIF ()

 ExternalProject_Add(libgit2
                    PREFIX "external"
@@ -253,7 +270,7 @@ include_directories ("${CMAKE_SOURCE_DIR}/src"
                     "${CMAKE_SOURCE_DIR}/include"
                     ${LIBLOADORDER_INCLUDE_DIRS}
                     ${LIBGIT2_INCLUDE_DIRS}
-                     ${LIBESPM_INCLUDE_DIRS}
+                     ${ESPLUGIN_INCLUDE_DIRS}
                     ${Boost_INCLUDE_DIRS}
                     ${YAML_CPP_INCLUDE_DIRS}
                     ${GTEST_INCLUDE_DIRS}
@@ -317,13 +334,13 @@ ENDIF ()

 # Build tests.
 add_executable       (loot_api_internals_tests ${LOOT_API_SRC} ${LOOT_API_HEADERS} ${LOOT_TESTS_SRC} ${LOOT_TESTS_HEADERS})
-add_dependencies     (loot_api_internals_tests libespm libgit2 libloadorder pseudosem yaml-cpp GTest testing-metadata testing-plugins)
-target_link_libraries(loot_api_internals_tests ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES} ${GTEST_LIBRARIES} ${LIBLOADORDER_LIBRARIES})
+add_dependencies     (loot_api_internals_tests esplugin libgit2 libloadorder pseudosem yaml-cpp GTest testing-metadata testing-plugins)
+target_link_libraries(loot_api_internals_tests ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${ESPLUGIN_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES} ${GTEST_LIBRARIES})

 # Build API.
 add_library          (loot_api ${LOOT_API_SRC} ${LOOT_API_HEADERS})
-add_dependencies     (loot_api libespm libgit2 libloadorder pseudosem yaml-cpp)
-target_link_libraries(loot_api ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES})
+add_dependencies     (loot_api esplugin libgit2 libloadorder pseudosem yaml-cpp)
+target_link_libraries(loot_api ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${ESPLUGIN_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES})

 # Build API tests.
 add_executable       (loot_api_tests ${LOOT_API_TESTS_SRC} ${LOOT_API_TESTS_HEADERS})
@@ -23,6 +23,10 @@ environment:
    secure: fBQJmUDK/EgEUWjjbo6bWcitczeFTJZT4OZ3ZZ4FoUT6soBsTWPQJnr8YEVMFhGP

 install:
+  - curl -sSf -o rustup-init.exe https://win.rustup.rs
+  - rustup-init.exe -y
+  - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+  - rustup target add i686-pc-windows-msvc
  - choco install -y doxygen.portable
  - python -m pip install -r docs/requirements.txt
  - ps: (New-Object Net.WebClient).DownloadFile('https://raw.githubusercontent.com/WrinklyNinja/ci-scripts/1.1.0/install_boost.py', "$env:APPVEYOR_BUILD_FOLDER\install_boost.py")
@@ -37,7 +37,6 @@
 #include "api/helpers/version.h"
 #include "loot/exception/file_access_error.h"

-using libespm::FormId;
 using std::set;
 using std::string;

@@ -48,7 +47,7 @@ Plugin::Plugin(const GameType gameType,
               const std::string& name,
               const bool headerOnly) :
  name_(name),
-  libespm::Plugin(Plugin::GetLibespmGameId(gameType)),
+  esPlugin(nullptr),
  isEmpty_(true),
  isActive_(false),
  loadsArchive_(false),
@@ -61,23 +60,26 @@ Plugin::Plugin(const GameType gameType,
    if (!boost::filesystem::exists(filepath) && boost::filesystem::exists(filepath.string() + ".ghost"))
      filepath += ".ghost";

-    load(filepath, headerOnly);
+    Load(filepath, gameType, headerOnly);

-    isEmpty_ = getRecordAndGroupCount() == 0;
+    auto ret = esp_plugin_is_empty(esPlugin.get(), &isEmpty_);
+    if (ret != ESP_OK) {
+      throw FileAccessError(name + " : Libespm error code: " + std::to_string(ret));
+    }

    if (!headerOnly) {
      BOOST_LOG_TRIVIAL(trace) << name_ << ": Caching CRC value.";
      crc_ = GetCrc32(filepath);
-    }

-    BOOST_LOG_TRIVIAL(trace) << name_ << ": Counting override FormIDs.";
-    for (const auto& formID : getFormIds()) {
-      if (!boost::iequals(formID.getPluginName(), name_))
-        ++numOverrideRecords_;
+      BOOST_LOG_TRIVIAL(trace) << name_ << ": Counting override FormIDs.";
+      ret = esp_plugin_count_override_records(esPlugin.get(), &numOverrideRecords_);
+      if (ret != ESP_OK) {
+        throw FileAccessError(name + " : Libespm error code: " + std::to_string(ret));
+      }
    }

    //Also read Bash Tags applied and version string in description.
-    string text = getDescription();
+    string text = GetDescription();
    BOOST_LOG_TRIVIAL(trace) << name_ << ": " << "Attempting to extract Bash Tags from the description.";
    size_t pos1 = text.find("{{BASH:");
    if (pos1 != string::npos && pos1 + 7 != text.length()) {
@@ -118,11 +120,21 @@ std::string Plugin::GetLowercasedName() const {
 }

 std::string Plugin::GetVersion() const {
-  return Version(getDescription()).AsString();
+  return Version(GetDescription()).AsString();
 }

 std::vector<std::string> Plugin::GetMasters() const {
-  return getMasters();
+  char ** masters;
+  uint8_t numMasters;
+  auto ret = esp_plugin_masters(esPlugin.get(), &masters, &numMasters);
+  if (ret != ESP_OK) {
+    throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret));
+  }
+
+  std::vector<std::string> mastersVec(masters, masters + numMasters);
+  esp_string_array_free(masters, numMasters);
+
+  return mastersVec;
 }

 std::set<Tag> Plugin::GetBashTags() const {
@@ -134,7 +146,13 @@ uint32_t Plugin::GetCRC() const {
 }

 bool Plugin::IsMaster() const {
-  return isMasterFile();
+  bool isMaster;
+  auto ret = esp_plugin_is_master(esPlugin.get(), &isMaster);
+  if (ret != ESP_OK) {
+    throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret));
+  }
+
+  return isMaster;
 }

 bool Plugin::IsEmpty() const {
@@ -149,22 +167,13 @@ bool Plugin::DoFormIDsOverlap(const PluginInterface& plugin) const {
  try {
    auto otherPlugin = dynamic_cast<const Plugin&>(plugin);

-    //Basically std::set_intersection except with an early exit instead of an append to results.
-    set<FormId> formIds(getFormIds());
-    set<FormId> otherFormIds(otherPlugin.getFormIds());
-    auto i = begin(formIds);
-    auto j = begin(otherFormIds);
-    auto iend = end(formIds);
-    auto jend = end(otherFormIds);
-
-    while (i != iend && j != jend) {
-      if (*i < *j)
-        ++i;
-      else if (*j < *i)
-        ++j;
-      else
-        return true;
+    bool doPluginsOverlap;
+    auto ret = esp_plugin_do_records_overlap(esPlugin.get(), otherPlugin.esPlugin.get(), &doPluginsOverlap);
+    if (ret != ESP_OK) {
+      throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret));
    }
+
+    return doPluginsOverlap;
  } catch (std::bad_cast&) {
    BOOST_LOG_TRIVIAL(error) << "Tried to check if FormIDs overlapped with a non-Plugin implementation of PluginInterface.";
  }
@@ -190,16 +199,16 @@ bool Plugin::IsValid(const std::string& filename, const GameType gameType, const
  if (!boost::iends_with(name, ".esm") && !boost::iends_with(name, ".esp"))
    return false;

-  // Add the ".ghost" file extension if the plugin is ghosted.
-  boost::filesystem::path filepath = dataPath / name;
-  if (!boost::filesystem::exists(filepath) && boost::filesystem::exists(filepath.string() + ".ghost"))
-    filepath += ".ghost";
+  bool isValid;
+  auto path = dataPath / filename;
+  int ret = esp_plugin_is_valid(GetEspluginGameId(gameType), path.string().c_str(), true, &isValid);

-  if (libespm::Plugin::isValid(filepath, GetLibespmGameId(gameType), true))
-    return true;
+  if (ret != ESP_OK || !isValid) {
+    BOOST_LOG_TRIVIAL(warning) << "The .es(p|m) file \"" << filename << "\" is not a valid plugin.";
+  }

-  BOOST_LOG_TRIVIAL(warning) << "The .es(p|m) file \"" << filename << "\" is not a valid plugin.";
-  return false;
+  return (ret == ESP_OK && isValid)
+    || Plugin::IsValid(filename + ".ghost", gameType, dataPath);
 }

 uintmax_t Plugin::GetFileSize(const std::string & filename, const boost::filesystem::path& dataPath) {
@@ -218,6 +227,37 @@ bool Plugin::IsActive() const {
  return isActive_;
 }

+void Plugin::Load(const boost::filesystem::path& path, GameType gameType, bool headerOnly) {
+  ::Plugin * plugin;
+  int ret = esp_plugin_new(&plugin, GetEspluginGameId(gameType), path.string().c_str());
+  if (ret != ESP_OK) {
+      throw FileAccessError(path.string() + " : Libespm error code: " + std::to_string(ret));
+  }
+
+  esPlugin = std::shared_ptr<std::remove_pointer<::Plugin>::type>(plugin, esp_plugin_free);
+
+  ret = esp_plugin_parse(esPlugin.get(), headerOnly);
+  if (ret != ESP_OK) {
+    throw FileAccessError(path.string() + " : Libespm error code: " + std::to_string(ret));
+  }
+}
+
+std::string Plugin::GetDescription() const {
+  char * description;
+  auto ret = esp_plugin_description(esPlugin.get(), &description);
+  if (ret != ESP_OK) {
+    throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret));
+  }
+  if (description == nullptr) {
+    return "";
+  }
+
+  string descriptionStr = description;
+  esp_string_free(description);
+
+  return descriptionStr;
+}
+
 std::string Plugin::GetArchiveFileExtension(const GameType gameType) {
  if (gameType == GameType::fo4)
    return ".ba2";
@@ -245,16 +285,16 @@ bool Plugin::LoadsArchive(const std::string& pluginName, const GameType gameType
  return false;
 }

-libespm::GameId Plugin::GetLibespmGameId(GameType gameType) {
+unsigned int Plugin::GetEspluginGameId(GameType gameType) {
  if (gameType == GameType::tes4)
-    return libespm::GameId::OBLIVION;
+    return ESP_GAME_OBLIVION;
  else if (gameType == GameType::tes5 || gameType == GameType::tes5se)
-    return libespm::GameId::SKYRIM;
+    return ESP_GAME_SKYRIM;
  else if (gameType == GameType::fo3)
-    return libespm::GameId::FALLOUT3;
+    return ESP_GAME_FALLOUT3;
  else if (gameType == GameType::fonv)
-    return libespm::GameId::FALLOUTNV;
+    return ESP_GAME_FALLOUTNV;
  else
-    return libespm::GameId::FALLOUT4;
+    return ESP_GAME_SKYRIM;
 }
 }
@@ -28,10 +28,11 @@
 #include <list>
 #include <set>
 #include <string>
+#include <type_traits>
 #include <vector>

 #include <boost/locale.hpp>
-#include <libespm/Plugin.h>
+#include <esplugin.hpp>

 #include "api/game/load_order_handler.h"
 #include "loot/metadata/plugin_metadata.h"
@@ -39,7 +40,7 @@
 #include "loot/plugin_interface.h"

 namespace loot {
-class Plugin : public PluginInterface, private libespm::Plugin {
+class Plugin : public PluginInterface {
 public:
  Plugin(const GameType gameType,
         const boost::filesystem::path& dataPath,
@@ -70,9 +71,12 @@ public:

  bool operator < (const Plugin& rhs) const;
 private:
+  void Load(const boost::filesystem::path& path, GameType gameType, bool headerOnly);
+  std::string GetDescription() const;
+
  static std::string GetArchiveFileExtension(const GameType gameType);
  static bool LoadsArchive(const std::string& pluginName, const GameType gameType, const boost::filesystem::path& dataPath);
-  static libespm::GameId GetLibespmGameId(GameType gameType);
+  static unsigned int GetEspluginGameId(GameType gameType);

  bool isEmpty_;  // Does the plugin contain any records other than the TES4 header?
  bool isActive_;
@@ -84,6 +88,8 @@ private:

  //Useful caches.
  size_t numOverrideRecords_;
+
+  std::shared_ptr<std::remove_pointer<::Plugin>::type> esPlugin;
 };
 }