From 02a2b8bd4c0c01c0b6a67f7c4887672ea48c4bf8 Mon Sep 17 00:00:00 2001 From: Oliver Hamlet Date: Thu, 5 Oct 2017 08:13:55 +0100 Subject: [PATCH] Replace libespm with esplugin esplugin is a Rust rewrite of libespm that's safer, more language-portable, better tested and more performant. However, it exposes a C API to C++, so it's not a straight swap. --- .travis.yml | 6 +- CMakeLists.txt | 37 +++++++++--- appveyor.yml | 4 ++ src/api/plugin/plugin.cpp | 124 +++++++++++++++++++++++++------------- src/api/plugin/plugin.h | 12 +++- 5 files changed, 125 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6eab186..d96ef23d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ sudo: false -language: cpp -compiler: gcc +language: rust addons: apt: @@ -16,9 +15,10 @@ cache: directories: - $HOME/boost_1_64_0/boost - $HOME/boost_1_64_0/stage/64/lib + cargo: true install: - - if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi + - export CXX="g++-6" CC="gcc-6" # Build Boost. - wget https://raw.githubusercontent.com/WrinklyNinja/ci-scripts/1.5.0/install_boost.py - python install_boost.py --directory ~ --boost-version 1.64.0 -a 64 -t gcc-6 atomic chrono date_time filesystem iostreams locale log regex system thread diff --git a/CMakeLists.txt b/CMakeLists.txt index eab7e184..9d71cfcd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,16 @@ IF (NOT Boost_USE_STATIC_LIBS) add_definitions(-DBOOST_LOG_DYN_LINK) ENDIF () +IF (CMAKE_SYSTEM_NAME MATCHES "Windows") + IF (NOT "${CMAKE_GENERATOR}" MATCHES "(Win64|IA64)") + set(RUST_TARGET i686-pc-windows-msvc) + ELSE () + set(RUST_TARGET x86_64-pc-windows-msvc) + ENDIF () +ELSE () + set(RUST_TARGET x86_64-unknown-linux-gnu) +ENDIF () + find_package(Boost REQUIRED COMPONENTS atomic log log_setup regex locale thread date_time chrono filesystem system iostreams) ExternalProject_Add(GTest @@ -62,14 +72,21 @@ ExternalProject_Get_Property(GTest SOURCE_DIR BINARY_DIR) set (GTEST_INCLUDE_DIRS "${SOURCE_DIR}/googletest/include") set (GTEST_LIBRARIES "${BINARY_DIR}/googlemock/gtest/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") -ExternalProject_Add(libespm +ExternalProject_Add(esplugin PREFIX "external" - URL "https://github.com/WrinklyNinja/libespm/archive/2.5.5.tar.gz" + URL "https://github.com/WrinklyNinja/esplugin/archive/1.0.3.tar.gz" CONFIGURE_COMMAND "" - BUILD_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_COMMAND cargo build --release --all --all-features --target ${RUST_TARGET} INSTALL_COMMAND "") -ExternalProject_Get_Property(libespm SOURCE_DIR) -set (LIBESPM_INCLUDE_DIRS "${SOURCE_DIR}/include") +ExternalProject_Get_Property(esplugin SOURCE_DIR) +set (ESPLUGIN_INCLUDE_DIRS "${SOURCE_DIR}/ffi/include") +set (ESPLUGIN_LIBRARIES "${SOURCE_DIR}/target/${RUST_TARGET}/release/${CMAKE_STATIC_LIBRARY_PREFIX}esplugin_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}") +IF (CMAKE_SYSTEM_NAME MATCHES "Windows") + set (ESPLUGIN_LIBRARIES ${ESPLUGIN_LIBRARIES} Userenv) +ELSE () + set (ESPLUGIN_LIBRARIES ${ESPLUGIN_LIBRARIES} dl) +ENDIF () ExternalProject_Add(libgit2 PREFIX "external" @@ -253,7 +270,7 @@ include_directories ("${CMAKE_SOURCE_DIR}/src" "${CMAKE_SOURCE_DIR}/include" ${LIBLOADORDER_INCLUDE_DIRS} ${LIBGIT2_INCLUDE_DIRS} - ${LIBESPM_INCLUDE_DIRS} + ${ESPLUGIN_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${YAML_CPP_INCLUDE_DIRS} ${GTEST_INCLUDE_DIRS} @@ -317,13 +334,13 @@ ENDIF () # Build tests. add_executable (loot_api_internals_tests ${LOOT_API_SRC} ${LOOT_API_HEADERS} ${LOOT_TESTS_SRC} ${LOOT_TESTS_HEADERS}) -add_dependencies (loot_api_internals_tests libespm libgit2 libloadorder pseudosem yaml-cpp GTest testing-metadata testing-plugins) -target_link_libraries(loot_api_internals_tests ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES} ${GTEST_LIBRARIES} ${LIBLOADORDER_LIBRARIES}) +add_dependencies (loot_api_internals_tests esplugin libgit2 libloadorder pseudosem yaml-cpp GTest testing-metadata testing-plugins) +target_link_libraries(loot_api_internals_tests ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${ESPLUGIN_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES} ${GTEST_LIBRARIES}) # Build API. add_library (loot_api ${LOOT_API_SRC} ${LOOT_API_HEADERS}) -add_dependencies (loot_api libespm libgit2 libloadorder pseudosem yaml-cpp) -target_link_libraries(loot_api ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES}) +add_dependencies (loot_api esplugin libgit2 libloadorder pseudosem yaml-cpp) +target_link_libraries(loot_api ${Boost_LIBRARIES} ${LIBGIT2_LIBRARIES} ${ESPLUGIN_LIBRARIES} ${LIBLOADORDER_LIBRARIES} ${LOOT_LIBS} ${YAML_CPP_LIBRARIES}) # Build API tests. add_executable (loot_api_tests ${LOOT_API_TESTS_SRC} ${LOOT_API_TESTS_HEADERS}) diff --git a/appveyor.yml b/appveyor.yml index 9fb1bbd4..1710f144 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,6 +23,10 @@ environment: secure: fBQJmUDK/EgEUWjjbo6bWcitczeFTJZT4OZ3ZZ4FoUT6soBsTWPQJnr8YEVMFhGP install: + - curl -sSf -o rustup-init.exe https://win.rustup.rs + - rustup-init.exe -y + - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin + - rustup target add i686-pc-windows-msvc - choco install -y doxygen.portable - python -m pip install -r docs/requirements.txt - ps: (New-Object Net.WebClient).DownloadFile('https://raw.githubusercontent.com/WrinklyNinja/ci-scripts/1.1.0/install_boost.py', "$env:APPVEYOR_BUILD_FOLDER\install_boost.py") diff --git a/src/api/plugin/plugin.cpp b/src/api/plugin/plugin.cpp index 34fa2049..a5262d4a 100644 --- a/src/api/plugin/plugin.cpp +++ b/src/api/plugin/plugin.cpp @@ -37,7 +37,6 @@ #include "api/helpers/version.h" #include "loot/exception/file_access_error.h" -using libespm::FormId; using std::set; using std::string; @@ -48,7 +47,7 @@ Plugin::Plugin(const GameType gameType, const std::string& name, const bool headerOnly) : name_(name), - libespm::Plugin(Plugin::GetLibespmGameId(gameType)), + esPlugin(nullptr), isEmpty_(true), isActive_(false), loadsArchive_(false), @@ -61,23 +60,26 @@ Plugin::Plugin(const GameType gameType, if (!boost::filesystem::exists(filepath) && boost::filesystem::exists(filepath.string() + ".ghost")) filepath += ".ghost"; - load(filepath, headerOnly); + Load(filepath, gameType, headerOnly); - isEmpty_ = getRecordAndGroupCount() == 0; + auto ret = esp_plugin_is_empty(esPlugin.get(), &isEmpty_); + if (ret != ESP_OK) { + throw FileAccessError(name + " : Libespm error code: " + std::to_string(ret)); + } if (!headerOnly) { BOOST_LOG_TRIVIAL(trace) << name_ << ": Caching CRC value."; crc_ = GetCrc32(filepath); - } - BOOST_LOG_TRIVIAL(trace) << name_ << ": Counting override FormIDs."; - for (const auto& formID : getFormIds()) { - if (!boost::iequals(formID.getPluginName(), name_)) - ++numOverrideRecords_; + BOOST_LOG_TRIVIAL(trace) << name_ << ": Counting override FormIDs."; + ret = esp_plugin_count_override_records(esPlugin.get(), &numOverrideRecords_); + if (ret != ESP_OK) { + throw FileAccessError(name + " : Libespm error code: " + std::to_string(ret)); + } } //Also read Bash Tags applied and version string in description. - string text = getDescription(); + string text = GetDescription(); BOOST_LOG_TRIVIAL(trace) << name_ << ": " << "Attempting to extract Bash Tags from the description."; size_t pos1 = text.find("{{BASH:"); if (pos1 != string::npos && pos1 + 7 != text.length()) { @@ -118,11 +120,21 @@ std::string Plugin::GetLowercasedName() const { } std::string Plugin::GetVersion() const { - return Version(getDescription()).AsString(); + return Version(GetDescription()).AsString(); } std::vector Plugin::GetMasters() const { - return getMasters(); + char ** masters; + uint8_t numMasters; + auto ret = esp_plugin_masters(esPlugin.get(), &masters, &numMasters); + if (ret != ESP_OK) { + throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret)); + } + + std::vector mastersVec(masters, masters + numMasters); + esp_string_array_free(masters, numMasters); + + return mastersVec; } std::set Plugin::GetBashTags() const { @@ -134,7 +146,13 @@ uint32_t Plugin::GetCRC() const { } bool Plugin::IsMaster() const { - return isMasterFile(); + bool isMaster; + auto ret = esp_plugin_is_master(esPlugin.get(), &isMaster); + if (ret != ESP_OK) { + throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret)); + } + + return isMaster; } bool Plugin::IsEmpty() const { @@ -149,22 +167,13 @@ bool Plugin::DoFormIDsOverlap(const PluginInterface& plugin) const { try { auto otherPlugin = dynamic_cast(plugin); - //Basically std::set_intersection except with an early exit instead of an append to results. - set formIds(getFormIds()); - set otherFormIds(otherPlugin.getFormIds()); - auto i = begin(formIds); - auto j = begin(otherFormIds); - auto iend = end(formIds); - auto jend = end(otherFormIds); - - while (i != iend && j != jend) { - if (*i < *j) - ++i; - else if (*j < *i) - ++j; - else - return true; + bool doPluginsOverlap; + auto ret = esp_plugin_do_records_overlap(esPlugin.get(), otherPlugin.esPlugin.get(), &doPluginsOverlap); + if (ret != ESP_OK) { + throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret)); } + + return doPluginsOverlap; } catch (std::bad_cast&) { BOOST_LOG_TRIVIAL(error) << "Tried to check if FormIDs overlapped with a non-Plugin implementation of PluginInterface."; } @@ -190,16 +199,16 @@ bool Plugin::IsValid(const std::string& filename, const GameType gameType, const if (!boost::iends_with(name, ".esm") && !boost::iends_with(name, ".esp")) return false; - // Add the ".ghost" file extension if the plugin is ghosted. - boost::filesystem::path filepath = dataPath / name; - if (!boost::filesystem::exists(filepath) && boost::filesystem::exists(filepath.string() + ".ghost")) - filepath += ".ghost"; + bool isValid; + auto path = dataPath / filename; + int ret = esp_plugin_is_valid(GetEspluginGameId(gameType), path.string().c_str(), true, &isValid); - if (libespm::Plugin::isValid(filepath, GetLibespmGameId(gameType), true)) - return true; + if (ret != ESP_OK || !isValid) { + BOOST_LOG_TRIVIAL(warning) << "The .es(p|m) file \"" << filename << "\" is not a valid plugin."; + } - BOOST_LOG_TRIVIAL(warning) << "The .es(p|m) file \"" << filename << "\" is not a valid plugin."; - return false; + return (ret == ESP_OK && isValid) + || Plugin::IsValid(filename + ".ghost", gameType, dataPath); } uintmax_t Plugin::GetFileSize(const std::string & filename, const boost::filesystem::path& dataPath) { @@ -218,6 +227,37 @@ bool Plugin::IsActive() const { return isActive_; } +void Plugin::Load(const boost::filesystem::path& path, GameType gameType, bool headerOnly) { + ::Plugin * plugin; + int ret = esp_plugin_new(&plugin, GetEspluginGameId(gameType), path.string().c_str()); + if (ret != ESP_OK) { + throw FileAccessError(path.string() + " : Libespm error code: " + std::to_string(ret)); + } + + esPlugin = std::shared_ptr::type>(plugin, esp_plugin_free); + + ret = esp_plugin_parse(esPlugin.get(), headerOnly); + if (ret != ESP_OK) { + throw FileAccessError(path.string() + " : Libespm error code: " + std::to_string(ret)); + } +} + +std::string Plugin::GetDescription() const { + char * description; + auto ret = esp_plugin_description(esPlugin.get(), &description); + if (ret != ESP_OK) { + throw FileAccessError(name_ + " : Libespm error code: " + std::to_string(ret)); + } + if (description == nullptr) { + return ""; + } + + string descriptionStr = description; + esp_string_free(description); + + return descriptionStr; +} + std::string Plugin::GetArchiveFileExtension(const GameType gameType) { if (gameType == GameType::fo4) return ".ba2"; @@ -245,16 +285,16 @@ bool Plugin::LoadsArchive(const std::string& pluginName, const GameType gameType return false; } -libespm::GameId Plugin::GetLibespmGameId(GameType gameType) { +unsigned int Plugin::GetEspluginGameId(GameType gameType) { if (gameType == GameType::tes4) - return libespm::GameId::OBLIVION; + return ESP_GAME_OBLIVION; else if (gameType == GameType::tes5 || gameType == GameType::tes5se) - return libespm::GameId::SKYRIM; + return ESP_GAME_SKYRIM; else if (gameType == GameType::fo3) - return libespm::GameId::FALLOUT3; + return ESP_GAME_FALLOUT3; else if (gameType == GameType::fonv) - return libespm::GameId::FALLOUTNV; + return ESP_GAME_FALLOUTNV; else - return libespm::GameId::FALLOUT4; + return ESP_GAME_SKYRIM; } } diff --git a/src/api/plugin/plugin.h b/src/api/plugin/plugin.h index 7971c3c5..c9110c92 100644 --- a/src/api/plugin/plugin.h +++ b/src/api/plugin/plugin.h @@ -28,10 +28,11 @@ #include #include #include +#include #include #include -#include +#include #include "api/game/load_order_handler.h" #include "loot/metadata/plugin_metadata.h" @@ -39,7 +40,7 @@ #include "loot/plugin_interface.h" namespace loot { -class Plugin : public PluginInterface, private libespm::Plugin { +class Plugin : public PluginInterface { public: Plugin(const GameType gameType, const boost::filesystem::path& dataPath, @@ -70,9 +71,12 @@ public: bool operator < (const Plugin& rhs) const; private: + void Load(const boost::filesystem::path& path, GameType gameType, bool headerOnly); + std::string GetDescription() const; + static std::string GetArchiveFileExtension(const GameType gameType); static bool LoadsArchive(const std::string& pluginName, const GameType gameType, const boost::filesystem::path& dataPath); - static libespm::GameId GetLibespmGameId(GameType gameType); + static unsigned int GetEspluginGameId(GameType gameType); bool isEmpty_; // Does the plugin contain any records other than the TES4 header? bool isActive_; @@ -84,6 +88,8 @@ private: //Useful caches. size_t numOverrideRecords_; + + std::shared_ptr::type> esPlugin; }; }