Add "virtual" texture replacement API

include/aurora/texture.hpp

@@ -3,6 +3,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <optional>
 #include <span>
 #include <string_view>
 #include <variant>
@@ -10,6 +11,10 @@
 
 namespace aurora::texture {
 
+/// Wildcard hash values used by the replacement filename convention ("$" fields).
+inline constexpr uint64_t kWildcardTextureHash = 0xFFFFFFFFFFFFFFFFull;
+inline constexpr uint64_t kWildcardTlutHash = 0xFFFFFFFFFFFFFFFEull;
+
 struct TextureSourceKey {
   uint64_t textureHash = 0;
   uint64_t tlutHash = 0;
@@ -51,6 +56,31 @@ struct ReplacementGroup {
   std::vector<ReplacementRegistration> registrations;
 };
 
+/// Parses a replacement filename of the form "tex1_{w}x{h}[_m]_{texhash}[_{tluthash}]_{fmt}[_arb].dds|.png"
+/// into the source key it addresses. Hash fields may be "$" (see the wildcard constants above).
+/// Returns nullopt if the filename is invalid.
+std::optional<TextureSourceKey> parse_replacement_filename(std::string_view filename) noexcept;
+
+/// Callback source for replacement files that do not live on the filesystem.
+///
+/// read() loads the entire encoded file at `path` into outBytes, returning false if the file is
+/// unavailable. It must be thread safe: it is invoked lazily, from arbitrary threads, at any point
+/// between registration and unregistration, with internal registry locks held. It must not call
+/// back into aurora::texture.
+struct VirtualFileSource {
+  bool (*read)(void* userData, const char* path, std::vector<uint8_t>& outBytes) = nullptr;
+  void* userData = nullptr;
+};
+
+/// Registers an encoded (.dds/.png) replacement served through callbacks instead of the filesystem.
+/// `path` is a relative '/'-separated virtual path whose final component follows the replacement
+/// filename convention and determines the key. Mip sidecars are probed lazily by deriving
+/// "{stem}_mipN{ext}" from `path` and calling source.read for each level until it returns false.
+///
+/// Returns ID 0 (invalid) if the filename does not parse or source.read is null.
+ReplacementRegistration register_virtual_replacement(std::string_view path, VirtualFileSource source,
+                                                     ReplacementOptions options = {});
+
 ReplacementRegistration register_replacement(ReplacementKey key, RawTextureReplacement replacement,
                                               ReplacementOptions options = {});
 void unregister_replacement(const ReplacementRegistration& registration);

lib/gfx/png_io.cpp

@@ -1,6 +1,6 @@
 #include "png_io.hpp"
 
-#include <fstream>
+#include <cstring>
 
 #include "dds_io.hpp"
 #include "png.h"
@@ -11,33 +11,31 @@ static aurora::Module Log("aurora::gfx::png");
 namespace aurora::gfx::png {
 
 struct PngStructs {
-  png_structp pStruct;
-  png_infop pInfo;
-  std::ifstream file;
+  png_structp pStruct = nullptr;
+  png_infop pInfo = nullptr;
 
   ~PngStructs() {
     png_destroy_read_struct(&pStruct, &pInfo, nullptr);
   }
 };
 
-static void readPngData(png_structp png, png_bytep data, const size_t length) {
-  auto structs = static_cast<PngStructs*>(png_get_io_ptr(png));
-  structs->file.read(reinterpret_cast<char*>(data), static_cast<std::streamsize>(length));
+struct MemoryCursor {
+  ArrayRef<uint8_t> bytes;
+  size_t pos = 0;
+};
 
-  if (structs->file.eof() || structs->file.fail()) {
-    png_error(png, "file read failed!");
+static void readPngData(png_structp png, png_bytep data, const size_t length) {
+  auto* cursor = static_cast<MemoryCursor*>(png_get_io_ptr(png));
+  if (length > cursor->bytes.size() - cursor->pos) {
+    png_error(png, "unexpected end of data");
   }
+  std::memcpy(data, cursor->bytes.data() + cursor->pos, length);
+  cursor->pos += length;
 }
 
-std::optional<ConvertedTexture>
-load_png_file(const std::filesystem::path& path) noexcept {
+std::optional<ConvertedTexture> parse_png_bytes(ArrayRef<uint8_t> bytes) noexcept {
   PngStructs structs{};
-
-  structs.file = std::move(std::ifstream(path, std::ifstream::in | std::ifstream::binary));
-  if (!structs.file) {
-    Log.error("failed to open file: {}", fs_path_to_string(path));
-    return std::nullopt;
-  }
+  MemoryCursor cursor{bytes};
 
   structs.pStruct = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr);
   if (!structs.pStruct) {
@@ -64,7 +62,7 @@ load_png_file(const std::filesystem::path& path) noexcept {
     return std::nullopt;
   }
 
-  png_set_read_fn(structs.pStruct, &structs, readPngData);
+  png_set_read_fn(structs.pStruct, &cursor, readPngData);
   png_read_info(structs.pStruct, structs.pInfo);
 
   if (!png_get_IHDR(structs.pStruct, structs.pInfo, &width, &height, &bit_depth, &color_type, &interlace_type, &compression_type, &filter_type)) {
@@ -99,4 +97,14 @@ load_png_file(const std::filesystem::path& path) noexcept {
     .data = std::move(imageData)
   };
 }
-}
+
+std::optional<ConvertedTexture>
+load_png_file(const std::filesystem::path& path) noexcept {
+  const auto bytes = dds::read_binary_file(path);
+  if (!bytes.has_value()) {
+    Log.error("failed to open file: {}", fs_path_to_string(path));
+    return std::nullopt;
+  }
+  return parse_png_bytes(*bytes);
+}
+}

lib/gfx/png_io.hpp

@@ -3,8 +3,10 @@
 #include <filesystem>
 #include <optional>
 
+#include "common.hpp"
 #include "texture_convert.hpp"
 
 namespace aurora::gfx::png {
+std::optional<ConvertedTexture> parse_png_bytes(ArrayRef<uint8_t> bytes) noexcept;
 std::optional<ConvertedTexture> load_png_file(const std::filesystem::path& path) noexcept;
 }

lib/gfx/texture_replacement.cpp

@@ -36,12 +36,13 @@ namespace {
 aurora::Module Log("aurora::texture");
 
 constexpr uint64_t kReplacementCacheBudgetBytes = 4294967296; // 4GB
-constexpr uint64_t kReplacementWildcardTextureHash = 0xFFFFFFFFFFFFFFFFull;
-constexpr uint64_t kReplacementWildcardTlutHash = 0xFFFFFFFFFFFFFFFEull;
+constexpr uint64_t kReplacementWildcardTextureHash = aurora::texture::kWildcardTextureHash;
+constexpr uint64_t kReplacementWildcardTlutHash = aurora::texture::kWildcardTlutHash;
 
 enum class EntryKind {
   Raw,
   File,
+  Virtual,
 };
 
 struct ReplacementEntry {
@@ -56,6 +57,8 @@ struct ReplacementEntry {
   uint32_t gxFormat = 0;
   std::string label;
   std::filesystem::path path;
+  std::string virtualPath;
+  aurora::texture::VirtualFileSource source;
 };
 
 struct SelectedCache {
@@ -260,7 +263,8 @@ aurora::ArrayRef<uint8_t> tlut_bytes(const GXTlutObj_& tlut) noexcept {
   return {static_cast<const uint8_t*>(tlut.data), static_cast<size_t>(tlut.numEntries) * sizeof(uint16_t)};
 }
 
-std::optional<uint64_t> compute_referenced_tlut_hash(const GXTexObj_& obj, aurora::ArrayRef<uint8_t> tlutData) noexcept {
+std::optional<uint64_t> compute_referenced_tlut_hash(const GXTexObj_& obj,
+                                                     aurora::ArrayRef<uint8_t> tlutData) noexcept {
   const uint32_t textureSize = texture_base_level_size(obj);
   const auto* textureData = static_cast<const uint8_t*>(obj.data);
   if (!is_palette_format(obj.format()) || !obj.has_data() || textureSize == 0 || tlutData.empty()) {
@@ -385,7 +389,16 @@ std::string format_source_key_for_log(const aurora::texture::TextureSourceKey& k
   return fmt::format("{}x{} tex={} tlut={} fmt={}", key.width, key.height, textureHash, tlutHash, key.format);
 }
 
-std::optional<aurora::texture::TextureSourceKey> parse_replacement_filename(std::string_view filename) noexcept {
+std::optional<aurora::gfx::ConvertedTexture> load_texture_file(const std::filesystem::path& path) {
+  if (iequals_ascii(fs_path_to_string(path.extension()), ".png")) {
+    return aurora::gfx::png::load_png_file(path);
+  }
+  return aurora::gfx::dds::load_dds_file(path);
+}
+} // namespace
+
+namespace aurora::texture {
+std::optional<TextureSourceKey> parse_replacement_filename(std::string_view filename) noexcept {
   const size_t dot = filename.rfind('.');
   if (dot == std::string_view::npos) {
     return std::nullopt;
@@ -469,7 +482,7 @@ std::optional<aurora::texture::TextureSourceKey> parse_replacement_filename(std:
     }
   }
 
-  return aurora::texture::TextureSourceKey{
+  return TextureSourceKey{
       .textureHash = textureHash,
       .tlutHash = tlutHash,
       .width = dimensions->first,
@@ -478,14 +491,9 @@ std::optional<aurora::texture::TextureSourceKey> parse_replacement_filename(std:
       .hasTlut = hasTlut,
   };
 }
+} // namespace aurora::texture
 
-std::optional<aurora::gfx::ConvertedTexture> load_texture_file(const std::filesystem::path& path) {
-  if (iequals_ascii(fs_path_to_string(path.extension()), ".png")) {
-    return aurora::gfx::png::load_png_file(path);
-  }
-  return aurora::gfx::dds::load_dds_file(path);
-}
-
+namespace {
 bool remove_mipmaps(aurora::gfx::ConvertedTexture& texture) noexcept {
   if (texture.mips <= 1) {
     return true;
@@ -568,18 +576,76 @@ bool validate_texture_size(wgpu::TextureFormat format, uint32_t width, uint32_t
   return false;
 }
 
-std::optional<aurora::gfx::ConvertedTexture> load_file_replacement(const ReplacementEntry& entry) noexcept {
-  auto base = load_texture_file(entry.path);
+struct FileTextureSource {
+  const std::filesystem::path& path;
+  std::filesystem::path mipPath;
+
+  std::string name() const { return fs_path_to_string(path); }
+  std::optional<aurora::gfx::ConvertedTexture> load_base() { return load_texture_file(path); }
+  bool open_mip(uint32_t mipLevel) {
+    mipPath = path.parent_path() /
+              fmt::format("{}_mip{}{}", fs_path_to_string(path.stem()), mipLevel, fs_path_to_string(path.extension()));
+    std::error_code ec;
+    return std::filesystem::is_regular_file(mipPath, ec);
+  }
+  std::string mip_name() const { return fs_path_to_string(mipPath); }
+  std::optional<aurora::gfx::ConvertedTexture> load_mip() { return load_texture_file(mipPath); }
+};
+
+std::string derive_virtual_mip_name(std::string_view path, uint32_t mipLevel) {
+  const size_t slash = path.rfind('/');
+  const size_t nameStart = slash == std::string_view::npos ? 0 : slash + 1;
+  size_t dot = path.rfind('.');
+  if (dot == std::string_view::npos || dot < nameStart) {
+    dot = path.size();
+  }
+  return fmt::format("{}_mip{}{}", path.substr(0, dot), mipLevel, path.substr(dot));
+}
+
+struct VirtualTextureSource {
+  std::string_view path;
+  const aurora::texture::VirtualFileSource& source;
+  std::vector<uint8_t> bytes;
+  std::string mipPath;
+
+  std::optional<aurora::gfx::ConvertedTexture> decode() const {
+    const aurora::ArrayRef<uint8_t> data{bytes.data(), bytes.size()};
+    const size_t dot = path.rfind('.');
+    if (dot != std::string_view::npos && iequals_ascii(path.substr(dot), ".png")) {
+      return aurora::gfx::png::parse_png_bytes(data);
+    }
+    return aurora::gfx::dds::parse_dds_bytes(data);
+  }
+  std::string name() const { return std::string{path}; }
+  std::optional<aurora::gfx::ConvertedTexture> load_base() {
+    bytes.clear();
+    if (!source.read(source.userData, std::string{path}.c_str(), bytes)) {
+      return std::nullopt;
+    }
+    return decode();
+  }
+  bool open_mip(uint32_t mipLevel) {
+    mipPath = derive_virtual_mip_name(path, mipLevel);
+    bytes.clear();
+    return source.read(source.userData, mipPath.c_str(), bytes);
+  }
+  std::string mip_name() const { return mipPath; }
+  std::optional<aurora::gfx::ConvertedTexture> load_mip() { return decode(); }
+};
+
+template <typename Source>
+std::optional<aurora::gfx::ConvertedTexture> load_encoded_replacement(Source&& src) noexcept {
+  auto base = src.load_base();
   if (!base.has_value()) {
-    Log.warn("texture_replacement: failed to load texture {}", fs_path_to_string(entry.path));
+    Log.warn("texture_replacement: failed to load texture {}", src.name());
     return std::nullopt;
   }
   if (is_unsupported_texture_format(base->format)) {
-    Log.warn("texture_replacement: failed to load texture {} due to unsupported format: {}",
-             fs_path_to_string(entry.path), static_cast<uint32_t>(base->format));
+    Log.warn("texture_replacement: failed to load texture {} due to unsupported format: {}", src.name(),
+             static_cast<uint32_t>(base->format));
     return std::nullopt;
   }
-  if (!validate_texture_size(base->format, base->width, base->height, fs_path_to_string(entry.path))) {
+  if (!validate_texture_size(base->format, base->width, base->height, src.name())) {
     return std::nullopt;
   }
 
@@ -588,31 +654,28 @@ std::optional<aurora::gfx::ConvertedTexture> load_file_replacement(const Replace
   }
 
   std::vector<aurora::gfx::ConvertedTexture> more;
-  std::error_code ec;
   for (uint32_t mipLevel = 1;; ++mipLevel) {
-    const auto mipPath = entry.path.parent_path() / fmt::format("{}_mip{}{}", fs_path_to_string(entry.path.stem()),
-                                                                mipLevel, fs_path_to_string(entry.path.extension()));
-    if (!std::filesystem::is_regular_file(mipPath, ec)) {
+    if (!src.open_mip(mipLevel)) {
       break;
     }
 
-    auto lvl = load_texture_file(mipPath);
+    auto lvl = src.load_mip();
     const uint32_t ew = std::max(base->width >> mipLevel, 1u);
     const uint32_t eh = std::max(base->height >> mipLevel, 1u);
     const bool ok = lvl.has_value() && lvl->format == base->format && lvl->width == ew && lvl->height == eh;
     if (!ok) {
       if (!lvl.has_value()) {
-        Log.warn("texture_replacement: could not load mip {}", fs_path_to_string(mipPath));
+        Log.warn("texture_replacement: could not load mip {}", src.mip_name());
       } else {
-        Log.warn("texture_replacement: expected {}x{} for mip {}, got {}x{}", ew, eh, fs_path_to_string(mipPath),
-                 lvl->width, lvl->height);
+        Log.warn("texture_replacement: expected {}x{} for mip {}, got {}x{}", ew, eh, src.mip_name(), lvl->width,
+                 lvl->height);
       }
 
       break;
     }
     // If a sidecar mip file contains mipmaps, keep only the top level mip.
     if (!remove_mipmaps(*lvl)) {
-      Log.warn("texture_replacement: could not slice first mip {}", fs_path_to_string(mipPath));
+      Log.warn("texture_replacement: could not slice first mip {}", src.mip_name());
       break;
     }
     more.push_back(std::move(*lvl));
@@ -660,6 +723,18 @@ std::optional<aurora::gfx::ConvertedTexture> load_file_replacement(const Replace
   };
 }
 
+std::optional<aurora::gfx::ConvertedTexture> load_file_replacement(const ReplacementEntry& entry) noexcept {
+  return load_encoded_replacement(FileTextureSource{.path = entry.path});
+}
+
+std::optional<aurora::gfx::ConvertedTexture> load_virtual_replacement(const ReplacementEntry& entry) noexcept {
+  return load_encoded_replacement(VirtualTextureSource{.path = entry.virtualPath, .source = entry.source});
+}
+
+std::string entry_path_for_log(const ReplacementEntry& entry) {
+  return entry.kind == EntryKind::Virtual ? entry.virtualPath : fs_path_to_string(entry.path);
+}
+
 aurora::gfx::TextureHandle create_converted_texture_handle(const aurora::texture::ReplacementKey& key,
                                                            const ReplacementEntry& entry,
                                                            const aurora::gfx::ConvertedTexture& replacement) noexcept {
@@ -797,8 +872,9 @@ aurora::gfx::TextureHandle load_entry_handle(const aurora::texture::ReplacementK
   }
 
   aurora::gfx::TextureHandle handle;
-  if (entry.kind == EntryKind::File) {
-    const auto replacement = load_file_replacement(entry);
+  if (entry.kind == EntryKind::File || entry.kind == EntryKind::Virtual) {
+    const auto replacement =
+        entry.kind == EntryKind::File ? load_file_replacement(entry) : load_virtual_replacement(entry);
     if (!replacement.has_value()) {
       s_failedIds.insert(entry.id);
       return {};
@@ -905,7 +981,7 @@ bool report_missing_key(const aurora::texture::TextureSourceKey& key, const GXTe
     const auto* selected = select_entry(entries);
     if (loggedCandidates < 8) {
       Log.warn("texture_replacement: candidate ({}) {} path={}", reason, format_source_key_for_log(*candidate),
-               selected != nullptr ? fs_path_to_string(selected->path) : std::string{});
+               selected != nullptr ? entry_path_for_log(*selected) : std::string{});
       ++loggedCandidates;
     } else {
       ++omittedCandidates;
@@ -1058,6 +1134,42 @@ void clear_replacements() {
   clear_static_texture_cache();
 }
 
+ReplacementRegistration register_virtual_replacement(std::string_view path, VirtualFileSource source,
+                                                     ReplacementOptions options) {
+  if (source.read == nullptr) {
+    return {};
+  }
+
+  const size_t slash = path.rfind('/');
+  const auto filename = slash == std::string_view::npos ? path : path.substr(slash + 1);
+  const auto parsed = parse_replacement_filename(filename);
+  if (!parsed.has_value()) {
+    return {};
+  }
+
+  std::lock_guard lk(s_registryMutex);
+  ReplacementKey replacementKey{*parsed};
+  ReplacementRegistration registration{
+      .id = s_nextRegistrationId++,
+      .key = replacementKey,
+  };
+
+  auto& entries = s_entriesByKey[replacementKey];
+  entries.push_back({
+      .id = registration.id,
+      .priority = options.priority,
+      .sequence = s_nextSequence++,
+      .kind = EntryKind::Virtual,
+      .label = fmt::format("TextureReplacement {}", filename),
+      .virtualPath = std::string{path},
+      .source = source,
+  });
+  ++s_sourceEntryCount;
+  erase_cache_locked(replacementKey);
+  clear_static_texture_cache();
+  return registration;
+}
+
 ReplacementGroup load_replacement_directory(const std::filesystem::path& root, ReplacementOptions options) {
   ReplacementGroup group;
   if (root.empty() || !ensure_directory(root)) {
@@ -1147,7 +1259,9 @@ std::optional<TextureHandle> find_source_replacement_locked(const GXTexObj_& obj
   const auto replacementKey = find_source_replacement_key_locked(sourceKey);
   if (!replacementKey.has_value()) {
     const bool alwaysReportMissingKey = false; // Enable for debugging
-    if (g_config.allowTextureDumps || alwaysReportMissingKey) { report_missing_key(sourceKey, obj); }
+    if (g_config.allowTextureDumps || alwaysReportMissingKey) {
+      report_missing_key(sourceKey, obj);
+    }
     return std::nullopt;
   }