Starting CUBE-OS v0.1...

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+build-*/
+cmake-build-*/
+*.otr
+*.log
+.DS_Store

.vscode/settings.json

@@ -0,0 +1,81 @@
+{
+    "files.associations": {
+        "__bit_reference": "cpp",
+        "__bits": "cpp",
+        "__config": "cpp",
+        "__debug": "cpp",
+        "__errc": "cpp",
+        "__hash_table": "cpp",
+        "__locale": "cpp",
+        "__mutex_base": "cpp",
+        "__node_handle": "cpp",
+        "__nullptr": "cpp",
+        "__split_buffer": "cpp",
+        "__string": "cpp",
+        "__threading_support": "cpp",
+        "__tree": "cpp",
+        "__tuple": "cpp",
+        "array": "cpp",
+        "atomic": "cpp",
+        "bit": "cpp",
+        "bitset": "cpp",
+        "cctype": "cpp",
+        "charconv": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "compare": "cpp",
+        "complex": "cpp",
+        "concepts": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "exception": "cpp",
+        "fstream": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "ios": "cpp",
+        "iosfwd": "cpp",
+        "iostream": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "locale": "cpp",
+        "map": "cpp",
+        "memory": "cpp",
+        "mutex": "cpp",
+        "new": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "ostream": "cpp",
+        "ratio": "cpp",
+        "set": "cpp",
+        "sstream": "cpp",
+        "stack": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "typeinfo": "cpp",
+        "unordered_map": "cpp",
+        "variant": "cpp",
+        "vector": "cpp",
+        "__verbose_abort": "cpp",
+        "algorithm": "cpp",
+        "filesystem": "cpp",
+        "any": "cpp",
+        "forward_list": "cpp",
+        "ranges": "cpp",
+        "valarray": "cpp",
+        "span": "cpp"
+    }
+}

CMakeLists.txt

@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.12)
+project(CubeOS)
+
+set(CMAKE_CXX_STANDARD 20)
+
+# Source files
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
+file(GLOB CXX_FILES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/**/*.cpp)
+file(GLOB C_FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib/**/*.c)
+set(SRC_DIR ${CXX_FILES} ${C_FILES})
+
+set(VPKTOOL_BUILD_GUI OFF)
+set(VPKTOOL_BUILD_TESTS OFF)
+set(VPKTOOL_BUILD_INSTALLER OFF)
+
+# Build
+add_executable(${PROJECT_NAME} ${SRC_DIR})
+
+# Fetch Dependencies
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/lib/binarytools)
+
+# Link BinaryTools
+add_dependencies(${PROJECT_NAME} BinaryTools)
+target_link_libraries(${PROJECT_NAME} PRIVATE BinaryTools)
+
+# Link StormLib
+
+set(STORMLIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/StormLib)
+add_subdirectory(${STORMLIB_DIR})
+target_link_libraries(${PROJECT_NAME} PRIVATE storm)
+
+if((CMAKE_SYSTEM_NAME MATCHES "Windows") AND ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
+    include(../cmake/HandleCompilerRT.cmake)
+    find_compiler_rt_library(builtins CLANG_RT_BUILTINS_LIBRARY)
+    get_filename_component(LIBDIR "${CLANG_RT_BUILTINS_LIBRARY}" DIRECTORY)
+    if(IS_DIRECTORY "${LIBDIR}")
+        target_link_libraries(storm ${CLANG_RT_BUILTINS_LIBRARY})
+    endif()
+endif()
+if (CMAKE_SYSTEM_NAME STREQUAL "NintendoSwitch")
+    target_compile_definitions(storm PRIVATE -D_POSIX_C_SOURCE=200809L)
+endif()

src/Companion.cpp

@@ -0,0 +1,60 @@
+#include "Companion.h"
+
+#include "storm/SWrapper.h"
+#include "utils/MIODecoder.h"
+#include "factories/RawFactory.h"
+#include "factories/TextureFactory.h"
+
+#include <fstream>
+#include <iostream>
+#include <filesystem>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+namespace fs = std::filesystem;
+
+static const std::unordered_map<std::string, RawFactory*> gFactories = {
+    { ".png", new TextureFactory() }
+};
+
+void Companion::Start() {
+    std::cout << "Super Mario 64 Rom Path: " << this->gRomPath << '\n';
+
+    std::ifstream input( this->gRomPath, std::ios::binary );
+    this->gRomData = std::vector<uint8_t>( std::istreambuf_iterator<char>( input ), {} );
+    input.close();
+
+    // TODO: Validate hash
+
+    std::cout << "Detected Rom Size: " << this->gRomData.size() << '\n';
+
+    this->ProcessAssets();
+}
+
+void Companion::ProcessAssets() {
+    json assets = json::parse( std::ifstream( "assets.json" ) );
+
+    SWrapper wrapper = SWrapper("smcube.otr");
+
+    for( auto& [asset, data] : assets.items() ) {
+        std::string extension = fs::path(asset).extension().string();
+        if( gFactories.find(extension) == gFactories.end() ) {
+            std::cout << "No factory found for " << asset << '\n';
+            continue;
+        }
+        LUS::BinaryWriter write = LUS::BinaryWriter();
+        std::string path = asset.substr(0, asset.find_last_of('.'));
+        json entry = {
+            { "path", path },
+            { "offsets", data }
+        };
+        gFactories.at(extension)->process(&write, entry, this->gRomData);
+
+        auto buffer = write.ToVector();
+        wrapper.CreateFile(path, buffer);
+        write.Close();
+    }
+
+    MIO0Decoder::ClearCache();
+    wrapper.Close();
+}

src/Companion.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <filesystem>
+
+class Companion {
+public:
+    static Companion* Instance;
+    Companion(const std::filesystem::path& rom) : gRomPath(rom) {}
+    void Start();
+    void ProcessAssets();
+private:
+    std::filesystem::path gRomPath;
+    std::vector<uint8_t> gRomData;
+};

src/factories/RawFactory.cpp

@@ -0,0 +1,18 @@
+#include "RawFactory.h"
+
+void RawFactory::WriteHeader(LUS::BinaryWriter* writer, LUS::ResourceType resType, int32_t version){
+    writer->Write((uint8_t)LUS::Endianness::Little); // 0x00
+	writer->Write((uint8_t)0); // 0x01
+	writer->Write((uint8_t)0); // 0x02
+	writer->Write((uint8_t)0); // 0x03
+
+	writer->Write((uint32_t) resType); // 0x04
+	writer->Write((uint32_t) version); // 0x08
+	writer->Write((uint64_t) 0xDEADBEEFDEADBEEF); // id, 0x0C
+	writer->Write((uint32_t) 0); // 0x10
+	writer->Write((uint64_t) 0); // ROM CRC, 0x14
+	writer->Write((uint32_t) 0); // ROM Enum, 0x1C
+
+	while (writer->GetBaseAddress() < 0x40)
+		writer->Write((uint32_t)0); // To be used at a later date!
+}

src/factories/RawFactory.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "ResourceType.h"
+#include <string>
+#include <vector>
+#include <binarytools/BinaryWriter.h>
+#include <nlohmann/json.hpp>
+
+#define WRITE_HEADER(type, version) this->WriteHeader(writer, type, version)
+#define WRITE_U32(value) writer->Write((uint32_t) value)
+#define WRITE_U64(value) writer->Write((uint64_t) value)
+#define WRITE_DATA(vec) writer->Write((char*) vec.data(), data.size())
+#define WRITE_ARRAY(data, size) writer->Write((char*) data, size)
+
+class RawFactory {
+public:
+    RawFactory() = default;
+    virtual void process(LUS::BinaryWriter* write, nlohmann::json& data, std::vector<uint8_t>& buffer) = 0;
+    void WriteHeader(LUS::BinaryWriter* write, LUS::ResourceType resType, int32_t version);
+};

src/factories/ResourceType.h

@@ -0,0 +1,18 @@
+#pragma once
+
+namespace LUS {
+
+enum class ResourceType {
+    // Not set
+    None = 0x00000000,
+
+    // Common
+    Archive = 0x4F415243,     // OARC (UNUSED)
+    DisplayList = 0x4F444C54, // ODLT
+    Vertex = 0x4F565458,      // OVTX
+    Matrix = 0x4F4D5458,      // OMTX
+    Array = 0x4F415252,       // OARR
+    Blob = 0x4F424C42,        // OBLB
+    Texture = 0x4F544558,     // OTEX
+};
+} // namespace LUS

src/factories/TextureFactory.cpp

@@ -0,0 +1,78 @@
+#include "TextureFactory.h"
+
+#include <iostream>
+#include <filesystem>
+#include "Companion.h"
+#include "utils/MIODecoder.h"
+
+namespace fs = std::filesystem;
+
+enum class TextureType {
+	Error,
+	RGBA32bpp,
+	RGBA16bpp,
+	Palette4bpp,
+	Palette8bpp,
+	Grayscale4bpp,
+	Grayscale8bpp,
+	GrayscaleAlpha4bpp,
+	GrayscaleAlpha8bpp,
+	GrayscaleAlpha16bpp,
+};
+
+static const std::unordered_map <std::string, TextureType> gTextureTypes = {
+    { ".rgba16", TextureType::RGBA16bpp },
+    { ".rgba32", TextureType::RGBA32bpp },
+    { ".ia1", TextureType::GrayscaleAlpha4bpp },
+	{ ".ia4", TextureType::GrayscaleAlpha4bpp },
+	{ ".ia8", TextureType::GrayscaleAlpha8bpp },
+	{ ".ia16", TextureType::GrayscaleAlpha16bpp },
+};
+
+void TextureFactory::process(LUS::BinaryWriter* writer, nlohmann::json& data, std::vector<uint8_t>& buffer) {
+	std::string path = data["path"];
+	std::string ext = fs::path(path).extension().string();
+
+	bool isTileTexture = path.find("cake") != std::string::npos || path.find("skyboxes") != std::string::npos;
+
+	if(isTileTexture) {
+		return;
+	}
+
+	if(!gTextureTypes.contains(ext)) {
+		throw std::runtime_error("Invalid texture type: " + ext);
+	}
+
+    auto metadata = data["offsets"];
+	if(!metadata[3].contains("us")){
+		return;
+	}
+
+	// Path: [Width, Height, Size, { Country : [Rom Offset, MIO0 Size] }],
+
+	TextureType type = isTileTexture ? TextureType::RGBA32bpp : gTextureTypes.at(ext);
+
+    WRITE_HEADER(LUS::ResourceType::Texture, 1);
+
+    WRITE_U32(type); // Texture Type
+    WRITE_U32(metadata[0]); // Width
+    WRITE_U32(metadata[1]); // Height
+
+	size_t size = metadata[2];
+	auto offsets = metadata[3]["us"];
+
+	auto* texture = new uint8_t[size];
+
+    if(offsets.size() > 1){
+        auto mio0 = MIO0Decoder::Decode(buffer, offsets[0]);
+        memcpy(texture, mio0.data() + offsets[1], size);
+    } else {
+        memcpy(texture, buffer.data() + offsets[0], size);
+    }
+
+	WRITE_U32(size); // Texture Data Size
+	WRITE_ARRAY(texture, size); // Texture Data
+
+	delete[] texture;
+	std::cout << "Processed " << path << '\n';
+}

src/factories/TextureFactory.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "RawFactory.h"
+
+class TextureFactory : public RawFactory {
+public:
+    TextureFactory() = default;
+    void process(LUS::BinaryWriter* write, nlohmann::json& data, std::vector<uint8_t>& buffer) override;
+};

src/main.cpp

@@ -0,0 +1,22 @@
+#include <iostream>
+#include "CLI11.hpp"
+#include "Companion.h"
+
+Companion* Companion::Instance;
+
+int main(int argc, char *argv[]) {
+    CLI::App app{"CubeOS - Rom extractor"};
+
+    std::string filename;
+    app.add_option("path", filename, "sm64 us rom")->required()->check(CLI::ExistingFile);
+
+    try {
+        app.parse(argc, argv);
+    } catch (const CLI::ParseError &e) {
+        return app.exit(e);
+    }
+
+    Companion::Instance = new Companion(filename);
+    Companion::Instance->Start();
+    return 0;
+}

src/storm/SWrapper.cpp

@@ -0,0 +1,67 @@
+#include "SWrapper.h"
+#include <filesystem>
+#include <iostream>
+
+namespace fs = std::filesystem;
+
+SWrapper::SWrapper(const std::string& path) {
+    if(fs::exists(path)) {
+        fs::remove(path);
+    }
+
+    if(!SFileCreateArchive(path.c_str(), MPQ_CREATE_LISTFILE | MPQ_CREATE_ATTRIBUTES | MPQ_CREATE_ARCHIVE_V2, 4096, &this->hMpq)){
+        std::cout << "Failed to create archive: " << path << std::endl;
+        std::cout << GetLastError() << std::endl;
+        return;
+    }
+}
+
+SWrapper::~SWrapper() {
+    SFileCloseArchive(this->hMpq);
+}
+
+bool SWrapper::CreateFile(std::string path, std::vector<char> data) {
+    HANDLE hFile;
+#ifdef _WIN32
+    SYSTEMTIME sysTime;
+    GetSystemTime(&sysTime);
+    FILETIME t;
+    SystemTimeToFileTime(&sysTime, &t);
+    ULONGLONG theTime = static_cast<uint64_t>(t.dwHighDateTime) << (sizeof(t.dwHighDateTime) * 8) | t.dwLowDateTime;
+#else
+    time_t theTime;
+    time(&theTime);
+#endif
+
+    char* raw = (char*) data.data();
+    size_t size = data.size();
+
+    if(size >> 32){
+        std::cout << "File too large: " << path << std::endl;
+        return false;
+    }
+
+    if(!SFileCreateFile(this->hMpq, path.c_str(), theTime, size, 0, MPQ_FILE_COMPRESS, &hFile)){
+        std::cout << "Failed to create file: " << path << std::endl;
+        std::cout << GetLastError() << std::endl;
+        return false;
+    }
+
+    if(!SFileWriteFile(hFile, (void*) raw, size, MPQ_COMPRESSION_ZLIB)){
+        std::cout << "Failed to write file: " << path << std::endl;
+        std::cout << GetLastError() << std::endl;
+        return false;
+    }
+
+    if(!SFileCloseFile(hFile)){
+        std::cout << "Failed to close file: " << path << std::endl;
+        std::cout << GetLastError() << std::endl;
+        return false;
+    }
+
+    return true;
+}
+
+void SWrapper::Close() {
+    SFileCloseArchive(this->hMpq);
+}

src/storm/SWrapper.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <StormLib/src/StormLib.h>
+
+class SWrapper {
+public:
+    SWrapper(const std::string& path);
+    ~SWrapper();
+
+    std::vector<char> ReadFile(std::string path);
+    bool CreateFile(std::string path, std::vector<char> data);
+    void Close();
+private:
+    HANDLE hMpq;
+};

src/utils/MIODecoder.cpp

@@ -0,0 +1,27 @@
+#include "MIODecoder.h"
+
+extern "C" {
+#include <libmio0/mio0.h>
+}
+
+std::unordered_map<uint32_t, std::vector<char>> MIO0Decoder::gCachedChunks;
+
+std::vector<char>& MIO0Decoder::Decode(std::vector<uint8_t>& buffer, uint32_t offset) {
+    const unsigned char* in_buf = buffer.data() + offset;
+
+	mio0_header_t head;
+	if(!mio0_decode_header(in_buf, &head)){
+        throw std::runtime_error("Invalid MIO0 header");
+    }
+
+    uint8_t* decompressed = new uint8_t[head.dest_size];
+	mio0_decode(in_buf, decompressed, nullptr);
+    gCachedChunks[offset] = std::vector<char>(decompressed, decompressed + head.dest_size);
+
+    delete[] decompressed;
+    return gCachedChunks[offset];
+}
+
+void MIO0Decoder::ClearCache() {
+    gCachedChunks.clear();
+}

src/utils/MIODecoder.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <vector>
+#include <unordered_map>
+
+class MIO0Decoder {
+public:
+    static std::unordered_map<uint32_t, std::vector<char>> gCachedChunks;
+    static std::vector<char>& Decode(std::vector<uint8_t>& buffer, uint32_t offset);
+    static void ClearCache();
+};