From 4d23c365b9e21399e94b59377e8bc318fa6e18b4 Mon Sep 17 00:00:00 2001 From: WrinklyNinja Date: Wed, 1 May 2013 21:26:40 +0100 Subject: [PATCH] Updated notes in README. Tidied up code a bit, improvements to sorting algorithm. --- README | 467 ++++++++--------------------------------- examples/settings.yaml | 21 +- src/globals.h | 4 - src/metadata.cpp | 160 ++++++++++++-- src/metadata.h | 98 ++++----- src/parsers.h | 2 +- src/tester.cpp | 117 ++++++++--- 7 files changed, 357 insertions(+), 512 deletions(-) diff --git a/README b/README index 2cdd04f7..bcb184dd 100644 --- a/README +++ b/README @@ -1,430 +1,95 @@ -LOOT -==== +BOSSv3 +====== Introduction ============ -LOOT is BOSS v3. See BOSS: http://code.google.com/p/better-oblivion-sorting-software/ -LOOT has been given a different working name because it will be a very -different program to the one that people know and love as BOSS. +BOSSv3 is being developed on a separate repository from the rest of BOSS's code as it is essentially a new program, and is being designed to address the shortcomings that are inherent in how BOSS approaches load order optimisation. -LOOT is being designed from the ground up to address the shortcomings -that are inherent in how BOSS approaches load order optimisation. That's -another reason why it's being hosted on a different site, under a -different name, even though it is intended to be released as an update -to BOSS. While ideas and some code will be transferred, the distancing -will hopefully produce better results. +So what's wrong with BOSS? Why do I feel the need to go back to the drawing board? -So what's wrong with BOSS? Why do I feel the need to go back to the -drawing board? +Well, modding for Bethesda's games has proved to be incredibly popular, with over 25,000 mods each available for Oblivion and Skyrim. -Well, Oblivion has accumulated around 25,000 mods on -Oblivion Nexus since its launch in 2006. Skyrim has accumulated around -19,000 mods on Skyrim Nexus since its launch in 2011. Growth models aside, -Skyrim has proved incredibly popular for modding. +For BOSS to be as useful as possible, it needs to be able to sort as many mods as possible. For Skyrim, the backlog of mods that need adding to its masterlist is at 5,800+ and growing. Recruiting extra team members isn't a real solution, as I've already done that and frankly copy/pasting filenames into a massive text file is not a whole load of fun and any analysis takes a while to do properly. -For BOSS to be as useful as possible, it needs to be able to sort as many -mods as possible. For Skyrim, we're getting user submissions of mods to -be sorted faster than we can add them, and our backlog is at 3,000+ -plugins and growing. That's after I've doubled the effective team size. +The other obvious solution is to cut down on the number of mods that need to be added to the masterlist - instead of having every single mod listed, BOSS could be made to sort the 'simple' mods on its own, and then we could add to the masterlist only those that proved too tricky for it to position itself. -The BOSS team's operational model is therefore unsustainable, and has -to change. The main issue is that it takes time for us to process all -these reports, and we only have so much free time: the team itself is the -weak point in the operating chain. It's a good idea, then, to automate -as much as possible, and so reduce the need for the team to do things. - -While that's being done, I might as well also make some improvements to -other areas of BOSS. +While that's being done, I might as well also make some improvements to other areas of BOSS. -LOOT Design Overview -==================== +Design Notes +============ -LOOT's central design idea is that its functionality should be as independent of -external systems as possible. A system might be a human, a group of humans, or a -server somewhere. This rather vague idea can be broken down into a few design -goals: +BOSS should be able to identify a reasonably correct load order for a group of arbitrary plugins based on their contents. However, there are always some plugins that must be positioned in certain ways relative to other plugins due to author intent that cannot be determined from their contents. There is also generally an element of user preference in setting a load order. BOSS should also be able to provide mechanisms for dealing with such plugins and preferences in addition to its basic 'reasonably correct' sorting. - * LOOT should be able to figure out an optimum load order without any inputs - other than the plugins it will order. - * LOOT should be able to figure out a correct load order with a minimum of - supplementary data on author intent. - * LOOT should be able to obtain supplementary data from an arbitrary source, - or arbitrary sources, depending on the number of types of data. - * LOOT's effects should be customiseable by the user. - * LOOT should be scaleable, focussed and extensible. +BOSSv3's masterlist will therefore be used to provide unordered data sets for plugins that require them, and the userlist used to allow users to override the automatic sorting and to provide additional metadata themselves. They will be quite different to the current masterlist and userlist, and will use a new language/format. -Note that an optimum load order is not necessarily a correct load order. If -we define the impact of a plugin as the number of its edits that get applied to -a person's game, then an optimum load order is one with the highest average -impact from its constituent plugins whilst satisfiying all explicit dependencies. -However, some plugins may be made with the intent that they are to be overridden -by others, and so a correct load order is one that also takes into account -author intent. Given a set of plugins, their correct load order may have a lower -average impact than their optimum load order. +If the 'impact' of a plugin is defined as the number of its edits that get applied to a person's game divided by the total number of edits it makes, then an 'optimum' load order is one that maximises the average impact of its constituent plugins whilst satisfying all explicit dependencies. A 'correct' load order is not necessarily an optimum load order though, as some plugins may be made with the intent that they are overridden by other plugins. -The design goal for a correct load order specifies the use of supplementary data -because the determination of author intent is not generally possible given only -a plugin. Such supplementary data might include any implicit masters, and -their intended order relative to any explicit masters. It might also include -whether the plugin was intended to be high or low priority. +For flexibility, the 'masterlist' data file that gets updated by the BOSS Team should have its online location as a configurable option within BOSS, so that should its location change for whatever reason, users would be able to redirect their copy of BOSS to look in the new location. It might be a good idea to download it via a diff, or allow compression, somehow as the Skyrim masterlist is 1.7 MB. -Supplementary data can also include data not related to load ordering: one of -BOSS's great successes has been the highlighting of the issue of dirty plugins, -and the distribution of generally applicable information such as that is an -important feature to retain. +BOSSv3 will also have an API for accessing BOSS data and functionality. The more general functionality found in v2's API has already been forked to libloadorder, which v3 shall make use of. -The goal of arbitrary data sources is with LOOT's long-term success in mind. It -may be that the team responsible for it may suddenly drop off the face of the -Internet, as has happened with other utilities' teams in the past. In such a -circumstance, getting access to the data source LOOT uses may be difficult or -impossible, and so anyone wishing to pick the project up would have to set up -a new source, but if LOOT can't handle arbitrary data sources, then those -picking it up would also have to release a new version of LOOT and distribute -that. This may take a while if there is nobody available with the skills or -experience required. If LOOT can handle arbitrary data sources, then all that -would need to be done is for LOOT users to be made aware of the situation, and -for them to be given instructions on how to change the source LOOT looks at in -their copies. +Unlike v2, v3 will not have a built-in updater/update checker. It's just too much of a headache to code and manage for the payoff it gives, especially since notifications can be sent to users via the masterlist. -The goal of customiseability is because LOOT will never be able to automatically -get a load order quite how everyone wants it. There will always be an element of -choice involved in setting up a load order, so LOOT should accommodate for that. +When run, BOSS shall output its results to a "log.yaml" file, which shall then be interpreted by the UI to display the BOSS Log. In addition, before running it shall create a copy of the current load order in the same directory with the filename "loadorder.old". -The final goal is basically to prevent the problems BOSS has happening again, by -implementing good systems design and coding practice. I'm always learning, and -now I realise some of the decisions I made working on BOSS weren't the best. I -probably won't make the best decisions this time around either, but they should -end up better. - -I anticipate that by having LOOT be able to figure out an optimum load order -without any input from humans, ordering the vast majority of plugins correctly -can be fully automated, as most plugins make a small number of changes that -are highly compatible. More complex plugins will still require the maintainance -team to provide LOOT with supplementary data, but they are relatively few. +BOSSv3 won't have a command line interface. Command line options may be added to the GUI if requested. Users need to be made aware that what BOSS sets isn't the absolutely perfect load order, and is open to tweaking, which is being lost on some people currently, in part due to the 'one click' nature of BOSS. -LOOT User Experience -==================== +Roadmap +======= -LOOT will require a greater degree of user interaction than BOSS does, -so an interface that facilitates this is required: as such, LOOT will -not have a CLI. A GUI that accepts command line parameters may be an option if -people want one. - -A general workflow for a user running LOOT to sort would be: - -1. Run LOOT: it auto-sorts based on plugin content plus supplementary data. -2. LOOT finishes sorting, and displays the load order it suggests. At this point - the load order has not yet been applied. - - Clicking on a plugin in the load order will highlight all those it - conflicts with. This provides a simple way for users to gauge conflicts - and so decide which plugin's changes they wish to apply to their games. - - Dragging a plugin will display the positions to which it can be moved - (by dropping it there) whilst satisfying dependencies. - - Plugins which have no conflicts are greyed out and cannot be selected, - to simplify this process. - - There will be an information panel which will display the selected - plugin's requirements, incompatibilities, priority and messages. These - will be editable, and upon exit of the load order window, the user will be - asked if they wish to submit any edits they make to these fields. Edits to - plugin positions will not be submitted. - - If LOOT encounters any errors while sorting, they are displayed instead - of the load order. -3. The user exits the load order display, choosing either to apply or - discard changes. -4. LOOT asks the user if they want to submit their plugin data edits. If - so, it then asks the user to supply links for each of the plugins - with edited data. -4. LOOT then applies the resulting load order, saves any user choices, and - exits. - -LOOT must also provide a means to manage past user choices, to undo its -changes (only back one run though), and to edit its settings. - -LOOT should ask for permission to send plugin metadata whenever there are -metadata changes to send, unless the user selects a "do not ask me again" -option. This setting should be configurable at any time. - -Documentation will be split between three readmes: one general readme, one API -readme and one syntax readme. The version history can be split into a -fourth readme when it grows sufficiently long. +1. Write new masterlist, userlist, settings file parsers. DONE +2. Implement per-game handling. DONE +3. Write API. DONE +4. Develop sorting algorithm. +5. Tie together automatic 'simple' sorting with masterlist, userlist data parsing and usage. +6. Write masterlist updating code. +7. Develop graphical user interface. -Implementation -============== +Sorting Algorithm +================= -LOOT will be an ambitious project: to keep its code as simple as possible, -it will make use of external libraries to supply functionality where -possible, for example: - * yaml-cpp: for masterlist/userlist parsing. - * libespm: for plugin parsing. - * libloadorder: for reading and writing load order. - * libsvn: for accessing the Subversion repositories used. - * cURL: for misc. networking. - * wxWidgets: for the interface. - * Boost: for filesystem interaction, various helpful functions. +Given two plugins A and B, their load order only matters if they both change a third plugin's records or if one changes the other's records. In such a case, the plugins can be said to 'overlap'. -As a reflection of this, LOOT's API will provide functions only for -accessing LOOT-specific functionality and data, eg. masterlist / userlist reading, -masterlist download and user edits submission. +The hard rules of load ordering are: + 1. Master files always load before non-master files. + 2. A plugin must load after all its masters. -The support for arbitrary data sources will be implemented by having the source -LOOT uses specified in a settings file, which will be editable via the GUI. +Beyond that, the following can be used to maximise the impact of two plugins: + 1. Compute the overlap between A and B. Divide the overlap size by the number of records in each plugin, and the plugin with the larger ratio should load later, as more of its overall changes would be lost if it loaded first than if the other plugin does. -Because auto-sorting requires LOOT to know what is inside the plugins it sorts, -it must scan the plugins, and determine which of its records are new and which -are edited (new records aren't used for determining load order). +There are issues related to author intent: for example, the Unofficial Patches contain a large number of mostly edited records, as does a Bashed Patch, but the former is intended to load early and be overridden by other plugins while the latter is designed to load late. It's not clear how such situations can be resolved. -Optimisations can be made to skip load order -calculation. For example, if the only changes from the last load order set by -LOOT is that a plugin or plugins have been removed, then the last load order -may be used. It might also be possible to only perform partial calculation -when plugins are added, though this is not certain. It might not also be worth -the effort, since preliminary testing shows that scanning is considerably -faster than first anticipated. +One possibility is to have a 'priority' metadata field, which has a default value, a value signifying the plugin should load late, and a value signifying the plugin should load early. If left at the default value, the plugin would be sorted by the algorithm normally, but if set to the early value, the plugin would be moved as far up the load order as the hard rules allow, and if set to the late value, would be moved as far down the load order as the hard rules allow. If multiple plugins are present that load early or late, then within that set of plugins the algorithm could be applied normally. -Note: For clarity, the he list of metadata that is maintained by the -LOOT team will be referred to as the "masterlist", and the list of metadata that -is individual to each user will be referred to as the "userlist". +It's not a great solution though, being far too imprecise for my liking, but then the loading intention is also vague. + +Optimisations may also be made, such as not performing any new sorting if the only differences since previous sorting are the removal of plugins, or if the plugins added only add new records they can just be placed at the end. -LOOT Masterlist & Userlist -========================== +Data Files Format +================= -The purpose of the masterlist is to store the following data for plugins: +BOSSv3's settings file, masterlist, userlist and previous run logs will all be written in YAML. This allows me to take advantage of existing parsing libraries and the format's flexibility. It's also not that much more verbose than v2's masterlist format, once data structure changes are taken into account. - * Implicit and explicit requirements in load order. - * Incompatibilities, for catching unresolveable conflicts. - * Messages, for displaying information about the plugins. - * The intended priority of the plugin relative to others that make the same - changes. - * Possibly also Bash Tags to be applied. +A custom parser is required for the evaluation of conditions though. The syntax has been made more human readable and compound conditions now evaluate according to standard logic. -It is also useful to store some data not associated with plugins, such as: +The masterlist and userlist will use the same parser, and simply combined by merging or replacing plugin metadata where depending on the type of metadata, though the details haven't yet been decided. - * Global messages. - * Message objects, for reference instead of rewriting the same message - multiple times. - * File objects, for reference instead of rewriting the same condition - argument multiple times. - -All messages and files should also be optionally conditional on an arbitrary -number of conditions. - -The purpose of the userlist is to store modifications to masterlist plugin data, -and to also store user-specified positions of plugins. The userlist must also -support the disabling of plugin entries within it, so that users can turn off -their changes without deleting them. - -As the masterlist and userlist are very similar, it makes sense for them to use -the same format. The userlist could then simply support two data fields that are -ignored by the masterlist parser. I think that YAML is a suitable format for -these files as it is simple, human-readable, yet quite powerful. - -The drag 'n' drop plugin positioning will likely require userlist rules -to support positioning of plugins precisely relative to other plugins, but it -may be possible to achieve the positioning using only the priority setting. If -possible, that is preferred. - -When the masterlist/userlist is loaded, it is scanned for entries for each of -the installed plugins. If more than one match is found, they should be merged. - -Further details can be found in the 'docs/BOSS Metadata File Syntax.html' file. - - -Networking -========== - -LOOT has two target networking functionalities: - - * Masterlist updating. - * User feedback submission. - -Neither of these functionalities are critical to usage of LOOT. - -The masterlist repository will use Subversion, as this -allows users to most easily download only the file they require, and only the -diffs of that file. - -The masterlists would be stored using a folder structure that mirrors LOOT's -install structure, for example: - -repo/ - Oblivion/ - masterlist.yaml - -"repo" does not have to be the repository root, but LOOT's settings should -use the URL pointing to that location. The repo will be required to use the same -game folder names as LOOT's install structure. - -The masterlist update mechanism will feature a validator such that if the -masterlist downloaded cannot be parsed, it will not replace the current -masterlist (if exists). The log will display notification of validation failure -with instructions to inform the maintainers of this issue. This is a fairly -major improvement over BOSS currently, which becomes unusable when a masterlist -update is occassionally broken. - -The final bit of networking to be considered is the feedback mechanism. It's not -an initial development target, as initially I think it would be better for all -communication between users and the team to be two-way, which is easier -accomplished through our forum threads. It's probably best to leave design of -this until we see what sort of feedback users are giving. - -A program update checker is notably absent, as a fully automated updater is -not worth the trouble of implementing well, and notification of an available -update is easily performed through a global masterlist message and so does not -require an additional transport mechanism. +Further details can be found in the 'docs/BOSS Metadata File Syntax.html' file. An example settings file can be found at 'examples/settings.yaml'. Install Structure ================= -LOOT will be a self-contained installation that can be dropped anywhere. It will -have an installer option that also installs some Start menu shortcuts and a -Registry entry, but these will not be required for LOOT to function. +LOOT will be a self-contained installation that can be dropped anywhere. It will have an installer option that also installs some Start menu shortcuts and a Registry entry, but these will not be required for LOOT to function. -The directory structure will be as follows: - -LOOT/ - Morrowind/ - .svn/ - ... - Cache/ - .svn/ - ... - m/ - o/ - morrowind.esm.lmdb - ... - masterlist.yaml - userlist.yaml - lastrun.yaml - loadorder.old - Oblivion/ - .svn/ - ... - Cache/ - .svn/ - ... - o/ - b/ - oblivion.esm.lmdb - ... - masterlist.yaml - userlist.yaml - lastrun.yaml - loadorder.old - Skyrim/ - .svn/ - ... - Cache/ - .svn/ - ... - s/ - k/ - skyrim.esm.lmdb - ... - masterlist.yaml - userlist.yaml - lastrun.yaml - loadorder.old - Fallout3/ - .svn/ - ... - Cache/ - .svn/ - ... - f/ - a/ - fallout3.esm.lmdb - ... - masterlist.yaml - userlist.yaml - lastrun.yaml - loadorder.old - FalloutNewVegas/ - .svn/ - ... - Cache/ - .svn/ - ... - f/ - a/ - falloutNV.esm.lmdb - ... - masterlist.yaml - userlist.yaml - lastrun.yaml - loadorder.old - LOOT.exe - LOOT.ini - -lastrun.yaml stores the last load order set by LOOT, with the messages -displayed at that time, and uses the same format as the masterlist and userlist. - -loadorder.old stores the load order before LOOT was last run. - -The game caches are for storing database files so that they can be used when -there is no Internet connection. Each cache acts as separate working copy of -the database repository, holding the files it needs. They are separate to -help prevent name clashes between games. Each game folder also acts as a working -copy of the folder which stores the masterlist for that game on the repository. - - -Configuration -============= - -LOOT will store its settings in a LOOT.ini file in its install directory. This -file should contain the following settings: - -bEnableNetworking Boolean -bUseOnlineCache Boolean Cache = Database -bGetMetadataUpdates Boolean Metadata = Masterlist -bPromptMetadataSubmit Boolean -bAlwaysSubmitMetadata Boolean Only has effect if the above is false. - -sOnlineCacheURL String URL to the base of the database repository. -sMetadataURL String -sVersionCheckerURL String URL to the file holding version info. -sFeedbackURL String URL to the issue tracker used for feedback. - -sGame String Same as in BOSS. -sLastGame String Same as in BOSS. -iDebugVerbosity Integer Debug output is logged if > 0. -bDoTrialRun Boolean Same as in BOSS. -sLanguage String Same as in BOSS. - - -Cross-Platform Support -====================== - -I develop on Linux, cross-compiling for Windows. LOOT will use the CMake build -system generator so that it can be compiled on multiple platforms, though some -functionality will require Windows-specific things, such as Registry queries. - - -Backwards Compatibility -======================= - -No backwards compatibility with previous versions of BOSS is planned: there are -simply too many changes. To ease the transition though, it may be beneficial for -there to be converters for the userlist and masterlist. - -Such converters would be limited in scope, however, due to the differences in -capabilities: only userlist rules which sort plugins after others and perform no -message changes have direct conversions. The masterlist may be adapted by having -rules created for those plugins with messages attached, but the content of those -messages would have to be adjusted manually, and all specific positioning -information would be lost too. +The directory structure will be identical to that which BOSS currently has, with the exception that the text and ini files will be replaced by YAML files. Misc @@ -438,3 +103,39 @@ ZiggyX200, RiddlingLynx, AliTheLord, Tokc.D.K., Valda, Space Oden69, Televator, Leandro Conde, Psymon, Loucifer, Torrello, Malonn, Skyline, Sharlikran, Red Eye, iyumichan, Peste, Calen Ellefson, SilentSpike, Arkangel, zyp, v111, Chevenga, rowynyew + + +Backwards Compatibility +======================= + +BOSSv3 will not be backwards compatible with previous versions of BOSS: that is, it will not be able to use masterlists, userlists, translation files and ini files from previous versions of BOSS. + +It may be possible to perform limited conversion from v2 masterlists and userlists to their v3 equivalents, but as their capabilities have substantially changed, much of them may be untransferrable or at least not automatically transferable. + +A userlist converter would be most useful, since that would make most people's switch between v2 and v3 easier. The masterlists will probably have to be manually scanned for useful information. + +Userlist Conversion +------------------- + +Differences between BOSSv2 and v3: + + - In v2, plugins can be sorted before or after other plugins, but v3 only supports sorting after other plugins. + - In v2, it is possible to replace or append to masterlist messages. In v3, the user can only append messages. + - In v2, it is possible to sort groups or relative to groups, but groups do not exist in v3. + - In v2, the order of rules affects their execution. In v3 the order of rules doesn't matter. + - In v2, requirements and incompatibilities are added in messages, but in v3 they are added in a separate data structure. + +Conversion for syntax is just a matter of reading in using the old parser and writing out using the new generator. If any of the following are detected in a rule, conversion should be aborted and error messages detailing why displayed, asking the user to make the relevant changes before running the converter again. + + - If the rule object is a group. + - If the sort keyword is 'BEFORE', 'TOP' or 'BOTTOM'. + - If the sort object is a group. + - If the message keyword is 'REPLACE'. + - If the message object's keyword is 'REQ' or 'INC'. + +If any of the following are detected, then BOSS should display a warning that the result may differ from the original: + + - If a message object's keyword is 'TAG'. + - If more than one rule references the same sort object or the same plugin appears as a sort object in one rule and a rule object in another. + +Additionally, a warning should be displayed that all comments will be ignored. diff --git a/examples/settings.yaml b/examples/settings.yaml index 5184cb53..144e9a86 100644 --- a/examples/settings.yaml +++ b/examples/settings.yaml @@ -3,18 +3,13 @@ # If you don't know YAML, just think of a normal ini file, but replace the # equals signs with colons followed by a space. No tabs allowed. -bEnableNetworking: true -bUseOnlineCache: true -bGetMetadataUpdates: true -bPromptMetadataSubmit: true -bAlwaysSubmitMetadata: true +bGetMetadataUpdates: true -sOnlineCacheURL: "http://better-oblivion-sorting-software.googlecode.com/svn/data/boss-nehrim/masterlist.txt" -sMetadataURL: "http://better-oblivion-sorting-software.googlecode.com/svn/data/boss-nehrim/masterlist.txt" -sVersionCheckerURL: "http://better-oblivion-sorting-software.googlecode.com/svn/data/boss-nehrim/masterlist.txt" +sMetadataURL: "http://better-oblivion-sorting-software.googlecode.com/svn/data/boss-nehrim/masterlist.txt" -sGame: auto # auto, oblivion, skyrim, fallout3, falloutnv -sLastGame: auto # auto, oblivion, skyrim, fallout3, falloutnv -iDebugVerbosity: 0 # 0, 1, 2, 3. Logging takes place if > 0. -bDoTrialRun: false -sLanguage: eng # An ISO 639-3 language string. +sGame: auto # auto, oblivion, skyrim, fallout3, falloutnv +sLastGame: auto # auto, oblivion, skyrim, fallout3, falloutnv +iDebugVerbosity: 0 # 0, 1, 2, 3. Logging takes place if > 0. +bDoTrialRun: false +sLanguage: eng # An ISO 639-3 language string. + diff --git a/src/globals.h b/src/globals.h index 1110805b..c574624a 100644 --- a/src/globals.h +++ b/src/globals.h @@ -34,10 +34,6 @@ namespace boss { const unsigned int VERSION_MAJOR = 3; const unsigned int VERSION_MINOR = 0; const unsigned int VERSION_PATCH = 0; - - // Temporary hack. - const char * const libespm_options_path = "../../libespm/opts/input"; - const char * const libespm_game = "Skyrim"; } diff --git a/src/metadata.cpp b/src/metadata.cpp index ee70b207..067c2625 100644 --- a/src/metadata.cpp +++ b/src/metadata.cpp @@ -24,6 +24,7 @@ #include "metadata.h" #include "helpers.h" #include +#include #include #include @@ -32,6 +33,54 @@ using namespace std; namespace boss { + FormID::FormID() : id(0) {} + + FormID::FormID(const std::string& sourcePlugin, const uint32_t objectID) : plugin(sourcePlugin), id(objectID) {} + + FormID::FormID(const std::vector& sourcePlugins, const uint32_t formID) { + int index = formID >> 24; + id = formID & ~((uint32_t)index << 24); + + if (index >= sourcePlugins.size()) { + cout << hex << formID << dec << " in " << sourcePlugins.back() << " has a higher modIndex than expected." << endl; + index = sourcePlugins.size() - 1; + } + + plugin = sourcePlugins[index]; + } + + bool FormID::operator == (const FormID& rhs) const { + + } + + bool FormID::operator != (const FormID& rhs) const { + return !(*this == rhs); + } + + bool FormID::operator < (const FormID& rhs) const { + return (id < rhs.Id() || plugin < rhs.Plugin()); + } + + bool FormID::operator > (const FormID& rhs) const { + return !(*this == rhs || *this < rhs); + } + + bool FormID::operator <= (const FormID& rhs) const { + return (*this == rhs || *this < rhs); + } + + bool FormID::operator >= (const FormID& rhs) const { + return !(*this < rhs); + } + + std::string FormID::Plugin() const { + return plugin; + } + + uint32_t FormID::Id() const { + return id; + } + ConditionalData::ConditionalData() {} ConditionalData::ConditionalData(const string& c) : condition(c) {} @@ -81,6 +130,10 @@ namespace boss { File::File(const std::string& n, const std::string& d, const std::string& c) : display(d), ConditionalData(c, n) {} + bool File::operator < (const File& rhs) const { + return Name() < rhs.Name(); + } + std::string File::Name() const { return Data(); } @@ -115,6 +168,10 @@ namespace boss { Data(data); } + bool Tag::operator < (const Tag& rhs) const { + return Name() < rhs.Name(); + } + bool Tag::IsAddition() const { return addTag; } @@ -130,20 +187,20 @@ namespace boss { return Data(); } - Plugin::Plugin() : enabled(true), priority(0), crc(0), isMaster(false) {} - Plugin::Plugin(const std::string& n) : name(n), enabled(true), priority(0), crc(0), isMaster(false) {} + Plugin::Plugin() : enabled(true), priority(0), isMaster(false) {} + Plugin::Plugin(const std::string& n) : name(n), enabled(true), priority(0), isMaster(false) {} Plugin::Plugin(const std::string& n, const std::string& path) : name(n), enabled(true), priority(0) { // Get data from file contents using libespm. Assumes libespm has already been initialised. boost::filesystem::path filepath = boost::filesystem::path(path) / n; + ifstream input(filepath.string().c_str(), ios::binary); espm::file File; espm::readFile(input, File); input.close(); - - crc = GetCrc32(filepath); + isMaster = espm::isMaster(File); vector rawMasters = espm::getMasters(File); for (size_t i=0,max=rawMasters.size(); i < max; ++i) { @@ -151,8 +208,11 @@ namespace boss { } vector records = espm::getRecords(File); + vector plugins = masters; + plugins.push_back(name); for (vector::const_iterator it = records.begin(),endIt = records.end(); it != endIt; ++it){ - formIDs.insert(*reinterpret_cast(it->record.recID)); + uint32_t id = *reinterpret_cast(it->record.recID); + formIDs.insert(FormID(plugins, id)); } } @@ -176,7 +236,7 @@ namespace boss { return requirements; } - std::set Plugin::Incs() const { + std::set Plugin::Incs() const { return incompatibilities; } @@ -208,7 +268,7 @@ namespace boss { requirements = r; } - void Plugin::Incs(const std::set& i) { + void Plugin::Incs(const std::set& i) { incompatibilities = i; } @@ -235,7 +295,7 @@ namespace boss { ++it; } - for (set::iterator it = incompatibilities.begin(); it != incompatibilities.end();) { + for (set::iterator it = incompatibilities.begin(); it != incompatibilities.end();) { if (!it->EvalCondition(game)) incompatibilities.erase(it++); else @@ -265,36 +325,90 @@ namespace boss { return boost::iends_with(name, "\\.esm") || boost::iends_with(name, "\\.esp"); } - bool Plugin::operator == (Plugin rhs) { - return name == rhs.Name(); + bool Plugin::operator == (const Plugin& rhs) const { + return boost::iequals(name, rhs.Name()); + } + + bool Plugin::operator != (const Plugin& rhs) const { + return !(*this == rhs); + } + + bool Plugin::operator < (const Plugin& rhs) const { + + /* If this plugin is a master and the other isn't, this plugin should load first. + * If this plugin is a master of the other plugin, this plugin should load first. + * If this plugin conflicts with the other plugin, then the plugin which contains the most records overall should load first. + */ + + if (IsMaster() && !rhs.IsMaster()) + return true; + + if (!IsMaster() && rhs.IsMaster()) + return false; + + if (rhs.IsChildOf(*this)) + return true; + + if (IsChildOf(rhs)) // Should probably also check for cyclic masters. + return false; + + if (!OverlapFormIDs(rhs).empty() && formIDs.size() != rhs.FormIDs().size()) + return formIDs.size() > rhs.FormIDs().size(); + + //return false; + return name < rhs.Name(); + } + + bool Plugin::operator > (const Plugin& rhs) const { + return !(*this == rhs || *this < rhs); + } + + bool Plugin::operator <= (const Plugin& rhs) const { + return (*this == rhs || *this < rhs); + } + + bool Plugin::operator >= (const Plugin& rhs) const { + return !(*this < rhs); } - std::set Plugin::FormIDs() const { + std::set Plugin::FormIDs() const { return formIDs; } - std::set Plugin::OverrideFormIDs() const { - int modIndex = masters.size() << 24; - - set fidSubset; - - for (set::const_iterator it = formIDs.begin(), endIt=formIDs.end(); it != endIt; ++it) { - if (*it < modIndex) + std::set Plugin::OverrideFormIDs() const { + set fidSubset; + for (set::const_iterator it = formIDs.begin(), endIt=formIDs.end(); it != endIt; ++it) { + if (boost::iequals(it->Plugin(), name)) fidSubset.insert(*it); } - return fidSubset; } + std::set Plugin::OverlapFormIDs(const Plugin& plugin) const { + std::set otherFormIDs = plugin.FormIDs(); + std::set overlap; + for (std::set::const_iterator it=formIDs.begin(), endIt=formIDs.end(); it != endIt; ++it) { + if (otherFormIDs.find(*it) != otherFormIDs.end()) + overlap.insert(*it); + } + + return overlap; + } + std::vector Plugin::Masters() const { return masters; } - uint32_t Plugin::Crc() const { - return crc; - } - bool Plugin::IsMaster() const { return isMaster; } + + //Checks if the object plugin is explicitly dependent on the given plugin. + bool Plugin::IsChildOf(const Plugin& plugin) const { + for (vector::const_iterator it=masters.begin(), endIt=masters.end(); it != endIt; ++it) { + if (boost::iequals(*it, plugin.Name())) + return true; + } + return false; + } } diff --git a/src/metadata.h b/src/metadata.h index ee5325af..3824ba92 100644 --- a/src/metadata.h +++ b/src/metadata.h @@ -31,10 +31,31 @@ #include #include -#include - namespace boss { + //A FormID is a 32 bit unsigned integer of the form xxYYYYYY in hex. The xx is the position in the masters list of the plugin that the FormID is from, and the YYYYYY is the rest of the FormID. Here the xx bit is stored as the corresponding filename to allow comparison between FormIDs from different plugins. + class FormID { + public: + FormID(); + FormID(const std::string& pluginName, const uint32_t objectID); + + //The masters here also includes the plugin that they are masters of as the last element. + FormID(const std::vector& masters, const uint32_t formID); + + bool operator == (const FormID& rhs) const; + bool operator != (const FormID& rhs) const; + bool operator < (const FormID& rhs) const; + bool operator > (const FormID& rhs) const; + bool operator <= (const FormID& rhs) const; + bool operator >= (const FormID& rhs) const; + + std::string Plugin() const; + uint32_t Id() const; + private: + std::string plugin; + uint32_t id; + }; + class ConditionalData { public: ConditionalData(); @@ -75,6 +96,8 @@ namespace boss { File(const std::string& name, const std::string& display, const std::string& condition); + bool operator < (const File& rhs) const; + std::string Name() const; std::string DisplayName() const; private: @@ -87,6 +110,8 @@ namespace boss { Tag(const std::string& tag); Tag(const std::string& tag, const std::string& condition); + bool operator < (const Tag& rhs) const; + bool IsAddition() const; std::string Name() const; std::string PrefixedName() const; //Name with '-' in front if suggested for removal. @@ -94,18 +119,6 @@ namespace boss { bool addTag; }; - struct file_comp { - bool operator() (const File& lhs, const File& rhs) const { - return lhs.Name() < rhs.Name(); - } - }; - - struct tag_comp { - bool operator() (const Tag& lhs, const Tag& rhs) const { - return lhs.Name() < rhs.Name(); - } - }; - class Plugin { public: Plugin(); @@ -117,7 +130,7 @@ namespace boss { int Priority() const; std::list LoadAfter() const; std::list Reqs() const; - std::set Incs() const; + std::set Incs() const; std::list Messages() const; std::list Tags() const; @@ -126,7 +139,7 @@ namespace boss { void Priority(const int priority); void LoadAfter(const std::list& after); void Reqs(const std::list& reqs); - void Incs(const std::set& incs); + void Incs(const std::set& incs); void Messages(const std::list& messages); void Tags(const std::list& tags); @@ -134,57 +147,36 @@ namespace boss { bool HasNameOnly() const; bool IsRegexPlugin() const; - bool operator == (Plugin rhs); + //Compare name strings. + bool operator == (const Plugin& rhs) const; + bool operator != (const Plugin& rhs) const; + + //Compare load order positions. + bool operator < (const Plugin& rhs) const; + bool operator > (const Plugin& rhs) const; + bool operator <= (const Plugin& rhs) const; + bool operator >= (const Plugin& rhs) const; - std::set FormIDs() const; - std::set OverrideFormIDs() const; + std::set FormIDs() const; + std::set OverrideFormIDs() const; + std::set OverlapFormIDs(const Plugin& plugin) const; std::vector Masters() const; - uint32_t Crc() const; - bool IsMaster() const; + bool IsMaster() const; //Checks master bit flag. + bool IsChildOf(const Plugin& plugin) const; //Checks masters for given plugin. private: std::string name; bool enabled; //Default to true. int priority; //Default to 0 : >0 is higher, <0 is lower priorities. std::list loadAfter; std::list requirements; - std::set incompatibilities; + std::set incompatibilities; std::list messages; std::list tags; std::vector masters; - std::set formIDs; - uint32_t crc; + std::set formIDs; bool isMaster; }; - - inline bool isMasterOf(const Plugin& servant, const Plugin& master) { - std::vector masters = servant.Masters(); - for (int i = 0; i < masters.size(); ++i) { - if (masters[i] == master.Name()) - return true; - } - return false; - } - - struct plugin_comp { - bool operator() (const Plugin& lhs, const Plugin& rhs) const { - if (lhs.IsMaster() && !rhs.IsMaster()) - return true; - else if (!lhs.IsMaster() && rhs.IsMaster()) - return false; - else if (isMasterOf(rhs, lhs)) - return true; - else if (isMasterOf(lhs, rhs)) // Should probably also check for cyclic masters. - return false; - else if (lhs.OverrideFormIDs().size() > rhs.OverrideFormIDs().size()) - return true; - else if (lhs.OverrideFormIDs().size() < rhs.OverrideFormIDs().size()) - return false; - else - return lhs.Name() < rhs.Name(); - } - }; - } diff --git a/src/parsers.h b/src/parsers.h index 6d1903d7..02dafc88 100644 --- a/src/parsers.h +++ b/src/parsers.h @@ -189,7 +189,7 @@ namespace YAML { if (node["req"]) rhs.Reqs(node["req"].as< std::list >()); if (node["inc"]) - rhs.Incs(node["inc"].as< std::set >()); + rhs.Incs(node["inc"].as< std::set >()); if (node["msg"]) rhs.Messages(node["msg"].as< std::list >()); if (node["tag"]) diff --git a/src/tester.cpp b/src/tester.cpp index 4ef57981..6f273617 100644 --- a/src/tester.cpp +++ b/src/tester.cpp @@ -1,3 +1,25 @@ +/* BOSS + + A plugin load order optimiser for games that use the esp/esm plugin system. + + Copyright (C) 2012 WrinklyNinja + + This file is part of BOSS. + + BOSS is free software: you can redistribute + it and/or modify it under the terms of the GNU General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + BOSS is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with BOSS. If not, see + . +*/ #include #include "metadata.h" @@ -11,25 +33,24 @@ #include #include +#include +#include + #include #include #include +#include namespace fs = boost::filesystem; using namespace std; -//It would be neater if this could be done by just skipping bits of the output... -void minimisePluginList(set& plugins) { - set out; - for (set::iterator it=plugins.begin(), endIt=plugins.end(); it != endIt; ++it) { - boss::Plugin p(it->Name()); - p.Tags(it->Tags()); - out.insert(p); - } - plugins.swap(out); -} +// Temporary hack. +const char * const libespm_options_path = "libespm.cfg"; +const char * const libespm_game = "Skyrim"; + + int main(int argc, char *argv[]) { @@ -78,51 +99,77 @@ int main(int argc, char *argv[]) { out2 << yout2.c_str(); out2.close(); */ - cout << "Testing plugin reading." << endl; + cout << "Setting up libespm and BOSS..." << endl; + + // Set up libesm. + common::options::setGame(libespm_game); + ifstream input(libespm_options_path); + common::readOptions(input); + input.close(); + + //Set up BOSS vars. + boss::Game game(boss::GAME_TES5); + + cout << "Reading plugins in Data folder..." << endl; time_t start, end; start = time(NULL); - // Set up libesm. - common::options::setGame(boss::libespm_game); - ifstream input(boss::libespm_options_path); - common::readOptions(input); - input.close(); - // Get a list of the plugins. list plugins; - for (fs::directory_iterator it(argv[1]); it != fs::directory_iterator(); ++it) { + for (fs::directory_iterator it(game.DataPath()); it != fs::directory_iterator(); ++it) { if (fs::is_regular_file(it->status()) && (it->path().extension().string() == ".esp" || it->path().extension().string() == ".esm")) { + + string filename = it->path().filename().string(); + if (filename == "Skyrim.esm" + || filename == "Tamriel Compendium.esp" + || filename == "Tamriel Compendium - Skill Books.esp") + continue; // Libespm crashes with these plugins. - if (it->path().filename().string() == "Skyrim.esm") - continue; // Libespm crashes with v1.9 Skyrim.esm. - - cout << "Found plugin: " << it->path().string() << endl; - boss::Plugin plugin(it->path().filename().string(),it->path().parent_path().string()); - plugins.push_back(plugin); + cout << "Reading plugin: " << filename << endl; + boss::Plugin plugin(filename, it->path().parent_path().string()); + plugins.push_back(plugin); } } - - cout << "Finished looking for plugins." << endl; - - plugins.sort(boss::plugin_comp()); - + + end = time(NULL); + cout << "Time taken to read plugins: " << (end - start) << " seconds." << endl; + start = time(NULL); + for (list::iterator it=plugins.begin(), endIt = plugins.end(); it != endIt; ++it) { cout << it->Name() << endl + << '\t' << "Number of records: " << it->FormIDs().size() << endl << '\t' << "Is Master: " << it->IsMaster() << endl << '\t' << "Masters:" << endl; vector masters = it->Masters(); for(int i = 0; i < masters.size(); ++i) cout << '\t' << '\t' << i << ": " << masters[i] << endl; - - cout << '\t' << "Number of Records: " << it->FormIDs().size() << endl; - - cout << '\t' << "Number of Override Records: " << it->OverrideFormIDs().size() << endl; + + cout << '\t' << "Conflicts with:" << endl; + for (list::iterator jt=plugins.begin(), endJt = plugins.end(); jt != endJt; ++jt) { + if (*jt != *it && !jt->IsChildOf(*it)) { + size_t overlap = jt->OverlapFormIDs(*it).size(); + if (overlap > 0) + cout << '\t' << '\t' << jt->Name() << " (" << overlap << " records)" << endl; + } + } } + + end = time(NULL); + cout << "Time taken to print plugins' details: " << (end - start) << " seconds." << endl; + start = time(NULL); + + cout << "Sorting plugins..." << endl; + + plugins.sort(); - end = time(NULL); cout << "Time taken to sort plugins: " << (end - start) << " seconds." << endl; - + + for (list::iterator it=plugins.begin(), endIt = plugins.end(); it != endIt; ++it) + cout << it->Name() << endl; + + cout << "Tester finished." << endl; + start = time(NULL); return 0; }