loot/libloot
0
0
Fork 0
mirror of https://github.com/loot/libloot.git synced 2026-07-27 14:16:01 -07:00
Code Issues Packages Projects Releases Wiki Activity

Check plugin sizes when loading again.

Browse Source 
Plugin size checking was removed a couple of commits ago when hardware
concurrency usage was implemented, but it makes sense to spread the data
between the threads as evenly as possible, as randomly assigning plugins
to threads could lead to all the big plugins being loaded in one thread,
which is inefficient.
This commit is contained in:
Oliver Hamlet
2015-06-25 15:06:01 +01:00
parent 322bd0c382
commit 7c45b9bc8d
1 changed files with 21 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backend/game/game.cpp
+21 -10
View File
@@ -123,17 +123,26 @@ namespace loot {
}
void Game::LoadPlugins(bool headersOnly) {
// First find out how many plugins there are.
uintmax_t meanFileSize = 0;
map<uintmax_t, string> sizeMap;
// First find out how many plugins there are, and their sizes.
BOOST_LOG_TRIVIAL(trace) << "Scanning for plugins in " << this->DataPath();
for (fs::directory_iterator it(this->DataPath()); it != fs::directory_iterator(); ++it) {
if (fs::is_regular_file(it->status()) && Plugin(it->path().filename().string()).IsValid(*this)) {
Plugin temp(it->path().filename().string());
BOOST_LOG_TRIVIAL(info) << "Found plugin: " << temp.Name();
uintmax_t fileSize = fs::file_size(it->path());
meanFileSize += fileSize;
//Insert the lowercased name as a key for case-insensitive matching.
plugins.insert(pair<string, Plugin>(boost::locale::to_lower(temp.Name()), temp));
std::string name = boost::locale::to_lower(temp.Name());
plugins.insert(pair<string, Plugin>(name, temp));
sizeMap.insert(pair<uintmax_t, string>(fileSize, name));
}
}
meanFileSize /= sizeMap.size(); //Rounding error, but not important.
// Get the number of threads to use.
// hardware_concurrency() may be zero, if so then use only one thread.
@@ -142,16 +151,18 @@ namespace loot {
// Divide the plugins up by thread.
unsigned int pluginsPerThread = ceil((double)plugins.size() / threadsToUse);
std::vector<std::vector<std::unordered_map<std::string, Plugin>::iterator>> pluginGroups(threadsToUse);
BOOST_LOG_TRIVIAL(info) << "Loading " << plugins.size() << " plugins using " << threadsToUse << " threads, with up to " << pluginsPerThread << " plugins per thread.";
std::vector<std::vector<std::unordered_map<std::string, Plugin>::iterator>> pluginGroups(threadsToUse);
size_t pluginGroup = 0;
for (auto it = plugins.begin(); it != plugins.end(); ++it) {
if (pluginGroups[pluginGroup].size() == pluginsPerThread) {
++pluginGroup;
}
BOOST_LOG_TRIVIAL(trace) << "Adding plugin " << it->second.Name() << " to loading group " << pluginGroup;
pluginGroups[pluginGroup].push_back(it);
// The plugins should be split between the threads so that the data
// load is as evenly spread as possible.
size_t currentGroup = 0;
for (auto& plugin : sizeMap) {
if (currentGroup == threadsToUse)
currentGroup = 0;
BOOST_LOG_TRIVIAL(trace) << "Adding plugin " << plugin.second << " to loading group " << currentGroup;
pluginGroups[currentGroup].push_back(plugins.find(plugin.second));
++currentGroup;
}
// Load the plugins.