From 16ac349e8b48c901d97e28d78e34f0f61af0fef3 Mon Sep 17 00:00:00 2001 From: Oliver Hamlet Date: Thu, 25 Jun 2015 13:59:24 +0100 Subject: [PATCH] Multithread plugin loading more effectively. Using hardware concurrency instead of the pretty arbitrary mean file size criterion previously used. File sizes are now used to load the threads as evenly as possible. While this coincidently doesn't have much effect on my computer (4 threads now vs. 5 before), it should help performance for users with CPUs that have less hardware concurrency support. --- src/backend/game/game.cpp | 83 +++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/src/backend/game/game.cpp b/src/backend/game/game.cpp index 5fd263a3..918c8a9c 100644 --- a/src/backend/game/game.cpp +++ b/src/backend/game/game.cpp @@ -120,63 +120,68 @@ namespace loot { } void Game::LoadPlugins(bool headersOnly) { - vector threads; uintmax_t meanFileSize = 0; - unordered_map tempMap; - std::vector groupPlugins; - //First calculate the mean plugin size. Store it temporarily in a map to reduce filesystem lookups and file size recalculation. + multimap sizeMap; + + // First find out how many plugins there are, and their sizes. BOOST_LOG_TRIVIAL(trace) << "Scanning for plugins in " << this->DataPath(); for (fs::directory_iterator it(this->DataPath()); it != fs::directory_iterator(); ++it) { if (fs::is_regular_file(it->status()) && Plugin(it->path().filename().string()).IsValid(*this)) { + Plugin temp(it->path().filename().string()); + BOOST_LOG_TRIVIAL(info) << "Found plugin: " << temp.Name(); + uintmax_t fileSize = fs::file_size(it->path()); meanFileSize += fileSize; - tempMap.insert(pair(it->path().filename().string(), fileSize)); + //Insert the lowercased name as a key for case-insensitive matching. + string name = boost::locale::to_lower(temp.Name()); + plugins.insert(pair(name, temp)); + sizeMap.insert(pair(fileSize, name)); } } - meanFileSize /= tempMap.size(); //Rounding error, but not important. + meanFileSize /= sizeMap.size(); //Rounding error, but not important. - //Now load plugins. - for (const auto &pluginPair : tempMap) { - BOOST_LOG_TRIVIAL(info) << "Found plugin: " << pluginPair.first; + // Get the number of threads to use. + // hardware_concurrency() may be zero, if so then use only one thread. + size_t threadsToUse = std::min((size_t)thread::hardware_concurrency(), plugins.size()); + threadsToUse = std::max(threadsToUse, (size_t)1); - //Insert the lowercased name as a key for case-insensitive matching. - Plugin temp(pluginPair.first); - auto plugin = plugins.insert(pair(boost::locale::to_lower(temp.Name()), temp)); + // Divide the plugins up by thread. + unsigned int pluginsPerThread = ceil((double)plugins.size() / threadsToUse); + vector::iterator>> pluginGroups(threadsToUse); + BOOST_LOG_TRIVIAL(info) << "Loading " << plugins.size() << " plugins using " << threadsToUse << " threads, with up to " << pluginsPerThread << " plugins per thread."; - if (pluginPair.second > meanFileSize) { - BOOST_LOG_TRIVIAL(trace) << "Creating individual loading thread for: " << pluginPair.first; - threads.push_back(thread([this, plugin, headersOnly]() { - BOOST_LOG_TRIVIAL(trace) << "Loading " << plugin.first->second.Name() << " individually."; + // The plugins should be split between the threads so that the data + // load is as evenly spread as possible. + size_t currentGroup = 0; + for (const auto& plugin : sizeMap) { + if (currentGroup == threadsToUse) + currentGroup = 0; + BOOST_LOG_TRIVIAL(trace) << "Adding plugin " << plugin.second << " to loading group " << currentGroup; + pluginGroups[currentGroup].push_back(plugins.find(plugin.second)); + ++currentGroup; + } + + // Load the plugins. + BOOST_LOG_TRIVIAL(trace) << "Starting plugin loading."; + vector threads; + while (threads.size() < threadsToUse) { + vector::iterator>& pluginGroup = pluginGroups[threads.size()]; + threads.push_back(thread([this, &pluginGroup, headersOnly]() { + for (auto it : pluginGroup) { + BOOST_LOG_TRIVIAL(trace) << "Loading " << it->second.Name(); try { - plugin.first->second = Plugin(*this, plugin.first->second.Name(), headersOnly); + it->second = Plugin(*this, it->second.Name(), headersOnly); } catch (exception &e) { - BOOST_LOG_TRIVIAL(error) << plugin.first->second.Name() << ": Exception occurred: " << e.what(); - Plugin p; + BOOST_LOG_TRIVIAL(error) << it->second.Name() << ": Exception occurred: " << e.what(); + Plugin p(it->second.Name()); p.Messages(list(1, Message(Message::error, lc::translate("An exception occurred while loading this plugin. Details:").str() + " " + e.what()))); - plugin.first->second = p; + it->second = p; } - })); - } - else { - groupPlugins.push_back(&plugin.first->second); - } + } + })); } - threads.push_back(thread([this, &groupPlugins, headersOnly]() { - for (auto plugin : groupPlugins) { - BOOST_LOG_TRIVIAL(trace) << "Loading " << plugin->Name() << " as part of a group."; - try { - *plugin = Plugin(*this, plugin->Name(), headersOnly); - } - catch (exception &e) { - BOOST_LOG_TRIVIAL(error) << plugin->Name() << ": Exception occurred: " << e.what(); - Plugin p; - p.Messages(list(1, Message(Message::error, lc::translate("An exception occurred while loading this plugin. Details:").str() + " " + e.what()))); - *plugin = p; - } - } - })); // Join all threads. for (auto& thread : threads) {