added DownloadInfo for headers and user agent

downloads now have an IRepositoryDownload to manage the actual downloads
curl downloader uses IDownload and IDownloader
download stats, allow downloading to buffer

src/CMakeLists.txt

@@ -65,6 +65,7 @@ add_filter(NAME src/downloads GROUPS
 	downloadlist
 	downloadlistview
 	downloadmanager
+	downloadmanager2
 )
 
 add_filter(NAME src/env GROUPS

src/curldownloader.h

@@ -0,0 +1,212 @@
+#ifndef MODORGANIZER_CURLDOWNLOADER_INCLUDED
+#define MODORGANIZER_CURLDOWNLOADER_INCLUDED
+
+#include <ipluginrepository.h>
+
+namespace dm::curl
+{
+
+namespace fs = std::filesystem;
+using hr_clock = std::chrono::high_resolution_clock;
+
+struct defer_t {};
+extern const defer_t defer;
+
+class GlobalHandle
+{
+public:
+  GlobalHandle();
+  ~GlobalHandle();
+
+  GlobalHandle(const GlobalHandle&) = delete;
+  GlobalHandle& operator=(const GlobalHandle) = delete;
+};
+
+
+class EasyHandle
+{
+public:
+  EasyHandle();
+  EasyHandle(defer_t);
+  ~EasyHandle();
+
+  EasyHandle(const EasyHandle&) = delete;
+  EasyHandle& operator=(const EasyHandle&) = delete;
+
+  bool create();
+  CURL* get() const;
+
+private:
+  CURL* m_handle;
+};
+
+
+class MultiHandle
+{
+public:
+  MultiHandle();
+  MultiHandle(defer_t);
+  ~MultiHandle();
+
+  MultiHandle(const MultiHandle&) = delete;
+  MultiHandle& operator=(const MultiHandle&) = delete;
+
+  bool create();
+  CURLM* get() const;
+
+private:
+  CURLM* m_handle;
+};
+
+
+class FileHandle
+{
+public:
+  FileHandle();
+  ~FileHandle();
+
+  FileHandle(const FileHandle&) = delete;
+  FileHandle& operator=(const FileHandle&) = delete;
+
+  bool opened() const;
+  std::size_t open(fs::path p, bool append);
+  void close();
+
+  bool write(std::string_view sv);
+
+private:
+  HANDLE m_handle;
+  fs::path m_path;
+
+  bool doOpen(bool append);
+};
+
+
+struct SListDeleter
+{
+  void operator()(curl_slist* p)
+  {
+    if (p) {
+      curl_slist_free_all(p);
+    }
+  }
+};
+
+using SList = std::unique_ptr<curl_slist, SListDeleter>;
+
+
+class Download : public MOBase::IDownload
+{
+public:
+  Download(std::string url, Info info);
+
+  CURL* setup(curl_off_t maxSpeed);
+
+  CURL* handle() const;
+  States state() const override;
+  Stats stats() const override;
+  std::string stealBuffer();
+  QByteArray buffer() const override;
+  int httpCode() const override;
+  std::string error() const;
+  std::string debugName() const;
+
+  void start();
+  void stop() override;
+  bool finish(CURLcode code);
+
+  bool xfer(
+    curl_off_t dltotal, curl_off_t dlnow,
+    curl_off_t ultotal, curl_off_t ulnow);
+
+  bool header(std::string_view sv);
+  bool write(std::string_view data);
+  void debug(curl_infotype t, std::string_view data);
+
+private:
+  std::string m_url;
+  Info m_info;
+  EasyHandle m_handle;
+  FileHandle m_out;
+  SList m_headers;
+  std::string m_buffer;
+  States m_state;
+  std::string m_error;
+
+  hr_clock::time_point m_lastCheck;
+  std::size_t m_bytes;
+  std::atomic<double> m_bytesPerSecond;
+  std::atomic<double> m_progress;
+
+  std::size_t resumeFrom();
+  bool rename();
+  fs::path outputFile() const;
+
+  static int s_xfer(
+    void* p,
+    curl_off_t dltotal, curl_off_t dlnow,
+    curl_off_t ultotal, curl_off_t ulnow);
+
+  static size_t s_header(char* data, size_t size, size_t n, void* p);
+  static size_t s_write(char* data, size_t size, size_t n, void* p);
+  static int s_debug(CURL* h, curl_infotype t, char* data, size_t n, void *p);
+};
+
+
+class Downloader : public MOBase::IDownloader
+{
+public:
+  static const std::size_t NoLimit =
+    std::numeric_limits<std::size_t>::max();
+
+  Downloader();
+  ~Downloader();
+
+  void cancel();
+  void stop();
+  void join();
+
+  void maxSpeed(std::size_t bytesPerSecond);
+
+  bool finished() const;
+
+  std::shared_ptr<MOBase::IDownload> add(
+    const QUrl& url, const MOBase::IDownload::Info& info={}) override;
+
+private:
+  using DownloadList = std::list<std::shared_ptr<Download>>;
+
+  std::shared_ptr<GlobalHandle> m_global;
+  MultiHandle m_handle;
+
+  std::vector<std::shared_ptr<Download>> m_temp;
+  std::mutex m_tempMutex;
+
+  std::thread m_thread;
+  std::atomic<bool> m_cancel, m_stop, m_finished;
+  std::condition_variable m_cv;
+  DownloadList m_list;
+  std::map<CURL*, DownloadList::iterator> m_map;
+
+  std::atomic<std::size_t> m_maxSpeed;
+
+
+  void run();
+  void checkTemp();
+  void perform();
+  void poll();
+  bool start(std::shared_ptr<Download> d);
+  void setLimits();
+  void checkCancel();
+
+  void checkQueue();
+  bool cleanupActive();
+  DownloadList::iterator removeFromActive(DownloadList::iterator itor);
+  void stopOverMax(std::size_t max);
+  bool addFromQueue(std::size_t max);
+  curl_off_t maxSpeedPer() const;
+};
+
+} // namespace
+
+#endif // MODORGANIZER_CURLDOWNLOADER_INCLUDED

src/downloadmanager2.h

@@ -0,0 +1,114 @@
+#ifndef MODORGANIZER_DOWNLOADEDMANAGER2_INCLUDED
+#define MODORGANIZER_DOWNLOADEDMANAGER2_INCLUDED
+
+#include <ipluginrepository.h>
+
+class PluginContainer;
+namespace dm::curl { class Downloader; class Download; }
+
+
+namespace dm
+{
+
+namespace fs = std::filesystem;
+
+class DownloadManager2;
+
+class Download
+{
+public:
+  enum States
+  {
+    None = 0,
+
+    Queueing,
+    Queued,
+
+    Finished,
+    Errored,
+
+    Running,
+
+    Pausing,
+    Paused,
+
+    Cancelling,
+    Cancelled
+  };
+
+  Download(DownloadManager2& dm, MOBase::IPluginRepository& repo, QString what);
+
+  const QString& what() const;
+  const QString& error() const;
+
+  States state() const;
+
+  bool start();
+  void cancel();
+  void pause();
+  void queue();
+  void tick();
+
+  QString debugName() const;
+
+private:
+  DownloadManager2& m_dm;
+  MOBase::IPluginRepository& m_repo;
+  std::unique_ptr<MOBase::IRepositoryDownload> m_download;
+  QString m_what;
+  States m_state;
+  QString m_error;
+
+  void setState(States s);
+  void next();
+};
+
+
+class DownloadManager2
+{
+public:
+  static const std::size_t NoLimit =
+    std::numeric_limits<std::size_t>::max();
+
+  DownloadManager2(PluginContainer& pc);
+  ~DownloadManager2();
+
+  void add(QString what);
+
+  void maxActive(std::size_t n);
+  void maxSpeed(std::size_t bytesPerSecond);
+
+  bool hasActive() const;
+
+  curl::Downloader& downloader();
+
+private:
+  using DownloadList = std::list<std::shared_ptr<Download>>;
+
+  PluginContainer& m_pc;
+
+  std::thread m_thread;
+  std::atomic<bool> m_stop;
+
+  std::unique_ptr<curl::Downloader> m_downloader;
+
+  std::vector<std::shared_ptr<Download>> m_temp;
+  std::mutex m_tempMutex;
+  std::condition_variable m_cv;
+
+  DownloadList m_queued, m_active, m_inactive;
+  std::atomic<std::size_t> m_maxActive, m_maxSpeed;
+  std::atomic<bool> m_hasActive;
+
+
+  void run();
+  void checkTemp();
+  void checkQueue();
+  void cleanupActive();
+  void stopOverMax(std::size_t max);
+  void addFromQueue(std::size_t max);
+};
+
+} // namespace
+
+#endif // MODORGANIZER_DOWNLOADEDMANAGER2_INCLUDED

src/main.cpp

@@ -7,6 +7,7 @@
 #include "instancemanager.h"
 #include "thread_utils.h"
 #include "shared/util.h"
+#include "downloadmanager2.h"
 #include <report.h>
 #include <log.h>
 

src/pch.h

@@ -7,6 +7,7 @@
 #include <ctime>
 #include <deque>
 #include <exception>
+#include <filesystem>
 #include <functional>
 #include <fstream>
 #include <iomanip>
@@ -267,3 +268,6 @@
 #include <QtPlugin>
 #include <QtTest/QtTest>
 #include <QStandardPaths>
+
+// curl
+#include <curl/curl.h>

src/plugincontainer.cpp

@@ -81,6 +81,7 @@ template <> struct PluginTypeName<MOBase::IPluginPreview> { static QString value
 template <> struct PluginTypeName<MOBase::IPluginTool> { static QString value() { return QT_TR_NOOP("Tool"); } };
 template <> struct PluginTypeName<MOBase::IPluginProxy> { static QString value() { return QT_TR_NOOP("Proxy"); } };
 template <> struct PluginTypeName<MOBase::IPluginFileMapper> { static QString value() { return QT_TR_NOOP("File Mapper"); } };
+template <> struct PluginTypeName<MOBase::IPluginRepository> { static QString value() { return QT_TR_NOOP("Repository"); } };
 
 
 QStringList PluginContainer::pluginInterfaces()
@@ -510,6 +511,16 @@ IPlugin* PluginContainer::registerPlugin(QObject *plugin, const QString& filepat
       return modPage;
     }
   }
+  { // repo plugin
+    IPluginRepository *repo = qobject_cast<IPluginRepository*>(plugin);
+    if (repo) {
+      if (initPlugin(repo, pluginProxy, skipInit)) {
+        bf::at_key<IPluginRepository>(m_Plugins).push_back(repo);
+        emit pluginRegistered(repo);
+        return repo;
+      }
+    }
+  }
   { // game plugin
     IPluginGame *game = qobject_cast<IPluginGame*>(plugin);
     if (game) {

src/plugincontainer.h

@@ -13,6 +13,7 @@ class IUserInterface;
 #include <ipluginproxy.h>
 #include <iplugininstaller.h>
 #include <ipluginfilemapper.h>
+#include <ipluginrepository.h>
 #include <QtPlugin>
 #include <QPluginLoader>
 #include <QFile>
@@ -142,7 +143,8 @@ private:
     boost::fusion::pair<MOBase::IPluginPreview, std::vector<MOBase::IPluginPreview*>>,
     boost::fusion::pair<MOBase::IPluginTool, std::vector<MOBase::IPluginTool*>>,
     boost::fusion::pair<MOBase::IPluginProxy, std::vector<MOBase::IPluginProxy*>>,
-    boost::fusion::pair<MOBase::IPluginFileMapper, std::vector<MOBase::IPluginFileMapper*>>
+    boost::fusion::pair<MOBase::IPluginFileMapper, std::vector<MOBase::IPluginFileMapper*>>,
+    boost::fusion::pair<MOBase::IPluginRepository, std::vector<MOBase::IPluginRepository*>>
   >;
 
   using AccessPluginMap = boost::fusion::map<
@@ -166,9 +168,9 @@ private:
     boost::mp11::mp_transform<
       std::add_pointer_t,
       boost::mp11::mp_list<
-        MOBase::IPluginGame, MOBase::IPluginInstaller, MOBase::IPluginModPage, MOBase::IPluginPreview,
-        MOBase::IPluginProxy, MOBase::IPluginTool, MOBase::IPluginDiagnose, MOBase::IPluginFileMapper,
-        MOBase::IPlugin
+        MOBase::IPluginGame, MOBase::IPluginInstaller, MOBase::IPluginModPage, MOBase::IPluginRepository,
+        MOBase::IPluginPreview, MOBase::IPluginProxy, MOBase::IPluginTool, MOBase::IPluginDiagnose,
+        MOBase::IPluginFileMapper, MOBase::IPlugin
       >
     >;