Initial commit

.gitignore

@@ -0,0 +1,3 @@
+build
+downloads
+*.pyc

config.py

@@ -0,0 +1,16 @@
+config = {
+    'paths': {
+        'download': "{base_dir}/downloads",
+        'build': "{base_dir}/build",
+        'progress': "{base_dir}/progress",
+        'graphviz': "D:/Graphviz2.38/bin/dot.exe",
+        'cmake': "C:/Program Files (x86)/CMake/bin",
+        'visual_studio': "C:/Program Files (x86)/Microsoft Visual Studio 12.0/VC",
+    },
+    'tools': {
+        'make': "nmake",
+    },
+    'architecture': 'x86_64',
+}
+
+

eggs/__init__.py

@@ -0,0 +1,24 @@
+import os.path
+import sys
+import urllib2
+
+
+def download(url, filename):
+    if os.path.exists(filename):
+        return
+
+    data = urllib2.urlopen(url)
+    with open(filename, 'wb') as outfile:
+        while True:
+            block = data.read(4096)
+            if not block:
+                break
+            outfile.write(block)
+
+path = os.path.abspath(os.path.join(os.path.realpath(__file__), os.pardir))
+
+for dep in ["https://pypi.python.org/packages/2.7/n/networkx/networkx-1.10-py2.7.egg",
+            "https://pypi.python.org/packages/2.5/p/pydot/pydot-1.0.2-py2.5.egg"]:
+    eggpath = os.path.join(path, os.path.basename(dep))
+    download(dep, eggpath)
+    sys.path.append(eggpath)

makefile.uni.py

@@ -0,0 +1,100 @@
+# Copyright (C) 2015 Sebastian Herbord. All rights reserved.
+#
+# This file is part of Mod Organizer.
+#
+# Mod Organizer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Mod Organizer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from unibuild import Project
+from unibuild.modules import github, cmake
+from config import config
+
+
+"""
+Settings
+"""
+
+config["build_type"] = "RelWithDebInfo"
+modorganizer_branch = "master"
+
+loot_version = "v0.8.0"
+
+
+"""
+Projects
+"""
+
+
+from unibuild.projects import sevenzip, qt5, boost, zlib
+
+
+Project("Spdlog") \
+    .depend(github.Source("gabime", "spdlog", "master"))
+
+Project("CppFormat") \
+    .depend(github.Source("cppformat", "cppformat", "master"))
+
+Project("LootApi") \
+    .depend(github.Release("loot", "loot", loot_version, "LOOT.API.{}".format(loot_version), "7z")
+            .set_destination("lootapi"))
+
+
+for git_path, path, install, dependencies in [
+    ("modorganizer-archive",          "archive",                 True,  ["7zip"]),
+    ("modorganizer-uibase",           "uibase",                  True,  ["Qt5", "boost"]),
+    ("modorganizer-lootcli",          "lootcli",                 True,  ["LootApi", "Qt5", "boost"]),
+    ("modorganizer-esptk",            "esptk",                   False, ["boost"]),
+    ("modorganizer-bsatk",            "bsatk",                   False, ["zlib"]),
+    ("modorganizer-nxmhandler",       "nxmhandler",              True,  ["Qt5"]),
+    ("modorganizer-python_runner",    "python_runner",           True,  ["boost"]),
+    ("modorganizer-game_features",    "plugin/game_features",    False, ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-game_gamebryo",    "plugin/game_gamebryo",    False, ["Qt5", "modorganizer-uibase",
+                                                                         "modorganizer-game_features"]),
+    ("modorganizer-game_skyrim",      "plugin/game_skyrim",      True,  ["Qt5", "modorganizer-uibase",
+                                                                         "modorganizer-game_gamebryo",
+                                                                         "modorganizer-game_features"]),
+    ("modorganizer-tool_nmmimport",   "plugin/tool_nmmimport",   True,  ["Qt5", "modorganizer-uibase",
+                                                                         "modorganizer-archive"]),
+    ("modorganizer-tool_inieditor",   "plugin/tool_inieditor",   True,  ["Qt5", "modorganizer-uibase"]),
+#    ("modorganizer-preview_dds",      "plugin/preview_dds",      True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-preview_base",     "plugin/preview_base",     True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-diagnose_basic",   "plugin/diagnose_basic",   True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-check_fnis",       "plugin/check_fnis",       True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-installer_bain",   "plugin/installer_bain",   True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-installer_manual", "plugin/installer_manual", True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-installer_bundle", "plugin/installer_bundle", True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-installer_quick",  "plugin/installer_quick",  True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-installer_fomod",  "plugin/installer_fomod",  True,  ["Qt5", "modorganizer-uibase"]),
+    ("modorganizer-plugin_python",    "plugin/plugin_python",    True,  ["Qt5", "boost", "modorganizer-uibase"]),
+    ("modorganizer",                  "modorganizer",            True,  ["Qt5", "boost",
+                                                                         "modorganizer-uibase", "modorganizer-archive",
+                                                                         "modorganizer-bsatk", "modorganizer-esptk",
+                                                                         "modorganizer-game_features"]),
+]:
+    build_step = cmake.CMake().arguments([
+                                         "-DCMAKE_BUILD_TYPE={}".format(config["build_type"]),
+                                         "-DDEPENDENCIES_DIR={}/build".format(config["__build_base_path"]),
+                                         "-DCMAKE_INSTALL_PREFIX:PATH={}/install".format(config["__build_base_path"])
+                                         ])
+    for dep in dependencies:
+        build_step.depend(dep)
+
+    if install:
+        build_step.install()
+
+    Project(git_path)\
+        .depend(build_step
+                .depend(github.Source("TanninOne", git_path, modorganizer_branch)
+                        .set_destination(path))
+                )

unibuild/__init__.py

@@ -0,0 +1,7 @@
+__author__ = 'Tannin'
+
+
+from project import Project
+from dependency import Dependency
+from version import Version
+from task import Task

unibuild/builder.py

@@ -0,0 +1,19 @@
+__author__ = 'Tannin'
+
+
+from task import Task
+
+
+class Builder(Task):
+
+    def __init__(self):
+        super(Builder, self).__init__()
+
+    def applies(self, parameters):
+        return True
+
+    def name(self):
+        return
+
+    def process(self, progress):
+        return

unibuild/dependency.py

@@ -0,0 +1,17 @@
+__author__ = 'Tannin'
+
+
+from project import Project
+
+
+class Dependency(Project):
+
+    def __init__(self, name):
+        super(Dependency, self).__init__(name)
+
+    def applies(self, parameters):
+        return True
+
+    def version_eq(self, version):
+        return self
+

unibuild/manager.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2015 Sebastian Herbord. All rights reserved.
+#
+# This file is part of Mod Organizer.
+#
+# Mod Organizer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Mod Organizer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import networkx as nx
+from utility.singleton import Singleton
+
+
+class TaskManager(object):
+    """
+    manages task dependency graph
+    """
+    __metaclass__ = Singleton
+
+    def __init__(self):
+        self.__topLevelTask = []
+
+    def add_task(self, task):
+        self.__topLevelTask.append(task)
+
+    def get_task(self, name):
+        for task in self.__topLevelTask:
+            if task.name == name:
+                return task
+        return None
+
+    def create_graph(self, parameters):
+        graph = nx.DiGraph()
+        for task in self.__topLevelTask:
+            self.__add_task(graph, task, parameters)
+        return graph
+
+    def enable(self, graph, node):
+        """
+        recursively enable the node
+        :param graph:
+        :param node:
+        :return:
+        """
+        for suc in graph.successors_iter(node):
+            self.enable(graph, suc)
+        graph.node[node]["enable"] = True
+
+    def enable_all(self, graph):
+        for node in graph.nodes_iter():
+            if graph.in_degree(node) == 0:
+                self.enable(graph, node)
+
+    def __add_task(self, graph, task, parameters):
+        if not graph.has_node(task.name):
+            graph.add_node(task.name, task=task, enable=False)
+
+        for dependency in task.dependencies:
+            self.__add_task(graph, dependency, parameters)
+            graph.add_edge(task.name, dependency.name)
+
+
+def register_project(task):
+    TaskManager().add_task(task)

unibuild/modules/__init__.py

@@ -0,0 +1,7 @@
+__author__ = 'Tannin'
+
+import os
+import glob
+
+modules = glob.glob(os.path.join(os.path.dirname(__file__), "*.py"))
+__all__ = [os.path.basename(f)[:-3] for f in modules]

unibuild/modules/b2.py

@@ -0,0 +1,62 @@
+__author__ = 'Tannin'
+
+
+from unibuild.builder import Builder
+from subprocess import Popen
+import os
+import logging
+
+
+class B2(Builder):
+
+    def __init__(self):
+        super(B2, self).__init__()
+        self.__arguments = []
+
+    @property
+    def name(self):
+        if self._context is None:
+            return "b2"
+        else:
+            return "b2 {0}".format(self._context.name)
+
+    def applies(self, parameters):
+        return True
+
+    def fulfilled(self):
+        return False
+
+    def arguments(self, arguments):
+        if arguments is None:
+            self.__arguments = []
+        else:
+            self.__arguments = arguments
+        return self
+
+    def process(self, progress):
+        if "build_path" not in self._context:
+            logging.error("source path not known for {},"
+                          " are you missing a matching retrieval script?".format(self.name()))
+        soutpath = os.path.join(self._context["build_path"], "stdout.log")
+        serrpath = os.path.join(self._context["build_path"], "stderr.log")
+        with open(soutpath, "a") as sout:
+            with open(serrpath, "a") as serr:
+                proc = Popen(["cmd.exe", "/C", "bootstrap.bat"], cwd=self._context["build_path"],
+                             stdout=sout, stderr=serr)
+                proc.communicate()
+                if proc.returncode != 0:
+                    logging.error("failed to bootstrap (returncode %s), see %s and %s",
+                                  proc.returncode, soutpath, serrpath)
+                    return False
+
+                cmdline = ["b2.exe"]
+                if self.__arguments:
+                    cmdline.extend(self.__arguments)
+
+                proc = Popen(cmdline, cwd=self._context["build_path"], stdout=sout, stderr=serr, shell=True)
+                proc.communicate()
+                if proc.returncode != 0:
+                    logging.error("failed to build (returncode %s), see %s and %s",
+                                  proc.returncode, soutpath, serrpath)
+                    return False
+        return True

unibuild/modules/build.py

@@ -0,0 +1,179 @@
+
+__author__ = 'Tannin'
+
+
+from unibuild.builder import Builder
+from subprocess import Popen
+from config import config
+from unibuild import Task
+import os.path
+import logging
+
+STATIC_LIB = 1
+SHARED_LIB = 2
+EXECUTABLE = 3
+
+
+class CPP(Builder):
+
+    def __init__(self):
+        super(CPP, self).__init__()
+        self.__type = EXECUTABLE
+        self.__targets = []
+
+    @property
+    def name(self):
+        if self._context is None:
+            return "custom build"
+        else:
+            return "custom build {0}".format(self._context.name)
+
+    def fulfilled(self):
+        return False
+
+    def type(self, build_type):
+        self.__type = build_type
+        return self
+
+    def __gen_build_cmd(self, target, files):
+        if self.__type == STATIC_LIB:
+            return "link.exe /lib /nologo /out:{0}.lib {1}".format(
+                target, " ".join([self.__to_obj(f) for f in files]))
+        else:
+            raise NotImplementedError("type {} not yet implemented", self.__type)
+
+    def sources(self, target, files, top_level=True):
+        self.__targets.append((target, files, self.__gen_build_cmd(target, files), top_level))
+        return self
+
+    def custom(self, target, dependencies=None, cmd=None, top_level=False):
+        self.__targets.append((target, dependencies, cmd, top_level))
+        return self
+
+    @staticmethod
+    def __to_obj(filename):
+        return "{}.obj".format(os.path.splitext(os.path.basename(filename))[0])
+
+    def gen_makefile(self, path):
+        with open(os.path.join(path, "unimakefile"), "w") as mf:
+            for target in self.__targets:
+                files = target[1] or []
+                for f in files:
+                    mf.write("{0}: {1}\n\n".format(self.__to_obj(f), f))
+
+                mf.write("{0}: {1}\n\t{2}\n\n".format(target[0],
+                                                      " ".join([self.__to_obj(f) for f in files]),
+                                                      target[2] or ""))
+
+            mf.write("all: {}\n".format(" ".join([target[0]
+                                                  for target in self.__targets
+                                                  if target[3]])))
+
+    def process(self, progress):
+        path = self._context["build_path"]
+
+        self.gen_makefile(path)
+
+        soutpath = os.path.join(self._context["build_path"], "stdout.log")
+        serrpath = os.path.join(self._context["build_path"], "stderr.log")
+        with open(soutpath, "a") as sout:
+            with open(serrpath, "a") as serr:
+                proc = Popen("{} /f unimakefile all".format(config["tools"]["make"]),
+                             env=config["__environment"],
+                             cwd=self._context["build_path"],
+                             shell=True,
+                             stdout=sout, stderr=serr)
+                proc.communicate()
+                if proc.returncode != 0:
+                    logging.error("failed to build custom makefile (returncode %s), see %s and %s",
+                                  proc.returncode, soutpath, serrpath)
+                    return False
+        return True
+
+
+class Make(Builder):
+    def __init__(self, make_tool=None):
+        super(Make, self).__init__()
+        self.__install = False
+        self.__make_tool = make_tool or config['tools']['make']
+
+    @property
+    def name(self):
+        if self._context is None:
+            return "make"
+        else:
+            return "make {0}".format(self._context.name)
+
+    def install(self):
+        self.__install = True
+        return self
+
+    def process(self, progress):
+        if "build_path" not in self._context:
+            logging.error("source path not known for {},"
+                          " are you missing a matching retrieval script?".format(self.name()))
+        soutpath = os.path.join(self._context["build_path"], "stdout.log")
+        serrpath = os.path.join(self._context["build_path"], "stderr.log")
+
+        with open(soutpath, "a") as sout:
+            with open(serrpath, "a") as serr:
+                proc = Popen(self.__make_tool,
+                             env=config["__environment"],
+                             cwd=self._context["build_path"],
+                             shell=True,
+                             stdout=sout, stderr=serr)
+                proc.communicate()
+                #-debug-and-release -force-debug-info -opensource -confirm-license -mp -no-compile-examples -nomake tests -nomake examples -no-angle -opengl desktop -no-icu -skip qtactiveqt -skip qtandroidextras -skip qtenginio -skip qtsensors -skip qtserialport -skip qtsvg -skip qtwebkit -skip qtpim -skip qttools -skip qtwebchannel -skip qtwayland -skip qtdoc -skip qtconnectivity -skip qtwebkit-examples
+                if proc.returncode != 0:
+                    logging.error("failed to run make (returncode %s), see %s and %s",
+                                  proc.returncode, soutpath, serrpath)
+                    return False
+
+                if self.__install:
+                    proc = Popen([config['tools']['make'], "install"],
+                                 shell=True,
+                                 env=config["__environment"],
+                                 cwd=self._context["build_path"],
+                                 stdout=sout, stderr=serr)
+                    proc.communicate()
+                    if proc.returncode != 0:
+                        logging.error("failed to install (returncode %s), see %s and %s",
+                                      proc.returncode, soutpath, serrpath)
+                        return False
+
+        return True
+
+
+class Run(Builder):
+    def __init__(self, command, fail_behaviour=Task.FailBehaviour.FAIL):
+        super(Run, self).__init__()
+        self.__command = command
+        self.__fail_behaviour = fail_behaviour
+
+    @property
+    def name(self):
+        return "run {}".format(self.__command.split()[0])
+
+    def process(self, progress):
+        if "build_path" not in self._context:
+            logging.error("source path not known for {},"
+                          " are you missing a matching retrieval script?".format(self.name))
+        soutpath = os.path.join(self._context["build_path"], "stdout.log")
+        serrpath = os.path.join(self._context["build_path"], "stderr.log")
+        with open(soutpath, "w") as sout:
+            with open(serrpath, "w") as serr:
+                sout.write("running {}".format(self.__command))
+                proc = Popen(self.__command,
+                             env=config["__environment"],
+                             cwd=self._context["build_path"],
+                             shell=True,
+                             stdout=sout, stderr=serr)
+                proc.communicate()
+                if proc.returncode != 0:
+                    logging.error("failed to run %s (returncode %s), see %s and %s",
+                                  self.__command, proc.returncode, soutpath, serrpath)
+                    return False
+
+        return True
+
+

unibuild/modules/cmake.py

@@ -0,0 +1,91 @@
+__author__ = 'Tannin'
+
+
+from unibuild.builder import Builder
+from subprocess import Popen
+from config import config
+import os.path
+import logging
+import shutil
+
+
+class CMake(Builder):
+
+    def __init__(self):
+        super(CMake, self).__init__()
+        self.__arguments = []
+        self.__install = False
+
+    @property
+    def name(self):
+        if self._context is None:
+            return "cmake"
+        else:
+            return "cmake {0}".format(self._context.name)
+
+    def applies(self, parameters):
+        return True
+
+    def fulfilled(self):
+        return False
+
+    def arguments(self, arguments):
+        self.__arguments = arguments
+        return self
+
+    def install(self):
+        self.__install = True
+        return self
+
+    def process(self, progress):
+        if "build_path" not in self._context:
+            logging.error("source path not known for {},"
+                          " are you missing a matching retrieval script?".format(self._context.name))
+            return False
+
+        print(self._context["build_path"])
+        build_path = os.path.join(self._context["build_path"], "build")
+        if os.path.exists(build_path):
+            shutil.rmtree(build_path)
+        os.mkdir(build_path)
+
+        soutpath = os.path.join(self._context["build_path"], "stdout.log")
+        serrpath = os.path.join(self._context["build_path"], "stderr.log")
+
+        with open(soutpath, "w") as sout:
+            with open(serrpath, "w") as serr:
+                proc = Popen(
+                    [os.path.join(config["paths"]["cmake"], "cmake"), "-G", "NMake Makefiles", ".."] + self.__arguments,
+                    cwd=build_path,
+                    env=config["__environment"],
+                    stdout=sout, stderr=serr)
+                proc.communicate()
+                if proc.returncode != 0:
+                    logging.error("failed to generate makefile (returncode %s), see %s and %s",
+                                  proc.returncode, soutpath, serrpath)
+                    return False
+
+                proc = Popen([config['tools']['make'], "verbose=1"],
+                             shell=True,
+                             env=config["__environment"],
+                             cwd=build_path,
+                             stdout=sout, stderr=serr)
+                proc.communicate()
+                if proc.returncode != 0:
+                    logging.error("failed to build (returncode %s), see %s and %s",
+                                  proc.returncode, soutpath, serrpath)
+                    return False
+
+                if self.__install:
+                    proc = Popen([config['tools']['make'], "install"],
+                                 shell=True,
+                                 env=config["__environment"],
+                                 cwd=build_path,
+                                 stdout=sout, stderr=serr)
+                    proc.communicate()
+                    if proc.returncode != 0:
+                        logging.error("failed to install (returncode %s), see %s and %s",
+                                      proc.returncode, soutpath, serrpath)
+                        return False
+
+        return True

unibuild/modules/git.py

@@ -0,0 +1,36 @@
+from subprocess import Popen
+from config import config
+import os
+import logging
+from repository import Repository
+
+
+class Clone(Repository):
+    def __init__(self, url, branch):
+        super(Clone, self).__init__(url, branch)
+
+    def prepare(self):
+        self._context["build_path"] = self._output_file_path
+
+    def process(self, progress):
+        if os.path.isdir(self._output_file_path):
+            proc = Popen(["git", "pull"],
+                         cwd=self._output_file_path,
+                         env=config["__environment"])
+        else:
+            proc = Popen(["git", "clone", "-b", self._branch, self._url, self._context["build_path"]],
+                         env=config["__environment"])
+        proc.communicate()
+        if proc.returncode != 0:
+            logging.error("failed to clone repository %s (returncode %s)", self._url, proc.returncode)
+            return False
+
+        return True
+
+    @staticmethod
+    def _expiration():
+        return 60 * 60 * 24  # one day
+
+    def set_destination(self, destination_name):
+        self._output_file_path = os.path.join(config["paths"]["build"], destination_name)
+        return self

unibuild/modules/github.py

@@ -0,0 +1,26 @@
+__author__ = 'Tannin'
+
+
+from urldownload import URLDownload
+from git import Clone
+
+
+class Release(URLDownload):
+    def __init__(self, author, project, version, filename, extension="zip"):
+        super(Release, self) \
+            .__init__("https://github.com/{author}/{project}/releases/download/{version}/"
+                      "{filename}.{extension}".format(author=author,
+                                                      project=project,
+                                                      version=version,
+                                                      filename=filename,
+                                                      extension=extension))
+
+
+class Source(Clone):
+    def __init__(self, author, project, tag):
+        super(Source, self).__init__("https://github.com/{author}/{project}.git".format(author=author,
+                                                                                        project=project,
+                                                                                        tag=tag), "master")
+        #super(Source, self).__init__("https://github.com/{author}/{project}/archive/{tag}.zip".format(), 1)
+        # don't use the tag as the file name, otherwise we get name collisions on "master" or other generic names
+        #self.set_destination(project)

unibuild/modules/googlecode.py

@@ -0,0 +1,11 @@
+__author__ = 'Tannin'
+
+
+from urldownload import URLDownload
+
+
+class Release(URLDownload):
+    def __init__(self, project, filename):
+        super(Release, self)\
+            .__init__("http://{project}.googlecode.com/files/{filename}".format(project=project,
+                                                                                filename=filename))

unibuild/modules/patch.py

@@ -0,0 +1,26 @@
+from unibuild.task import Task
+import os.path
+
+
+class Replace(Task):
+
+    def __init__(self, filename, search, substitute):
+        super(Replace, self).__init__()
+        self.__file = filename
+        self.__search = search
+        self.__substitute = substitute
+
+    @property
+    def name(self):
+        return "Replace in {}".format(self.__file)
+
+    def process(self, progress):
+        full_path = os.path.join(self._context["build_path"], self.__file)
+        with open(full_path, "r") as f:
+            data = f.read()
+
+        data = data.replace(self.__search, self.__substitute)
+
+        with open(full_path, "w") as f:
+            f.write(data)
+        return True

unibuild/modules/repository.py

@@ -0,0 +1,17 @@
+from unibuild.retrieval import Retrieval
+from config import config
+import os
+
+
+class Repository(Retrieval):
+    def __init__(self, url, branch):
+        super(Repository, self).__init__()
+        self._url = url
+        self._branch = branch
+        self._dir_name = os.path.basename(self._url)
+        self._output_file_path = os.path.join(config["paths"]["build"], self._dir_name)
+
+    @property
+    def name(self):
+        return "retrieve {0}".format(self._dir_name)
+

unibuild/modules/sourceforge.py

@@ -0,0 +1,11 @@
+__author__ = 'Tannin'
+
+
+from urldownload import URLDownload
+
+
+class Release(URLDownload):
+    def __init__(self, project, path):
+        super(Release, self)\
+            .__init__("http://downloads.sourceforge.net/project/{project}/{path}".format(project=project,
+                                                                                         path=path))

unibuild/modules/urldownload.py

@@ -0,0 +1,116 @@
+from unibuild.retrieval import Retrieval
+from config import config
+import os
+import sys
+import logging
+from urlparse import urlparse
+import urllib2
+import tarfile
+import zipfile
+import subprocess
+import shutil
+
+
+class URLDownload(Retrieval):
+
+    BLOCK_SIZE = 8192
+
+    def __init__(self, url, tree_depth=0):
+        super(URLDownload, self).__init__()
+        self.__url = url
+        self.__tree_depth = tree_depth
+        self.__file_name = os.path.basename(urlparse(self.__url).path)
+
+    @property
+    def name(self):
+        return "download {0}".format(self.__file_name)
+
+    def set_destination(self, destination_name):
+        name, ext = os.path.splitext(self.__file_name)
+        if name.lower().endswith(".tar"):
+            ext = ".tar" + ext
+
+        self.__file_name = destination_name + ext
+        return self
+
+    def prepare(self):
+        name, ext = os.path.splitext(self.__file_name)
+        if name.lower().endswith(".tar"):
+            name, e2 = os.path.splitext(name)
+
+        output_file_path = os.path.join(config["paths"]["build"], name)
+        self._context["build_path"] = output_file_path
+
+    def process(self, progress):
+        output_file_path = self._context["build_path"]
+        archive_file_path = os.path.join(config["paths"]["download"], self.__file_name)
+
+        if os.path.isdir(output_file_path):
+            logging.info("File already extracted: {0}".format(output_file_path))
+        else:
+            if os.path.isfile(archive_file_path):
+                logging.info("File already downloaded: {0}".format(archive_file_path))
+            else:
+                logging.info("File not yet downloaded: {0}".format(archive_file_path))
+                self.download(archive_file_path, progress)
+
+            self.extract(archive_file_path, output_file_path, progress)
+
+        builddir = os.listdir(self._context["build_path"])
+        if len(builddir) == 1:
+            self._context["build_path"] = os.path.join(self._context["build_path"], builddir[0])
+        return True
+
+    def download(self, output_file_path, progress):
+        logging.info("Downloading {} to {}".format(self.__url, output_file_path))
+        data = urllib2.urlopen(self.__url)
+        with open(output_file_path, 'wb') as outfile:
+            meta = data.info()
+            length_str = meta.getheaders("Content-Length")
+            if length_str:
+                progress.maximum = int(length_str[0]) * 2
+            else:
+                progress.maximum = sys.maxint
+
+            bytes_read = 0
+            while True:
+                block = data.read(URLDownload.BLOCK_SIZE)
+                if not block:
+                    break
+                bytes_read += len(block)
+                outfile.write(block)
+                progress.value = bytes_read
+
+    def extract(self, archive_file_path, output_file_path, progress):
+        output_file_path = u"\\\\?\\" + os.path.abspath(output_file_path)
+
+        logging.info("Extracting {0}".format(self.__url))
+
+        os.makedirs(output_file_path)
+        filename, extension = os.path.splitext(self.__file_name)
+        print("{}".format(filename))
+        if extension == ".gz":
+            with tarfile.open(archive_file_path, 'r:gz') as arch:
+                arch.extractall(output_file_path)
+        elif extension == ".bz2":
+            with tarfile.open(archive_file_path, 'r:bz2') as arch:
+                arch.extractall(output_file_path)
+        elif extension == ".zip":
+            with zipfile.ZipFile(archive_file_path) as arch:
+                arch.extractall(output_file_path)
+        elif extension == ".7z":
+            subprocess.call(["7za", "x", archive_file_path, "-o{}".format(output_file_path)])
+        else:
+            logging.error("unsupported file extension {0}".format(extension))
+
+        for i in range(self.__tree_depth):
+            sub_dirs = os.listdir(output_file_path)
+            if len(sub_dirs) != 1:
+                raise ValueError("unexpected archive structure,"
+                                 " expected exactly one directory in {}".format(output_file_path))
+            source_dir = os.path.join(output_file_path, sub_dirs[0])
+
+            for src in os.listdir(source_dir):
+                shutil.move(os.path.join(source_dir, src), output_file_path)
+
+            shutil.rmtree(source_dir)

unibuild/progress.py

@@ -0,0 +1,39 @@
+__author__ = 'Tannin'
+
+
+class Progress(object):
+
+    def __init__(self):
+        self.__minimum = 0
+        self.__maximum = 100
+        self.__value = 0
+        self.__changeCallback = None
+
+    @property
+    def maximum(self):
+        return self.__maximum
+
+    @maximum.setter
+    def maximum(self, new_value):
+        self.__maximum = new_value
+
+    @property
+    def minimum(self):
+        return self.__minimum
+
+    @minimum.setter
+    def minimum(self, new_value):
+        self.__minimum = new_value
+
+    @property
+    def value(self):
+        return self.__value
+
+    @value.setter
+    def value(self, new_value):
+        self.__value = new_value
+        if self.__changeCallback is not None:
+            self.__changeCallback(self.__value * 100 / self.__maximum)
+
+    def set_change_callback(self, callback):
+        self.__changeCallback = callback