2014-09-01 12:32:23 -07:00
|
|
|
#!/usr/bin/python2
|
2014-07-25 12:34:03 -07:00
|
|
|
#
|
2015-01-07 00:49:06 -08:00
|
|
|
# Automatic patch dependency checker and apply script/README.md generator.
|
2014-07-25 12:34:03 -07:00
|
|
|
#
|
2015-01-07 00:49:06 -08:00
|
|
|
# Copyright (C) 2014-2015 Sebastian Lackner
|
2014-07-25 12:34:03 -07:00
|
|
|
#
|
|
|
|
# This library is free software; you can redistribute it and/or
|
|
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
|
|
# License as published by the Free Software Foundation; either
|
|
|
|
# version 2.1 of the License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This library is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
# Lesser General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
|
|
# License along with this library; if not, write to the Free Software
|
|
|
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
|
|
|
|
#
|
|
|
|
|
2014-11-28 23:17:16 -08:00
|
|
|
import binascii
|
|
|
|
import cPickle as pickle
|
2015-03-05 15:23:05 -08:00
|
|
|
import contextlib
|
2015-02-22 11:46:03 -08:00
|
|
|
import fnmatch
|
2014-07-24 18:32:01 -07:00
|
|
|
import hashlib
|
|
|
|
import itertools
|
2014-11-28 23:17:16 -08:00
|
|
|
import math
|
|
|
|
import multiprocessing.pool
|
2014-11-14 21:35:26 -08:00
|
|
|
import operator
|
2014-07-25 12:34:03 -07:00
|
|
|
import os
|
2014-07-24 18:32:01 -07:00
|
|
|
import patchutils
|
2014-11-28 23:17:16 -08:00
|
|
|
import progressbar
|
2014-07-11 12:25:18 -07:00
|
|
|
import re
|
2014-11-28 23:17:16 -08:00
|
|
|
import signal
|
2014-07-25 12:34:03 -07:00
|
|
|
import subprocess
|
2014-11-28 23:17:16 -08:00
|
|
|
import sys
|
|
|
|
import tempfile
|
2014-07-25 12:34:03 -07:00
|
|
|
import textwrap
|
2015-03-05 15:23:05 -08:00
|
|
|
import urllib
|
|
|
|
import xml.dom.minidom
|
2014-07-11 09:51:03 -07:00
|
|
|
|
2014-11-28 23:17:16 -08:00
|
|
|
_devnull = open(os.devnull, 'wb')
|
|
|
|
|
2014-07-25 07:39:08 -07:00
|
|
|
# Cached information to speed up patch dependency checks
|
2014-07-27 17:22:58 -07:00
|
|
|
latest_wine_commit = None
|
2014-07-25 07:39:08 -07:00
|
|
|
cached_patch_result = {}
|
|
|
|
|
2014-07-27 17:41:50 -07:00
|
|
|
class config(object):
|
2014-08-12 16:59:01 -07:00
|
|
|
path_patches = "patches"
|
|
|
|
path_changelog = "debian/changelog"
|
|
|
|
path_wine = "debian/tools/wine"
|
2014-07-27 17:41:50 -07:00
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
path_template_script = "debian/tools/patchinstall.sh.in"
|
|
|
|
path_script = "patches/patchinstall.sh"
|
2014-07-27 17:41:50 -07:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
path_template_README_md = "debian/tools/README.md.in"
|
2015-01-07 00:49:06 -08:00
|
|
|
path_README_md = "README.md"
|
2014-07-27 17:41:50 -07:00
|
|
|
|
2014-12-13 21:03:05 -08:00
|
|
|
path_IfDefined = "9999-IfDefined.patch"
|
|
|
|
|
2014-07-25 12:54:58 -07:00
|
|
|
class PatchUpdaterError(RuntimeError):
|
|
|
|
"""Failed to update patches."""
|
|
|
|
pass
|
|
|
|
|
2014-07-11 09:51:03 -07:00
|
|
|
class PatchSet(object):
|
2014-12-13 21:03:05 -08:00
|
|
|
def __init__(self, name, directory):
|
2014-07-24 18:32:01 -07:00
|
|
|
self.name = name
|
2015-01-07 00:49:06 -08:00
|
|
|
self.variable = None
|
2014-12-13 21:03:05 -08:00
|
|
|
self.directory = directory
|
2014-07-24 18:32:01 -07:00
|
|
|
self.fixes = []
|
|
|
|
self.changes = []
|
2014-11-03 11:12:34 -08:00
|
|
|
self.disabled = False
|
2014-12-13 21:03:05 -08:00
|
|
|
self.ifdefined = None
|
2014-07-11 09:51:03 -07:00
|
|
|
|
2014-07-24 18:32:01 -07:00
|
|
|
self.files = []
|
|
|
|
self.patches = []
|
|
|
|
self.modified_files = set()
|
|
|
|
self.depends = set()
|
2015-02-22 11:46:03 -08:00
|
|
|
self.auto_depends = set()
|
2014-07-11 09:51:03 -07:00
|
|
|
|
|
|
|
self.verify_time = None
|
|
|
|
|
2014-07-27 17:32:13 -07:00
|
|
|
def _pairs(a):
|
|
|
|
"""Iterate over all pairs of elements contained in the list a."""
|
|
|
|
for i, j in enumerate(a):
|
|
|
|
for k in a[i+1:]:
|
|
|
|
yield (j, k)
|
|
|
|
|
2014-11-14 21:35:26 -08:00
|
|
|
def _unique(iterable, key=None):
|
|
|
|
"List unique elements, preserving order. Remember only the element just seen."
|
2014-11-23 10:45:14 -08:00
|
|
|
# _unique('AAAABBBCCDAABBB') --> A B C D A B
|
|
|
|
# _unique('ABBCcAD', str.lower) --> A B C A D
|
2014-11-14 21:35:26 -08:00
|
|
|
return itertools.imap(next, itertools.imap(operator.itemgetter(1), itertools.groupby(iterable, key)))
|
|
|
|
|
2014-11-28 23:17:16 -08:00
|
|
|
def _split_seq(iterable, size):
|
|
|
|
"""Split an iterator into chunks of a given size."""
|
|
|
|
it = iter(iterable)
|
|
|
|
items = list(itertools.islice(it, size))
|
|
|
|
while items:
|
|
|
|
yield items
|
|
|
|
items = list(itertools.islice(it, size))
|
|
|
|
|
2014-11-14 21:35:26 -08:00
|
|
|
def _escape(s):
|
|
|
|
"""Escape string inside of '...' quotes."""
|
2014-11-23 10:45:14 -08:00
|
|
|
return s.replace("\\", "\\\\\\\\").replace("\"", "\\\"").replace("'", "'\\''")
|
2014-11-14 21:35:26 -08:00
|
|
|
|
2014-07-27 17:32:13 -07:00
|
|
|
def _load_dict(filename):
|
|
|
|
"""Load a Python dictionary object from a file."""
|
|
|
|
try:
|
|
|
|
with open(filename) as fp:
|
|
|
|
return pickle.load(fp)
|
|
|
|
except IOError:
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def _save_dict(filename, value):
|
|
|
|
"""Save a Python dictionary object to a file."""
|
|
|
|
with open(filename, "wb") as fp:
|
|
|
|
pickle.dump(value, fp, pickle.HIGHEST_PROTOCOL)
|
|
|
|
|
2014-11-28 23:17:16 -08:00
|
|
|
def _sha256(fp):
|
|
|
|
"""Calculate sha256sum from a file descriptor."""
|
|
|
|
m = hashlib.sha256()
|
|
|
|
fp.seek(0)
|
|
|
|
while True:
|
|
|
|
buf = fp.read(16384)
|
|
|
|
if buf == "": break
|
|
|
|
m.update(buf)
|
|
|
|
return m.digest()
|
|
|
|
|
|
|
|
def _parse_int(val, default=0):
|
2014-11-03 11:12:34 -08:00
|
|
|
"""Parse an integer or boolean value."""
|
|
|
|
r = re.match("^[0-9]+$", val)
|
|
|
|
if r:
|
|
|
|
return int(val)
|
|
|
|
try:
|
|
|
|
return {'true': 1, 'yes': 1, 'false': 0, 'no': 0}[val.lower()]
|
|
|
|
except AttributeError:
|
|
|
|
return default
|
|
|
|
|
2015-03-05 15:23:05 -08:00
|
|
|
def _winebugs_query(bugids):
|
|
|
|
"""Query short_desc from multiple wine bugzilla bugs at the same time."""
|
|
|
|
bugids = list(set(bugids)) # Remove duplicates and convert to fixed order
|
|
|
|
if len(bugids) == 0: return {}
|
|
|
|
|
|
|
|
# Query bugzilla
|
|
|
|
url = "http://bugs.winehq.org/show_bug.cgi?%s&ctype=xml&field=short_desc&field=bug_status&field=resolution" % \
|
|
|
|
"&".join(["id=%d" % bugid for bugid in bugids])
|
|
|
|
with contextlib.closing(urllib.urlopen(url)) as fp:
|
|
|
|
data = xml.dom.minidom.parseString(fp.read())
|
|
|
|
|
|
|
|
# convert xml in a dictionary containing all bugs we found
|
|
|
|
result = {}
|
|
|
|
for element in data.getElementsByTagName('bug_id'):
|
|
|
|
bugids.remove(int(element.firstChild.data))
|
|
|
|
|
|
|
|
def _get_text(n):
|
|
|
|
return n.firstChild.data if (n and n.firstChild) else None
|
|
|
|
|
|
|
|
for bugid, element in zip(bugids, data.getElementsByTagName('bug')):
|
|
|
|
if element.getElementsByTagName('bug_id'): continue
|
|
|
|
result[bugid] = {
|
|
|
|
'short_desc': _get_text(element.getElementsByTagName('short_desc')[0]),
|
|
|
|
'bug_status': _get_text(element.getElementsByTagName('bug_status')[0]),
|
|
|
|
'resolution': _get_text(element.getElementsByTagName('resolution')[0]) }
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
def _read_changelog():
|
2014-11-28 23:17:16 -08:00
|
|
|
"""Read information from changelog."""
|
2014-08-12 16:59:01 -07:00
|
|
|
with open(config.path_changelog) as fp:
|
|
|
|
for line in fp:
|
|
|
|
r = re.match("^([a-zA-Z0-9][^(]*)\((.*)\) ([^;]*)", line)
|
|
|
|
if r: yield (r.group(1).strip(), r.group(2).strip(), r.group(3).strip())
|
|
|
|
|
|
|
|
def _stable_compholio_version():
|
2014-11-28 23:17:16 -08:00
|
|
|
"""Get version number of the latest stable release."""
|
2014-08-12 16:59:01 -07:00
|
|
|
for package, version, distro in _read_changelog():
|
|
|
|
if distro.lower() != "unreleased":
|
|
|
|
return version
|
|
|
|
|
2015-01-15 20:05:54 -08:00
|
|
|
def _latest_wine_commit(commit=None):
|
2014-11-28 23:17:16 -08:00
|
|
|
"""Get latest wine commit."""
|
2014-08-12 16:59:01 -07:00
|
|
|
if not os.path.isdir(config.path_wine):
|
|
|
|
raise PatchUpdaterError("Please create a symlink to the wine repository in %s" % config.path_wine)
|
2015-01-15 20:05:54 -08:00
|
|
|
if commit is None:
|
|
|
|
commit = subprocess.check_output(["git", "rev-parse", "origin/master"], cwd=config.path_wine).strip()
|
2014-08-12 16:59:01 -07:00
|
|
|
assert len(commit) == 40
|
|
|
|
return commit
|
|
|
|
|
|
|
|
def enum_directories(revision, path):
|
|
|
|
"""Enumerate all subdirectories of 'path' at a specific revision."""
|
|
|
|
dirs = []
|
|
|
|
|
|
|
|
if path[0:2] == "./":
|
|
|
|
path = path[2:]
|
|
|
|
elif path[0] == "/":
|
|
|
|
raise RuntimeError("Expected relative path, not an absolute path")
|
|
|
|
|
|
|
|
if revision is None:
|
|
|
|
for name in os.listdir(path):
|
|
|
|
if name in [".", ".."]: continue
|
2014-12-13 21:03:05 -08:00
|
|
|
directory = os.path.join(path, name)
|
|
|
|
if not os.path.isdir(directory):
|
2014-08-12 16:59:01 -07:00
|
|
|
continue
|
2014-12-13 21:03:05 -08:00
|
|
|
dirs.append((name, directory))
|
2014-08-12 16:59:01 -07:00
|
|
|
else:
|
|
|
|
filename = "%s:%s" % (revision, path)
|
2014-08-12 17:45:21 -07:00
|
|
|
try:
|
2014-11-28 23:17:16 -08:00
|
|
|
content = subprocess.check_output(["git", "show", filename], stderr=_devnull)
|
2014-08-12 17:45:21 -07:00
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
if e.returncode != 128: raise
|
|
|
|
return [] # ignore error
|
2014-08-12 16:59:01 -07:00
|
|
|
lines = content.split("\n")
|
|
|
|
if not lines[0].startswith("tree ") or lines[1] != "":
|
|
|
|
raise RuntimeError("Unexpected output from 'git show %s'" % filename)
|
|
|
|
for name in lines[2:]:
|
|
|
|
if name == "" or name[-1] != "/": continue
|
|
|
|
name = name[:-1]
|
|
|
|
dirs.append((name, os.path.join(path, name)))
|
|
|
|
|
|
|
|
return dirs
|
|
|
|
|
|
|
|
def read_patchset(revision = None):
|
|
|
|
"""Read information about all patchsets for a specific revision."""
|
2014-07-25 12:34:03 -07:00
|
|
|
unique_id = itertools.count()
|
|
|
|
all_patches = {}
|
|
|
|
name_to_id = {}
|
2015-03-05 15:23:05 -08:00
|
|
|
all_bugids = set()
|
2014-07-25 12:34:03 -07:00
|
|
|
|
|
|
|
# Read in sorted order (to ensure created Makefile doesn't change too much)
|
2014-12-13 21:03:05 -08:00
|
|
|
for name, directory in sorted(enum_directories(revision, config.path_patches)):
|
|
|
|
patch = PatchSet(name, directory)
|
2014-07-25 12:34:03 -07:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
if revision is None:
|
|
|
|
|
|
|
|
# If its the latest revision, then request additional information
|
2014-12-13 21:03:05 -08:00
|
|
|
if not os.path.isdir(directory):
|
|
|
|
raise RuntimeError("Unable to open directory %s" % directory)
|
2014-08-12 16:59:01 -07:00
|
|
|
|
|
|
|
# Enumerate .patch files in the given directory, enumerate individual patches and affected files
|
2014-12-13 21:03:05 -08:00
|
|
|
for f in sorted(os.listdir(directory)):
|
2014-12-13 19:54:34 -08:00
|
|
|
if not re.match("^[0-9]{4}-.*\\.patch$", f):
|
|
|
|
continue
|
2014-12-13 21:03:05 -08:00
|
|
|
if f.startswith(config.path_IfDefined):
|
|
|
|
continue
|
|
|
|
if not os.path.isfile(os.path.join(directory, f)):
|
2014-08-12 16:59:01 -07:00
|
|
|
continue
|
|
|
|
patch.files.append(f)
|
2014-12-13 21:03:05 -08:00
|
|
|
for p in patchutils.read_patch(os.path.join(directory, f)):
|
2014-08-12 16:59:01 -07:00
|
|
|
patch.modified_files.add(p.modified_file)
|
|
|
|
patch.patches.append(p)
|
|
|
|
|
|
|
|
# No single patch within this directory, ignore it
|
|
|
|
if len(patch.patches) == 0:
|
|
|
|
del patch
|
2014-07-25 12:34:03 -07:00
|
|
|
continue
|
|
|
|
|
|
|
|
i = next(unique_id)
|
|
|
|
all_patches[i] = patch
|
|
|
|
name_to_id[name] = i
|
|
|
|
|
|
|
|
# Now read the definition files in a second step
|
|
|
|
for i, patch in all_patches.iteritems():
|
2014-08-12 16:59:01 -07:00
|
|
|
try:
|
2015-02-22 11:46:03 -08:00
|
|
|
filename = os.path.join(os.path.join(config.path_patches, patch.name), "definition")
|
|
|
|
if revision is None:
|
|
|
|
with open(filename) as fp:
|
|
|
|
content = fp.read()
|
|
|
|
else:
|
|
|
|
filename = "%s:%s" % (revision, filename)
|
|
|
|
content = subprocess.check_output(["git", "show", filename], stderr=_devnull)
|
|
|
|
except (IOError, subprocess.CalledProcessError):
|
|
|
|
continue # Skip this definition file
|
|
|
|
|
|
|
|
for line in content.split("\n"):
|
|
|
|
if line.startswith("#"):
|
|
|
|
continue
|
|
|
|
tmp = line.split(":", 1)
|
|
|
|
if len(tmp) != 2:
|
|
|
|
continue
|
|
|
|
|
|
|
|
key, val = tmp[0].lower(), tmp[1].strip()
|
|
|
|
if key == "depends":
|
|
|
|
if not name_to_id.has_key(val):
|
|
|
|
raise PatchUpdaterError("Definition file %s references unknown dependency %s" % (filename, val))
|
|
|
|
patch.depends.add(name_to_id[val])
|
|
|
|
|
|
|
|
elif key == "apply-after":
|
|
|
|
for j, other_patch in all_patches.iteritems():
|
|
|
|
if i != j and any([fnmatch.fnmatch(f, val) for f in other_patch.modified_files]):
|
|
|
|
patch.auto_depends.add(j)
|
|
|
|
|
|
|
|
elif key == "fixes":
|
|
|
|
r = re.match("^[0-9]+$", val)
|
|
|
|
if r:
|
2015-03-05 15:23:05 -08:00
|
|
|
bugid = int(val)
|
|
|
|
patch.fixes.append((bugid, None))
|
|
|
|
all_bugids.add(bugid)
|
2015-02-22 11:46:03 -08:00
|
|
|
continue
|
|
|
|
r = re.match("^\\[ *([0-9]+) *\\](.*)$", val)
|
|
|
|
if r:
|
2015-03-05 15:23:05 -08:00
|
|
|
bugid = int(r.group(1))
|
|
|
|
patch.fixes.append((bugid, r.group(2).strip()))
|
|
|
|
all_bugids.add(bugid)
|
2015-02-22 11:46:03 -08:00
|
|
|
continue
|
|
|
|
patch.fixes.append((None, val))
|
|
|
|
|
|
|
|
elif key == "disabled":
|
|
|
|
patch.disabled = _parse_int(val)
|
|
|
|
|
|
|
|
elif key == "ifdefined":
|
|
|
|
patch.ifdefined = val
|
|
|
|
|
|
|
|
elif revision is None:
|
|
|
|
print "WARNING: Ignoring unknown command in definition file %s: %s" % (filename, line)
|
2014-07-25 12:34:03 -07:00
|
|
|
|
2015-03-05 15:23:05 -08:00
|
|
|
# To simplify the task of keeping the bug list up-to-date, list all bugs
|
|
|
|
# which might require attention.
|
|
|
|
if revision is None:
|
|
|
|
once = True
|
|
|
|
for bugid, bug in sorted(_winebugs_query(all_bugids).items()):
|
|
|
|
if bug['bug_status'] not in ["UNCONFIRMED", "NEW", "REOPENED"]:
|
|
|
|
if once:
|
|
|
|
print ""
|
|
|
|
print "WARNING: The following bugs might require attention:"
|
|
|
|
print ""
|
|
|
|
once = False
|
|
|
|
print " #%d - \"%s\" - %s %s" % (bugid, bug['short_desc'], bug['bug_status'],
|
|
|
|
bug['resolution'] if bug['resolution'] else "")
|
|
|
|
print ""
|
|
|
|
|
2014-07-25 12:34:03 -07:00
|
|
|
return all_patches
|
2014-07-11 09:51:03 -07:00
|
|
|
|
|
|
|
def causal_time_combine(a, b):
|
2014-07-25 12:34:03 -07:00
|
|
|
"""Combines two timestamps into a new one."""
|
2014-07-11 16:33:04 -07:00
|
|
|
return [max(a, b) for a, b in zip(a, b)]
|
2014-07-11 09:51:03 -07:00
|
|
|
|
|
|
|
def causal_time_smaller(a, b):
|
2014-07-25 12:34:03 -07:00
|
|
|
"""Checks if timestamp a is smaller than timestamp b."""
|
2014-07-11 09:51:03 -07:00
|
|
|
return all([i <= j for i, j in zip(a,b)]) and any([i < j for i, j in zip(a,b)])
|
|
|
|
|
2014-07-26 14:19:39 -07:00
|
|
|
def causal_time_relation(all_patches, indices):
|
|
|
|
"""Checks if the dependencies of patches are compatible with a specific apply order."""
|
2014-07-25 12:34:03 -07:00
|
|
|
for i, j in _pairs(indices):
|
2014-07-26 14:19:39 -07:00
|
|
|
if causal_time_smaller(all_patches[j].verify_time, all_patches[i].verify_time):
|
2014-07-25 07:39:08 -07:00
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
2014-07-26 14:19:39 -07:00
|
|
|
def causal_time_relation_any(all_patches, indices):
|
|
|
|
"""Similar to causal_time_relation(), but also check all possible permutations of indices."""
|
|
|
|
for i, j in _pairs(indices):
|
|
|
|
if not (causal_time_smaller(all_patches[i].verify_time, all_patches[j].verify_time) or \
|
|
|
|
causal_time_smaller(all_patches[j].verify_time, all_patches[i].verify_time)):
|
2014-07-26 14:13:27 -07:00
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
2014-07-25 07:39:08 -07:00
|
|
|
def contains_binary_patch(all_patches, indices, filename):
|
2014-07-25 12:34:03 -07:00
|
|
|
"""Checks if any patch with given indices affecting filename is a binary patch."""
|
2014-07-25 07:39:08 -07:00
|
|
|
for i in indices:
|
|
|
|
for patch in all_patches[i].patches:
|
|
|
|
if patch.modified_file == filename and patch.is_binary():
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
2014-11-28 23:17:16 -08:00
|
|
|
def get_wine_file(filename):
|
2014-07-27 17:32:13 -07:00
|
|
|
"""Return the hash and optionally the content of a file."""
|
2014-11-28 23:17:16 -08:00
|
|
|
entry = "%s:%s" % (latest_wine_commit, filename)
|
|
|
|
result = tempfile.NamedTemporaryFile()
|
2014-07-27 17:22:58 -07:00
|
|
|
try:
|
2014-11-28 23:17:16 -08:00
|
|
|
content = subprocess.check_call(["git", "show", entry], cwd=config.path_wine, \
|
|
|
|
stdout=result, stderr=_devnull)
|
2014-07-27 17:22:58 -07:00
|
|
|
except subprocess.CalledProcessError as e:
|
2014-08-12 17:45:21 -07:00
|
|
|
if e.returncode != 128: raise
|
2014-11-28 23:17:16 -08:00
|
|
|
result.flush() # shouldn't be necessary because the subprocess writes directly to the fd
|
|
|
|
return result
|
|
|
|
|
2014-12-13 19:57:16 -08:00
|
|
|
def extract_patch(patchset, filename):
|
|
|
|
"""Extract all changes to a specific file from a patchset."""
|
|
|
|
p = tempfile.NamedTemporaryFile()
|
|
|
|
m = hashlib.sha256()
|
|
|
|
|
|
|
|
for patch in patchset.patches:
|
|
|
|
if patch.modified_file != filename:
|
|
|
|
continue
|
|
|
|
for chunk in patch.read_chunks():
|
|
|
|
p.write(chunk)
|
|
|
|
m.update(chunk)
|
|
|
|
p.write("\n")
|
|
|
|
m.update("\n")
|
|
|
|
|
|
|
|
p.flush()
|
|
|
|
return (m.digest(), p)
|
|
|
|
|
2014-11-28 23:17:16 -08:00
|
|
|
def select_patches(all_patches, indices, filename):
|
|
|
|
"""Create a temporary patch file for each patchset and calculate the checksum."""
|
|
|
|
selected_patches = {}
|
|
|
|
for i in indices:
|
2014-12-13 19:57:16 -08:00
|
|
|
selected_patches[i] = extract_patch(all_patches[i], filename)
|
2014-11-28 23:17:16 -08:00
|
|
|
return selected_patches
|
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
def resolve_dependencies(all_patches, index = None, depends = None):
|
2014-12-13 21:03:05 -08:00
|
|
|
"""Returns a sorted list with all dependencies for a given patch."""
|
|
|
|
|
|
|
|
def _resolve(depends):
|
|
|
|
for i in depends:
|
2015-01-07 00:49:06 -08:00
|
|
|
# Check for disabled patch
|
|
|
|
if all_patches[i].disabled:
|
|
|
|
raise PatchUpdaterError("Encountered dependency on disabled patchset %s" % all_patches[i].name)
|
2014-12-13 21:03:05 -08:00
|
|
|
# Dependencies already resolved
|
|
|
|
if all_patches[i].verify_resolved > 0:
|
|
|
|
continue
|
|
|
|
# Detect circular dependency
|
|
|
|
if all_patches[i].verify_resolved < 0:
|
2015-01-07 00:49:06 -08:00
|
|
|
raise PatchUpdaterError("Circular dependency while trying to resolve %s" % all_patches[i].name)
|
2014-12-13 21:03:05 -08:00
|
|
|
|
|
|
|
# Recusively resolve dependencies
|
|
|
|
all_patches[i].verify_resolved = -1
|
2014-12-14 12:45:45 -08:00
|
|
|
_resolve(all_patches[i].depends)
|
2015-02-22 11:46:03 -08:00
|
|
|
_resolve(all_patches[i].auto_depends)
|
2014-12-13 21:03:05 -08:00
|
|
|
all_patches[i].verify_resolved = 1
|
|
|
|
resolved.append(i)
|
|
|
|
|
|
|
|
for _, patch in all_patches.iteritems():
|
|
|
|
patch.verify_resolved = 0
|
|
|
|
|
|
|
|
resolved = []
|
2015-01-07 00:49:06 -08:00
|
|
|
if depends is None:
|
|
|
|
depends = all_patches[index].depends
|
|
|
|
_resolve(depends)
|
2014-12-13 21:03:05 -08:00
|
|
|
return resolved
|
|
|
|
|
|
|
|
def generate_ifdefined(all_patches):
|
|
|
|
"""Update autogenerated ifdefined patches, which can be used to selectively disable features at compile time."""
|
|
|
|
enabled_patches = dict([(i, patch) for i, patch in all_patches.iteritems() if not patch.disabled])
|
|
|
|
|
|
|
|
for i, patch in enabled_patches.iteritems():
|
|
|
|
if patch.ifdefined is None:
|
|
|
|
continue
|
|
|
|
|
|
|
|
filename = os.path.join(patch.directory, config.path_IfDefined)
|
|
|
|
with open(filename, "wb") as fp:
|
|
|
|
fp.write("From: Wine Staging Team <webmaster@fds-team.de>\n")
|
|
|
|
fp.write("Subject: Autogenerated #ifdef patch for %s.\n" % patch.name)
|
|
|
|
fp.write("\n")
|
|
|
|
|
|
|
|
depends = resolve_dependencies(enabled_patches, i)
|
|
|
|
for f in patch.modified_files:
|
|
|
|
|
|
|
|
# Reconstruct the state after applying the dependencies
|
|
|
|
original = get_wine_file(f)
|
2014-12-13 21:20:48 -08:00
|
|
|
for _, (_, p) in select_patches(enabled_patches, depends, f).iteritems():
|
|
|
|
original = patchutils.apply_patch(original, p, fuzz=0)
|
2014-12-13 21:03:05 -08:00
|
|
|
|
|
|
|
# Now apply the main patch
|
2014-12-13 21:20:48 -08:00
|
|
|
p = extract_patch(patch, f)[1]
|
|
|
|
patched = patchutils.apply_patch(original, p, fuzz=0)
|
2014-12-13 21:03:05 -08:00
|
|
|
|
|
|
|
# Now get the diff between both
|
|
|
|
diff = patchutils.generate_ifdef_patch(original, patched, ifdef=patch.ifdefined)
|
|
|
|
if diff is not None:
|
|
|
|
fp.write("diff --git a/%s b/%s\n" % (f, f))
|
|
|
|
fp.write("--- a/%s\n" % f)
|
|
|
|
fp.write("+++ b/%s\n" % f)
|
|
|
|
while True:
|
|
|
|
buf = diff.read(16384)
|
|
|
|
if buf == "": break
|
|
|
|
fp.write(buf)
|
|
|
|
diff.close()
|
|
|
|
|
|
|
|
# Close the file
|
|
|
|
fp.close()
|
|
|
|
|
2014-12-13 21:29:32 -08:00
|
|
|
# Add changes to git
|
|
|
|
subprocess.call(["git", "add", filename])
|
|
|
|
|
2014-12-13 21:03:05 -08:00
|
|
|
# Add the autogenerated file as a last patch
|
|
|
|
patch.files.append(os.path.basename(filename))
|
|
|
|
for p in patchutils.read_patch(filename):
|
|
|
|
assert p.modified_file in patch.modified_files
|
|
|
|
patch.patches.append(p)
|
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
def generate_script(all_patches):
|
|
|
|
"""Resolve dependencies, and afterwards check if everything applies properly."""
|
|
|
|
depends = sorted([i for i, patch in all_patches.iteritems() if not patch.disabled])
|
|
|
|
resolved = resolve_dependencies(all_patches, depends=depends)
|
|
|
|
max_patches = max(resolved) + 1
|
2014-07-11 09:51:03 -07:00
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
# Generate timestamps based on dependencies, still required for binary patches
|
2014-07-11 09:51:03 -07:00
|
|
|
# Find out which files are modified by multiple patches
|
|
|
|
modified_files = {}
|
2015-01-07 00:49:06 -08:00
|
|
|
for i, patch in [(i, all_patches[i]) for i in resolved]:
|
|
|
|
patch.verify_time = [0]*max_patches
|
|
|
|
patch.verify_time[i] += 1
|
|
|
|
for j in patch.depends:
|
|
|
|
patch.verify_time = causal_time_combine(patch.verify_time, all_patches[j].verify_time)
|
|
|
|
|
2014-07-24 18:32:01 -07:00
|
|
|
for f in patch.modified_files:
|
2014-07-11 09:51:03 -07:00
|
|
|
if f not in modified_files:
|
|
|
|
modified_files[f] = []
|
|
|
|
modified_files[f].append(i)
|
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
# Check dependencies
|
2015-02-23 10:14:25 -08:00
|
|
|
pool = multiprocessing.pool.ThreadPool(processes=4)
|
|
|
|
try:
|
|
|
|
for filename, indices in modified_files.iteritems():
|
2014-07-11 09:51:03 -07:00
|
|
|
|
2015-02-23 10:14:25 -08:00
|
|
|
# If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case
|
|
|
|
if contains_binary_patch(all_patches, indices, filename):
|
|
|
|
if not causal_time_relation_any(all_patches, indices):
|
|
|
|
raise PatchUpdaterError("Because of binary patch modifying file %s the following patches need explicit dependencies: %s" %
|
|
|
|
(filename, ", ".join([all_patches[i].name for i in indices])))
|
|
|
|
continue
|
|
|
|
|
|
|
|
original_content = get_wine_file(filename)
|
|
|
|
selected_patches = select_patches(all_patches, indices, filename)
|
|
|
|
|
|
|
|
# Show a progress bar while applying the patches - this task might take some time
|
|
|
|
chunk_size = 20
|
|
|
|
with progressbar.ProgressBar(desc=filename, total=2 ** len(indices) / chunk_size) as progress:
|
|
|
|
|
|
|
|
def test_apply(bitstring):
|
|
|
|
set_apply = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if u]
|
|
|
|
set_skip = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if not u]
|
|
|
|
|
|
|
|
# Check if there is any patch2 which depends directly or indirectly on patch1.
|
|
|
|
# If this is the case we found an impossible situation, we can be skipped in this test.
|
|
|
|
for i, patch1 in set_apply:
|
|
|
|
for j, patch2 in set_skip:
|
|
|
|
if causal_time_smaller(patch2.verify_time, patch1.verify_time):
|
|
|
|
return True # we can skip this test
|
2014-07-11 09:51:03 -07:00
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
try:
|
|
|
|
original = original_content
|
2015-01-07 17:12:38 -08:00
|
|
|
for i, patch in set_apply:
|
2015-01-07 00:49:06 -08:00
|
|
|
original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0)
|
|
|
|
except patchutils.PatchApplyError:
|
2015-02-23 10:14:25 -08:00
|
|
|
return False
|
|
|
|
|
|
|
|
return True # everything is fine
|
|
|
|
|
|
|
|
def test_apply_seq(bitstrings):
|
|
|
|
for bitstring in bitstrings:
|
|
|
|
if not test_apply(bitstring):
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
|
|
it = _split_seq(itertools.product([0,1], repeat=len(indices)), chunk_size)
|
|
|
|
for k, res in enumerate(pool.imap_unordered(test_apply_seq, it)):
|
|
|
|
if not res:
|
2015-01-07 00:49:06 -08:00
|
|
|
progress.finish("<failed to apply>")
|
|
|
|
raise PatchUpdaterError("Changes to file %s don't apply: %s" %
|
|
|
|
(filename, ", ".join([all_patches[i].name for i in indices])))
|
2015-02-23 10:14:25 -08:00
|
|
|
progress.update(k)
|
|
|
|
finally:
|
|
|
|
pool.close()
|
2015-01-07 00:49:06 -08:00
|
|
|
|
|
|
|
# Generate code for helper functions
|
|
|
|
lines = []
|
|
|
|
lines.append("# Enable or disable all patchsets\n")
|
|
|
|
lines.append("patch_enable_all ()\n")
|
|
|
|
lines.append("{\n")
|
2015-01-07 17:12:38 -08:00
|
|
|
for i, patch in sorted([(i, all_patches[i]) for i in resolved], key=lambda x:x[1].name):
|
2015-02-08 06:02:51 -08:00
|
|
|
patch.variable = "enable_%s" % patch.name.replace("-","_").replace(".","_")
|
2015-01-07 00:49:06 -08:00
|
|
|
lines.append("\t%s=\"$1\"\n" % patch.variable)
|
|
|
|
lines.append("}\n")
|
|
|
|
lines.append("\n")
|
|
|
|
lines.append("# Enable or disable a specific patchset\n")
|
|
|
|
lines.append("patch_enable ()\n")
|
|
|
|
lines.append("{\n")
|
|
|
|
lines.append("\tcase \"$1\" in\n")
|
2015-01-07 17:12:38 -08:00
|
|
|
for i, patch in sorted([(i, all_patches[i]) for i in resolved], key=lambda x:x[1].name):
|
2015-01-07 00:49:06 -08:00
|
|
|
lines.append("\t\t%s)\n" % patch.name)
|
|
|
|
lines.append("\t\t\t%s=\"$2\"\n" % patch.variable)
|
|
|
|
lines.append("\t\t\t;;\n")
|
|
|
|
lines.append("\t\t*)\n")
|
|
|
|
lines.append("\t\t\treturn 1\n")
|
|
|
|
lines.append("\t\t\t;;\n")
|
|
|
|
lines.append("\tesac\n")
|
|
|
|
lines.append("\treturn 0\n")
|
|
|
|
lines.append("}\n")
|
|
|
|
lines_helpers = lines
|
|
|
|
|
|
|
|
# Generate code for dependency resolver
|
|
|
|
lines = []
|
|
|
|
for i, patch in [(i, all_patches[i]) for i in reversed(resolved)]:
|
|
|
|
if len(patch.depends):
|
2015-01-10 06:30:31 -08:00
|
|
|
lines.append("if test \"$%s\" -eq 1; then\n" % patch.variable)
|
2015-01-07 14:06:07 -08:00
|
|
|
for j in sorted(patch.depends):
|
2015-01-10 06:30:31 -08:00
|
|
|
lines.append("\tif test \"$%s\" -gt 1; then\n" % all_patches[j].variable)
|
2015-01-07 14:06:07 -08:00
|
|
|
lines.append("\t\tabort \"Patchset %s disabled, but %s depends on that.\"\n" %
|
|
|
|
(all_patches[j].name, patch.name))
|
|
|
|
lines.append("\tfi\n")
|
|
|
|
for j in sorted(patch.depends):
|
2015-01-07 00:49:06 -08:00
|
|
|
lines.append("\t%s=1\n" % all_patches[j].variable)
|
|
|
|
lines.append("fi\n\n")
|
|
|
|
lines_resolver = lines
|
|
|
|
|
|
|
|
# Generate code for applying all patchsets
|
|
|
|
lines = []
|
|
|
|
for i, patch in [(i, all_patches[i]) for i in resolved]:
|
|
|
|
lines.append("# Patchset %s\n" % patch.name)
|
|
|
|
lines.append("# |\n")
|
|
|
|
|
|
|
|
# List all bugs fixed by this patchset
|
|
|
|
if any([bugid is not None for bugid, bugname in patch.fixes]):
|
|
|
|
lines.append("# | This patchset fixes the following Wine bugs:\n")
|
|
|
|
for bugid, bugname in patch.fixes:
|
|
|
|
if bugid is not None:
|
|
|
|
lines.append("# | *\t%s\n" % "\n# | \t".join(textwrap.wrap("[#%d] %s" % (bugid, bugname), 120)))
|
|
|
|
lines.append("# |\n")
|
|
|
|
|
|
|
|
# List all modified files
|
|
|
|
lines.append("# | Modified files:\n")
|
|
|
|
lines.append("# | *\t%s\n" % "\n# | \t".join(textwrap.wrap(", ".join(sorted(patch.modified_files)), 120)))
|
|
|
|
lines.append("# |\n")
|
2015-01-10 06:30:31 -08:00
|
|
|
lines.append("if test \"$%s\" -eq 1; then\n" % patch.variable)
|
2015-01-07 00:49:06 -08:00
|
|
|
for f in patch.files:
|
|
|
|
lines.append("\tpatch_apply %s\n" % os.path.join(patch.name, f))
|
|
|
|
if len(patch.patches):
|
|
|
|
lines.append("\t(\n")
|
|
|
|
for p in _unique(patch.patches, key=lambda p: (p.patch_author, p.patch_subject, p.patch_revision)):
|
2015-01-07 14:06:07 -08:00
|
|
|
lines.append("\t\techo '+ { \"%s\", \"%s\", %d },';\n" %
|
|
|
|
(_escape(p.patch_author), _escape(p.patch_subject), p.patch_revision))
|
2015-01-07 00:49:06 -08:00
|
|
|
lines.append("\t) >> \"$patchlist\"\n")
|
|
|
|
lines.append("fi\n\n")
|
|
|
|
lines_apply = lines
|
|
|
|
|
|
|
|
with open(config.path_template_script) as template_fp:
|
|
|
|
template = template_fp.read()
|
|
|
|
with open(config.path_script, "w") as fp:
|
|
|
|
fp.write(template.format(patch_helpers="".join(lines_helpers).rstrip("\n"),
|
|
|
|
patch_resolver="".join(lines_resolver).rstrip("\n"),
|
|
|
|
patch_apply="".join(lines_apply).rstrip("\n")))
|
2014-08-12 14:11:15 -07:00
|
|
|
|
2014-12-13 21:29:32 -08:00
|
|
|
# Add changes to git
|
2015-01-07 00:49:06 -08:00
|
|
|
subprocess.call(["git", "add", config.path_script])
|
2014-12-13 21:29:32 -08:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
def generate_markdown(all_patches, stable_patches, stable_compholio_version):
|
2014-11-29 18:54:53 -08:00
|
|
|
"""Generate README.md including information about specific patches and bugfixes."""
|
2014-07-11 12:25:18 -07:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
def _format_bug(mode, bugid, bugname):
|
|
|
|
if mode < 0: bugname = "~~%s~~" % bugname
|
|
|
|
if bugid is None: return "* %s" % bugname
|
2014-10-31 07:15:13 -07:00
|
|
|
return "* %s ([Wine Bug #%d](https://bugs.winehq.org/show_bug.cgi?id=%d))" % \
|
2014-09-21 21:27:47 -07:00
|
|
|
(bugname, bugid, bugid) #, short_desc.replace("\\", "\\\\").replace("\"", "\\\""))
|
2014-07-12 16:30:44 -07:00
|
|
|
|
2015-03-05 15:23:05 -08:00
|
|
|
all_fixes = {}
|
2014-08-12 14:11:15 -07:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
# Get fixes for current version
|
2014-11-28 23:17:16 -08:00
|
|
|
for _, patch in all_patches.iteritems():
|
2014-08-12 16:59:01 -07:00
|
|
|
for bugid, bugname in patch.fixes:
|
|
|
|
key = bugid if bugid is not None else bugname
|
|
|
|
all_fixes[key] = [1, bugid, bugname]
|
|
|
|
|
|
|
|
# Compare with fixes for latest stable version
|
2014-11-28 23:17:16 -08:00
|
|
|
for _, patch in stable_patches.iteritems():
|
2014-08-12 16:59:01 -07:00
|
|
|
for bugid, bugname in patch.fixes:
|
|
|
|
key = bugid if bugid is not None else bugname
|
|
|
|
if all_fixes.has_key(key):
|
|
|
|
all_fixes[key][0] = 0
|
|
|
|
else:
|
|
|
|
all_fixes[key] = [-1, bugid, bugname]
|
|
|
|
|
|
|
|
# Generate lists for all new and old fixes
|
|
|
|
new_fixes = [(mode, bugid, bugname) for dummy, (mode, bugid, bugname) in
|
|
|
|
all_fixes.iteritems() if mode > 0]
|
|
|
|
old_fixes = [(mode, bugid, bugname) for dummy, (mode, bugid, bugname) in
|
|
|
|
all_fixes.iteritems() if mode <= 0]
|
|
|
|
|
2014-08-12 17:45:21 -07:00
|
|
|
# List of old fixes is not available when releasing a new version
|
|
|
|
if len(old_fixes) == 0:
|
|
|
|
old_fixes = new_fixes
|
|
|
|
new_fixes = []
|
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
# Generate information for current version
|
|
|
|
lines = []
|
|
|
|
if len(new_fixes):
|
2014-08-12 17:45:21 -07:00
|
|
|
lines.append("**Bugfixes and features included in the next upcoming release [%d]:**" % len(new_fixes))
|
2014-08-12 16:59:01 -07:00
|
|
|
lines.append("")
|
|
|
|
for mode, bugid, bugname in sorted(new_fixes, key=lambda x: x[2]):
|
|
|
|
lines.append(_format_bug(mode, bugid, bugname))
|
|
|
|
lines.append("")
|
|
|
|
lines.append("")
|
2014-11-03 15:47:24 -08:00
|
|
|
lines.append("**Bugs fixed in Wine Staging %s [%d]:**" % (stable_compholio_version, len(old_fixes)))
|
2014-08-12 16:59:01 -07:00
|
|
|
lines.append("")
|
|
|
|
for mode, bugid, bugname in sorted(old_fixes, key=lambda x: x[2]):
|
|
|
|
lines.append(_format_bug(mode, bugid, bugname))
|
|
|
|
|
|
|
|
# Update README.md
|
2014-08-12 14:11:15 -07:00
|
|
|
with open(config.path_template_README_md) as template_fp:
|
2014-07-25 18:31:08 -07:00
|
|
|
template = template_fp.read()
|
2014-08-12 14:11:15 -07:00
|
|
|
with open(config.path_README_md, "w") as fp:
|
2014-08-12 16:59:01 -07:00
|
|
|
fp.write(template.format(fixes="\n".join(lines)))
|
2014-07-28 17:33:30 -07:00
|
|
|
|
2014-12-13 21:29:32 -08:00
|
|
|
# Add changes to git
|
|
|
|
subprocess.call(["git", "add", config.path_README_md])
|
|
|
|
|
2014-07-11 09:51:03 -07:00
|
|
|
if __name__ == "__main__":
|
2014-11-28 23:17:16 -08:00
|
|
|
|
2015-01-07 00:49:06 -08:00
|
|
|
# Hack to avoid KeyboardInterrupts on different threads
|
2014-11-28 23:17:16 -08:00
|
|
|
def _sig_int(signum=None, frame=None):
|
|
|
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
|
|
raise RuntimeError("CTRL+C pressed")
|
|
|
|
signal.signal(signal.SIGINT, _sig_int)
|
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
try:
|
2014-07-25 07:39:08 -07:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
# Get information about Wine and Compholio version
|
2015-01-15 20:05:54 -08:00
|
|
|
latest_wine_commit = _latest_wine_commit(sys.argv[1] if len(sys.argv) >= 2 else None)
|
2014-08-12 16:59:01 -07:00
|
|
|
stable_compholio_version = _stable_compholio_version()
|
2014-07-27 17:22:58 -07:00
|
|
|
|
2014-08-12 16:59:01 -07:00
|
|
|
# Read current and stable patches
|
|
|
|
all_patches = read_patchset()
|
|
|
|
stable_patches = read_patchset(revision="v%s" % stable_compholio_version)
|
|
|
|
|
2014-12-13 21:03:05 -08:00
|
|
|
generate_ifdefined(all_patches)
|
2015-01-07 00:49:06 -08:00
|
|
|
generate_script(all_patches)
|
2014-08-12 16:59:01 -07:00
|
|
|
generate_markdown(all_patches, stable_patches, stable_compholio_version)
|
|
|
|
|
2014-07-25 12:54:58 -07:00
|
|
|
except PatchUpdaterError as e:
|
|
|
|
print ""
|
|
|
|
print "ERROR: %s" % e
|
|
|
|
print ""
|
|
|
|
exit(1)
|