wine/wine-staging
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-staging.git synced 2024-09-13 09:17:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

patchupdate.py: Improve patch system and remove dependencies, when the order of patches doesn't matter.

Browse Source
This commit is contained in:
Sebastian Lackner 2014-07-25 16:39:08 +02:00
parent 80e51dcad6
commit e827cc078f
9 changed files with 212 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1 +1,2 @@
*.ok
*.ok
.depcache

124
debian/tools/patchupdate.py vendored
View File

@ -8,9 +8,14 @@ import sys
import hashlib
import itertools
import patchutils
import subprocess
import pickle
import os
import re
# Cached information to speed up patch dependency checks
cached_patch_result = {}
class AuthorInfo(object):
def __init__(self):
self.author = ""
@ -48,8 +53,110 @@ def causal_time_combine(a, b):
def causal_time_smaller(a, b):
return all([i <= j for i, j in zip(a,b)]) and any([i < j for i, j in zip(a,b)])
def causal_time_relation(a, b):
return causal_time_smaller(a, b) or causal_time_smaller(b, a)
def causal_time_relation(all_patches, indices):
for i, j in pairs(indices):
if not (causal_time_smaller(all_patches[i].verify_time, all_patches[j].verify_time) or \
causal_time_smaller(all_patches[j].verify_time, all_patches[i].verify_time)):
return False
return True
def causal_time_permutations(all_patches, indices, filename):
for perm in itertools.permutations(indices):
for i, j in zip(perm[:-1], perm[1:]):
if causal_time_smaller(all_patches[j].verify_time, all_patches[i].verify_time):
break
else:
selected_patches = []
for i in perm:
selected_patches += [patch for patch in all_patches[i].patches if patch.modified_file == filename]
yield selected_patches
def contains_binary_patch(all_patches, indices, filename):
for i in indices:
for patch in all_patches[i].patches:
if patch.modified_file == filename and patch.is_binary():
return True
return False
def load_patch_cache():
global cached_patch_result
try:
with open("./.depcache") as fp:
cached_patch_result = pickle.load(fp)
except IOError:
cached_patch_result = {}
def save_patch_cache():
with open("./.depcache", "wb") as fp:
pickle.dump(cached_patch_result, fp, pickle.HIGHEST_PROTOCOL)
def verify_patch_order(all_patches, indices, filename):
global cached_patch_result
# If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case
if contains_binary_patch(all_patches, indices, filename):
if not causal_time_relation(all_patches, indices):
abort("Missing dependency between %s and %s because of same file %s" % (all_patches[i].name, all_patches[j].name, filename))
return
# Grab original file from the wine git repository - please note we grab from origin/master, not the current branch
try:
with open(os.devnull, 'w') as devnull:
original_content = subprocess.check_output(["git", "show", "origin/master:%s" % filename],
cwd="./debian/tools/wine", stderr=devnull)
except subprocess.CalledProcessError as e:
if e.returncode != 128: raise # not found
original_content = ""
# Calculate hash of original content
original_content_hash = hashlib.sha256(original_content).digest()
# Check for possible ways to apply the patch
failed_to_apply = False
last_result_hash = None
for patches in causal_time_permutations(all_patches, indices, filename):
# Calculate unique hash based on the original content and the order in which the patches are applied
m = hashlib.sha256()
m.update(original_content_hash)
for patch in patches:
m.update(patch.hash())
unique_hash = m.digest()
# Fast path -> we know that it applies properly
if cached_patch_result.has_key(unique_hash):
result_hash = cached_patch_result[unique_hash]
else:
# Apply the patches (without fuzz)
try:
content = patchutils.apply_patch(original_content, patches, fuzz=0)
except patchutils.PatchApplyError:
if last_result_hash is not None:
break
# We failed to apply the patches, but don't know if it works at all - continue
failed_to_apply = True
continue
# Get hash of resulting file and add to cache
result_hash = hashlib.sha256(content).digest()
cached_patch_result[unique_hash] = result_hash
# First time we got a successful result
if last_result_hash is None:
last_result_hash = result_hash
if failed_to_apply: break
# All the other times: hash to match with previous attempt
elif last_result_hash != result_hash:
last_result_hash = None
break
if failed_to_apply and last_result_hash is None:
abort("Patches affecting file %s don't apply on source tree: %s" % (filename, ', '.join([all_patches[i].name for i in indices])))
elif failed_to_apply or last_result_hash is None:
abort("Missing dependency between patches affecting %s: %s" % (filename, ', '.join([all_patches[i].name for i in indices])))
else:
assert len(last_result_hash) == 32
def verify_dependencies(all_patches):
max_patches = max(all_patches.keys()) + 1
@ -87,10 +194,12 @@ def verify_dependencies(all_patches):
modified_files[f].append(i)
# Iterate over pairs of patches, check for existing causal relationship
for f, indices in modified_files.iteritems():
for i, j in pairs(indices):
if not causal_time_relation(all_patches[i].verify_time, all_patches[j].verify_time):
abort("Missing dependency between %s and %s because of same file %s" % (all_patches[i].name, all_patches[j].name, f))
load_patch_cache()
try:
for f, indices in modified_files.iteritems():
verify_patch_order(all_patches, indices, f)
finally:
save_patch_cache()
def download(url):
with contextlib.closing(urllib.urlopen(url)) as fp:
@ -361,6 +470,9 @@ def generate_readme(patches, fp):
fp.write(README_template.format(bugs=_enum(_all_bugs()), fixes=_enum(_all_fixes()), version=_latest_stable_version()))
if __name__ == "__main__":
if not os.path.isdir("./debian/tools/wine"):
raise RuntimeError("Please create a symlink to the wine repository in ./debian/tools/wine")
patches = read_patchsets("./patches")
verify_dependencies(patches)

289
debian/tools/patchutils.py vendored
View File

@ -5,50 +5,20 @@ import itertools
import difflib
import subprocess
import hashlib
import collections
class GeneralPatchError(RuntimeError):
pass
class PatchParserError(GeneralPatchError):
class PatchParserError(RuntimeError):
"""Unable to parse patch file - either an unimplemented feature, or corrupted patch."""
pass
class PatchApplyError(GeneralPatchError):
class PatchApplyError(RuntimeError):
"""Failed to apply/merge patch."""
pass
class Patch(object):
def __init__(self):
self.extracted_patch = None
def is_empty(self):
raise NotImplementedError("is_empty not implemented.")
def read_chunks(self):
raise NotImplementedError("read_chunks not implemented.")
def read_hunks(self, reverse=False):
raise NotImplementedError("read_hunks not implemented.")
def extract(self):
"""Create a temporary file containing the extracted patch."""
if not self.extracted_patch:
self.extracted_patch = tempfile.NamedTemporaryFile()
for chunk in self.read_chunks():
self.extracted_patch.write(chunk)
self.extracted_patch.flush()
return self.extracted_patch
def hash(self):
m = hashlib.md5()
for srcpos, srchunk, dsthunk in self.read_hunks():
m.update(hashlib.md5("\n".join(srchunk)).digest())
m.update(hashlib.md5("\n".join(dsthunk)).digest())
return m.digest()
class FilePatch(Patch):
class PatchObject(object):
def __init__(self, filename):
super(FilePatch, self).__init__()
self.extracted_patch = None
self.unique_hash = None
self.filename = filename
self.offset_begin = None
@ -62,15 +32,13 @@ class FilePatch(Patch):
self.newsha1 = None
self.newmode = None
self.hunks = [] # offset, srcpos, srclines, dstpos, dstlines
self.isbinary = False
self.binary_patch_offset = None
self.binary_patch_type = None
self.binary_patch_size = None
def is_empty(self):
return len(self.hunks) == 0
def is_binary(self):
return self.isbinary
def read_chunks(self):
"""Iterates over arbitrary sized chunks of this patch."""
@ -84,155 +52,73 @@ class FilePatch(Patch):
yield buf
i -= len(buf)
def read_hunks(self, reverse=False):
"""Iterates over hunks contained in this patch, reverse exchanges source and destination."""
if self.isbinary:
raise GeneralPatchError("Command not allowed for binary patches.")
def extract(self):
"""Create a temporary file containing the extracted patch."""
if not self.extracted_patch:
self.extracted_patch = tempfile.NamedTemporaryFile()
for chunk in self.read_chunks():
self.extracted_patch.write(chunk)
self.extracted_patch.flush()
return self.extracted_patch
with open(self.filename) as fp:
for offset, srcpos, srclines, dstpos, dstlines in self.hunks:
fp.seek(offset)
srchunk = []
dsthunk = []
while srclines > 0 or dstlines > 0:
line = fp.readline()
if line == "":
raise PatchParserError("Truncated patch.")
line = line.rstrip("\r\n")
if line.startswith(" "):
if srclines == 0 or dstlines == 0:
raise PatchParserError("Corrupted patch.")
srchunk.append(line[1:])
dsthunk.append(line[1:])
srclines -= 1
dstlines -= 1
elif line.startswith("-"):
if srclines == 0:
raise PatchParserError("Corrupted patch.")
srchunk.append(line[1:])
srclines -= 1
elif line.startswith("+"):
if dstlines == 0:
raise PatchParserError("Corrupted patch.")
dsthunk.append(line[1:])
dstlines -= 1
elif line.startswith("\\ "):
pass # ignore
else:
raise PatchParserError("Unexpected line in hunk.")
if reverse:
yield dstpos, dsthunk, srchunk
else:
yield srcpos, srchunk, dsthunk
class MemoryPatch(Patch):
def __init__(self):
super(MemoryPatch, self).__init__()
self.oldname = None
self.newname = None
self.modified_file = None
self.hunks = [] # srcpos, srclines, dstpos, dstlines, lines
def is_empty(self):
return len(self.hunks) == 0
def read_chunks(self):
"""Iterates over arbitrary sized chunks of this patch."""
if self.oldname is None or self.newname is None:
raise GeneralPatchError("Patch doesn't have old or new name.")
yield "diff --git a/%s b/%s\n" % (self.oldname, self.newname)
yield "--- a/%s\n" % self.oldname
yield "+++ b/%s\n" % self.newname
for srcpos, srclines, dstpos, dstlines, hunk in self.hunks:
yield "@@ -%d,%d +%d,%d @@\n" % (srcpos+1, srclines, dstpos+1, dstlines)
for mode, line in hunk:
if mode == 0:
yield " %s\n" % line
elif mode == 1:
yield "+%s\n" % line
elif mode == 2:
yield "-%s\n" % line
def read_hunks(self, reverse=False):
"""Iterates over hunks contained in this patch, reverse exchanges source and destination."""
for srcpos, srclines, dstpos, dstlines, hunk in self.hunks:
srchunk = []
dsthunk = []
for mode, line in hunk:
if mode == 0:
srchunk.append(line)
dsthunk.append(line)
elif mode == 1:
dsthunk.append(line)
elif mode == 2:
srchunk.append(line)
assert srclines == len(srchunk)
assert dstlines == len(dsthunk)
if reverse:
yield dstpos, dsthunk, srchunk
else:
yield srcpos, srchunk, dsthunk
class FileReader(object):
def __init__(self, filename):
self.filename = filename
self.fp = open(self.filename)
self.peeked = None
def close(self):
self.fp.close()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def seek(self, pos):
"""Change the file cursor position."""
self.fp.seek(pos)
self.peeked = None
def tell(self):
"""Return the current file cursor position."""
if self.peeked is None:
return self.fp.tell()
return self.peeked[0]
def peek(self):
"""Read one line without changing the file cursor."""
if self.peeked is None:
pos = self.fp.tell()
tmp = self.fp.readline()
if len(tmp) == 0: return None
self.peeked = (pos, tmp)
return self.peeked[1]
def read(self):
"""Read one line from the file, and move the file cursor to the next line."""
if self.peeked is None:
tmp = self.fp.readline()
if len(tmp) == 0: return None
return tmp
tmp, self.peeked = self.peeked, None
return tmp[1]
def hash(self):
"""Hash the content of the patch."""
if not self.unique_hash:
m = hashlib.sha256()
for chunk in self.read_chunks():
m.update(chunk)
self.unique_hash = m.digest()
return self.unique_hash
def read_patch(filename):
"""Iterates over all patches contained in a file, and returns FilePatch objects."""
"""Iterates over all patches contained in a file, and returns PatchObject objects."""
class _FileReader(object):
def __init__(self, filename):
self.filename = filename
self.fp = open(self.filename)
self.peeked = None
def close(self):
self.fp.close()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def seek(self, pos):
"""Change the file cursor position."""
self.fp.seek(pos)
self.peeked = None
def tell(self):
"""Return the current file cursor position."""
if self.peeked is None:
return self.fp.tell()
return self.peeked[0]
def peek(self):
"""Read one line without changing the file cursor."""
if self.peeked is None:
pos = self.fp.tell()
tmp = self.fp.readline()
if len(tmp) == 0: return None
self.peeked = (pos, tmp)
return self.peeked[1]
def read(self):
"""Read one line from the file, and move the file cursor to the next line."""
if self.peeked is None:
tmp = self.fp.readline()
if len(tmp) == 0: return None
return tmp
tmp, self.peeked = self.peeked, None
return tmp[1]
def _read_single_patch(fp, oldname=None, newname=None):
patch = FilePatch(fp.filename)
patch = PatchObject(fp.filename)
patch.offset_begin = fp.tell()
patch.oldname = oldname
patch.newname = newname
@ -313,7 +199,6 @@ def read_patch(filename):
raise PatchParserError("Empty hunk doesn't make sense.")
assert fp.read() == line
patch.hunks.append(( fp.tell(), srcpos, srclines, dstpos, dstlines ))
while srclines > 0 or dstlines > 0:
line = fp.read()
if line is None:
@ -370,7 +255,7 @@ def read_patch(filename):
patch.offset_end = fp.tell()
return patch
with FileReader(filename) as fp:
with _FileReader(filename) as fp:
while True:
line = fp.peek()
if line is None:
@ -386,34 +271,34 @@ def read_patch(filename):
else:
assert fp.read() == line
def apply_patch(lines, patch, reverse=False, fuzz=2):
def apply_patch(content, patches, reverse=False, fuzz=2):
"""Apply a patch with optional fuzz - uses the commandline 'patch' utility."""
if patch.is_empty():
return lines
if not isinstance(patches, collections.Sequence):
patches = [patches]
contentfile = tempfile.NamedTemporaryFile(delete=False)
try:
for line in lines:
contentfile.write("%s\n" % line)
contentfile.write(content)
contentfile.close()
patchfile = patch.extract()
cmdline = ["patch", "--batch", "-r", "-"] # "--silent" ?
if reverse: cmdline.append("--reverse")
if fuzz != 2: cmdline.append("--fuzz=%d" % fuzz)
cmdline += [contentfile.name, patchfile.name]
for patch in patches:
exitcode = subprocess.call(cmdline)
if exitcode != 0:
raise PatchApplyError("Failed to apply patch (exitcode %d)." % exitcode)
patchfile = patch.extract()
cmdline = ["patch", "--batch", "--silent", "-r", "-"]
if reverse: cmdline.append("--reverse")
if fuzz != 2: cmdline.append("--fuzz=%d" % fuzz)
cmdline += [contentfile.name, patchfile.name]
exitcode = subprocess.call(cmdline)
if exitcode != 0:
raise PatchApplyError("Failed to apply patch (exitcode %d)." % exitcode)
with open(contentfile.name) as fp:
lines = fp.read().split("\n")
if lines[-1] == "": lines.pop()
content = fp.read()
finally:
os.unlink(contentfile.name)
return lines
return content

3
patches/06-winepulse/definition
View File

@ -2,6 +2,3 @@ Author: Maarten Lankhorst
Subject: Winepulse patches extracted from https://launchpad.net/~ubuntu-wine/+archive/ubuntu/ppa/+files/wine1.7_1.7.21-0ubuntu1~ppa1.debian.tar.gz.
Revision: 4
Fixes: Support for PulseAudio backend for audio
# Both patches modify configure.ac
Depends: 02-ACL_Extended_Attributes

3
patches/08-Junction_Points/definition
View File

@ -2,6 +2,3 @@ Author: Erich E. Hoover
Subject: Support for junction points/reparse points.
Revision: 1
Fixes: [12401] Support for Junction Points
# Both patches modify dlls/kernel32/volume.c
Depends: 07-GetVolumePathName

3
patches/09-TransmitFile/definition
View File

@ -2,6 +2,3 @@ Author: Erich E. Hoover
Subject: Implement TransmitFile.
Revision: 1
Fixes: [5048] Support for TransmitFile
# both patches modify server/sock.c
Depends: 01-Address_Change_Notification

3
patches/13-Misc_ACL/definition
View File

@ -2,6 +2,3 @@ Author: Erich E. Hoover
Subject: Add default security descriptor ownership and DACLs for processes.
Revision: 1
Fixes: [15980] GetSecurityInfo returns NULL DACL for process object
# Both patches modify dlls/advapi32/tests/security.c
Depends: 02-ACL_Extended_Attributes

7
patches/16-server-CreateProcess_ACLs/definition
View File

@ -2,10 +2,3 @@ Author: Joris van der Wel
Subject: Implement passing ACLs to CreateProcess.
Revision: 1
Fixes: [22006] Support for process ACLs
# Both patches modify server/process.c
Depends: 13-Misc_ACL
# Both patches modify server/protocol.def
Depends: 09-TransmitFile

10
patches/Makefile
View File

@ -150,7 +150,7 @@ abort:
# | * configure.ac, dlls/mmdevapi/main.c, dlls/mmdevapi/tests/render.c, dlls/winepulse.drv/Makefile.in,
# | dlls/winepulse.drv/mmdevdrv.c, dlls/winepulse.drv/winepulse.drv.spec, dlls/winmm/lolvldrv.c
# |
06-winepulse.ok: 02-ACL_Extended_Attributes.ok
06-winepulse.ok:
$(PATCH) < 06-winepulse/0001-winmm-Load-winealsa-if-winepulse-is-found.patch
$(PATCH) < 06-winepulse/0002-winepulse-Add-initial-stub-for-pulseaudio-support.patch
$(PATCH) < 06-winepulse/0003-winepulse-Add-format-and-period-probing.patch
@ -209,7 +209,7 @@ abort:
# | Modified files:
# | * dlls/kernel32/path.c, dlls/kernel32/volume.c, dlls/ntdll/file.c, dlls/ntdll/tests/file.c, include/ntifs.h
# |
08-Junction_Points.ok: 07-GetVolumePathName.ok
08-Junction_Points.ok:
$(PATCH) < 08-Junction_Points/0001-ntdll-Add-support-for-junction-point-creation.patch
$(PATCH) < 08-Junction_Points/0002-ntdll-Add-support-for-reading-junction-points.patch
$(PATCH) < 08-Junction_Points/0003-ntdll-Add-support-for-deleting-junction-points.patch
@ -233,7 +233,7 @@ abort:
# | Modified files:
# | * dlls/ws2_32/socket.c, dlls/ws2_32/tests/sock.c, include/winsock.h, server/protocol.def, server/sock.c
# |
09-TransmitFile.ok: 01-Address_Change_Notification.ok
09-TransmitFile.ok:
$(PATCH) < 09-TransmitFile/0001-ws2_32-Add-stub-for-TransmitFile.patch
$(PATCH) < 09-TransmitFile/0002-ws2_32-Check-for-invalid-parameters-in-TransmitFile.patch
$(PATCH) < 09-TransmitFile/0003-ws2_32-Implement-a-basic-synchronous-TransmitFile.patch
@ -286,7 +286,7 @@ abort:
# | Modified files:
# | * dlls/advapi32/tests/security.c, server/process.c, server/security.h, server/token.c
# |
13-Misc_ACL.ok: 02-ACL_Extended_Attributes.ok
13-Misc_ACL.ok:
$(PATCH) < 13-Misc_ACL/0001-server-Add-default-security-descriptor-ownership-for.patch
$(PATCH) < 13-Misc_ACL/0002-server-Add-default-security-descriptor-DACL-for-proc.patch
( \
@ -338,7 +338,7 @@ abort:
# | * dlls/advapi32/tests/security.c, dlls/kernel32/process.c, server/object.c, server/object.h, server/process.c,
# | server/protocol.def
# |
16-server-CreateProcess_ACLs.ok: 09-TransmitFile.ok 13-Misc_ACL.ok
16-server-CreateProcess_ACLs.ok:
$(PATCH) < 16-server-CreateProcess_ACLs/0001-server-A-new-function-set_sd_defaults_from_token-try.patch
$(PATCH) < 16-server-CreateProcess_ACLs/0002-server-Support-sending-process-and-thread-security-d.patch
$(PATCH) < 16-server-CreateProcess_ACLs/0003-server-implement-passing-a-process-security-descript.patch