Rewrite of patch system to simplify mainting large patchsets.

This commit is contained in:
Sebastian Lackner
2014-07-25 22:08:08 +02:00
79 changed files with 1722 additions and 545 deletions

1
debian/changelog vendored
View File

@ -1,4 +1,5 @@
wine-compholio (1.7.23) UNRELEASED; urgency=low
* Rewrite of patch system to simplify mainting large patchsets.
* Fix failing Junction Point test.
* Fix possible race conditions in strmbase/quartz.
* Fix race condition between EndOfStream and Pause.

View File

@ -1,4 +1,23 @@
#!/usr/bin/env bash
#
# Wrapper to apply binary patches without git.
#
# Copyright (C) 2014 Sebastian Lackner
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
#
# Setup parser variables
nogit=0
@ -41,6 +60,12 @@ usage()
echo ""
}
gitsha1()
{
echo -en "blob $(du -b "$1" | cut -f1)\x00" | cat - "$1" | sha1sum | cut -d' ' -f1
}
# Parse environment variables
while [[ $# > 0 ]]; do
cmd="$1"; shift
@ -62,7 +87,8 @@ while [[ $# > 0 ]]; do
;;
-R)
abort "Reverse applying patches not supported yet with this patch tool."
echo "Reverse applying patches not supported yet with this tool." >&2
exit 1
;;
--help)
@ -82,10 +108,19 @@ if [ "$nogit" -eq 0 ] && command -v git >/dev/null 2>&1; then
exit 1
fi
# Check for missing depdencies
for dependency in awk chmod cut dd du gzip hexdump patch sha1sum; do
if ! command -v "$dependency" >/dev/null 2>&1; then
echo "Missing dependency: $dependency - please install this program and try again." >&2
exit 1
fi
done
# Detect BSD
if gzip -V 2>&1 | grep "BSD" &> /dev/null; then
echo "This script is not compatible with *BSD utilities." >&2
echo "Please install git, which provides the same functionality and will be used instead." >&2
exit 1;
echo "This script is not compatible with *BSD utilities. Please install git," >&2
echo "which provides the same functionality and will be used instead." >&2
exit 1
fi
# Decode base85 git data, prepend with a gzip header
@ -194,7 +229,7 @@ while IFS= read -r line; do
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^old\ mode ]] || [[ "$line" =~ ^deleted\ file\ mode ]]; then
elif [ "${line:0:8}" == "old mode" ] || [ "${line:0:17}" == "deleted file mode" ]; then
# ignore
echo "$line" >> "$patch_tmpfile"
continue
@ -204,31 +239,25 @@ while IFS= read -r line; do
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^new\ mode ]] || [[ "$line" =~ ^new\ file\ mode ]]; then
patch_errors+=("$lineno: unable to parse header line '$line'")
elif [ "${line:0:8}" == "new mode" ] || [ "${line:0:13}" == "new file mode" ]; then
patch_errors+=("$lineno: Unable to parse header line '$line'.")
patch_invalid=1
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^copy\ from ]] || [[ "$line" =~ ^copy\ to ]]; then
patch_errors+=("$lineno: copy header not implemented yet")
elif [ "${line:0:9}" == "copy from" ] || [ "${line:0:7}" == "copy to" ]; then
patch_errors+=("$lineno: Copy header not implemented yet.")
patch_invalid=1
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^rename\ old ]] || [[ "$line" =~ ^rename\ from ]]; then
patch_errors+=("$lineno: rename header not implemented yet")
elif [ "${line:0:7}" == "rename " ]; then
patch_errors+=("$lineno: Patch rename header not implemented yet.")
patch_invalid=1
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^rename\ new ]] || [[ "$line" =~ ^rename\ to ]]; then
patch_errors+=("$lineno: rename header not implemented yet")
patch_invalid=1
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^similarity\ index ]] || [[ "$line" =~ ^dissimilarity\ index ]]; then
elif [ "${line:0:16}" == "similarity index" ] || [ "${line:0:19}" == "dissimilarity index" ]; then
# ignore
echo "$line" >> "$patch_tmpfile"
continue
@ -239,8 +268,8 @@ while IFS= read -r line; do
echo "$line" >> "$patch_tmpfile"
continue
elif [[ "$line" =~ ^index\ ]]; then
patch_errors+=("$lineno: unable to parse header line '$line'")
elif [ "${line:0:6}" == "index " ]; then
patch_errors+=("$lineno: Unable to parse header line '$line'.")
patch_invalid=1
echo "$line" >> "$patch_tmpfile"
continue
@ -250,12 +279,12 @@ while IFS= read -r line; do
if [[ "$patch_oldname" =~ ^a/(.*)$ ]]; then
patch_oldname="${BASH_REMATCH[1]}"
elif [ "$patch_oldname" != "/dev/null" ]; then
abort "old name doesn't start with a/."
abort "Old name doesn't start with a/."
fi
if [[ "$patch_newname" =~ ^b/(.*)$ ]]; then
patch_newname="${BASH_REMATCH[1]}"
elif [ "$patch_newname" != "/dev/null" ]; then
abort "new name doesn't start with b/."
abort "New name doesn't start with b/."
fi
patch_mode=2
@ -269,12 +298,12 @@ while IFS= read -r line; do
if [[ "$line" == "GIT binary patch" ]]; then
if [ -z "$patch_oldsha1" ] || [ -z "$patch_newsha1" ]; then
patch_errors+=("$lineno: missing index header, sha1 sums required for binary patch")
patch_errors+=("$lineno: Missing index header, sha1 sums required for binary patch.")
patch_invalid=1
fi
if [ "$patch_oldname" != "$patch_newname" ]; then
patch_errors+=("$lineno: stripped old- and new name doesn't match")
patch_errors+=("$lineno: Stripped old- and new name doesn't match for binary patch.")
patch_invalid=1
fi
@ -286,7 +315,7 @@ while IFS= read -r line; do
patch_mode=100
continue
elif [[ "$line" =~ ^@@\ - ]]; then
elif [ "${line:0:4}" == "@@ -" ]; then
# We count the number of lines added/removed for informational purposes
patch_total_add=0
patch_total_rem=0
@ -294,10 +323,10 @@ while IFS= read -r line; do
patch_mode=200
# fall-through
elif [[ "$line" =~ ^diff\ --git\ ]]; then
elif [ "${line:0:11}" == "diff --git " ]; then
if [ "$patch_oldname" != "$patch_newname" ]; then
patch_errors+=("$lineno: stripped old- and new name doesn't match")
patch_errors+=("$lineno: Stripped old- and new name doesn't match.")
patch_invalid=1
fi
@ -328,7 +357,7 @@ while IFS= read -r line; do
# Check shasum if its not a patch creating a new file
if [ "$patch_oldsha1" != "0000000000000000000000000000000000000000" ] || [ "$binary_patch_type" == "delta" ] || [ -f "$patch_oldname" ]; then
if [ -f "$patch_oldname" ]; then
sha=$(echo -en "blob $(du -b "$patch_oldname" | cut -f1)\x00" | cat - "$patch_oldname" | sha1sum | cut -d' ' -f1)
sha=$(gitsha1 "$patch_oldname")
else
sha="0000000000000000000000000000000000000000"
fi
@ -403,13 +432,13 @@ while IFS= read -r line; do
while read cmd arg1 arg2; do
if [ "$cmd" == "S" ]; then
binary_patch_destsize="$arg2"
if [ "$arg1" -ne "$(du -b "$patch_oldname" | cut -f 1)" ]; then break; fi
[ "$arg1" -eq "$(du -b "$patch_oldname" | cut -f 1)" ] || break
elif [ "$cmd" == "1" ]; then
if ! dd if="$patch_oldname" bs=1 skip="$arg1" count="$arg2" >> "$decoded_tmpfile" 2>/dev/null; then break; fi
dd if="$patch_oldname" bs=1 skip="$arg1" count="$arg2" >> "$decoded_tmpfile" 2>/dev/null || break
elif [ "$cmd" == "2" ]; then
if ! dd if="$patch_tmpfile" bs=1 skip="$arg1" count="$arg2" >> "$decoded_tmpfile" 2>/dev/null; then break; fi
dd if="$patch_tmpfile" bs=1 skip="$arg1" count="$arg2" >> "$decoded_tmpfile" 2>/dev/null || break
elif [ "$cmd" == "E" ]; then
binary_patch_complete=1
@ -432,7 +461,7 @@ while IFS= read -r line; do
fi
# Check shasum if its not a patch creating a new file
sha=$(echo -en "blob $(du -b "$patch_tmpfile" | cut -f1)\x00" | cat - "$patch_tmpfile" | sha1sum | cut -d' ' -f1)
sha=$(gitsha1 "$patch_tmpfile")
if [ "$patch_newsha1" != "$sha" ]; then
echo "$lineno: Expected $patch_newsha1" >&2
echo "$lineno: Got $sha" >&2
@ -440,7 +469,7 @@ while IFS= read -r line; do
fi
if ! cp "$patch_tmpfile" "$patch_oldname"; then
abort "Unable to replace original file"
abort "Unable to replace original file."
fi
if [ ! -z "$patch_filemode" ]; then
chmod "${patch_filemode: -3}" "$patch_oldname" # we ignore failures for now
@ -471,6 +500,11 @@ while IFS= read -r line; do
patch_mode=201
continue
elif [ "${line:0:2}" == "\\ " ]; then
# ignore
echo "$line" >> "$patch_tmpfile"
continue
else
echo "patching $patch_newname"
@ -489,23 +523,23 @@ while IFS= read -r line; do
# These lines are part of a hunk, append it
echo "$line" >> "$patch_tmpfile"
if [ "$hunk_src_lines" -gt 0 ] && [ "$hunk_dst_lines" -gt 0 ] && [[ "$line" =~ ^\ ]]; then
if [ "${line:0:1}" == " " ] && [ "$hunk_src_lines" -gt 0 ] && [ "$hunk_dst_lines" -gt 0 ]; then
(( hunk_src_lines-- ))
(( hunk_dst_lines-- ))
elif [ "$hunk_src_lines" -gt 0 ] && [[ "$line" =~ ^- ]]; then
elif [ "${line:0:1}" == "-" ] && [ "$hunk_src_lines" -gt 0 ]; then
(( hunk_src_lines-- ))
(( patch_total_rem++ ))
elif [ "$hunk_dst_lines" -gt 0 ] && [[ "$line" =~ ^\+ ]]; then
elif [ "${line:0:1}" == "+" ] && [ "$hunk_dst_lines" -gt 0 ]; then
(( hunk_dst_lines-- ))
(( patch_total_add++ ))
elif [[ "$line" =~ ^\\\ ]]; then
elif [ "${line:0:2}" == "\\ " ]; then
continue # ignore "\\ No newline ..."
else
abort "Unexpected line in hunk"
abort "Unexpected line in hunk."
fi
# If it was the last line of this hunk then go back to mode 200
@ -544,7 +578,7 @@ while IFS= read -r line; do
patch_mode=1
continue
elif [[ "$line" =~ ^@@\ - ]] || [[ "$line" =~ ^---\ ]] || [[ "$line" =~ ^\+\+\+\ ]]; then
elif [ "${line:0:4}" == "@@ -" ] || [ "${line:0:4}" == "--- " ] || [ "${line:0:4}" == "+++ " ]; then
abort "Patch corrupted or not created with git."
fi
fi

View File

@ -1,21 +1,5 @@
#!/bin/sh
PATCH_DATA="";
for FILE in patches/*/*.def; do
UUID=$(echo "${FILE}" | sed -e 's|^.*/||g' -e 's|\.def$||g');
REVISION=$(cat "${FILE}" | sed -n 's|Revision: \(.*\)|\1|p');
AUTHOR=$(cat "${FILE}" | sed -n 's|Author: \(.*\)|\1|p');
TITLE=$(cat "${FILE}" | sed -n 's|Title: \(.*\)|\1|p');
if [ "${AUTHOR}" = "" ] && [ "${TITLE}" = "" ]; then
continue;
fi
if [ "${PATCH_DATA}" != "" ]; then
PATCH_DATA="${PATCH_DATA}
";
fi
PATCH_DATA="${PATCH_DATA}+ { \"${UUID}:${REVISION}\", \"${AUTHOR}\", \"${TITLE}\" },";
done
PATCH_DATA=$(cat);
PATCH_LINES=$(echo "${PATCH_DATA}" | wc -l);
PATCH_LINES=$((${PATCH_LINES}+20));

570
debian/tools/patchupdate.py vendored Executable file

File diff suppressed because it is too large Load Diff

320
debian/tools/patchutils.py vendored Normal file
View File

@ -0,0 +1,320 @@
#!/usr/bin/python
#
# Python functions to read, split and apply patches.
#
# Copyright (C) 2014 Sebastian Lackner
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
#
import collections
import difflib
import hashlib
import itertools
import os
import re
import subprocess
import tempfile
class PatchParserError(RuntimeError):
"""Unable to parse patch file - either an unimplemented feature, or corrupted patch."""
pass
class PatchApplyError(RuntimeError):
"""Failed to apply/merge patch."""
pass
class PatchObject(object):
def __init__(self, filename):
self.extracted_patch = None
self.unique_hash = None
self.filename = filename
self.offset_begin = None
self.offset_end = None
self.isbinary = False
self.oldname = None
self.newname = None
self.modified_file = None
self.oldsha1 = None
self.newsha1 = None
self.newmode = None
def is_binary(self):
return self.isbinary
def read_chunks(self):
"""Iterates over arbitrary sized chunks of this patch."""
assert self.offset_end >= self.offset_begin
with open(self.filename) as fp:
fp.seek(self.offset_begin)
i = self.offset_end - self.offset_begin
while i > 0:
buf = fp.read(4096 if i > 4096 else i)
if buf == "": raise IOError("Unable to extract patch.")
yield buf
i -= len(buf)
def extract(self):
"""Create a temporary file containing the extracted patch."""
if not self.extracted_patch:
self.extracted_patch = tempfile.NamedTemporaryFile()
for chunk in self.read_chunks():
self.extracted_patch.write(chunk)
self.extracted_patch.flush()
return self.extracted_patch
def hash(self):
"""Hash the content of the patch."""
if not self.unique_hash:
m = hashlib.sha256()
for chunk in self.read_chunks():
m.update(chunk)
self.unique_hash = m.digest()
return self.unique_hash
def read_patch(filename):
"""Iterates over all patches contained in a file, and returns PatchObject objects."""
class _FileReader(object):
def __init__(self, filename):
self.filename = filename
self.fp = open(self.filename)
self.peeked = None
def close(self):
self.fp.close()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def seek(self, pos):
"""Change the file cursor position."""
self.fp.seek(pos)
self.peeked = None
def tell(self):
"""Return the current file cursor position."""
if self.peeked is None:
return self.fp.tell()
return self.peeked[0]
def peek(self):
"""Read one line without changing the file cursor."""
if self.peeked is None:
pos = self.fp.tell()
tmp = self.fp.readline()
if len(tmp) == 0: return None
self.peeked = (pos, tmp)
return self.peeked[1]
def read(self):
"""Read one line from the file, and move the file cursor to the next line."""
if self.peeked is None:
tmp = self.fp.readline()
if len(tmp) == 0: return None
return tmp
tmp, self.peeked = self.peeked, None
return tmp[1]
def _read_single_patch(fp, oldname=None, newname=None):
"""Internal function to read a single patch from a file."""
patch = PatchObject(fp.filename)
patch.offset_begin = fp.tell()
patch.oldname = oldname
patch.newname = newname
# Skip over initial diff --git header
line = fp.peek()
if line.startswith("diff --git "):
assert fp.read() == line
# Read header
while True:
line = fp.peek()
if line is None:
break
elif line.startswith("--- "):
patch.oldname = line[4:].strip()
elif line.startswith("+++ "):
patch.newname = line[4:].strip()
elif line.startswith("old mode") or line.startswith("deleted file mode"):
pass # ignore
elif line.startswith("new mode "):
patch.newmode = line[9:].strip()
elif line.startswith("new file mode "):
patch.newmode = line[14:].strip()
elif line.startswith("new mode") or line.startswith("new file mode"):
raise PatchParserError("Unable to parse header line '%s'." % line)
elif line.startswith("copy from") or line.startswith("copy to"):
raise NotImplementedError("Patch copy header not implemented yet.")
elif line.startswith("rename "):
raise NotImplementedError("Patch rename header not implemented yet.")
elif line.startswith("similarity index") or line.startswith("dissimilarity index"):
pass # ignore
elif line.startswith("index "):
r = re.match("^index ([a-fA-F0-9]*)\.\.([a-fA-F0-9]*)", line)
if not r: raise PatchParserError("Unable to parse index header line '%s'." % line)
patch.oldsha1, patch.newsha1 = r.group(1), r.group(2)
else:
break
assert fp.read() == line
if patch.oldname is None or patch.newname is None:
raise PatchParserError("Missing old or new name.")
elif patch.oldname == "/dev/null" and patch.newname == "/dev/null":
raise PatchParserError("Old and new name is /dev/null?")
if patch.oldname.startswith("a/"):
patch.oldname = patch.oldname[2:]
elif patch.oldname != "/dev/null":
raise PatchParserError("Old name in patch doesn't start with a/.")
if patch.newname.startswith("b/"):
patch.newname = patch.newname[2:]
elif patch.newname != "/dev/null":
raise PatchParserError("New name in patch doesn't start with b/.")
if patch.newname != "/dev/null":
patch.modified_file = patch.newname
else:
patch.modified_file = patch.oldname
# Decide between binary and textual patch
if line is None or line.startswith("diff --git ") or line.startswith("--- "):
if oldname != newname:
raise PatchParserError("Stripped old- and new name doesn't match.")
elif line.startswith("@@ -"):
while True:
line = fp.peek()
if line is None or not line.startswith("@@ -"):
break
r = re.match("^@@ -(([0-9]+),)?([0-9]+) \+(([0-9]+),)?([0-9]+) @@", line)
if not r: raise PatchParserError("Unable to parse hunk header '%s'." % line)
srcpos = max(int(r.group(2)) - 1, 0) if r.group(2) else 0
dstpos = max(int(r.group(5)) - 1, 0) if r.group(5) else 0
srclines, dstlines = int(r.group(3)), int(r.group(6))
if srclines <= 0 and dstlines <= 0:
raise PatchParserError("Empty hunk doesn't make sense.")
assert fp.read() == line
while srclines > 0 or dstlines > 0:
line = fp.read()
if line is None:
raise PatchParserError("Truncated patch.")
elif line.startswith(" "):
if srclines == 0 or dstlines == 0:
raise PatchParserError("Corrupted patch.")
srclines -= 1
dstlines -= 1
elif line.startswith("-"):
if srclines == 0:
raise PatchParserError("Corrupted patch.")
srclines -= 1
elif line.startswith("+"):
if dstlines == 0:
raise PatchParserError("Corrupted patch.")
dstlines -= 1
elif line.startswith("\\ "):
pass # ignore
else:
raise PatchParserError("Unexpected line in hunk.")
while True:
line = fp.peek()
if line is None or not line.startswith("\\ "): break
assert fp.read() == line
elif line.rstrip() == "GIT binary patch":
if patch.oldsha1 is None or patch.newsha1 is None:
raise PatchParserError("Missing index header, sha1 sums required for binary patch.")
elif patch.oldname != patch.newname:
raise PatchParserError("Stripped old- and new name doesn't match for binary patch.")
assert fp.read() == line
line = fp.read()
if line is None: raise PatchParserError("Unexpected end of file.")
r = re.match("^(literal|delta) ([0-9]+)", line)
if not r: raise NotImplementedError("Only literal/delta patches are supported.")
patch.isbinary = True
# Skip over patch data
while True:
line = fp.read()
if line is None or line.strip() == "":
break
else:
raise PatchParserError("Unknown patch format.")
patch.offset_end = fp.tell()
return patch
with _FileReader(filename) as fp:
while True:
line = fp.peek()
if line is None:
break
elif line.startswith("diff --git "):
tmp = line.strip().split(" ")
if len(tmp) != 4: raise PatchParserError("Unable to parse git diff header line '%s'." % line)
yield _read_single_patch(fp, tmp[2].strip(), tmp[3].strip())
elif line.startswith("--- "):
yield _read_single_patch(fp)
elif line.startswith("@@ -") or line.startswith("+++ "):
raise PatchParserError("Patch didn't start with a git or diff header.")
else:
assert fp.read() == line
def apply_patch(content, patches, reverse=False, fuzz=2):
"""Apply a patch with optional fuzz - uses the commandline 'patch' utility."""
if not isinstance(patches, collections.Sequence):
patches = [patches]
contentfile = tempfile.NamedTemporaryFile(delete=False)
try:
contentfile.write(content)
contentfile.close()
for patch in patches:
patchfile = patch.extract()
cmdline = ["patch", "--batch", "--silent", "-r", "-"]
if reverse: cmdline.append("--reverse")
if fuzz != 2: cmdline.append("--fuzz=%d" % fuzz)
cmdline += [contentfile.name, patchfile.name]
with open(os.devnull, 'w') as devnull:
exitcode = subprocess.call(cmdline, stdout=devnull, stderr=devnull)
if exitcode != 0:
raise PatchApplyError("Failed to apply patch (exitcode %d)." % exitcode)
with open(contentfile.name) as fp:
content = fp.read()
finally:
os.unlink(contentfile.name)
return content