wine/wine-staging
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-staging.git synced 2024-11-21 16:46:54 -08:00
Code Issues Packages Projects Releases Wiki Activity

Added kernel32-NormalizeString patchset

Browse Source
This commit is contained in:
Alistair Leslie-Hughes 2018-10-25 08:47:22 +11:00
parent f3f4f80a01
commit 2802df41dc
7 changed files with 5707 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

488
patches/kernel32-NormalizeString/0001-tools-make_unicode-Implement-full-Unicode-character-.patch Normal file
View File

@ -0,0 +1,488 @@
From 83053c7a7379900ac0ca1922e7551298848216d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Del=20Real?=
<sdelreal@codeweavers.com>
Date: Wed, 4 Apr 2018 10:49:39 -0500
Subject: [PATCH 1/3] tools/make_unicode: Implement full Unicode character
decomposition.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Sergio Gómez Del Real <sdelreal@codeweavers.com>
---
libs/port/mbtowc.c | 14 +-
tools/make_unicode | 333 ++++++++++++++++++++++++++++++++++++---------
2 files changed, 279 insertions(+), 68 deletions(-)
diff --git a/libs/port/mbtowc.c b/libs/port/mbtowc.c
index 4977c82d8b..6976f89722 100644
--- a/libs/port/mbtowc.c
+++ b/libs/port/mbtowc.c
@@ -22,7 +22,7 @@
#include "wine/unicode.h"
-extern unsigned int wine_decompose( WCHAR ch, WCHAR *dst, unsigned int dstlen ) DECLSPEC_HIDDEN;
+extern int wine_unicode_decompose_string( int compat, const WCHAR *src, int srclen, WCHAR *dst, int dstlen );
/* check the code whether it is in Unicode Private Use Area (PUA). */
/* MB_ERR_INVALID_CHARS raises an error converting from 1-byte character to PUA. */
@@ -101,19 +101,19 @@ static int mbstowcs_sbcs_decompose( const struct sbcs_table *table, int flags,
WCHAR *dst, unsigned int dstlen )
{
const WCHAR * const cp2uni = (flags & MB_USEGLYPHCHARS) ? table->cp2uni_glyphs : table->cp2uni;
- unsigned int len;
+ int len;
if (!dstlen) /* compute length */
{
WCHAR dummy[4]; /* no decomposition is larger than 4 chars */
for (len = 0; srclen; srclen--, src++)
- len += wine_decompose( cp2uni[*src], dummy, 4 );
+ len += wine_unicode_decompose_string( 0, &cp2uni[*src], 1, dummy, 4 );
return len;
}
for (len = dstlen; srclen && len; srclen--, src++)
{
- unsigned int res = wine_decompose( cp2uni[*src], dst, len );
+ int res = wine_unicode_decompose_string( 0, &cp2uni[*src], 1, dst, len );
if (!res) break;
len -= res;
dst += res;
@@ -203,7 +203,7 @@ static int mbstowcs_dbcs_decompose( const struct dbcs_table *table,
{
const WCHAR * const cp2uni = table->cp2uni;
const unsigned char * const cp2uni_lb = table->cp2uni_leadbytes;
- unsigned int len, res;
+ int len, res;
WCHAR ch;
if (!dstlen) /* compute length */
@@ -219,7 +219,7 @@ static int mbstowcs_dbcs_decompose( const struct dbcs_table *table,
ch = cp2uni[(off << 8) + *src];
}
else ch = cp2uni[*src];
- len += wine_decompose( ch, dummy, 4 );
+ len += wine_unicode_decompose_string( 0, &ch, 1, dummy, 4 );
}
return len;
}
@@ -234,7 +234,7 @@ static int mbstowcs_dbcs_decompose( const struct dbcs_table *table,
ch = cp2uni[(off << 8) + *src];
}
else ch = cp2uni[*src];
- if (!(res = wine_decompose( ch, dst, len ))) break;
+ if (!(res = wine_unicode_decompose_string( 0, &ch, 1, dst, len ))) break;
dst += res;
len -= res;
}
diff --git a/tools/make_unicode b/tools/make_unicode
index 56d1905656..2aa063bff6 100755
--- a/tools/make_unicode
+++ b/tools/make_unicode
@@ -472,6 +472,28 @@ sub READ_DEFAULTS($)
next if $decomp eq ""; # no decomposition, skip it
+ # store decomposition table
+ if ($decomp =~ /^<([a-zA-Z]+)>(\s+[0-9a-fA-F]+)+$/)
+ {
+ my @seq = ();
+ for my $ch (split /\s+/, (split /\s+/, $decomp, 2)[1])
+ {
+ push @seq, (hex $ch);
+ }
+ $decomp_table[$src] = [1, \@seq];
+ }
+ elsif ($decomp =~ /^([0-9a-fA-F]+)(\s+([0-9a-fA-F]+))*$/)
+ {
+ my @seq = ();
+ for my $ch (split /\s+/, $decomp)
+ {
+ # we don't support surrogates at the moment
+ next if hex $ch > 0xffff;
+ push @seq, (hex $ch);
+ }
+ $decomp_table[$src] = [0, \@seq];
+ }
+
if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
{
# decomposition of the form "<foo> 1234" -> use char if type is known
@@ -506,16 +528,13 @@ sub READ_DEFAULTS($)
# decomposition contains only char values without prefix -> use first char
$dst = hex $1;
$category_table[$src] |= $category_table[$dst] if defined $category_table[$dst];
- # store decomposition if it contains two chars
if ($decomp =~ /^([0-9a-fA-F]+)\s+([0-9a-fA-F]+)$/)
{
- $decomp_table[$src] = [ hex $1, hex $2 ];
push @compose_table, [ hex $1, hex $2, $src ];
}
elsif ($decomp =~ /^(<[a-z]+>\s)*([0-9a-fA-F]+)$/ &&
(($src >= 0xf900 && $src < 0xfb00) || ($src >= 0xfe30 && $src < 0xfffd)))
{
- # Single char decomposition in the compatibility range
$compatmap_table[$src] = hex $2;
}
}
@@ -2264,6 +2283,51 @@ EOF
save_file($filename);
}
+sub do_decomp
+{
+ my ($char, $table_ref, $compat) = @_;
+
+ return ($char) unless defined $table_ref->[$char];
+ my $data = $table_ref->[$char];
+ return ($char) if $data->[0] && !$compat;
+ my @mapping = ();
+ for my $ch (@{$data->[1]})
+ {
+ push @mapping, $ch;
+ }
+ return @mapping;
+}
+
+sub expand_pairs
+{
+ my @data = @_;
+ my @result = ();
+
+ for my $ch (@data)
+ {
+ if ($ch <= 0xFFFF)
+ {
+ push @result, $ch;
+ }
+ elsif ($ch >= 2097152) # 2**21
+ {
+ die sprintf "Invalid Unicode character %04x\n", $ch;
+ }
+ else
+ {
+ my $hx = $ch & 0xFFFF;
+ my $hu = ($ch >> 16) & ((1 << 5) - 1);
+ my $hw = ($hu - 1) & 0xFFFF;
+ my $hi = 0xD800 | ($hw << 6) | ($hx >> 10);
+ my $lx = $ch & 0xFFFF;
+ my $lo = (0xDC00 | ($lx & ((1 << 10) - 1))) & 0xFFFF;
+ push @result, $hi;
+ push @result, $lo;
+ }
+ }
+ return @result;
+}
+
################################################################
# dump the char decomposition table
sub dump_decompose_table($)
@@ -2272,98 +2336,245 @@ sub dump_decompose_table($)
open OUTPUT,">$filename.new" or die "Cannot create $filename";
print "Building $filename\n";
- print OUTPUT "/* Unicode char composition */\n";
+ print OUTPUT "/* Unicode char decomposition */\n";
print OUTPUT "/* generated from $UNIDATA/UnicodeData.txt */\n";
print OUTPUT "/* DO NOT EDIT!! */\n\n";
print OUTPUT "#include \"wine/unicode.h\"\n\n";
- # first determine all the 16-char subsets that contain something
+ # limit code points to BMP
+ my $utflim = 65536;
+ my %nfd_lookup = ();
+ my %nfkd_lookup = ();
+ my %decomp_lookup = ();
+ my @decomp_data = (0);
+ my $pos = 1;
+ my $lastchar_decomp;
- my @filled = (0) x 4096;
- my $pos = 16*2; # for the null subset
- for (my $i = 0; $i < 65536; $i++)
+ for (my $i = 0; $i < $utflim; $i++)
{
next unless defined $decomp_table[$i];
- $filled[$i >> 4] = $pos;
- $pos += 16*2;
- $i |= 15;
+
+ if (defined $decomp_table[$i])
+ {
+ $lastchar_decomp = $i;
+ # fully expand input and mappings
+
+ my @char;
+ push @char, $i;
+ push @char, 0;
+ my $char = pack "n*", @char;
+
+ my @nfd = do_decomp( $i, \@decomp_table, 0 );
+ push @nfd, 0;
+ my $nfd = pack "n*", @nfd;
+
+ my @nfkd = do_decomp( $i, \@decomp_table, 1 );
+ push @nfkd, 0;
+ my $nfkd = pack "n*", @nfkd;
+
+ # lookup or add mappings
+
+ if ($nfd eq $char)
+ {
+ $nfd = undef;
+ }
+ elsif (exists $decomp_lookup{$nfd})
+ {
+ $nfd_lookup{$i} = $decomp_lookup{$nfd};
+ }
+ else
+ {
+ push @decomp_data, @nfd;
+ $decomp_lookup{$nfd} = $pos;
+ $nfd_lookup{$i} = $pos;
+ $pos += @nfd;
+ }
+
+ if ($nfkd eq $char)
+ {
+ $nfkd = undef;
+ }
+ elsif (exists $decomp_lookup{$nfkd})
+ {
+ $nfkd_lookup{$i} = $decomp_lookup{$nfkd};
+ }
+ else
+ {
+ push @decomp_data, @nfkd;
+ $decomp_lookup{$nfkd} = $pos;
+ $nfkd_lookup{$i} = $pos;
+ $pos += @nfkd;
+ }
+ }
}
- my $total = $pos;
- # now count the 256-char subsets that contain something
+ printf OUTPUT "static const UINT last_decomposable = 0x%x;\n\n", $lastchar_decomp;
- my @filled_idx = (256) x 256;
- $pos = 256 + 16;
- for (my $i = 0; $i < 4096; $i++)
+ # dump decomposition data
+
+ printf OUTPUT "static const WCHAR data_decomp[%d] =\n", $pos;
+ print OUTPUT "{\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @decomp_data );
+ print OUTPUT "\n};\n\n";
+
+ # find 256-char subsets that contain something
+
+ my $filled_pos = 1;
+ my $filled_lim = ($lastchar_decomp >> 8) + 1;
+ my @filled = (0) x $filled_lim;
+ for (my $i = 0; $i < $utflim; $i++)
+ {
+ last if $i > $lastchar_decomp;
+ next unless exists $nfd_lookup{$i} || exists $nfkd_lookup{$i};
+ $filled[$i >> 8] = $filled_pos++;
+ $i |= 255;
+ }
+
+ # dump index of 256-char subsets
+
+ printf OUTPUT "static const BYTE idx1_decomp[%d] =\n", $filled_lim;
+ print OUTPUT "{\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%02x", 0, @filled );
+ print OUTPUT "\n};\n\n";
+
+ # for 256-char subsets, find non-empty 16-char subsets
+
+ my $sub_filled_pos = 1;
+ my %sub_filled = ();
+ for (my $i = 0; $i < $filled_lim; $i++)
{
next unless $filled[$i];
- $filled_idx[$i >> 4] = $pos;
- $pos += 16;
- $i |= 15;
+ for (my $j = 0; $j < 256; $j++)
+ {
+ my $idx = ($i << 8) | $j;
+ next unless exists $nfd_lookup{$idx} || exists $nfkd_lookup{$idx};
+ $sub_filled{$idx >> 4} = $sub_filled_pos++;
+ $j |= 15;
+ }
}
- my $null_offset = $pos; # null mapping
- $total += $pos;
- # add the index offsets to the subsets positions
+ # dump index of 16-char subsets
- for (my $i = 0; $i < 4096; $i++)
+ printf OUTPUT "static const USHORT idx2_decomp[%d] =\n", $filled_pos * 16;
+ print OUTPUT "{\n";
+ my @null_idx = (0) x 16;
+ print OUTPUT " /* null sub-index */\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @null_idx );
+ for (my $i = 0; $i < $filled_lim; $i++)
{
next unless $filled[$i];
- $filled[$i] += $null_offset;
+ printf OUTPUT ",\n /* sub-index 0x%02x */\n", $filled[$i];
+
+ my @sub_idx;
+ for (my $j = 0; $j < 16; $j++)
+ {
+ my $idx = ($i << 4) | $j;
+ $sub_idx[$j] = $sub_filled{$idx} || 0;
+ }
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @sub_idx );
}
+ print OUTPUT "\n};\n\n";
- # dump the main index
+ # dump the 16-char subsets
- printf OUTPUT "static const WCHAR table[%d] =\n", $total;
- printf OUTPUT "{\n /* index */\n";
- printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @filled_idx );
- printf OUTPUT ",\n /* null sub-index */\n%s", DUMP_ARRAY( "0x%04x", 0, ($null_offset) x 16 );
+ printf OUTPUT "static const USHORT offsets_decomp[%d] =\n", 32 * $sub_filled_pos;
+ print OUTPUT "{\n";
+ print OUTPUT " /* (nfd, nfkd) x 16 */\n";
+ my @null_table = (0) x 32;
+ print OUTPUT " /* no decomposition */\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @null_table );
+ for my $key (sort {$a <=> $b} keys %sub_filled)
+ {
+ printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $key, $key;
+ my @sub_table;
+ for (my $j = 0; $j < 16; $j++)
+ {
+ my $idx = ($key << 4) | $j;
+ $sub_table[2 * $j] = $nfd_lookup{$idx} || 0;
+ $sub_table[2 * $j + 1] = $nfkd_lookup{$idx} || 0;
+ }
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @sub_table );
+ }
+ print OUTPUT "\n};\n\n";
- # dump the second-level indexes
+ print OUTPUT <<"EOF";
+static const WCHAR *unicode_table_lookup( UINT cp, int compat, const BYTE *idx1, UINT scale_idx1,
+ const USHORT *idx2, UINT scale_idx2, const USHORT *offsets,
+ UINT scale_off, const WCHAR *data, UINT scale_data )
+{
+ USHORT a, b, c, d;
- for (my $i = 0; $i < 256; $i++)
+ a = idx1[cp >> scale_idx1];
+ b = idx2[(a << scale_idx2) + ((cp >> scale_idx2) & 0xf)];
+ c = (b << scale_off) + ((cp & 0xf) << scale_data);
+ if (compat) ++c;
+ d = offsets[c];
+
+ return &data[d];
+}
+
+static int decompose_char_recursive( int compat, UINT ch, WCHAR *dst, int dstlen )
+{
+ int total_decomp = 0;
+ int size_decomp;
+ const WCHAR *map;
+
+ if (ch < 0xa0) /* fast path */
{
- next unless ($filled_idx[$i] > 256);
- my @table = @filled[($i<<4)..($i<<4)+15];
- for (my $j = 0; $j < 16; $j++) { $table[$j] ||= $null_offset; }
- printf OUTPUT ",\n /* sub-index %02x */\n", $i;
- printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
+ if (dstlen) *dst = (WCHAR)ch;
+ return 1;
+ }
+ else if (ch > last_decomposable ||
+ !*(map = unicode_table_lookup( ch, compat, idx1_decomp, 8,
+ idx2_decomp, 4, offsets_decomp, 5, data_decomp, 1 )))
+ {
+ if (dstlen) *dst = (WCHAR)ch;
+ return 1;
}
+ else {
+ while (*map)
+ {
+ size_decomp = decompose_char_recursive( compat, *map, dst, dstlen );
+ dstlen -= size_decomp;
+ if (dstlen < 0) dstlen = 0;
+ dst += size_decomp;
+ map++;
+ total_decomp += size_decomp;
+ }
+ return total_decomp;
+ }
+}
- # dump the 16-char subsets
+int wine_unicode_decompose_string( int compat, const WCHAR *src,
+ int srclen, WCHAR *dst, int dstlen )
+{
+ UINT ch;
+ int srcpos = 0, dstpos = 0;
+ int decomp_len;
- printf OUTPUT ",\n /* null mapping */\n";
- printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 32 );
+ if (dstlen < 0) dstlen = 0;
- for (my $i = 0; $i < 4096; $i++)
+ while (srcpos < srclen)
{
- next unless $filled[$i];
- my @table = (0) x 32;
- for (my $j = 0; $j < 16; $j++)
+ ch = src[srcpos];
+
+ decomp_len = decompose_char_recursive( compat, ch, dst+dstpos, dstlen );
+ dstpos += decomp_len;
+
+ if (dstlen > 0)
{
- if (defined $decomp_table[($i<<4) + $j])
+ dstlen -= decomp_len;
+ while (dstlen < 0)
{
- $table[2 * $j] = ${$decomp_table[($i << 4) + $j]}[0];
- $table[2 * $j + 1] = ${$decomp_table[($i << 4) + $j]}[1];
+ dstpos--;
+ dstlen++;
}
}
- printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $i, $i;
- printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
+
+ ++srcpos;
}
- printf OUTPUT "\n};\n\n";
- print OUTPUT <<"EOF";
-unsigned int DECLSPEC_HIDDEN wine_decompose( WCHAR ch, WCHAR *dst, unsigned int dstlen )
-{
- const WCHAR *ptr = table + table[table[ch >> 8] + ((ch >> 4) & 0x0f)] + 2 * (ch & 0xf);
- unsigned int res;
-
- *dst = ch;
- if (!*ptr) return 1;
- if (dstlen <= 1) return 0;
- /* apply the decomposition recursively to the first char */
- if ((res = wine_decompose( *ptr, dst, dstlen-1 ))) dst[res++] = ptr[1];
- return res;
+ return dstpos;
}
EOF
close OUTPUT;
--
2.17.1

400
patches/kernel32-NormalizeString/0002-tools-make_unicode-Implement-canonical-composition-f.patch Normal file
View File

@ -0,0 +1,400 @@
From 0f35e862eb3e0d2e230431b8d71c30ac34b6b329 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Del=20Real?=
<sdelreal@codeweavers.com>
Date: Tue, 27 Mar 2018 07:28:02 -0500
Subject: [PATCH 2/3] tools/make_unicode: Implement canonical composition for
use in normalization.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Sergio Gómez Del Real <sdelreal@codeweavers.com>
---
tools/make_unicode | 302 ++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 301 insertions(+), 1 deletion(-)
diff --git a/tools/make_unicode b/tools/make_unicode
index 2aa063bff6..4b1c46fde2 100755
--- a/tools/make_unicode
+++ b/tools/make_unicode
@@ -360,6 +360,8 @@ my @joining_table = ();
my @direction_table = ();
my @decomp_table = ();
my @compose_table = ();
+my @comb_class_table = ();
+my @full_comp_table = ();
my $default_char;
my $default_wchar;
@@ -470,6 +472,11 @@ sub READ_DEFAULTS($)
}
}
+ if ($comb != 0)
+ {
+ $comb_class_table[$src] = (hex $comb);
+ }
+
next if $decomp eq ""; # no decomposition, skip it
# store decomposition table
@@ -562,6 +569,25 @@ sub READ_DEFAULTS($)
my $flag = $ctype{$cat};
foreach my $i (@{$special_categories{$cat}}) { $category_table[$i] |= $flag; }
}
+
+ my $UNICODE_DERIVED = open_data_file( $UNIDATA, "DerivedNormalizationProps.txt" );
+ while (<$UNICODE_DERIVED>)
+ {
+ next unless (/^([0-9a-fA-F.]+)\s+;\s+Full_Composition_Exclusion/);
+ my ($first, $last) = split /\.\./,$1;
+ $first = hex $first;
+ if (defined $last)
+ {
+ $last = hex $last;
+ while ($last gt $first)
+ {
+ $full_comp_table[$last] = 1;
+ $last--;
+ }
+ }
+ $full_comp_table[$first] = 1;
+ }
+ close $UNICODE_DERIVED;
}
@@ -2255,6 +2281,8 @@ sub dump_compose_table($)
}
print OUTPUT "\n};\n\n";
print OUTPUT <<"EOF";
+#include "decompose.c"
+
static inline int binary_search( WCHAR ch, int low, int high )
{
while (low <= high)
@@ -2278,6 +2306,59 @@ WCHAR DECLSPEC_HIDDEN wine_compose( const WCHAR *str )
count = table[2 * pos + 3];
}
}
+
+static inline int is_blocked(WCHAR *ptr1, WCHAR *ptr2)
+{
+ if (ptr1 >= ptr2) return -1;
+
+ while (++ptr1 < ptr2)
+ {
+ const WCHAR *map1, *map2;
+ map1 = unicode_table_lookup( *ptr1, 0, idx1_comb, 8, idx2_comb, 4,
+ offsets_comb, 4, data_comb, 0 );
+ map2 = unicode_table_lookup( *ptr2, 0, idx1_comb, 8, idx2_comb, 4,
+ offsets_comb, 4, data_comb, 0 );
+ if (*map1 == 0 || *map2 <= *map1) return 1;
+ }
+ return 0;
+}
+
+static inline int is_fullexcl(WCHAR ch)
+{
+ const WCHAR *map = unicode_table_lookup( ch, 0, idx1_fullcomp, 8, idx2_fullcomp,
+ 4, offsets_fullcomp, 4, data_fullcomp, 0 );
+ return (int)*map;
+}
+
+int unicode_canonical_composition( WCHAR *str, int strlen )
+{
+ int i, j;
+ WCHAR dum[3] = {0};
+
+ if (strlen == 0) strlen = strlenW( str );
+
+ for (i = 1; i < strlen; i++)
+ {
+ WCHAR *ptr_comp = str+i-1, comp;
+ if (str[i] == 0) break;
+ while (ptr_comp - str > 0)
+ {
+ if (is_starter( *ptr_comp )) break;
+ --ptr_comp;
+ }
+ if (!is_starter( *ptr_comp ) || is_blocked( ptr_comp, str+i )) continue;
+ dum[0] = *ptr_comp;
+ dum[1] = str[i];
+ comp = wine_compose( dum );
+ if (!comp || is_fullexcl( comp )) continue;
+ *ptr_comp = comp;
+ for (j = i; j < strlen-1; j++) str[j] = str[j+1];
+ strlen--;
+ i--;
+ }
+
+ return strlen;
+}
EOF
close OUTPUT;
save_file($filename);
@@ -2346,13 +2427,21 @@ sub dump_decompose_table($)
my %nfd_lookup = ();
my %nfkd_lookup = ();
my %decomp_lookup = ();
+ my %comb_lookup = ();
+ my %fullcomp_lookup = ();
my @decomp_data = (0);
+ my @comb_data = (0);
+ my @full_comp_data = (0);
my $pos = 1;
+ my $pos_comb = 1;
+ my $pos_fullcomp = 1;
my $lastchar_decomp;
+ my $lastchar_comb;
+ my $lastchar_fullcomp;
for (my $i = 0; $i < $utflim; $i++)
{
- next unless defined $decomp_table[$i];
+ next unless defined $decomp_table[$i] || defined $comb_class_table[$i] || defined $full_comp_table[$i];
if (defined $decomp_table[$i])
{
@@ -2406,6 +2495,20 @@ sub dump_decompose_table($)
$pos += @nfkd;
}
}
+ if (defined $comb_class_table[$i])
+ {
+ push @comb_data, $comb_class_table[$i];
+ $lastchar_comb = $i;
+ $comb_lookup{$i} = $pos_comb;
+ $pos_comb++;
+ }
+ if (defined $full_comp_table[$i])
+ {
+ push @full_comp_data, $full_comp_table[$i];
+ $lastchar_fullcomp = $i;
+ $fullcomp_lookup{$i} = $pos_fullcomp;
+ $pos_fullcomp++;
+ }
}
printf OUTPUT "static const UINT last_decomposable = 0x%x;\n\n", $lastchar_decomp;
@@ -2497,6 +2600,154 @@ sub dump_decompose_table($)
}
print OUTPUT "\n};\n\n";
+ # now for Compatibility Class
+
+ printf OUTPUT "static const WCHAR data_comb[%d] =\n", $pos_comb;
+ print OUTPUT "{\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @comb_data );
+ print OUTPUT "\n};\n\n";
+
+ my $comb_pos = 1;
+ my $comb_lim = ($lastchar_comb >> 8) + 1;
+ my @comb_filled = (0) x $comb_lim;
+ for (my $i = 0; $i < $utflim; $i++)
+ {
+ last if $i > $lastchar_comb;
+ next unless defined $comb_class_table[$i];
+ $comb_filled[$i >> 8] = $comb_pos++;
+ $i |= 255;
+ }
+ printf OUTPUT "static const BYTE idx1_comb[%d] =\n", $comb_lim;
+ print OUTPUT "{\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%02x", 0, @comb_filled );
+ print OUTPUT "\n};\n\n";
+
+ my $sub_comb_filled_pos = 1;
+ my %sub_comb_filled = ();
+ for (my $i = 0; $i < $comb_lim; $i++)
+ {
+ next unless $comb_filled[$i];
+ for (my $j = 0; $j < 256; $j++)
+ {
+ my $idx = ($i << 8) | $j;
+ next unless defined $comb_class_table[$idx];
+ $sub_comb_filled{$idx >> 4} = $sub_comb_filled_pos++;
+ $j |= 15;
+ }
+ }
+
+ printf OUTPUT "static const USHORT idx2_comb[%d] =\n", $comb_pos * 16;
+ print OUTPUT "{\n";
+ @null_idx = (0) x 16;
+ print OUTPUT " /* all-zero 256-char blocks get mapped to here */\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @null_idx );
+ for (my $i = 0; $i < $comb_lim; $i++)
+ {
+ next unless $comb_filled[$i];
+ printf OUTPUT ",\n /* sub-index 0x%02x */\n", $comb_filled[$i];
+
+ my @sub_idx;
+ for (my $j = 0; $j < 16; $j++)
+ {
+ my $idx = ($i << 4) | $j;
+ $sub_idx[$j] = $sub_comb_filled{$idx} || 0;
+ }
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @sub_idx );
+ }
+ print OUTPUT "\n};\n\n";
+
+ printf OUTPUT "static const USHORT offsets_comb[%d] =\n", 16 * $sub_comb_filled_pos;
+ print OUTPUT "{\n";
+ @null_table = (0) x 16;
+ print OUTPUT " /* all-zero 16-char blocks get mapped to here */\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @null_table );
+ for my $key (sort {$a <=> $b} keys %sub_comb_filled)
+ {
+ printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $key, $key;
+ my @sub_table;
+ for (my $j = 0; $j < 16; $j++)
+ {
+ my $idx = ($key << 4) | $j;
+ $sub_table[$j] = $comb_lookup{$idx} || 0;
+ }
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @sub_table );
+ }
+ print OUTPUT "\n};\n\n";
+
+ # now for Full Composition Exclusion
+
+ printf OUTPUT "const WCHAR data_fullcomp[%d] =\n", $pos_fullcomp;
+ print OUTPUT "{\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @full_comp_data );
+ print OUTPUT "\n};\n\n";
+
+ my $fullcomp_pos = 1;
+ my $fullcomp_lim = ($lastchar_fullcomp >> 8) + 1;
+ my @fullcomp_filled = (0) x $fullcomp_lim;
+ for (my $i = 0; $i < $utflim; $i++)
+ {
+ last if $i > $lastchar_fullcomp;
+ next unless defined $full_comp_table[$i];
+ $fullcomp_filled[$i >> 8] = $fullcomp_pos++;
+ $i |= 255;
+ }
+ printf OUTPUT "const BYTE idx1_fullcomp[%d] =\n", $fullcomp_lim;
+ print OUTPUT "{\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%02x", 0, @fullcomp_filled );
+ print OUTPUT "\n};\n\n";
+
+ my $sub_fullcomp_filled_pos = 1;
+ my %sub_fullcomp_filled = ();
+ for (my $i = 0; $i < $fullcomp_lim; $i++)
+ {
+ next unless $fullcomp_filled[$i];
+ for (my $j = 0; $j < 256; $j++)
+ {
+ my $idx = ($i << 8) | $j;
+ next unless defined $full_comp_table[$idx];
+ $sub_fullcomp_filled{$idx >> 4} = $sub_fullcomp_filled_pos++;
+ $j |= 15;
+ }
+ }
+
+ printf OUTPUT "const USHORT idx2_fullcomp[%d] =\n", $fullcomp_pos * 16;
+ print OUTPUT "{\n";
+ @null_idx = (0) x 16;
+ print OUTPUT " /* all-zero 256-char blocks get mapped to here */\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @null_idx );
+ for (my $i = 0; $i < $fullcomp_lim; $i++)
+ {
+ next unless $fullcomp_filled[$i];
+ printf OUTPUT ",\n /* sub-index 0x%02x */\n", $fullcomp_filled[$i];
+
+ my @sub_idx;
+ for (my $j = 0; $j < 16; $j++)
+ {
+ my $idx = ($i << 4) | $j;
+ $sub_idx[$j] = $sub_fullcomp_filled{$idx} || 0;
+ }
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @sub_idx );
+ }
+ print OUTPUT "\n};\n\n";
+
+ printf OUTPUT "const USHORT offsets_fullcomp[%d] =\n", 16 * $sub_fullcomp_filled_pos;
+ print OUTPUT "{\n";
+ @null_table = (0) x 16;
+ print OUTPUT " /* all-zero 16-char blocks get mapped to here */\n";
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @null_table );
+ for my $key (sort {$a <=> $b} keys %sub_fullcomp_filled)
+ {
+ printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $key, $key;
+ my @sub_table;
+ for (my $j = 0; $j < 16; $j++)
+ {
+ my $idx = ($key << 4) | $j;
+ $sub_table[$j] = $fullcomp_lookup{$idx} || 0;
+ }
+ printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @sub_table );
+ }
+ print OUTPUT "\n};\n\n";
+
print OUTPUT <<"EOF";
static const WCHAR *unicode_table_lookup( UINT cp, int compat, const BYTE *idx1, UINT scale_idx1,
const USHORT *idx2, UINT scale_idx2, const USHORT *offsets,
@@ -2513,6 +2764,20 @@ static const WCHAR *unicode_table_lookup( UINT cp, int compat, const BYTE *idx1,
return &data[d];
}
+static inline int reorderable_pair( WCHAR ch1, WCHAR ch2 )
+{
+ const WCHAR *cc1, *cc2;
+
+ if (ch1 == 0 || ch2 == 0) return 0;
+
+ cc1 = unicode_table_lookup( ch1, 0, idx1_comb, 8, idx2_comb, 4,
+ offsets_comb, 4, data_comb, 0 );
+ cc2 = unicode_table_lookup( ch2, 0, idx1_comb, 8, idx2_comb, 4,
+ offsets_comb, 4, data_comb, 0 );
+ if (*cc2 < *cc1) return 1;
+ else return 0;
+}
+
static int decompose_char_recursive( int compat, UINT ch, WCHAR *dst, int dstlen )
{
int total_decomp = 0;
@@ -2576,6 +2841,41 @@ int wine_unicode_decompose_string( int compat, const WCHAR *src,
return dstpos;
}
+
+int is_starter( WCHAR ch )
+{
+ const WCHAR *map = unicode_table_lookup( ch, 0, idx1_comb, 8, idx2_comb, 4,
+ offsets_comb, 4, data_comb, 0 );
+ return (*map == 0) ? 1 : 0;
+}
+
+void unicode_canon_order( WCHAR *str, int strlen )
+{
+ int i, j, m;
+ int sublen = 0, tot_sublen = 0;
+ WCHAR *substr = str;
+
+ for (m = 1; m <= strlen; m++)
+ {
+ if (m == strlen || is_starter( str[m] )) sublen = m - tot_sublen;
+ else continue;
+
+ for (i = 0; i < sublen; i++)
+ {
+ for (j = 1; j < sublen; j++)
+ {
+ if (reorderable_pair( substr[j-1], substr[j] ))
+ {
+ WCHAR swp = substr[j-1];
+ substr[j-1] = substr[j];
+ substr[j] = swp;
+ }
+ }
+ }
+ tot_sublen += m;
+ substr = str+m;
+ }
+}
EOF
close OUTPUT;
save_file($filename);
--
2.17.1

183
patches/kernel32-NormalizeString/0003-kernel32-Implement-NormalizeString-API-function.patch Normal file
View File

@ -0,0 +1,183 @@
From 965604cac1de3c3a6db44f99094da3a6ec91fc11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Del=20Real?=
<sdelreal@codeweavers.com>
Date: Fri, 6 Apr 2018 10:29:39 -0500
Subject: [PATCH 3/3] kernel32: Implement NormalizeString API function.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Sergio Gómez Del Real <sdelreal@codeweavers.com>
---
dlls/kernel32/locale.c | 88 ++++++++++++++++++++++++++++++++++--
dlls/kernel32/tests/locale.c | 44 ++++++++----------
2 files changed, 103 insertions(+), 29 deletions(-)
diff --git a/dlls/kernel32/locale.c b/dlls/kernel32/locale.c
index 4d1eac6ebe..607008fcc7 100644
--- a/dlls/kernel32/locale.c
+++ b/dlls/kernel32/locale.c
@@ -5358,15 +5358,93 @@ INT WINAPI GetUserDefaultLocaleName(LPWSTR localename, int buffersize)
return LCIDToLocaleName(userlcid, localename, buffersize, 0);
}
+static inline int is_valid_norm(NORM_FORM norm)
+{
+ if (norm == NormalizationC || norm == NormalizationD ||
+ norm == NormalizationKC || norm == NormalizationKD)
+ return 1;
+ else
+ return 0;
+}
+
/******************************************************************************
* NormalizeString (KERNEL32.@)
+ *
+ * Normalizes a string according to a Unicode Normalization Form.
+ *
+ * PARAMS
+ * norm [I] Normalization Form
+ * src [I] Source string to normalize
+ * srclen [I] Length of source string (if -1, source string is null-terminated)
+ * dst [O] Buffer to write normalized source string (can be NULL)
+ * dstlen [I] Length of dst string (can be 0)
+ *
+ * RETURNS
+ * Success: If dstlen is 0, return size needed, else return size of normalized string.
+ * Failure: ret <= 0. Use GetLastError to determine error.
*/
-INT WINAPI NormalizeString(NORM_FORM NormForm, LPCWSTR lpSrcString, INT cwSrcLength,
- LPWSTR lpDstString, INT cwDstLength)
+INT WINAPI NormalizeString(NORM_FORM norm, LPCWSTR src, INT srclen,
+ LPWSTR dst, INT dstlen)
{
- FIXME("%x %p %d %p %d\n", NormForm, lpSrcString, cwSrcLength, lpDstString, cwDstLength);
- SetLastError(ERROR_CALL_NOT_IMPLEMENTED);
- return 0;
+ extern int wine_unicode_decompose_string( int compat, const WCHAR *src,
+ int srclen, WCHAR *dst, int dstlen );
+ extern int unicode_canonical_composition( WCHAR *str, UINT strlen );
+ extern void unicode_canon_order( WCHAR *str, int strlen );
+
+ WCHAR *decomp = NULL;
+ INT compat = 0;
+ INT needed_len;
+
+ if (src == NULL || !is_valid_norm( norm ))
+ {
+ SetLastError(ERROR_INVALID_PARAMETER);
+ return 0;
+ }
+
+ if (norm == NormalizationKC || norm == NormalizationKD) compat++;
+
+ if (srclen == -1) srclen = strlenW( src ) + 1;
+
+ needed_len = wine_unicode_decompose_string( compat, src, srclen, NULL, 0 );
+
+ if (needed_len < 0)
+ {
+ SetLastError(ERROR_NO_UNICODE_TRANSLATION);
+ return needed_len;
+ }
+
+ if (norm == NormalizationC || norm == NormalizationKC)
+ {
+ decomp = HeapAlloc( GetProcessHeap(), 0, needed_len * sizeof( WCHAR ) );
+ wine_unicode_decompose_string( compat, src, srclen, decomp, needed_len );
+ unicode_canon_order( decomp, needed_len );
+ needed_len = unicode_canonical_composition( decomp, needed_len );
+ }
+
+ if (dstlen < needed_len && dstlen > 0)
+ {
+ if (decomp) HeapFree(GetProcessHeap(), 0, decomp);
+ SetLastError(ERROR_INSUFFICIENT_BUFFER);
+ return -1;
+ }
+ else if (dstlen <= 0)
+ {
+ if (decomp) HeapFree(GetProcessHeap(), 0, decomp);
+ return needed_len;
+ }
+
+ if (norm == NormalizationC || norm == NormalizationKC)
+ {
+ memcpy( dst, decomp, sizeof(WCHAR) * needed_len );
+ HeapFree(GetProcessHeap(), 0, decomp);
+ return needed_len;
+ }
+ else
+ {
+ int decomp_len = wine_unicode_decompose_string( compat, src, srclen, dst, needed_len );
+ unicode_canon_order( dst, needed_len );
+ return decomp_len;
+ }
}
/******************************************************************************
diff --git a/dlls/kernel32/tests/locale.c b/dlls/kernel32/tests/locale.c
index d3eb2ecf15..50c3f07a20 100644
--- a/dlls/kernel32/tests/locale.c
+++ b/dlls/kernel32/tests/locale.c
@@ -5656,10 +5656,8 @@ static void test_NormalizeString(void)
return;
}
- todo_wine {
- dstlen = pNormalizeString( NormalizationD, ptest->str, -1, dst, 1 );
- ok(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "Should have failed with ERROR_INSUFFICIENT_BUFFER\n");
- }
+ dstlen = pNormalizeString( NormalizationD, ptest->str, -1, dst, 1 );
+ ok(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "Should have failed with ERROR_INSUFFICIENT_BUFFER");
/*
* For each string, first test passing -1 as srclen to NormalizeString,
@@ -5673,26 +5671,24 @@ static void test_NormalizeString(void)
for (i = 0; i < 4; i++)
{
- todo_wine {
- dstlen = pNormalizeString( norm_forms[i], ptest->str, -1, NULL, 0 );
- if (dstlen)
- {
- dstlen = pNormalizeString( norm_forms[i], ptest->str, -1, dst, dstlen );
- ok(dstlen == strlenW( ptest->expected[i] )+1, "Copied length differed: was %d, should be %d\n",
- dstlen, strlenW( ptest->expected[i] )+1);
- str_cmp = strncmpW( ptest->expected[i], dst, dstlen+1 );
- ok( str_cmp == 0, "test failed: returned value was %d\n", str_cmp );
- }
-
- dstlen = pNormalizeString( norm_forms[i], ptest->str, strlenW(ptest->str), NULL, 0 );
- if (dstlen)
- {
- dstlen = pNormalizeString( norm_forms[i], ptest->str, strlenW(ptest->str), dst, dstlen );
- ok(dstlen == strlenW( ptest->expected[i] ), "Copied length differed: was %d, should be %d\n",
- dstlen, strlenW( ptest->expected[i] ));
- str_cmp = strncmpW( ptest->expected[i], dst, dstlen );
- ok( str_cmp == 0, "test failed: returned value was %d\n", str_cmp );
- }
+ dstlen = pNormalizeString( norm_forms[i], ptest->str, -1, NULL, 0 );
+ if (dstlen)
+ {
+ dstlen = pNormalizeString( norm_forms[i], ptest->str, -1, dst, dstlen );
+ ok(dstlen == strlenW( ptest->expected[i] )+1, "Copied length differed: was %d, should be %d\n",
+ dstlen, strlenW( ptest->expected[i] )+1);
+ str_cmp = strncmpW( ptest->expected[i], dst, dstlen+1 );
+ ok( str_cmp == 0, "test failed: returned value was %d\n", str_cmp );
+ }
+
+ dstlen = pNormalizeString( norm_forms[i], ptest->str, strlenW(ptest->str), NULL, 0 );
+ if (dstlen)
+ {
+ dstlen = pNormalizeString( norm_forms[i], ptest->str, strlenW(ptest->str), dst, dstlen );
+ ok(dstlen == strlenW( ptest->expected[i] ), "Copied length differed: was %d, should be %d\n",
+ dstlen, strlenW( ptest->expected[i] ));
+ str_cmp = strncmpW( ptest->expected[i], dst, dstlen );
+ ok( str_cmp == 0, "test failed: returned value was %d\n", str_cmp );
}
}
ptest++;
--
2.17.1

4598
patches/kernel32-NormalizeString/0004-libs-Generated-make_unicode-files.patch Normal file
View File

File diff suppressed because it is too large Load Diff

1
patches/kernel32-NormalizeString/definition Normal file
View File

@ -0,0 +1 @@
Fixes: [42734] Implement NormalizeString

1
patches/libs-Unicode_Collation/definition
View File

@ -1,2 +1,3 @@
Fixes: [10767] Fix comparison of punctuation characters in lstrcmp
Fixes: [32490] Graphical issues in Inquisitor
Depends: kernel32-NormalizeString

36
patches/patchinstall.sh
View File

@ -164,6 +164,7 @@ patch_enable_all ()
enable_kernel32_K32GetPerformanceInfo="$1"
enable_kernel32_MoveFile="$1"
enable_kernel32_NeedCurrentDirectoryForExePath="$1"
enable_kernel32_NormalizeString="$1"
enable_kernel32_PE_Loader_Fixes="$1"
enable_kernel32_Processor_Group="$1"
enable_kernel32_Profile="$1"
@ -657,6 +658,9 @@ patch_enable ()
kernel32-NeedCurrentDirectoryForExePath)
enable_kernel32_NeedCurrentDirectoryForExePath="$2"
;;
kernel32-NormalizeString)
enable_kernel32_NormalizeString="$2"
;;
kernel32-PE_Loader_Fixes)
enable_kernel32_PE_Loader_Fixes="$2"
;;
@ -2150,6 +2154,13 @@ if test "$enable_loader_OSX_Preloader" -eq 1; then
enable_configure_Absolute_RPATH=1
fi
if test "$enable_libs_Unicode_Collation" -eq 1; then
if test "$enable_kernel32_NormalizeString" -gt 1; then
abort "Patchset kernel32-NormalizeString disabled, but libs-Unicode_Collation depends on that."
fi
enable_kernel32_NormalizeString=1
fi
if test "$enable_kernel32_Processor_Group" -eq 1; then
if test "$enable_api_ms_win_Stub_DLLs" -gt 1; then
abort "Patchset api-ms-win-Stub_DLLs disabled, but kernel32-Processor_Group depends on that."
@ -3973,6 +3984,28 @@ if test "$enable_kernel32_NeedCurrentDirectoryForExePath" -eq 1; then
) >> "$patchlist"
fi
# Patchset kernel32-NormalizeString
# |
# | This patchset fixes the following Wine bugs:
# | * [#42734] Implement NormalizeString
# |
# | Modified files:
# | * dlls/kernel32/locale.c, dlls/kernel32/tests/locale.c, libs/port/compose.c, libs/port/decompose.c, libs/port/mbtowc.c,
# | tools/make_unicode
# |
if test "$enable_kernel32_NormalizeString" -eq 1; then
patch_apply kernel32-NormalizeString/0001-tools-make_unicode-Implement-full-Unicode-character-.patch
patch_apply kernel32-NormalizeString/0002-tools-make_unicode-Implement-canonical-composition-f.patch
patch_apply kernel32-NormalizeString/0003-kernel32-Implement-NormalizeString-API-function.patch
patch_apply kernel32-NormalizeString/0004-libs-Generated-make_unicode-files.patch
(
printf '%s\n' '+ { "Sergio Gómez Del Real", "tools/make_unicode: Implement full Unicode character decomposition.", 1 },';
printf '%s\n' '+ { "Sergio Gómez Del Real", "tools/make_unicode: Implement canonical composition for use in normalization.", 1 },';
printf '%s\n' '+ { "Sergio Gómez Del Real", "kernel32: Implement NormalizeString API function.", 1 },';
printf '%s\n' '+ { "Alistair Leslie-Hughes", "libs: Generated make_unicode files.", 1 },';
) >> "$patchlist"
fi
# Patchset kernel32-PE_Loader_Fixes
# |
# | This patchset fixes the following Wine bugs:
@ -4148,6 +4181,9 @@ fi
# Patchset libs-Unicode_Collation
# |
# | This patchset has the following (direct or indirect) dependencies:
# | * kernel32-NormalizeString
# |
# | This patchset fixes the following Wine bugs:
# | * [#10767] Fix comparison of punctuation characters in lstrcmp
# | * [#32490] Graphical issues in Inquisitor