Bug 1142953 - Update Brotli decompressor to latest upstream revision from GitHub; now at ca29aa22c295daac15baf5d85427ecc7808b515c. r=jdaggett

modules/brotli/README.mozilla

@@ -1,17 +1,17 @@
 This is the Brotli data compression library from
-https://code.google.com/p/font-compression-reference/.
+https://github.com/google/brotli.
 
 Currently, we import only the Brotli decoder (the /dec/ subdirectory), not the
 encoder (/enc/ subdirectory).
 
 Upstream code can be viewed at
-  https://code.google.com/p/font-compression-reference/source/browse/#git%2Fbrotli%2Fdec
+  https://github.com/google/brotli/tree/master/dec
 
 and cloned by
-  git clone https://code.google.com/p/font-compression-reference/ 
+  git clone https://github.com/google/brotli
 
 The in-tree copy is updated by running
   sh update.sh
 from within the modules/brotli directory.
 
-Current version: [commit d9a74803fa884559879e3205cfe6f257a2d85519].
+Current version: [commit ca29aa22c295daac15baf5d85427ecc7808b515c].

modules/brotli/dec/Makefile

@@ -1,6 +1,8 @@
 #brotli/dec
 
-include ../../shared.mk
+include ../shared.mk
+
+CPPFLAGS += -Wall
 
 OBJS = bit_reader.o decode.o huffman.o safe_malloc.o streams.o
 

modules/brotli/dec/README

@@ -1,3 +0,0 @@
-This directory holds the decoder for brotli compression format.
-
-Brotli is proposed to be used at the byte-compression level in WOFF 2.0 format.

modules/brotli/dec/bit_reader.h

@@ -26,12 +26,19 @@
 extern "C" {
 #endif
 
+#if (defined(__x86_64__) || defined(_M_X64))
+/* This should be set to 1 only on little-endian machines. */
+#define BROTLI_USE_64_BITS 1
+#else
+#define BROTLI_USE_64_BITS 0
+#endif
 #define BROTLI_MAX_NUM_BIT_READ   25
 #define BROTLI_READ_SIZE          4096
 #define BROTLI_IBUF_SIZE          (2 * BROTLI_READ_SIZE + 32)
 #define BROTLI_IBUF_MASK          (2 * BROTLI_READ_SIZE - 1)
 
 #define UNALIGNED_COPY64(dst, src) memcpy(dst, src, 8)
+#define UNALIGNED_MOVE64(dst, src) memmove(dst, src, 8)
 
 static const uint32_t kBitMask[BROTLI_MAX_NUM_BIT_READ] = {
   0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767,
@@ -44,7 +51,11 @@ typedef struct {
   uint8_t buf_[BROTLI_IBUF_SIZE];
   uint8_t*    buf_ptr_;      /* next input will write here */
   BrotliInput input_;        /* input callback */
+#if (BROTLI_USE_64_BITS)
   uint64_t    val_;          /* pre-fetched bits */
+#else
+  uint32_t    val_;          /* pre-fetched bits */
+#endif
   uint32_t    pos_;          /* byte position in stream */
   uint32_t    bit_pos_;      /* current bit-reading position in val_ */
   uint32_t    bit_end_pos_;  /* bit-reading end position from LSB of val_ */
@@ -71,11 +82,14 @@ static BROTLI_INLINE void BrotliSetBitPos(BrotliBitReader* const br,
   br->bit_pos_ = val;
 }
 
-/* Reload up to 64 bits byte-by-byte */
-static BROTLI_INLINE void ShiftBytes(BrotliBitReader* const br) {
+/*
+ * Reload up to 32 bits byte-by-byte.
+ * This function works on both little and big endian.
+ */
+static BROTLI_INLINE void ShiftBytes32(BrotliBitReader* const br) {
   while (br->bit_pos_ >= 8) {
     br->val_ >>= 8;
-    br->val_ |= ((uint64_t)br->buf_[br->pos_ & BROTLI_IBUF_MASK]) << 56;
+    br->val_ |= ((uint32_t)br->buf_[br->pos_ & BROTLI_IBUF_MASK]) << 24;
     ++br->pos_;
     br->bit_pos_ -= 8;
     br->bit_end_pos_ -= 8;
@@ -108,7 +122,7 @@ static BROTLI_INLINE int BrotliReadMoreInput(BrotliBitReader* const br) {
     if (bytes_read < BROTLI_READ_SIZE) {
       br->eos_ = 1;
       /* Store 32 bytes of zero after the stream end. */
-#if (defined(__x86_64__) || defined(_M_X64))
+#if (BROTLI_USE_64_BITS)
       *(uint64_t*)(dst + bytes_read) = 0;
       *(uint64_t*)(dst + bytes_read + 8) = 0;
       *(uint64_t*)(dst + bytes_read + 16) = 0;
@@ -119,7 +133,7 @@ static BROTLI_INLINE int BrotliReadMoreInput(BrotliBitReader* const br) {
     }
     if (dst == br->buf_) {
       /* Copy the head of the ringbuffer to the slack region. */
-#if (defined(__x86_64__) || defined(_M_X64))
+#if (BROTLI_USE_64_BITS)
       UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 32, br->buf_);
       UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 24, br->buf_ + 8);
       UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 16, br->buf_ + 16);
@@ -136,30 +150,45 @@ static BROTLI_INLINE int BrotliReadMoreInput(BrotliBitReader* const br) {
   }
 }
 
-/* Advances the Read buffer by 5 bytes to make room for reading next 24 bits. */
+/* Guarantees that there are at least 24 bits in the buffer. */
 static BROTLI_INLINE void BrotliFillBitWindow(BrotliBitReader* const br) {
+#if (BROTLI_USE_64_BITS)
   if (br->bit_pos_ >= 40) {
-#if (defined(__x86_64__) || defined(_M_X64))
+    /*
+     * Advances the Read buffer by 5 bytes to make room for reading next
+     * 24 bits.
+     * The expression below needs a little-endian arch to work correctly.
+     * This gives a large speedup for decoding speed.
+     */
     br->val_ >>= 40;
-    /* The expression below needs a little-endian arch to work correctly. */
-    /* This gives a large speedup for decoding speed. */
     br->val_ |= *(const uint64_t*)(
         br->buf_ + (br->pos_ & BROTLI_IBUF_MASK)) << 24;
     br->pos_ += 5;
     br->bit_pos_ -= 40;
     br->bit_end_pos_ -= 40;
-#else
-    ShiftBytes(br);
-#endif
   }
+#else
+  ShiftBytes32(br);
+#endif
 }
 
 /* Reads the specified number of bits from Read Buffer. */
 static BROTLI_INLINE uint32_t BrotliReadBits(
     BrotliBitReader* const br, int n_bits) {
   uint32_t val;
+#if (BROTLI_USE_64_BITS)
   BrotliFillBitWindow(br);
   val = (uint32_t)(br->val_ >> br->bit_pos_) & kBitMask[n_bits];
+#else
+  /*
+   * The if statement gives 2-4% speed boost on Canterbury data set with
+   * asm.js/firefox/x86-64.
+   */
+  if ((32 - br->bit_pos_) < ((uint32_t) n_bits)) {
+    BrotliFillBitWindow(br);
+  }
+  val = (br->val_ >> br->bit_pos_) & kBitMask[n_bits];
+#endif
 #ifdef BROTLI_DECODE_DEBUG
   printf("[BrotliReadBits]  %010d %2d  val: %6x\n",
          (br->pos_ << 3) + br->bit_pos_ - 64, n_bits, val);

modules/brotli/dec/decode.c

@@ -50,8 +50,9 @@ static const int kDistanceContextBits = 2;
 
 #define HUFFMAN_TABLE_BITS      8
 #define HUFFMAN_TABLE_MASK      0xff
-/* This is a rough estimate, not an exact bound. */
-#define HUFFMAN_MAX_TABLE_SIZE  2048
+/* Maximum possible Huffman table size for an alphabet size of 704, max code
+ * length 15 and root table bits 8. */
+#define HUFFMAN_MAX_TABLE_SIZE  1080
 
 #define CODE_LENGTH_CODES 18
 static const uint8_t kCodeLengthCodeOrder[CODE_LENGTH_CODES] = {
@@ -531,7 +532,7 @@ static BROTLI_INLINE void IncrementalCopyFastPath(
     uint8_t* dst, const uint8_t* src, int len) {
   if (src < dst) {
     while (dst - src < 8) {
-      UNALIGNED_COPY64(dst, src);
+      UNALIGNED_MOVE64(dst, src);
       len -= (int)(dst - src);
       dst += dst - src;
     }
@@ -552,6 +553,7 @@ int CopyUncompressedBlockToOutput(BrotliOutput output, int len, int pos,
   int rb_pos = pos & ringbuffer_mask;
   int br_pos = br->pos_ & BROTLI_IBUF_MASK;
   int nbytes;
+  uint32_t remaining_bits;
 
   /* For short lengths copy byte-by-byte */
   if (len < 8 || br->bit_pos_ + (uint32_t)(len << 3) < br->bit_end_pos_) {
@@ -574,8 +576,16 @@ int CopyUncompressedBlockToOutput(BrotliOutput output, int len, int pos,
     return 0;
   }
 
-  /* Copy remaining 0-8 bytes from br->val_ to ringbuffer. */
-  while (br->bit_pos_ < 64) {
+  /*
+   * Copy remaining 0-4 in 32-bit case or 0-8 bytes in the 64-bit case
+   * from br->val_ to ringbuffer.
+   */
+#if (BROTLI_USE_64_BITS)
+  remaining_bits = 64;
+#else
+  remaining_bits = 32;
+#endif
+  while (br->bit_pos_ < remaining_bits) {
     ringbuffer[rb_pos] = (uint8_t)(br->val_ >> br->bit_pos_);
     br->bit_pos_ += 8;
     ++rb_pos;
@@ -633,22 +643,62 @@ int CopyUncompressedBlockToOutput(BrotliOutput output, int len, int pos,
 int BrotliDecompressedSize(size_t encoded_size,
                            const uint8_t* encoded_buffer,
                            size_t* decoded_size) {
-  BrotliMemInput memin;
-  BrotliInput input = BrotliInitMemInput(encoded_buffer, encoded_size, &memin);
-  BrotliBitReader br;
-  int meta_block_len;
-  int input_end;
-  int is_uncompressed;
-  if (!BrotliInitBitReader(&br, input)) {
+  int i;
+  uint64_t val = 0;
+  int bit_pos = 0;
+  int is_last;
+  int is_uncompressed = 0;
+  int size_nibbles;
+  int meta_block_len = 0;
+  if (encoded_size == 0) {
     return 0;
   }
-  DecodeWindowBits(&br);
-  DecodeMetaBlockLength(&br, &meta_block_len, &input_end, &is_uncompressed);
-  if (!input_end) {
-    return 0;
+  /* Look at the first 8 bytes, it is enough to decode the length of the first
+     meta-block. */
+  for (i = 0; (size_t)i < encoded_size && i < 8; ++i) {
+    val |= (uint64_t)encoded_buffer[i] << (8 * i);
   }
-  *decoded_size = (size_t)meta_block_len;
-  return 1;
+  /* Skip the window bits. */
+  bit_pos += (val & 1) ? 4 : 1;
+  /* Decode the ISLAST bit. */
+  is_last = (val >> bit_pos) & 1;
+  ++bit_pos;
+  if (is_last) {
+    /* Decode the ISEMPTY bit, if it is set to 1, we are done. */
+    if ((val >> bit_pos) & 1) {
+      *decoded_size = 0;
+      return 1;
+    }
+    ++bit_pos;
+  }
+  /* Decode the length of the first meta-block. */
+  size_nibbles = (int)((val >> bit_pos) & 3) + 4;
+  bit_pos += 2;
+  for (i = 0; i < size_nibbles; ++i) {
+    meta_block_len |= (int)((val >> bit_pos) & 0xf) << (4 * i);
+    bit_pos += 4;
+  }
+  ++meta_block_len;
+  if (is_last) {
+    /* If this meta-block is the only one, we are done. */
+    *decoded_size = (size_t)meta_block_len;
+    return 1;
+  }
+  is_uncompressed = (val >> bit_pos) & 1;
+  ++bit_pos;
+  if (is_uncompressed) {
+    /* If the first meta-block is uncompressed, we skip it and look at the
+       first two bits (ISLAST and ISEMPTY) of the next meta-block, and if
+       both are set to 1, we have a stream with an uncompressed meta-block
+       followed by an empty one, so the decompressed size is the size of the
+       first meta-block. */
+    size_t offset = (size_t)((bit_pos + 7) >> 3) + (size_t)meta_block_len;
+    if (offset < encoded_size && ((encoded_buffer[offset] & 3) == 3)) {
+      *decoded_size = (size_t)meta_block_len;
+      return 1;
+    }
+  }
+  return 0;
 }
 
 int BrotliDecompressBuffer(size_t encoded_size,
@@ -945,7 +995,6 @@ int BrotliDecompress(BrotliInput input, BrotliOutput output) {
                           block_type_rb_index, &br);
           block_length[2] = ReadBlockLength(
               &block_len_trees[2 * HUFFMAN_MAX_TABLE_SIZE], &br);
-          dist_htree_index = (uint8_t)block_type[2];
           dist_context_offset = block_type[2] << kDistanceContextBits;
           dist_context_map_slice = dist_context_map + dist_context_offset;
         }

modules/brotli/dec/decode.h

@@ -27,7 +27,8 @@ extern "C" {
 
 /* Sets *decoded_size to the decompressed size of the given encoded stream. */
 /* This function only works if the encoded buffer has a single meta block, */
-/* and this meta block must have the "is last" bit set. */
+/* or if it has two meta-blocks, where the first is uncompressed and the */
+/* second is empty. */
 /* Returns 1 on success, 0 on failure. */
 int BrotliDecompressedSize(size_t encoded_size,
                            const uint8_t* encoded_buffer,

modules/brotli/dec/safe_malloc.h

@@ -15,8 +15,8 @@
    Size-checked memory allocation.
 */
 
-#ifndef BROTLI_UTILS_UTILS_H_
-#define BROTLI_UTILS_UTILS_H_
+#ifndef BROTLI_DEC_SAFE_MALLOC_H_
+#define BROTLI_DEC_SAFE_MALLOC_H_
 
 #include <assert.h>
 
@@ -42,4 +42,4 @@ void* BrotliSafeMalloc(uint64_t nmemb, size_t size);
 }    /* extern "C" */
 #endif
 
-#endif  /* BROTLI_UTILS_UTILS_H_ */
+#endif  /* BROTLI_DEC_SAFE_MALLOC_H_ */

modules/brotli/dec/streams.c

@@ -71,6 +71,7 @@ BrotliOutput BrotliInitMemOutput(uint8_t* buffer, size_t length,
 }
 
 int BrotliStdinInputFunction(void* data, uint8_t* buf, size_t count) {
+  (void) data; /* Shut up LLVM */
 #ifndef _WIN32
   return (int)read(STDIN_FILENO, buf, count);
 #else
@@ -86,6 +87,7 @@ BrotliInput BrotliStdinInput() {
 }
 
 int BrotliStdoutOutputFunction(void* data, const uint8_t* buf, size_t count) {
+  (void) data; /* Shut up LLVM */
 #ifndef _WIN32
   return (int)write(STDOUT_FILENO, buf, count);
 #else
@@ -100,6 +102,17 @@ BrotliOutput BrotliStdoutOutput() {
   return out;
 }
 
+int BrotliFileInputFunction(void* data, uint8_t* buf, size_t count) {
+  return (int)fread(buf, 1, count, (FILE*)data);
+}
+
+BrotliInput BrotliFileInput(FILE* f) {
+  BrotliInput in;
+  in.cb_ = BrotliFileInputFunction;
+  in.data_ = f;
+  return in;
+}
+
 int BrotliFileOutputFunction(void* data, const uint8_t* buf, size_t count) {
   return (int)fwrite(buf, 1, count, (FILE*)data);
 }

modules/brotli/dec/streams.h

@@ -92,6 +92,10 @@ BrotliInput BrotliStdinInput();
 int BrotliStdoutOutputFunction(void* data, const uint8_t* buf, size_t count);
 BrotliOutput BrotliStdoutOutput();
 
+/* Input callback that reads from a file. */
+int BrotliFileInputFunction(void* data, uint8_t* buf, size_t count);
+BrotliInput BrotliFileInput(FILE* f);
+
 /* Output callback that writes to a file. */
 int BrotliFileOutputFunction(void* data, const uint8_t* buf, size_t count);
 BrotliOutput BrotliFileOutput(FILE* f);

modules/brotli/dec/transform.h

@@ -182,7 +182,7 @@ static const Transform kTransforms[] = {
 
 static const int kNumTransforms = sizeof(kTransforms) / sizeof(kTransforms[0]);
 
-static int ToUpperCase(uint8_t *p, int len) {
+static int ToUpperCase(uint8_t *p) {
   if (p[0] < 0xc0) {
     if (p[0] >= 'a' && p[0] <= 'z') {
       p[0] ^= 32;
@@ -220,10 +220,10 @@ static BROTLI_INLINE int TransformDictionaryWord(
   while (i < len) { dst[idx++] = word[i++]; }
   uppercase = &dst[idx - len];
   if (t == kUppercaseFirst) {
-    ToUpperCase(uppercase, len);
+    ToUpperCase(uppercase);
   } else if (t == kUppercaseAll) {
     while (len > 0) {
-      int step = ToUpperCase(uppercase, len);
+      int step = ToUpperCase(uppercase);
       uppercase += step;
       len -= step;
     }

modules/brotli/dec/types.h

@@ -22,10 +22,11 @@
 
 #ifndef _MSC_VER
 #include <inttypes.h>
-#ifdef __STRICT_ANSI__
-#define BROTLI_INLINE
-#else  /* __STRICT_ANSI__ */
+#if defined(__cplusplus) || !defined(__STRICT_ANSI__) \
+    || __STDC_VERSION__ >= 199901L
 #define BROTLI_INLINE inline
+#else
+#define BROTLI_INLINE
 #endif
 #else
 typedef signed   char int8_t;

modules/brotli/update.sh

@@ -5,9 +5,9 @@
 
 MY_TEMP_DIR=`mktemp -d -t brotli_update` || exit 1
 
-git clone https://code.google.com/p/font-compression-reference/ ${MY_TEMP_DIR}
+git clone https://github.com/google/brotli ${MY_TEMP_DIR}/brotli
 
-COMMIT=`(cd ${MY_TEMP_DIR} && git log | head -n 1)`
+COMMIT=`(cd ${MY_TEMP_DIR}/brotli && git log | head -n 1)`
 perl -p -i -e "s/\[commit [0-9a-f]{40}\]/[${COMMIT}]/" README.mozilla;
 
 rm -rf dec
@@ -15,7 +15,7 @@ mv ${MY_TEMP_DIR}/brotli/dec dec
 rm -rf ${MY_TEMP_DIR}
 hg add dec
 
-echo '###'
-echo '### Updated brotli/dec to $COMMIT.'
-echo '### Remember to verify and commit the changes to source control!'
-echo '###'
+echo "###"
+echo "### Updated brotli/dec to $COMMIT."
+echo "### Remember to verify and commit the changes to source control!"
+echo "###"