Plug buffer head leak on submit for the 7.2 kernel

Starting with the 7.2 kernel, buffer heads no longer lose a reference
during the submit-wait procedure. I didn't notice that change, so now I
would expect all buffer heads to be leaking. The fact that I still can't
tell the difference makes me believe, like I mentioned before, that
somehow the kernel is still disposing of the buffer heads. Either way,
fix this.

Signed-off-by: Ernesto A. Fernández <ernesto@corellium.com>

.gitignore

@@ -9,3 +9,4 @@
 *.a
 *.mod
 tags
+version.h

Makefile

@@ -13,9 +13,16 @@ apfs-y := btree.o compress.o dir.o extents.o file.o inode.o key.o libzbitmap.o \
 	  lzfse/lzvn_decode_base.o message.o namei.o node.o object.o snapshot.o \
 	  spaceman.o super.o symlink.o transaction.o unicode.o xattr.o xfield.o
 
+# If you want mounts to be writable by default, run the build as:
+#   make APFS_CONFIG=-DCONFIG_APFS_RW_ALWAYS
+# This is risky and not generally recommended.
+ccflags-y += $(APFS_CONFIG)
+
 default:
+	./genver.sh
 	make -C $(KERNEL_DIR) M=$(PWD)
 install:
 	make -C $(KERNEL_DIR) M=$(PWD) modules_install
 clean:
+	rm -f version.h
 	make -C $(KERNEL_DIR) M=$(PWD) clean

README.rst

@@ -4,13 +4,8 @@ Apple File System for Linux
 
 The Apple File System (APFS) is the copy-on-write filesystem currently used on
 all Apple devices. This module provides a degree of experimental support on
-Linux.
+Linux. It's intended to work with all kernel versions since 4.12, but testing
-
+is focused on 5.18 and above.
-It's supposed to work with a range of kernel versions starting at 4.9 or before,
-but only a few of those have actually been tested. Also, kernel versions below
-4.12 will be slower for some operations. If you run into any problem, please
-send a report to <linux-apfs@googlegroups.com> or file a github issue at
-https://github.com/eafer/linux-apfs-rw/issues.
 
 To help test write support, a set of userland tools is also under development.
 The git tree can be retrieved from https://github.com/eafer/apfsprogs.git.
@@ -19,14 +14,32 @@ Known limitations
 =================
 
 This module is the result of reverse engineering and testing has been limited.
-If you make use of the write support, expect data corruption. Please report any
+If you make use of the write support, there is a real risk of data corruption,
-issues that you find, but I can't promise a quick resolution at this stage.
+so mounts are read-only by default. Please report any issues that you find.
 
-Encryption is not yet implemented even in read-only mode.
+Writes to fusion drives will likely never be supported, though I might
+reconsider this if people actually need it. Encryption is not yet implemented
+even in read-only mode.
+
+Reporting bugs
+==============
+
+If you encounter any problem, the first thing you should do is run (as root)::
+
+	dmesg | grep -i apfs
+
+to see all the error messages. If that doesn't help you, please file a github
+issue at https://github.com/eafer/linux-apfs-rw/issues. Or send me an email if
+you prefer.
 
 Build
 =====
 
+Note that at this point the driver has been packaged for several distros, so
+it may be more convenient to install it from there instead of building it
+yourself. The distro packages are often outdated though, so be sure to check
+the version.
+
 In order to build a module out-of-tree, you will first need the Linux kernel
 headers. Depending on your distro, you can get them by running (as root)::
 
@@ -44,10 +57,15 @@ Now you can just cd to the linux-apfs-rw directory and run::
 	make
 
 The resulting module is the apfs.ko file. Before you can use it you must insert
-it into the kernel, as well as its dependencies. Again as root::
+it into the kernel. Again as root::
+
+	insmod apfs.ko
+
+If you get an "unknown symbol in module" error, run::
 
 	modprobe libcrc32c
-	insmod apfs.ko
+
+and then try the insmod command again.
 
 Mount
 =====
@@ -64,12 +82,14 @@ vol=n	       Volume number to mount. The default is volume 0.
 
 snap=label     Volume snapshot to mount (in read-only mode).
 
+tier2=path     Path to the tier 2 device. For fusion drives only.
+
 uid=n, gid=n   Override on-disk inode ownership data with given uid/gid.
 
 cknodes	       Verify the checksum on all metadata nodes. Right now this has a
 	       severe performance cost, so it's not recommended.
 
-readwrite      Enable the experimental write support. This **will** corrupt your
+readwrite      Enable the experimental write support. This may corrupt your
 	       container.
 ============   =================================================================
 

apfs_raw.h

@@ -268,6 +268,10 @@ struct apfs_phys_ext_val {
 #define APFS_FILE_EXTENT_LEN_MASK	0x00ffffffffffffffULL
 #define APFS_FILE_EXTENT_FLAG_MASK	0xff00000000000000ULL
 #define APFS_FILE_EXTENT_FLAG_SHIFT	56
+#define APFS_FILE_EXTENT_CRYPTO_FLAG	0x01	/* Made-up name */
+#define APFS_FILE_EXTENT_PREALLOCATED	0x02	/* Made-up name */
+#define APFS_VALID_FILE_EXTENT_FLAGS	(APFS_FILE_EXTENT_CRYPTO_FLAG \
+					| APFS_FILE_EXTENT_PREALLOCATED)
 
 /*
  * Put a bound on maximum file size so that a growing truncation will always
@@ -306,7 +310,7 @@ struct apfs_wrapped_crypto_state {
 	__le32 key_os_version;
 	__le16 key_revision;
 	__le16 key_len;
-	u8 persistent_key[0];
+	u8 persistent_key[];
 } __packed;
 
 /*
@@ -503,7 +507,7 @@ struct apfs_dir_stats_val {
 struct apfs_sibling_val {
 	__le64 parent_id;
 	__le16 name_len;
-	u8 name[0];
+	u8 name[];
 } __packed;
 
 /*
@@ -622,7 +626,7 @@ enum {
 struct apfs_drec_key {
 	struct apfs_key_header hdr;
 	__le16 name_len;
-	u8 name[0];
+	u8 name[];
 } __packed;
 
 /*
@@ -632,7 +636,7 @@ struct apfs_drec_key {
 struct apfs_drec_hashed_key {
 	struct apfs_key_header hdr;
 	__le32 name_len_and_hash;
-	u8 name[0];
+	u8 name[];
 } __packed;
 
 /*
@@ -641,7 +645,7 @@ struct apfs_drec_hashed_key {
 struct apfs_xattr_key {
 	struct apfs_key_header hdr;
 	__le16 name_len;
-	u8 name[0];
+	u8 name[];
 } __packed;
 
 /*
@@ -660,7 +664,7 @@ struct apfs_snap_metadata_key {
 struct apfs_snap_name_key {
 	struct apfs_key_header hdr;
 	__le16 name_len;
-	u8 name[0];
+	u8 name[];
 } __packed;
 
 /*
@@ -1051,13 +1055,19 @@ struct apfs_checkpoint_map_phys {
 #define APFS_FS_SPILLEDOVER			0x00000010LL
 #define APFS_FS_RUN_SPILLOVER_CLEANER		0x00000020LL
 #define APFS_FS_ALWAYS_CHECK_EXTENTREF		0x00000040LL
+#define APFS_FS_PREVIOUSLY_SEALED		0x00000080LL /* Made-up name */
+#define APFS_FS_PFK				0x00000100LL /* Made-up name */
+#define APFS_FS_UNKNOWN_200			0x00000200LL
 #define APFS_FS_FLAGS_VALID_MASK		(APFS_FS_UNENCRYPTED \
-					       | APFS_FS_EFFACEABLE \
+						| APFS_FS_EFFACEABLE \
-					       | APFS_FS_RESERVED_4 \
+						| APFS_FS_RESERVED_4 \
-					       | APFS_FS_ONEKEY \
+						| APFS_FS_ONEKEY \
-					       | APFS_FS_SPILLEDOVER \
+						| APFS_FS_SPILLEDOVER \
-					       | APFS_FS_RUN_SPILLOVER_CLEANER \
+						| APFS_FS_RUN_SPILLOVER_CLEANER \
-					       | APFS_FS_ALWAYS_CHECK_EXTENTREF)
+						| APFS_FS_ALWAYS_CHECK_EXTENTREF \
+						| APFS_FS_PREVIOUSLY_SEALED \
+						| APFS_FS_PFK \
+						| APFS_FS_UNKNOWN_200)
 
 #define APFS_FS_CRYPTOFLAGS			(APFS_FS_UNENCRYPTED \
 						| APFS_FS_EFFACEABLE \
@@ -1124,7 +1134,9 @@ struct apfs_checkpoint_map_phys {
 #define APFS_INCOMPAT_NORMALIZATION_INSENSITIVE	0x00000008LL
 #define APFS_INCOMPAT_INCOMPLETE_RESTORE	0x00000010LL
 #define APFS_INCOMPAT_SEALED_VOLUME		0x00000020LL
-#define APFS_INCOMPAT_RESERVED_40		0x00000040LL
+#define APFS_INCOMPAT_PFK			0x00000040LL /* Made-up name */
+#define APFS_INCOMPAT_EXTENT_PREALLOC_FLAG	0x00000080LL /* Made-up name */
+#define APFS_INCOMPAT_SECONDARY_FSROOT		0x00000100LL /* Made-up name */
 
 #define APFS_SUPPORTED_INCOMPAT_MASK (APFS_INCOMPAT_CASE_INSENSITIVE \
 				     | APFS_INCOMPAT_DATALESS_SNAPS \
@@ -1132,7 +1144,9 @@ struct apfs_checkpoint_map_phys {
 				     | APFS_INCOMPAT_NORMALIZATION_INSENSITIVE \
 				     | APFS_INCOMPAT_INCOMPLETE_RESTORE \
 				     | APFS_INCOMPAT_SEALED_VOLUME \
-				     | APFS_INCOMPAT_RESERVED_40)
+				     | APFS_INCOMPAT_PFK \
+				     | APFS_INCOMPAT_EXTENT_PREALLOC_FLAG \
+				     | APFS_INCOMPAT_SECONDARY_FSROOT)
 
 #define APFS_MODIFIED_NAMELEN	      32
 
@@ -1162,6 +1176,18 @@ struct apfs_modified_by {
 #define APFS_CRYPTO_RESERVED_5		5
 #define APFS_UNASSIGNED_CRYPTO_ID	(~0ULL)
 
+/* Doc id index flags. I'm making up the names for now. */
+#define APFS_DOC_ID_HAS_PREV_TREE	0x00000001
+#define APFS_DOC_ID_UNKNOWN_02		0x00000002
+#define APFS_DOC_ID_UNKNOWN_04		0x00000004
+#define APFS_DOC_ID_UNKNOWN_08		0x00000008
+#define APFS_DOC_ID_UNKNOWN_10		0x00000010
+#define APFS_DOC_ID_VALID_FLAGS		(APFS_DOC_ID_HAS_PREV_TREE \
+					| APFS_DOC_ID_UNKNOWN_02 \
+					| APFS_DOC_ID_UNKNOWN_04 \
+					| APFS_DOC_ID_UNKNOWN_08 \
+					| APFS_DOC_ID_UNKNOWN_10)
+
 /*
  * Structure used to store the encryption state
  */
@@ -1241,15 +1267,24 @@ struct apfs_superblock {
 
 	__le64 apfs_snap_meta_ext_oid;
 
-	char apfs_volume_group_id[UUID_SIZE];
+/*3F0*/	char apfs_volume_group_id[UUID_SIZE];
 
-	__le64 apfs_integrity_meta_oid;
+/*400*/	__le64 apfs_integrity_meta_oid;
 
 	__le64 apfs_fext_tree_oid;
-	__le32 apfs_fext_tree_type;
+/*410*/	__le32 apfs_fext_tree_type;
 
 	__le32 reserved_type;
 	__le64 reserved_oid;
+
+/*420*/	__le64 apfs_doc_id_index_xid;
+	__le32 apfs_doc_id_index_flags;
+	__le32 apfs_doc_id_tree_type;
+/*430*/	__le64 apfs_doc_id_tree_oid; /* Made-up name */
+	__le64 apfs_prev_doc_id_tree_oid;
+	__le64 apfs_doc_id_fixup_cursor;
+	__le64 apfs_sec_root_tree_oid;
+/*450*/	__le32 apfs_sec_root_tree_type;
 } __packed;
 
 /* Extended attributes constants */
@@ -1268,9 +1303,10 @@ enum {
 	APFS_XATTR_DATA_EMBEDDED	= 0x00000002,
 	APFS_XATTR_FILE_SYSTEM_OWNED	= 0x00000004,
 	APFS_XATTR_RESERVED_8		= 0x00000008,
+	APFS_XATTR_UNKNOWN_10		= 0x00000010,
 };
 
-#define APFS_XATTR_VALID_FLAGS 0x0000000f
+#define APFS_XATTR_VALID_FLAGS 0x0000001f
 
 /*
  * Structure of the value of an extended attributes record
@@ -1278,7 +1314,7 @@ enum {
 struct apfs_xattr_val {
 	__le16 flags;
 	__le16 xdata_len;
-	u8 xdata[0];
+	u8 xdata[];
 } __packed;
 
 /*
@@ -1372,7 +1408,7 @@ struct apfs_file_info_key {
 struct apfs_file_data_hash_val {
 	__le16	hashed_len;
 	u8	hash_size;
-	u8	hash[0];
+	u8	hash[];
 } __packed;
 
 #define APFS_FILE_INFO_DATA_HASH	1
@@ -1435,9 +1471,11 @@ struct apfs_compress_rsrc_data {
 	struct apfs_compress_rsrc_block {
 		__le32 offs;
 		__le32 size;
-	} __packed block[0];
+	} __packed block[];
 } __packed;
 
+#define APFS_FUSION_TIER2_DEVICE_BYTE_ADDR	0x4000000000000000ULL
+
 /*
  * TODO: this is a placeholder, create some snapshots with the official
  * implementation to find the actual limit.
@@ -1456,9 +1494,15 @@ struct apfs_snap_metadata_val {
 	__le32 extentref_tree_type;
 	__le32 flags;
 	__le16 name_len;
-	u8 name[0];
+	u8 name[];
 } __packed;
 
+/* Snapshot metadata record flags */
+enum {
+	APFS_SNAP_META_PENDING_DATALESS	= 0x00000001,
+	APFS_SNAP_MERGE_IN_PROGRESS	= 0x00000002,
+};
+
 /*
  * Structure of the value of a snapshot name record
  */
@@ -1475,7 +1519,7 @@ struct apfs_snap_meta_ext {
 	__le32	sme_version;
 	__le32	sme_flags;
 	__le64	sme_snap_xid;
-	char 	sme_uuid[UUID_SIZE];
+	char	sme_uuid[UUID_SIZE];
 	__le64	sme_token;
 } __packed;
 
@@ -1503,11 +1547,11 @@ enum {
  * Structure of a single entry in the keybag
  */
 struct apfs_keybag_entry {
-	char 	ke_uuid[UUID_SIZE];
+	char	ke_uuid[UUID_SIZE];
 	__le16	ke_tag;
 	__le16	ke_keylen;
 	__le32	padding;
-	u8	ke_keydata[0];
+	u8	ke_keydata[];
 } __packed;
 
 #define APFS_KEYBAG_VERSION	2
@@ -1520,7 +1564,7 @@ struct apfs_kb_locker {
 	__le16				kl_nkeys;
 	__le32				kl_nbytes;
 	__le64				padding;
-	struct apfs_keybag_entry	kl_entries[0];
+	struct apfs_keybag_entry	kl_entries[];
 } __packed;
 
 #endif	/* _APFS_RAW_H */

dkms.conf

@@ -1,7 +1,8 @@
 PACKAGE_NAME="linux-apfs-rw"
-PACKAGE_VERSION="0.3.1"
+PACKAGE_VERSION="0.3.21"
 
 BUILT_MODULE_NAME[0]="apfs"
 DEST_MODULE_LOCATION[0]="/extra"
 
 AUTOINSTALL="yes"
+PRE_BUILD="genver.sh"

file.c

@@ -5,10 +5,20 @@
 
 #include "apfs.h"
 
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0)
+#include <linux/splice.h>
+#endif
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)
 typedef int vm_fault_t;
 #endif
 
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
+static void wait_for_stable_page(struct page *page)
+{
+	return folio_wait_stable(page_folio(page));
+}
+#endif
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
 static vm_fault_t apfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
@@ -18,12 +28,13 @@ static vm_fault_t apfs_page_mkwrite(struct vm_fault *vmf)
 	struct vm_area_struct *vma = vmf->vma;
 #endif
 	struct page *page = vmf->page;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 7, 0)
+	struct folio *folio;
+#endif
 	struct inode *inode = file_inode(vma->vm_file);
 	struct super_block *sb = inode->i_sb;
 	struct buffer_head *bh, *head;
 	vm_fault_t ret = VM_FAULT_LOCKED;
-	struct apfs_max_ops maxops;
-	int blkcount = PAGE_SIZE >> inode->i_blkbits;
 	unsigned int blocksize, block_start, len;
 	u64 size;
 	int err = 0;
@@ -31,19 +42,16 @@ static vm_fault_t apfs_page_mkwrite(struct vm_fault *vmf)
 	sb_start_pagefault(inode->i_sb);
 	file_update_time(vma->vm_file);
 
-	/* Placeholder values, I need to get back to this in the future */
+	err = apfs_transaction_start(sb, APFS_TRANS_REG);
-	maxops.cat = APFS_UPDATE_INODE_MAXOPS() +
-		     blkcount * APFS_GET_NEW_BLOCK_MAXOPS();
-	maxops.blks = blkcount;
-
-	err = apfs_transaction_start(sb, maxops);
 	if (err)
 		goto out;
 	apfs_inode_join_transaction(sb, inode);
 
 	err = apfs_inode_create_exclusive_dstream(inode);
-	if (err)
+	if (err) {
+		apfs_err(sb, "dstream creation failed for ino 0x%llx", apfs_ino(inode));
 		goto out_abort;
+	}
 
 	lock_page(page);
 	wait_for_stable_page(page);
@@ -52,11 +60,23 @@ static vm_fault_t apfs_page_mkwrite(struct vm_fault *vmf)
 		goto out_unlock;
 	}
 
-	if (!page_has_buffers(page))
+	if (!page_has_buffers(page)) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0)
 		create_empty_buffers(page, sb->s_blocksize, 0);
+#else
+		folio = page_folio(page);
+		bh = folio_buffers(folio);
+		if (!bh)
+			bh = create_empty_buffers(folio, sb->s_blocksize, 0);
+#endif
+	}
 
 	size = i_size_read(inode);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 16, 0)
+	if (page_folio(page)->index == size >> PAGE_SHIFT)
+#else
 	if (page->index == size >> PAGE_SHIFT)
+#endif
 		len = size & ~PAGE_MASK;
 	else
 		len = PAGE_SIZE;
@@ -77,8 +97,10 @@ static vm_fault_t apfs_page_mkwrite(struct vm_fault *vmf)
 	unlock_page(page); /* XXX: race? */
 
 	err = block_page_mkwrite(vma, vmf, apfs_get_new_block);
-	if (err)
+	if (err) {
+		apfs_err(sb, "mkwrite failed for ino 0x%llx", apfs_ino(inode));
 		goto out_abort;
+	}
 	set_page_dirty(page);
 
 	/* An immediate commit would leave the page unlocked */
@@ -95,7 +117,11 @@ out_abort:
 	apfs_transaction_abort(sb);
 out:
 	if (err)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0) && !RHEL_VERSION_GE(9, 5)
 		ret = block_page_mkwrite_return(err);
+#else
+		ret = vmf_fs_error(err);
+#endif
 	sb_end_pagefault(inode->i_sb);
 	return ret;
 }
@@ -106,11 +132,11 @@ static const struct vm_operations_struct apfs_file_vm_ops = {
 	.page_mkwrite	= apfs_page_mkwrite,
 };
 
-static int apfs_file_mmap(struct file * file, struct vm_area_struct * vma)
+int apfs_file_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct address_space *mapping = file->f_mapping;
 
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) || RHEL_VERSION_GE(9, 3)
 	if (!mapping->a_ops->read_folio)
 #else
 	if (!mapping->a_ops->readpage)
@@ -133,6 +159,16 @@ int apfs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	return apfs_sync_fs(sb, true /* wait */);
 }
 
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0)
+static ssize_t apfs_copy_file_range(struct file *src_file, loff_t src_off,
+				    struct file *dst_file, loff_t dst_off,
+				    size_t len, unsigned int flags)
+{
+	return (splice_copy_file_range(src_file, src_off,
+		dst_file, dst_off, len));
+}
+#endif
+
 const struct file_operations apfs_file_operations = {
 	.llseek			= generic_file_llseek,
 	.read_iter		= generic_file_read_iter,
@@ -141,8 +177,25 @@ const struct file_operations apfs_file_operations = {
 	.open			= generic_file_open,
 	.fsync			= apfs_fsync,
 	.unlocked_ioctl		= apfs_file_ioctl,
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0)
+	.copy_file_range	= apfs_copy_file_range,
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
 	.copy_file_range	= generic_copy_file_range,
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)
 	.remap_file_range	= apfs_remap_file_range,
+#else
+	.clone_file_range	= apfs_clone_file_range,
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0)
+	.splice_read		= generic_file_splice_read,
+#else
+	.splice_read		= filemap_splice_read,
+#endif
+	.splice_write		= iter_file_splice_write,
 };
 
 #if LINUX_VERSION_CODE == KERNEL_VERSION(5, 3, 0)

genver.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Script to generate the module version header
+#
+
+if command -v git >/dev/null 2>&1 && [ -d .git ]; then
+	GIT_COMMIT=$(git describe HEAD | tail -c 9)
+else
+	GIT_COMMIT="$(grep PACKAGE_VERSION dkms.conf | cut -d '"' -f2)?"
+fi
+
+printf '#define GIT_COMMIT\t"%s"\n' "$GIT_COMMIT" > version.h

key.c

@@ -81,8 +81,10 @@ int apfs_keycmp(struct apfs_key *k1, struct apfs_key *k2)
  */
 int apfs_read_cat_key(void *raw, int size, struct apfs_key *key, bool hashed)
 {
-	if (size < sizeof(struct apfs_key_header))
+	if (size < sizeof(struct apfs_key_header)) {
+		apfs_err(NULL, "bad key length (%d)", size);
 		return -EFSCORRUPTED;
+	}
 	key->id = apfs_cat_cnid((struct apfs_key_header *)raw);
 	key->type = apfs_cat_type((struct apfs_key_header *)raw);
 
@@ -92,6 +94,7 @@ int apfs_read_cat_key(void *raw, int size, struct apfs_key *key, bool hashed)
 			if (size < sizeof(struct apfs_drec_hashed_key) + 1 ||
 			    *((char *)raw + size - 1) != 0) {
 				/* Filename must have NULL-termination */
+				apfs_err(NULL, "invalid drec key (%d)", size);
 				return -EFSCORRUPTED;
 			}
 			/* Name length is not used in key comparisons, only the hash */
@@ -103,6 +106,7 @@ int apfs_read_cat_key(void *raw, int size, struct apfs_key *key, bool hashed)
 			if (size < sizeof(struct apfs_drec_key) + 1 ||
 			    *((char *)raw + size - 1) != 0) {
 				/* Filename must have NULL-termination */
+				apfs_err(NULL, "invalid drec key (%d)", size);
 				return -EFSCORRUPTED;
 			}
 			/* There's no hash */
@@ -114,21 +118,26 @@ int apfs_read_cat_key(void *raw, int size, struct apfs_key *key, bool hashed)
 		if (size < sizeof(struct apfs_xattr_key) + 1 ||
 		    *((char *)raw + size - 1) != 0) {
 			/* xattr name must have NULL-termination */
+			apfs_err(NULL, "invalid xattr key (%d)", size);
 			return -EFSCORRUPTED;
 		}
 		key->number = 0;
 		key->name = ((struct apfs_xattr_key *)raw)->name;
 		break;
 	case APFS_TYPE_FILE_EXTENT:
-		if (size != sizeof(struct apfs_file_extent_key))
+		if (size != sizeof(struct apfs_file_extent_key)) {
+			apfs_err(NULL, "bad key length (%d)", size);
 			return -EFSCORRUPTED;
+		}
 		key->number = le64_to_cpu(
 			((struct apfs_file_extent_key *)raw)->logical_addr);
 		key->name = NULL;
 		break;
 	case APFS_TYPE_SIBLING_LINK:
-		if (size != sizeof(struct apfs_sibling_link_key))
+		if (size != sizeof(struct apfs_sibling_link_key)) {
+			apfs_err(NULL, "bad key length (%d)", size);
 			return -EFSCORRUPTED;
+		}
 		key->number = le64_to_cpu(
 			((struct apfs_sibling_link_key *)raw)->sibling_id);
 		key->name = NULL;
@@ -146,8 +155,10 @@ int apfs_read_fext_key(void *raw, int size, struct apfs_key *key)
 {
 	struct apfs_fext_tree_key *raw_key;
 
-	if (size != sizeof(*raw_key))
+	if (size != sizeof(*raw_key)) {
+		apfs_err(NULL, "bad key length (%d)", size);
 		return -EFSCORRUPTED;
+	}
 	raw_key = raw;
 
 	key->id = le64_to_cpu(raw_key->private_id);
@@ -169,8 +180,10 @@ int apfs_read_free_queue_key(void *raw, int size, struct apfs_key *key)
 {
 	struct apfs_spaceman_free_queue_key *raw_key;
 
-	if (size < sizeof(struct apfs_spaceman_free_queue_key))
+	if (size != sizeof(*raw_key)) {
+		apfs_err(NULL, "bad key length (%d)", size);
 		return -EFSCORRUPTED;
+	}
 	raw_key = raw;
 
 	key->id = le64_to_cpu(raw_key->sfqk_xid);
@@ -191,12 +204,17 @@ int apfs_read_free_queue_key(void *raw, int size, struct apfs_key *key)
  */
 int apfs_read_omap_key(void *raw, int size, struct apfs_key *key)
 {
-	if (size < sizeof(struct apfs_omap_key))
+	struct apfs_omap_key *raw_key;
-		return -EFSCORRUPTED;
 
-	key->id = le64_to_cpu(((struct apfs_omap_key *)raw)->ok_oid);
+	if (size != sizeof(*raw_key)) {
+		apfs_err(NULL, "bad key length (%d)", size);
+		return -EFSCORRUPTED;
+	}
+	raw_key = raw;
+
+	key->id = le64_to_cpu(raw_key->ok_oid);
 	key->type = 0;
-	key->number = le64_to_cpu(((struct apfs_omap_key *)raw)->ok_xid);
+	key->number = le64_to_cpu(raw_key->ok_xid);
 	key->name = NULL;
 
 	return 0;
@@ -212,8 +230,10 @@ int apfs_read_omap_key(void *raw, int size, struct apfs_key *key)
  */
 int apfs_read_extentref_key(void *raw, int size, struct apfs_key *key)
 {
-	if (size != sizeof(struct apfs_phys_ext_key))
+	if (size != sizeof(struct apfs_phys_ext_key)) {
+		apfs_err(NULL, "bad key length (%d)", size);
 		return -EFSCORRUPTED;
+	}
 	key->id = apfs_cat_cnid((struct apfs_key_header *)raw);
 	key->type = apfs_cat_type((struct apfs_key_header *)raw);
 	key->number = 0;
@@ -223,21 +243,26 @@ int apfs_read_extentref_key(void *raw, int size, struct apfs_key *key)
 
 int apfs_read_snap_meta_key(void *raw, int size, struct apfs_key *key)
 {
-	if (size < sizeof(struct apfs_key_header))
+	if (size < sizeof(struct apfs_key_header)) {
+		apfs_err(NULL, "bad key length (%d)", size);
 		return -EFSCORRUPTED;
+	}
 	key->id = apfs_cat_cnid((struct apfs_key_header *)raw);
 	key->type = apfs_cat_type((struct apfs_key_header *)raw);
 	key->number = 0;
 
 	switch (key->type) {
 	case APFS_TYPE_SNAP_METADATA:
-		if (size != sizeof(struct apfs_snap_metadata_key))
+		if (size != sizeof(struct apfs_snap_metadata_key)) {
+			apfs_err(NULL, "bad key length (%d)", size);
 			return -EFSCORRUPTED;
+		}
 		key->name = NULL;
 		return 0;
 	case APFS_TYPE_SNAP_NAME:
 		if (size < sizeof(struct apfs_snap_name_key) + 1 || *((char *)raw + size - 1) != 0) {
 			/* snapshot name must have NULL-termination */
+			apfs_err(NULL, "invalid snap name key (%d)", size);
 			return -EFSCORRUPTED;
 		}
 		key->name = ((struct apfs_snap_name_key *)raw)->name;
@@ -251,8 +276,10 @@ int apfs_read_omap_snap_key(void *raw, int size, struct apfs_key *key)
 {
 	__le64 *xid = NULL;
 
-	if (size != sizeof(*xid))
+	if (size != sizeof(*xid)) {
+		apfs_err(NULL, "bad key length (%d)", size);
 		return -EFSCORRUPTED;
+	}
 	xid = raw;
 
 	key->id = le64_to_cpup(xid);

libzbitmap.c

@@ -12,8 +12,6 @@
 #include <linux/string.h>
 #include "libzbitmap.h"
 
-#define MIN(x, y)   ((x) > (y) ? (y) : (x))
-
 #define ZBM_MAGIC       "ZBM\x09"
 #define ZBM_MAGIC_SZ    4
 

lzfse/lzfse.h

@@ -1,23 +1,24 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
 /*
-Copyright (c) 2015-2016, Apple Inc. All rights reserved.
+ * Copyright (c) 2015-2016, Apple Inc. All rights reserved.
-
+ *
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
+ *
-1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ * 1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
+ *
-2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
+ * 2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the distribution.
+ *     in the documentation and/or other materials provided with the distribution.
-
+ *
-3.  Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
+ * 3.  Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
-    from this software without specific prior written permission.
+ *     from this software without specific prior written permission.
-
+ *
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+ */
 
 #ifndef LZFSE_H
 #define LZFSE_H
@@ -25,71 +26,6 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSI
 #include <linux/stddef.h>
 #include <linux/types.h>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#  define __attribute__(X)
-#  pragma warning(disable : 4068)
-#endif
-
-#if defined(LZFSE_DLL)
-#  if defined(_WIN32) || defined(__CYGWIN__)
-#    if defined(LZFSE_DLL_EXPORTS)
-#      define LZFSE_API __declspec(dllexport)
-#    else
-#      define LZFSE_API __declspec(dllimport)
-#    endif
-#  endif
-#endif
-
-#if !defined(LZFSE_API)
-#  if __GNUC__ >= 4
-#    define LZFSE_API __attribute__((visibility("default")))
-#  else
-#    define LZFSE_API
-#  endif
-#endif
-
-/*! @abstract Get the required scratch buffer size to compress using LZFSE.   */
-size_t lzfse_encode_scratch_size(void);
-
-/*! @abstract Compress a buffer using LZFSE.
- *
- *  @param dst_buffer
- *  Pointer to the first byte of the destination buffer.
- *
- *  @param dst_size
- *  Size of the destination buffer in bytes.
- *
- *  @param src_buffer
- *  Pointer to the first byte of the source buffer.
- *
- *  @param src_size
- *  Size of the source buffer in bytes.
- *
- *  @param scratch_buffer
- *  If non-NULL, a pointer to scratch space for the routine to use as workspace;
- *  the routine may use up to lzfse_encode_scratch_size( ) bytes of workspace
- *  during its operation, and will not perform any internal allocations. If
- *  NULL, the routine may allocate its own memory to use during operation via
- *  a single call to malloc( ), and will release it by calling free( ) prior
- *  to returning. For most use, passing NULL is perfectly satisfactory, but if
- *  you require strict control over allocation, you will want to pass an
- *  explicit scratch buffer.
- *
- *  @return
- *  The number of bytes written to the destination buffer if the input is
- *  successfully compressed. If the input cannot be compressed to fit into
- *  the provided buffer, or an error occurs, zero is returned, and the
- *  contents of dst_buffer are unspecified.                                   */
-size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer,
-                           size_t dst_size,
-                           const uint8_t *__restrict src_buffer,
-                           size_t src_size,
-                           void *__restrict scratch_buffer);
-
 /*! @abstract Get the required scratch buffer size to decompress using LZFSE. */
 size_t lzfse_decode_scratch_size(void);
 
@@ -122,15 +58,10 @@ size_t lzfse_decode_scratch_size(void);
  *  successfully decompressed. If there is not enough space in the destination
  *  buffer to hold the entire expanded output, only the first dst_size bytes
  *  will be written to the buffer and dst_size is returned. Note that this
- *  behavior differs from that of lzfse_encode_buffer.                        */
+ *  behavior differs from that of lzfse_encode_buffer.
-size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer,
+ */
-                           size_t dst_size,
+size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
-                           const uint8_t *__restrict src_buffer,
+			   const uint8_t *__restrict src_buffer, size_t src_size,
-                           size_t src_size,
+			   void *__restrict scratch_buffer);
-                           void *__restrict scratch_buffer);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
 
 #endif /* LZFSE_H */

lzfse/lzfse_decode.c

@@ -1,74 +1,80 @@
+// SPDX-License-Identifier: BSD-3-Clause
 /*
-Copyright (c) 2015-2016, Apple Inc. All rights reserved.
+ * Copyright (c) 2015-2016, Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+ *
+ * 1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ *
+ * 2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
+ *     in the documentation and/or other materials provided with the distribution.
+ *
+ * 3.  Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
 
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+/* LZFSE decode API */
-
-1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the distribution.
-
-3.  Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
-    from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-// LZFSE decode API
 
 #include <linux/slab.h>
 #include "lzfse.h"
 #include "lzfse_internal.h"
 
-size_t lzfse_decode_scratch_size(void) { return sizeof(lzfse_decoder_state); }
+size_t lzfse_decode_scratch_size(void)
+{
+	return sizeof(lzfse_decoder_state);
+}
 
-static size_t lzfse_decode_buffer_with_scratch(uint8_t *__restrict dst_buffer,
+static size_t lzfse_decode_buffer_with_scratch(uint8_t *__restrict dst_buffer, size_t dst_size,
-                         size_t dst_size, const uint8_t *__restrict src_buffer,
+					       const uint8_t *__restrict src_buffer,
-                         size_t src_size, void *__restrict scratch_buffer) {
+					       size_t src_size, void *__restrict scratch_buffer)
-  int status;
+{
-  lzfse_decoder_state *s = (lzfse_decoder_state *)scratch_buffer;
+	int status;
-  memset(s, 0x00, sizeof(*s));
+	lzfse_decoder_state *s = (lzfse_decoder_state *)scratch_buffer;
 
-  // Initialize state
+	memset(s, 0x00, sizeof(*s));
-  s->src = src_buffer;
-  s->src_begin = src_buffer;
-  s->src_end = s->src + src_size;
-  s->dst = dst_buffer;
-  s->dst_begin = dst_buffer;
-  s->dst_end = dst_buffer + dst_size;
 
-  // Decode
+	/* Initialize state */
-  status = lzfse_decode(s);
+	s->src = src_buffer;
-  if (status == LZFSE_STATUS_DST_FULL)
+	s->src_begin = src_buffer;
-    return dst_size;
+	s->src_end = s->src + src_size;
-  if (status != LZFSE_STATUS_OK)
+	s->dst = dst_buffer;
-    return 0;                           // failed
+	s->dst_begin = dst_buffer;
-  return (size_t)(s->dst - dst_buffer); // bytes written
+	s->dst_end = dst_buffer + dst_size;
+
+	/* Decode */
+	status = lzfse_decode(s);
+	if (status == LZFSE_STATUS_DST_FULL)
+		return dst_size;
+	if (status != LZFSE_STATUS_OK)
+		return 0; /* failed */
+	return (size_t)(s->dst - dst_buffer); /* bytes written */
 }
 
 size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
-                           const uint8_t *__restrict src_buffer,
+			   const uint8_t *__restrict src_buffer, size_t src_size,
-                           size_t src_size, void *__restrict scratch_buffer) {
+			   void *__restrict scratch_buffer)
-  int has_malloc = 0;
+{
-  size_t ret = 0;
+	int has_malloc = 0;
+	size_t ret = 0;
 
-  // Deal with the possible NULL pointer
+	/* Deal with the possible NULL pointer */
-  if (scratch_buffer == NULL) {
+	if (scratch_buffer == NULL) {
-    // +1 in case scratch size could be zero
+		/* +1 in case scratch size could be zero */
-    scratch_buffer = kmalloc(lzfse_decode_scratch_size() + 1, GFP_KERNEL);
+		scratch_buffer = kmalloc(lzfse_decode_scratch_size() + 1, GFP_KERNEL);
-    has_malloc = 1;
+		has_malloc = 1;
-  }
+	}
-  if (scratch_buffer == NULL)
+	if (scratch_buffer == NULL)
-    return 0;
+		return 0;
-  ret = lzfse_decode_buffer_with_scratch(dst_buffer,
+	ret = lzfse_decode_buffer_with_scratch(dst_buffer, dst_size, src_buffer, src_size,
-                               dst_size, src_buffer,
+					       scratch_buffer);
-                               src_size, scratch_buffer);
+	if (has_malloc)
-  if (has_malloc)
+		kfree(scratch_buffer);
-    kfree(scratch_buffer);
+	return ret;
-  return ret;
 }

lzfse/lzfse_encode.c

@@ -1,163 +0,0 @@
-/*
-Copyright (c) 2015-2016, Apple Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the distribution.
-
-3.  Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
-    from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-// LZFSE encode API
-
-#include "lzfse.h"
-#include "lzfse_internal.h"
-
-size_t lzfse_encode_scratch_size() {
-  size_t s1 = sizeof(lzfse_encoder_state);
-  size_t s2 = lzvn_encode_scratch_size();
-  return (s1 > s2) ? s1 : s2; // max(lzfse,lzvn)
-}
-
-size_t lzfse_encode_buffer_with_scratch(uint8_t *__restrict dst_buffer,
-                       size_t dst_size, const uint8_t *__restrict src_buffer,
-                       size_t src_size, void *__restrict scratch_buffer) {
-  const size_t original_size = src_size;
-
-  // If input is really really small, go directly to uncompressed buffer
-  // (because LZVN will refuse to encode it, and we will report a failure)
-  if (src_size < LZVN_ENCODE_MIN_SRC_SIZE)
-    goto try_uncompressed;
-
-  // If input is too small, try encoding with LZVN
-  if (src_size < LZFSE_ENCODE_LZVN_THRESHOLD) {
-    // need header + end-of-stream marker
-    size_t extra_size = 4 + sizeof(lzvn_compressed_block_header);
-    if (dst_size <= extra_size)
-      goto try_uncompressed; // DST is really too small, give up
-
-    size_t sz = lzvn_encode_buffer(
-        dst_buffer + sizeof(lzvn_compressed_block_header),
-        dst_size - extra_size, src_buffer, src_size, scratch_buffer);
-    if (sz == 0 || sz >= src_size)
-      goto try_uncompressed; // failed, or no compression, fall back to
-                             // uncompressed block
-
-    // If we could encode, setup header and end-of-stream marker (we left room
-    // for them, no need to test)
-    lzvn_compressed_block_header header;
-    header.magic = LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC;
-    header.n_raw_bytes = (uint32_t)src_size;
-    header.n_payload_bytes = (uint32_t)sz;
-    memcpy(dst_buffer, &header, sizeof(header));
-    store4(dst_buffer + sizeof(lzvn_compressed_block_header) + sz,
-           LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
-
-    return sz + extra_size;
-  }
-
-  // Try encoding with LZFSE
-  {
-    lzfse_encoder_state *state = scratch_buffer;
-    memset(state, 0x00, sizeof *state);
-    if (lzfse_encode_init(state) != LZFSE_STATUS_OK)
-      goto try_uncompressed;
-    state->dst = dst_buffer;
-    state->dst_begin = dst_buffer;
-    state->dst_end = &dst_buffer[dst_size];
-    state->src = src_buffer;
-    state->src_encode_i = 0;
-
-    if (src_size >= 0xffffffffU) {
-      //  lzfse only uses 32 bits for offsets internally, so if the input
-      //  buffer is really huge, we need to process it in smaller chunks.
-      //  Note that we switch over to this path for sizes much smaller
-      //  2GB because it's actually faster to change algorithms well before
-      //  it's necessary for correctness.
-      //  The first chunk, we just process normally.
-      const lzfse_offset encoder_block_size = 262144;
-      state->src_end = encoder_block_size;
-      if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
-        goto try_uncompressed;
-      src_size -= encoder_block_size;
-      while (src_size >= encoder_block_size) {
-        //  All subsequent chunks require a translation to keep the offsets
-        //  from getting too big.  Note that we are always going from
-        //  encoder_block_size up to 2*encoder_block_size so that the
-        //  offsets remain positive (as opposed to resetting to zero and
-        //  having negative offsets).
-        state->src_end = 2 * encoder_block_size;
-        if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
-          goto try_uncompressed;
-        lzfse_encode_translate(state, encoder_block_size);
-        src_size -= encoder_block_size;
-      }
-      //  Set the end for the final chunk.
-      state->src_end = encoder_block_size + (lzfse_offset)src_size;
-    }
-    //  If the source buffer is small enough to use 32-bit offsets, we simply
-    //  encode the whole thing in a single chunk.
-    else
-      state->src_end = (lzfse_offset)src_size;
-    //  This is either the trailing chunk (if the source file is huge), or
-    //  the whole source file.
-    if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
-      goto try_uncompressed;
-    if (lzfse_encode_finish(state) != LZFSE_STATUS_OK)
-      goto try_uncompressed;
-    //  No error occured, return compressed size.
-    return state->dst - dst_buffer;
-  }
-
-try_uncompressed:
-  //  Compression failed for some reason.  If we can fit the data into the
-  //  output buffer uncompressed, go ahead and do that instead.
-  if (original_size + 12 <= dst_size && original_size < INT32_MAX) {
-    uncompressed_block_header header = {.magic = LZFSE_UNCOMPRESSED_BLOCK_MAGIC,
-                                        .n_raw_bytes = (uint32_t)src_size};
-    uint8_t *dst_end = dst_buffer;
-    memcpy(dst_end, &header, sizeof header);
-    dst_end += sizeof header;
-    memcpy(dst_end, src_buffer, original_size);
-    dst_end += original_size;
-    store4(dst_end, LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
-    dst_end += 4;
-    return dst_end - dst_buffer;
-  }
-
-  //  Otherwise, there's nothing we can do, so return zero.
-  return 0;
-}
-
-size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
-                           const uint8_t *__restrict src_buffer,
-                           size_t src_size, void *__restrict scratch_buffer) {
-  int has_malloc = 0;
-  size_t ret = 0;
-
-  // Deal with the possible NULL pointer
-  if (scratch_buffer == NULL) {
-    // +1 in case scratch size could be zero
-    scratch_buffer = malloc(lzfse_encode_scratch_size() + 1);
-    has_malloc = 1;
-  }
-  if (scratch_buffer == NULL)
-    return 0;
-  ret = lzfse_encode_buffer_with_scratch(dst_buffer,
-                        dst_size, src_buffer,
-                        src_size, scratch_buffer);
-  if (has_malloc)
-    free(scratch_buffer);
-  return ret;
-}