Merge pull request #48 from lumag/xz

tqftpserv: add support for xz decompression

.github/workflows/ci.yml

@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Loosely based on the cdba project CI:
+#   https://github.com/linux-msm/cdba
+#
+name: "Builds"
+on:
+  pull_request:
+  push:
+  schedule:
+    # Run at 1:01 PM, every Tuesday
+    - cron: '1 13 * * 2'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        compiler: [gcc, clang]
+        # zstd and xz support are autodetected: the "enabled" variant installs
+        # the corresponding development package, the "disabled" one does not.
+        zstd: [enabled, disabled]
+        xz: [enabled, disabled]
+        container:
+          - alpine:latest
+          - archlinux:latest
+          - debian:testing
+          - debian:stable
+          - fedora:latest
+          - ubuntu:latest
+          - ubuntu:noble
+        exclude:
+          # zstd and xz ship in the Arch base image and cannot be removed, so
+          # the "disabled" builds cannot be exercised there.
+          - container: archlinux:latest
+            zstd: disabled
+          - container: archlinux:latest
+            xz: disabled
+
+    container:
+      image: ${{ matrix.container }}
+      env:
+        CC: ${{ matrix.compiler }}
+        ZSTD: ${{ matrix.zstd }}
+        XZ: ${{ matrix.xz }}
+
+    steps:
+    - name: Show OS
+      run: cat /etc/os-release
+
+    - name: Git checkout
+      uses: actions/checkout@v5
+
+    - name: Install additional packages
+      run: |
+        INSTALL=${{ matrix.container }}
+        INSTALL="${INSTALL%%:*}"
+        INSTALL="${INSTALL%%/*}"
+        ./ci/$INSTALL.sh
+
+    - name: Compiler version
+      run: $CC --version
+
+    - name: Meson setup
+      run: meson setup --errorlogs --werror build .
+
+    - name: Compile
+      run: ninja -C build
+
+    - name: Verify zstd autodetection
+      run: ./ci/check-zstd.sh
+
+    - name: Verify xz autodetection
+      run: ./ci/check-xz.sh
+
+    - name: Install
+      run: ninja -C build install
+
+  android:
+    name: Android
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        abi: [arm64-v8a, armeabi-v7a, x86_64]
+
+    env:
+      NDK_VERSION: r27d
+      QRTR_REF: v1.2
+      API: "24"
+      ABI: ${{ matrix.abi }}
+      QRTR_SRC: ${{ github.workspace }}/qrtr
+      QRTR_PREFIX: ${{ github.workspace }}/qrtr-install/${{ matrix.abi }}
+
+    steps:
+    - name: Git checkout
+      uses: actions/checkout@v5
+
+    - name: Install build tools
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends meson ninja-build pkg-config
+
+    - name: Cache Android NDK
+      id: ndk-cache
+      uses: actions/cache@v4
+      with:
+        path: android-ndk-${{ env.NDK_VERSION }}
+        key: ndk-${{ env.NDK_VERSION }}-linux
+
+    - name: Download Android NDK
+      if: steps.ndk-cache.outputs.cache-hit != 'true'
+      run: |
+        curl -sSLO https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux.zip
+        unzip -q android-ndk-${NDK_VERSION}-linux.zip
+        rm android-ndk-${NDK_VERSION}-linux.zip
+
+    - name: Cache libqrtr
+      id: qrtr-cache
+      uses: actions/cache@v4
+      with:
+        path: qrtr-install/${{ matrix.abi }}
+        key: qrtr-android-${{ matrix.abi }}-${{ env.QRTR_REF }}-ndk-${{ env.NDK_VERSION }}
+
+    - name: Checkout qrtr
+      if: steps.qrtr-cache.outputs.cache-hit != 'true'
+      uses: actions/checkout@v5
+      with:
+        repository: linux-msm/qrtr
+        ref: ${{ env.QRTR_REF }}
+        path: qrtr
+
+    - name: Build for Android
+      run: |
+        export ANDROID_NDK="$GITHUB_WORKSPACE/android-ndk-${NDK_VERSION}"
+        ./ci/android.sh

ci/alpine.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Install build dependencies on Alpine Linux.
+
+set -ex
+
+# build-base pulls in binutils (the linker/assembler) and the musl headers,
+# which clang needs as well; add clang on top when it is the compiler.
+PKGS_CC=""
+case $CC in
+	clang*)
+		PKGS_CC="clang"
+	;;
+esac
+
+PKGS_ZSTD=""
+if [ "$ZSTD" = "enabled" ]; then
+	PKGS_ZSTD="zstd-dev"
+fi
+
+PKGS_XZ=""
+if [ "$XZ" = "enabled" ]; then
+	PKGS_XZ="xz-dev"
+fi
+
+apk add \
+	build-base \
+	pkgconf \
+	meson \
+	ninja \
+	qrtr-dev \
+	$PKGS_ZSTD \
+	$PKGS_XZ \
+	$PKGS_CC
+
+echo "Install finished: $0"

ci/android.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Cross-build tqftpserv (and its libqrtr dependency) for Android using the NDK.
+#
+# Expects the following environment variables:
+#   ABI          - Android ABI (arm64-v8a, armeabi-v7a, x86_64)
+#   API          - Android API level to target
+#   ANDROID_NDK  - path to an unpacked Android NDK
+#   QRTR_SRC     - path to a checked-out qrtr source tree
+#   QRTR_PREFIX  - prefix into which libqrtr is (or was) installed
+
+set -ex
+
+TOOLCHAIN="$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64"
+
+case "$ABI" in
+	arm64-v8a)
+		CPU_FAMILY=aarch64
+		CPU=aarch64
+		TRIPLE=aarch64-linux-android
+		ELF_MACHINE=AArch64
+	;;
+	armeabi-v7a)
+		CPU_FAMILY=arm
+		CPU=armv7a
+		TRIPLE=armv7a-linux-androideabi
+		ELF_MACHINE=ARM
+	;;
+	x86_64)
+		CPU_FAMILY=x86_64
+		CPU=x86_64
+		TRIPLE=x86_64-linux-android
+		ELF_MACHINE=X86-64
+	;;
+	*)
+		echo "Unknown ABI '$ABI'" >&2
+		exit 1
+	;;
+esac
+
+# The per-ABI clang wrapper already selects the correct target and sysroot, so
+# the cross file does not need to spell those out.
+cat > android-cross.txt <<EOF
+[binaries]
+c = '$TOOLCHAIN/bin/${TRIPLE}${API}-clang'
+ar = '$TOOLCHAIN/bin/llvm-ar'
+strip = '$TOOLCHAIN/bin/llvm-strip'
+pkg-config = 'pkg-config'
+
+[host_machine]
+system = 'android'
+cpu_family = '$CPU_FAMILY'
+cpu = '$CPU'
+endian = 'little'
+
+[properties]
+pkg_config_libdir = '$QRTR_PREFIX/lib/pkgconfig'
+EOF
+cat android-cross.txt
+
+# Build and install libqrtr, unless it was already restored from the cache.
+if [ ! -f "$QRTR_PREFIX/lib/pkgconfig/qrtr.pc" ]; then
+	meson setup --cross-file android-cross.txt \
+		--prefix="$QRTR_PREFIX" --libdir=lib \
+		"$QRTR_SRC/build-$ABI" "$QRTR_SRC"
+	ninja -C "$QRTR_SRC/build-$ABI" install
+fi
+
+meson setup --errorlogs --werror --cross-file android-cross.txt build .
+ninja -C build
+
+# Confirm the resulting binary really is for the requested ABI.
+"$TOOLCHAIN/bin/llvm-readelf" -h build/tqftpserv | grep -q "Machine:.*$ELF_MACHINE"
+echo "Android build for $ABI ($ELF_MACHINE) succeeded"

ci/archlinux.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Install build dependencies on Arch Linux.
+#
+# qrtr is only packaged in the AUR, which is not available in the CI
+# container, so build and install it from source.
+
+set -ex
+
+PKGS_CC="gcc"
+case $CC in
+	clang*)
+		PKGS_CC="clang"
+	;;
+esac
+
+# zstd and xz (liblzma) ship as part of the Arch base image and cannot be
+# removed, so there is no "disabled" variant to install here.
+pacman -Syu --noconfirm \
+	pkgconf \
+	meson \
+	ninja \
+	git \
+	zstd \
+	xz \
+	$PKGS_CC
+
+QRTR_SRC=$(mktemp -d)
+git clone --depth 1 https://github.com/linux-msm/qrtr "$QRTR_SRC"
+meson setup --prefix=/usr "$QRTR_SRC/build" "$QRTR_SRC"
+ninja -C "$QRTR_SRC/build"
+ninja -C "$QRTR_SRC/build" install
+rm -rf "$QRTR_SRC"
+
+echo "Install finished: $0"

ci/check-xz.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Verify that xz support was autodetected as expected: xz-decompress.c is
+# only compiled when liblzma is available.
+
+set -eu
+
+OBJ=build/tqftpserv.p/xz-decompress.c.o
+
+case "$XZ" in
+enabled)
+	if [ ! -f "$OBJ" ]; then
+		echo "ERROR: expected xz support, but xz-decompress.c was not built"
+		exit 1
+	fi
+	;;
+disabled)
+	if [ -f "$OBJ" ]; then
+		echo "ERROR: expected no xz support, but xz-decompress.c was built"
+		exit 1
+	fi
+	;;
+*)
+	echo "ERROR: unexpected XZ value '$XZ'"
+	exit 1
+	;;
+esac
+
+echo "xz support check ($XZ) passed"

ci/check-zstd.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Verify that zstd support was autodetected as expected: zstd-decompress.c is
+# only compiled when libzstd is available.
+
+set -eu
+
+OBJ=build/tqftpserv.p/zstd-decompress.c.o
+
+case "$ZSTD" in
+enabled)
+	if [ ! -f "$OBJ" ]; then
+		echo "ERROR: expected zstd support, but zstd-decompress.c was not built"
+		exit 1
+	fi
+	;;
+disabled)
+	if [ -f "$OBJ" ]; then
+		echo "ERROR: expected no zstd support, but zstd-decompress.c was built"
+		exit 1
+	fi
+	;;
+*)
+	echo "ERROR: unexpected ZSTD value '$ZSTD'"
+	exit 1
+	;;
+esac
+
+echo "zstd support check ($ZSTD) passed"

ci/debian.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Install build dependencies on Debian and Ubuntu.
+
+set -ex
+
+apt update
+
+# Some distros might pull tzdata which asks questions
+export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true
+
+PKGS_CC="build-essential"
+case $CC in
+	clang*)
+		PKGS_CC="clang"
+	;;
+esac
+
+PKGS_ZSTD=""
+if [ "$ZSTD" = "enabled" ]; then
+	PKGS_ZSTD="libzstd-dev"
+fi
+
+PKGS_XZ=""
+if [ "$XZ" = "enabled" ]; then
+	PKGS_XZ="liblzma-dev"
+fi
+
+apt install -y --no-install-recommends \
+	pkg-config \
+	meson \
+	ninja-build \
+	libqrtr-dev \
+	$PKGS_ZSTD \
+	$PKGS_XZ \
+	$PKGS_CC
+
+echo "Install finished: $0"

ci/fedora.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+#
+# Install build dependencies on Fedora.
+
+set -ex
+
+PKGS_CC="gcc"
+case $CC in
+	clang*)
+		PKGS_CC="clang"
+	;;
+esac
+
+PKGS_ZSTD=""
+if [ "$ZSTD" = "enabled" ]; then
+	PKGS_ZSTD="libzstd-devel"
+fi
+
+PKGS_XZ=""
+if [ "$XZ" = "enabled" ]; then
+	PKGS_XZ="xz-devel"
+fi
+
+dnf -y install \
+	pkgconf-pkg-config \
+	meson \
+	ninja-build \
+	qrtr-devel \
+	$PKGS_ZSTD \
+	$PKGS_XZ \
+	$PKGS_CC
+
+echo "Install finished: $0"

ci/ubuntu.sh

@@ -0,0 +1 @@
+debian.sh

meson.build

@@ -10,8 +10,27 @@ project('tqftpserv',
 
 prefix = get_option('prefix')
 
-zstd_dep = dependency('libzstd')
-add_project_arguments('-DHAVE_ZSTD', language : 'c')
+if host_machine.system() == 'android'
+  add_project_arguments('-DANDROID', language : 'c')
+endif
+
+tqftpserv_srcs = ['translate.c',
+                  'tqftpserv.c']
+tqftpserv_deps = []
+
+xz_dep = dependency('liblzma', required : false)
+if xz_dep.found()
+  add_project_arguments('-DHAVE_XZ', language : 'c')
+  tqftpserv_srcs += 'xz-decompress.c'
+  tqftpserv_deps += xz_dep
+endif
+
+zstd_dep = dependency('libzstd', required : false)
+if zstd_dep.found()
+  add_project_arguments('-DHAVE_ZSTD', language : 'c')
+  tqftpserv_srcs += 'zstd-decompress.c'
+  tqftpserv_deps += zstd_dep
+endif
 
 # Not required to build the executable,
 # only to automatically retrieve the install dir for unit files
@@ -22,16 +41,21 @@ elif systemd.found()
   systemd_system_unit_dir = systemd.get_variable(
     pkgconfig : 'systemdsystemunitdir',
     pkgconfig_define: ['prefix', prefix])
+else
+  systemd_system_unit_dir = ''
 endif
 
-qrtr_dep = dependency('qrtr')
+qrtr_dep = dependency('qrtr', required : false)
+if not qrtr_dep.found()
+  # Support old libqrtr that does not ship a qrtr.pc pkg-config file.
+  cc = meson.get_compiler('c')
+  qrtr_dep = cc.find_library('qrtr', has_headers : 'libqrtr.h')
+endif
+tqftpserv_deps += qrtr_dep
 
-tqftpserv_srcs = ['translate.c',
-                  'tqftpserv.c',
-                  'zstd-decompress.c']
 executable('tqftpserv',
            tqftpserv_srcs,
-           dependencies : [qrtr_dep, zstd_dep],
+           dependencies : tqftpserv_deps,
            install : true)
 
 if systemd_system_unit_dir != ''

populate-readwrite.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/sh
+# Populate tqftp readwrite data from persist partition
+
+set -eu
+
+PERSIST_DEV="/dev/disk/by-partlabel/persist"
+PERSIST_MOUNT="/mnt/persist"
+TQFTPSERV_TMP="/var/lib/tqftpserv"
+
+if [ -f $TQFTPSERV_TMP/readwrite_ready ]; then
+	exit 0
+fi
+
+mkdir -p $PERSIST_MOUNT
+mount $PERSIST_DEV -o ro,noatime $PERSIST_MOUNT
+mkdir -p $TQFTPSERV_TMP
+cp -r $PERSIST_MOUNT/rfs/msm/mpss/* $TQFTPSERV_TMP/
+touch $TQFTPSERV_TMP/readwrite_ready
+umount $PERSIST_MOUNT
+rmdir $PERSIST_MOUNT

tqftpserv-populate-readwrite.service.in

@@ -0,0 +1,11 @@
+[Unit]
+Description= Populate tqftp readwrite data from persist partition
+Requires=dev-disk-by\x2dpartlabel-persist.device
+After=dev-disk-by\x2dpartlabel-persist.device
+
+[Service]
+Type=oneshot
+ExecStart=@prefix@/bin/populate-readwrite.sh
+
+[Install]
+WantedBy=multi-user.target

tqftpserv.c

@@ -313,7 +313,7 @@ static int tftp_send_oack(int sock, size_t *blocksize, size_t *tsize,
 		memcpy(p, "seek", 5);
 		p += 5;
 
-		n = snprintf(p, end - p, "%zd", *seek);
+		n = snprintf(p, end - p, "%zd", (size_t)*seek);
 		if (n < 0 || n >= end - p)
 			return -1;
 		p += n;

translate.c

@@ -16,6 +16,7 @@
 #include <unistd.h>
 
 #include "translate.h"
+#include "xz-decompress.h"
 #include "zstd-decompress.h"
 
 #define READONLY_PATH	"/readonly/firmware/image/"
@@ -234,6 +235,8 @@ int translate_open(const char *path, int flags)
 	return -1;
 }
 
+/* linux-firmware uses .xz as file extension */
+#define XZ_EXTENSION ".xz"
 /* linux-firmware uses .zst as file extension */
 #define ZSTD_EXTENSION ".zst"
 
@@ -245,21 +248,33 @@ int translate_open(const char *path, int flags)
  */
 static int open_maybe_compressed(const char *path)
 {
-	char *path_with_zstd_extension = NULL;
+	char *path_with_extension = NULL;
 	int fd = -1;
 	int ret;
 
 	if (access(path, F_OK) == 0)
 		return open(path, O_RDONLY);
 
-	ret = asprintf(&path_with_zstd_extension, "%s%s", path, ZSTD_EXTENSION);
+	ret = asprintf(&path_with_extension, "%s%s", path, XZ_EXTENSION);
 	if (ret < 0)
 		return ret;
 
-	if (access(path_with_zstd_extension, F_OK) == 0)
-		fd = zstd_decompress_file(path_with_zstd_extension);
+	if (access(path_with_extension, F_OK) == 0)
+		fd = xz_decompress_file(path_with_extension);
 
-	free(path_with_zstd_extension);
+	free(path_with_extension);
+
+	if (fd != -1)
+		return fd;
+
+	ret = asprintf(&path_with_extension, "%s%s", path, ZSTD_EXTENSION);
+	if (ret < 0)
+		return ret;
+
+	if (access(path_with_extension, F_OK) == 0)
+		fd = zstd_decompress_file(path_with_extension);
+
+	free(path_with_extension);
 
 	return fd;
 }

xz-decompress.c

@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * Based on the code copyright by:
+ * Copyright (c) 2024, Stefan Hansson
+ * Copyright (c) 2024, Emil Velikov
+ */
+
+/* For memfd_create */
+#define _GNU_SOURCE
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <lzma.h>
+
+#include "xz-decompress.h"
+
+/**
+ * xz_decompress_file() - decompress an xz-compressed file
+ * @filename:	path to a file to decompress
+ *
+ * The xz format does not store the uncompressed size in an easily
+ * queryable header, so decode the stream incrementally and write the
+ * output to a memfd as it is produced.
+ *
+ * Return: opened fd on success, -1 on error
+ */
+int xz_decompress_file(const char *filename)
+{
+	/* Figure out the size of the file. */
+	struct stat file_stat;
+	if (stat(filename, &file_stat) == -1) {
+		fprintf(stderr, "stat %s failed (%s)\n", filename, strerror(errno));
+		return -1;
+	}
+
+	const size_t file_size = file_stat.st_size;
+
+	const int input_file_fd = open(filename, O_RDONLY);
+	if (input_file_fd == -1) {
+		perror("open failed");
+		return -1;
+	}
+
+	void* const compressed_buffer = mmap(NULL, file_size, PROT_READ, MAP_POPULATE | MAP_PRIVATE, input_file_fd, 0);
+	if (compressed_buffer == MAP_FAILED) {
+		perror("mmap failed");
+		close(input_file_fd);
+		return -1;
+	}
+	close(input_file_fd);
+
+	lzma_stream strm = LZMA_STREAM_INIT;
+	lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX, 0);
+	if (ret != LZMA_OK) {
+		fprintf(stderr, "lzma_stream_decoder failed for %s (error %d)\n", filename, ret);
+		munmap(compressed_buffer, file_size);
+		return -1;
+	}
+
+	const int output_file_fd = memfd_create(filename, 0);
+	if (output_file_fd == -1) {
+		perror("memfd_create failed");
+		lzma_end(&strm);
+		munmap(compressed_buffer, file_size);
+		return -1;
+	}
+
+	uint8_t output_buffer[BUFSIZ];
+
+	strm.next_in = compressed_buffer;
+	strm.avail_in = file_size;
+
+	do {
+		strm.next_out = output_buffer;
+		strm.avail_out = sizeof(output_buffer);
+
+		ret = lzma_code(&strm, LZMA_FINISH);
+		if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
+			fprintf(stderr, "lzma_code failed for %s (error %d)\n", filename, ret);
+			close(output_file_fd);
+			lzma_end(&strm);
+			munmap(compressed_buffer, file_size);
+			return -1;
+		}
+
+		const size_t produced = sizeof(output_buffer) - strm.avail_out;
+		if (write(output_file_fd, output_buffer, produced) != (ssize_t)produced) {
+			perror("write failed");
+			close(output_file_fd);
+			lzma_end(&strm);
+			munmap(compressed_buffer, file_size);
+			return -1;
+		}
+	} while (ret != LZMA_STREAM_END);
+
+	lzma_end(&strm);
+	munmap(compressed_buffer, file_size);
+
+	return output_file_fd;
+}

xz-decompress.h

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __XZ_DECOMPRESS_H__
+#define __XZ_DECOMPRESS_H__
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#ifdef HAVE_XZ
+int xz_decompress_file(const char *filename);
+#else
+static int xz_decompress_file(const char *filename)
+{
+	fprintf(stderr, "Built without XZ support\n");
+	return -1;
+}
+#endif
+
+#endif

zstd-decompress.h

@@ -7,6 +7,7 @@
 #define __ZSTD_DECOMPRESS_H__
 
 #include <stdbool.h>
+#include <stdio.h>
 
 #ifdef HAVE_ZSTD
 int zstd_decompress_file(const char *filename);