Merge pull request #6793 from RenjiSann/checksum-utf8

cksum/hashsum: Support for non-UTF-8 input in checksum files

src/uucore/src/lib/lib.rs

@@ -100,10 +100,14 @@ pub use crate::features::fsxattr;
 
 //## core functions
 
+use std::borrow::Cow;
 use std::ffi::OsStr;
 use std::ffi::OsString;
+use std::io::{BufRead, BufReader};
+use std::iter;
 #[cfg(unix)]
-use std::os::unix::ffi::OsStrExt;
+use std::os::unix::ffi::{OsStrExt, OsStringExt};
+use std::str;
 use std::sync::atomic::Ordering;
 
 use once_cell::sync::Lazy;
@@ -240,6 +244,72 @@ pub fn os_str_as_bytes(os_string: &OsStr) -> mods::error::UResult<&[u8]> {
     Ok(bytes)
 }
 
+/// Helper function for converting a slice of bytes into an &OsStr
+/// or OsString in non-unix targets.
+///
+/// It converts `&[u8]` to `Cow<OsStr>` for unix targets only.
+/// On non-unix (i.e. Windows), the conversion goes through the String type
+/// and thus undergo UTF-8 validation, making it fail if the stream contains
+/// non-UTF-8 characters.
+pub fn os_str_from_bytes(bytes: &[u8]) -> mods::error::UResult<Cow<'_, OsStr>> {
+    #[cfg(unix)]
+    let os_str = Cow::Borrowed(OsStr::from_bytes(bytes));
+    #[cfg(not(unix))]
+    let os_str = Cow::Owned(OsString::from(str::from_utf8(bytes).map_err(|_| {
+        mods::error::UUsageError::new(1, "Unable to transform bytes into OsStr")
+    })?));
+
+    Ok(os_str)
+}
+
+/// Helper function for making an `OsString` from a byte field
+/// It converts `Vec<u8>` to `OsString` for unix targets only.
+/// On non-unix (i.e. Windows) it may fail if the bytes are not valid UTF-8
+pub fn os_string_from_vec(vec: Vec<u8>) -> mods::error::UResult<OsString> {
+    #[cfg(unix)]
+    let s = OsString::from_vec(vec);
+    #[cfg(not(unix))]
+    let s = OsString::from(String::from_utf8(vec).map_err(|_| {
+        mods::error::UUsageError::new(1, "invalid UTF-8 was detected in one or more arguments")
+    })?);
+
+    Ok(s)
+}
+
+/// Equivalent to `std::BufRead::lines` which outputs each line as a `Vec<u8>`,
+/// which avoids panicking on non UTF-8 input.
+pub fn read_byte_lines<R: std::io::Read>(
+    mut buf_reader: BufReader<R>,
+) -> impl Iterator<Item = Vec<u8>> {
+    iter::from_fn(move || {
+        let mut buf = Vec::with_capacity(256);
+        let size = buf_reader.read_until(b'\n', &mut buf).ok()?;
+
+        if size == 0 {
+            return None;
+        }
+
+        // Trim (\r)\n
+        if buf.ends_with(b"\n") {
+            buf.pop();
+            if buf.ends_with(b"\r") {
+                buf.pop();
+            }
+        }
+
+        Some(buf)
+    })
+}
+
+/// Equivalent to `std::BufRead::lines` which outputs each line as an `OsString`
+/// This won't panic on non UTF-8 characters on Unix,
+/// but it still will on Windows.
+pub fn read_os_string_lines<R: std::io::Read>(
+    buf_reader: BufReader<R>,
+) -> impl Iterator<Item = OsString> {
+    read_byte_lines(buf_reader).map(|byte_line| os_string_from_vec(byte_line).expect("UTF-8 error"))
+}
+
 /// Prompt the user with a formatted string and returns `true` if they reply `'y'` or `'Y'`
 ///
 /// This macro functions accepts the same syntax as `format!`. The prompt is written to

tests/by-util/test_cksum.rs

@@ -1402,3 +1402,103 @@ fn test_zero_single_file() {
         .succeeds()
         .stdout_is_fixture("zero_single_file.expected");
 }
+
+#[test]
+fn test_check_trailing_space_fails() {
+    // If a checksum line has trailing spaces after the digest,
+    // it shall be considered improperly formatted.
+
+    let scene = TestScenario::new(util_name!());
+    let at = &scene.fixtures;
+
+    at.write("foo", "foo-content\n");
+    at.write(
+        "CHECKSUM",
+        "SHA1 (foo) = 058ab38dd3603703b3a7063cf95dc51a4286b6fe    \n",
+    );
+
+    scene
+        .ucmd()
+        .arg("--check")
+        .arg("CHECKSUM")
+        .fails()
+        .no_stdout()
+        .stderr_contains("CHECKSUM: no properly formatted checksum lines found");
+}
+
+/// Regroup tests related to the handling of non-utf-8 content
+/// in checksum files.
+/// These tests are excluded from Windows because it does not provide any safe
+/// conversion between `OsString` and byte sequences for non-utf-8 strings.
+#[cfg(not(windows))]
+mod check_utf8 {
+    use super::*;
+
+    #[test]
+    fn test_check_non_utf8_comment() {
+        let hashes =
+        b"MD5 (empty) = 1B2M2Y8AsgTpgAmY7PhCfg==\n\
+        # Comment with a non utf8 char: >>\xff<<\n\
+        SHA256 (empty) = 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\n\
+        BLAKE2b (empty) = eGoC90IBWQPGxv2FJVLScpEvR0DhWEdhiobiF/cfVBnSXhAxr+5YUxOJZESTTrBLkDpoWxRIt1XVb3Aa/pvizg==\n"
+    ;
+
+        let scene = TestScenario::new(util_name!());
+        let at = &scene.fixtures;
+
+        at.touch("empty");
+        at.write_bytes("check", hashes);
+
+        scene
+            .ucmd()
+            .arg("--check")
+            .arg(at.subdir.join("check"))
+            .succeeds()
+            .stdout_is("empty: OK\nempty: OK\nempty: OK\n")
+            .no_stderr();
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn test_check_non_utf8_filename() {
+        use std::{ffi::OsString, os::unix::ffi::OsStringExt};
+
+        let scene = TestScenario::new(util_name!());
+        let at = &scene.fixtures;
+        let filename: OsString = OsStringExt::from_vec(b"funky\xffname".to_vec());
+        at.touch(&filename);
+
+        // Checksum match
+        at.write_bytes("check",
+            b"SHA256 (funky\xffname) = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n");
+        scene
+            .ucmd()
+            .arg("--check")
+            .arg(at.subdir.join("check"))
+            .succeeds()
+            .stdout_is_bytes(b"funky\xffname: OK\n")
+            .no_stderr();
+
+        // Checksum mismatch
+        at.write_bytes("check",
+            b"SHA256 (funky\xffname) = ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\n");
+        scene
+            .ucmd()
+            .arg("--check")
+            .arg(at.subdir.join("check"))
+            .fails()
+            .stdout_is_bytes(b"funky\xffname: FAILED\n")
+            .stderr_contains("1 computed checksum did NOT match");
+
+        // file not found
+        at.write_bytes("check",
+            b"SHA256 (flakey\xffname) = ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\n");
+        scene
+            .ucmd()
+            .arg("--check")
+            .arg(at.subdir.join("check"))
+            .fails()
+            .stdout_is_bytes(b"flakey\xffname: FAILED open or read\n")
+            .stderr_contains("1 listed file could not be read");
+    }
+}