checksum: Rework the OutputFormat decision

src/uu/cksum/src/cksum.rs

@@ -9,7 +9,7 @@ use clap::builder::ValueParser;
 use clap::{Arg, ArgAction, Command};
 use std::ffi::{OsStr, OsString};
 use uucore::checksum::compute::{
-    ChecksumComputeOptions, figure_out_output_format, perform_checksum_computation,
+    ChecksumComputeOptions, OutputFormat, perform_checksum_computation,
 };
 use uucore::checksum::validate::{
     ChecksumValidateOptions, ChecksumVerbose, perform_checksum_validation,
@@ -208,18 +208,20 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
 
     let (tag, binary) = handle_tag_text_binary_flags(std::env::args_os())?;
 
+    let output_format = OutputFormat::from_cksum(
+        algo_kind,
+        tag,
+        binary,
+        matches.get_flag(options::RAW),
+        matches.get_flag(options::BASE64),
+    );
+
     let algo = SizedAlgoKind::from_unsized(algo_kind, length)?;
     let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO));
 
     let opts = ChecksumComputeOptions {
         algo_kind: algo,
-        output_format: figure_out_output_format(
-            algo,
-            tag,
-            binary,
-            matches.get_flag(options::RAW),
-            matches.get_flag(options::BASE64),
-        ),
+        output_format,
         line_ending,
     };
 

src/uu/hashsum/src/hashsum.rs

@@ -13,7 +13,7 @@ use clap::builder::ValueParser;
 use clap::{Arg, ArgAction, ArgMatches, Command};
 
 use uucore::checksum::compute::{
-    ChecksumComputeOptions, figure_out_output_format, perform_checksum_computation,
+    ChecksumComputeOptions, OutputFormat, perform_checksum_computation,
 };
 use uucore::checksum::validate::{
     ChecksumValidateOptions, ChecksumVerbose, perform_checksum_validation,
@@ -121,9 +121,6 @@ pub fn uumain(mut args: impl uucore::Args) -> UResult<()> {
 
     let args = iter::once(program.clone()).chain(args);
 
-    // Default binary in Windows, text mode otherwise
-    let binary_flag_default = cfg!(windows);
-
     let (command, is_hashsum_bin) = uu_app(&binary_name);
 
     // FIXME: this should use try_get_matches_from() and crash!(), but at the moment that just
@@ -148,13 +145,6 @@ pub fn uumain(mut args: impl uucore::Args) -> UResult<()> {
         (AlgoKind::from_bin_name(&binary_name)?, length)
     };
 
-    let binary = if matches.get_flag("binary") {
-        true
-    } else if matches.get_flag("text") {
-        false
-    } else {
-        binary_flag_default
-    };
     let check = matches.get_flag("check");
 
     let check_flag = |flag| match (check, matches.get_flag(flag)) {
@@ -204,16 +194,11 @@ pub fn uumain(mut args: impl uucore::Args) -> UResult<()> {
 
     let algo = SizedAlgoKind::from_unsized(algo_kind, length)?;
     let line_ending = LineEnding::from_zero_flag(matches.get_flag("zero"));
+    let output_format = OutputFormat::from_standalone(std::env::args_os())?;
 
     let opts = ChecksumComputeOptions {
         algo_kind: algo,
-        output_format: figure_out_output_format(
-            algo,
-            matches.get_flag(options::TAG),
-            binary,
-            /* raw */ false,
-            /* base64: */ false,
-        ),
+        output_format,
         line_ending,
     };
 

src/uucore/src/lib/features/checksum/compute.rs

@@ -5,12 +5,12 @@
 
 // spell-checker:ignore bitlen
 
-use std::ffi::OsStr;
+use std::ffi::{OsStr, OsString};
 use std::fs::File;
 use std::io::{self, BufReader, Read, Write};
 use std::path::Path;
 
-use crate::checksum::{ChecksumError, SizedAlgoKind, digest_reader, escape_filename};
+use crate::checksum::{AlgoKind, ChecksumError, SizedAlgoKind, digest_reader, escape_filename};
 use crate::error::{FromIo, UResult, USimpleError};
 use crate::line_ending::LineEnding;
 use crate::sum::DigestOutput;
@@ -103,42 +103,76 @@ impl OutputFormat {
     fn is_raw(&self) -> bool {
         *self == Self::Raw
     }
-}
 
-/// Use already-processed arguments to decide the output format.
-pub fn figure_out_output_format(
-    algo: SizedAlgoKind,
-    tag: bool,
-    binary: bool,
-    raw: bool,
-    base64: bool,
-) -> OutputFormat {
-    // Raw output format takes precedence over anything else.
-    if raw {
-        return OutputFormat::Raw;
-    }
+    /// Find the correct output format for cksum.
+    pub fn from_cksum(algo: AlgoKind, tag: bool, binary: bool, raw: bool, base64: bool) -> Self {
+        // Raw output format takes precedence over anything else.
+        if raw {
+            return Self::Raw;
+        }
 
-    // Then, if the algo is legacy, takes precedence over the rest
-    if algo.is_legacy() {
-        return OutputFormat::Legacy;
-    }
+        // Then, if the algo is legacy, takes precedence over the rest
+        if algo.is_legacy() {
+            return Self::Legacy;
+        }
 
-    let digest_format = if base64 {
-        DigestFormat::Base64
-    } else {
-        DigestFormat::Hexadecimal
-    };
-
-    // After that, decide between tagged and untagged output
-    if tag {
-        OutputFormat::Tagged(digest_format)
-    } else {
-        let reading_mode = if binary {
-            ReadingMode::Binary
+        let digest_format = if base64 {
+            DigestFormat::Base64
         } else {
-            ReadingMode::Text
+            DigestFormat::Hexadecimal
         };
-        OutputFormat::Untagged(digest_format, reading_mode)
+
+        // After that, decide between tagged and untagged output
+        if tag {
+            Self::Tagged(digest_format)
+        } else {
+            let reading_mode = if binary {
+                ReadingMode::Binary
+            } else {
+                ReadingMode::Text
+            };
+            Self::Untagged(digest_format, reading_mode)
+        }
+    }
+
+    /// Find the correct output format for a standalone checksum util (b2sum,
+    /// md5sum, etc)
+    ///
+    /// Since standalone utils can't use the Raw or Legacy output format, it is
+    /// decided only using the --tag, --binary and --text arguments.
+    pub fn from_standalone(args: impl Iterator<Item = OsString>) -> UResult<Self> {
+        let mut text = true;
+        let mut tag = false;
+
+        for arg in args {
+            if arg == "--" {
+                break;
+            } else if arg == "--tag" {
+                tag = true;
+                text = false;
+            } else if arg == "--binary" || arg == "-b" {
+                text = false;
+            } else if arg == "--text" || arg == "-t" {
+                // Finding a `--text` after `--tag` is an error.
+                if tag {
+                    return Err(ChecksumError::TextAfterTag.into());
+                }
+                text = true;
+            }
+        }
+
+        if tag {
+            Ok(Self::Tagged(DigestFormat::Hexadecimal))
+        } else {
+            Ok(Self::Untagged(
+                DigestFormat::Hexadecimal,
+                if text {
+                    ReadingMode::Text
+                } else {
+                    ReadingMode::Binary
+                },
+            ))
+        }
     }
 }
 

src/uucore/src/lib/features/checksum/mod.rs

@@ -397,6 +397,8 @@ pub enum ChecksumError {
     BinaryTextConflict,
     #[error("--text mode is only supported with --untagged")]
     TextWithoutUntagged,
+    #[error("--tag does not support --text mode")]
+    TextAfterTag,
     #[error("--check is not supported with --algorithm={{bsd,sysv,crc,crc32b}}")]
     AlgorithmNotSupportedWithCheck,
     #[error("You cannot combine multiple hash algorithms!")]