Disable T in POSIX mode

Implements the GNU `T LABEL` command: branch to LABEL if no s///
substitution has succeeded since the last input line was read or the
last t/T was taken. It is the inverse of `t` and shares the same
address and label parsing.

Wires 'T' into get_cmd_spec, resolve_branch_targets, and the processor
branch logic, resetting the substitution flag on evaluation just like
't'. Adds integration tests (simple branch + flag-clear) and a cmd-spec
unit test.

Closes #395

Co-authored-by: Diomidis Spinellis <dds@aueb.gr>

.github/workflows/benchmarks.yml

@@ -33,7 +33,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: Run sccache-cache
-        uses: mozilla-actions/sccache-action@v0.0.10
+        uses: mozilla-actions/sccache-action@v0.0.11
 
       - name: Install tools
         uses: taiki-e/install-action@v2
@@ -47,7 +47,7 @@ jobs:
           cargo codspeed build -p ${{ matrix.benchmark-target.package }}
 
       - name: Run benchmarks for ${{ matrix.benchmark-target.package }}
-        uses: CodSpeedHQ/action@v4
+        uses: CodSpeedHQ/action@v5
         env:
           CODSPEED_LOG: debug
         with:

.github/workflows/code-quality.yml

@@ -51,7 +51,7 @@ jobs:
           components: clippy
       - uses: Swatinem/rust-cache@v2
       - name: Run sccache-cache
-        uses: mozilla-actions/sccache-action@v0.0.10
+        uses: mozilla-actions/sccache-action@v0.0.11
       - name: Initialize workflow variables
         id: vars
         shell: bash

Cargo.lock

@@ -197,18 +197,18 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
 dependencies = [
  "anstream",
  "anstyle",
@@ -219,9 +219,9 @@ dependencies = [
 
 [[package]]
 name = "clap_complete"
-version = "4.6.7"
+version = "4.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db8b397918185f0161ff3d6fcaa9e4bfc09b8367caf6e1d4a2848e5477ed027b"
+checksum = "b1f84a88507dbd05c695f2cb5e8558e747179134005e9893882dec964190ed89"
 dependencies = [
  "clap",
 ]
@@ -384,7 +384,7 @@ version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -426,9 +426,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -450,9 +450,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "1.0.8"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb"
+checksum = "2d83cb7e7a873830708d6b02a78cd36a592c6fa14bf267b68725103b85c0d77f"
 dependencies = [
  "link-section",
  "linktime-proc-macro",
@@ -543,7 +543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -830,7 +830,7 @@ dependencies = [
  "portable-atomic",
  "portable-atomic-util",
  "serde_core",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -883,21 +883,21 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "link-section"
-version = "0.19.0"
+version = "0.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e333fe507b738576d6da5bb3f1a7d7a1c80307ed9ef31624c057d844c19c93e9"
+checksum = "5ee1a0d6e252afe82e7bc2db42fba60e02ddf3b1accaf8cb21d96e34ba61f3d4"
 
 [[package]]
 name = "linktime-proc-macro"
-version = "0.2.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c7b0a3383c2a1002d11349c92c85a666a5fb679e96c79d782cf0dbe557fd6ee"
+checksum = "348d0075b1fc163b26d72a7f75fc5141daf2fd1bdf128d873cbaf6785d495bdf"
 
 [[package]]
 name = "linux-raw-sys"
@@ -913,9 +913,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
 [[package]]
 name = "memchr"
-version = "2.8.2"
+version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
 [[package]]
 name = "memmap2"
@@ -1179,9 +1179,9 @@ checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
 
 [[package]]
 name = "regex"
-version = "1.12.4"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1191,9 +1191,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.14"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1243,7 +1243,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1445,7 +1445,7 @@ dependencies = [
  "getrandom 0.4.2",
  "once_cell",
  "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1455,7 +1455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
 dependencies = [
  "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1831,7 +1831,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]

README.md

@@ -116,12 +116,19 @@ cargo test
   sequences.
 
 ### Incompatibilities
-* The input is assumed to be valid UTF-8 (this includes 7-bit ASCII).
-  If the input is in another code page, consider converting it through UTF-8
-  in order to avoid errors on invalid UTF-8 sequences and for the correct
-  handling of regular expressions.
-  This _sed_ program can also handle arbitrary byte sequences if no part of the
-  input is treated as string.
+* Similarly to GNU _sed_, input is processed as raw bytes or as valid UTF-8
+  (this includes 7-bit ASCII) based on the locale as specified by the
+  `LC_ALL`, `LC_CTYPE`, and `LANG` environment variables,
+  with the default being byte processing.
+  However, in contrast with GNU _sed_, other locales (e.g. ISO-8859-1)
+  are not supported. If the input is in another code page or encoding
+  and requires locale-specific processing (e.g. ignore/map case,
+  character classes), consider converting it through UTF-8 to ensure
+  the correct handling of locale-specific regular expressions.
+  This _sed_ program can also handle arbitrary byte sequences
+  if no part of the input requires treating it as a Rust String.
+* Back-references aren't supported when input is processed as bytes
+  (`LC_ALL=C`).
 * The command will report an error and fail if duplicate labels are found
   in the script.
   This matches the BSD behavior. The GNU version accepts duplicate labels.

fuzz/Cargo.lock

@@ -571,9 +571,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libfuzzer-sys"

src/sed/command.rs

@@ -52,6 +52,8 @@ pub struct ProcessingContext {
     pub last_file: bool,
     /// Stop processing further input.
     pub stop_processing: bool,
+    /// Whether sed operates on bytes or UTF-8 characters
+    pub character_mode: CharacterMode,
     /// Previously compiled RE, saved for reuse when specifying an empty RE
     pub saved_regex: Option<Regex>,
     /// Modification of input processing action
@@ -59,7 +61,7 @@ pub struct ProcessingContext {
     // command.
     pub input_action: Option<InputAction>,
     /// Hold space
-    pub hold: StringSpace,
+    pub hold: ByteSpace,
     /// Nesting of { } at compile time
     pub parsed_block_nesting: usize,
     /// Command associated with each label
@@ -75,14 +77,22 @@ pub struct ProcessingContext {
 #[derive(Clone, Debug)]
 /// Elements that shall be appended at the end of each command processing cycle
 pub enum AppendElement {
-    Text(Rc<str>), // The specified text string
-    Path(PathBuf), // The contents of the specified file path
+    Text(Rc<[u8]>), // The specified text bytes
+    Path(PathBuf),  // The contents of the specified file path
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+/// Whether sed operates on bytes or UTF-8 characters
+pub enum CharacterMode {
+    Byte, // Interpret data as arbitrary bytes (C/POSIX locale).
+    #[default]
+    Utf8, // Interpret data as UTF-8 characters (UTF-8 locale).
 }
 
 #[derive(Clone, Debug, Default, PartialEq)]
-/// A space mirroring IOChunk, but only with a String
-pub struct StringSpace {
-    pub content: String,   // Line content without newline
+/// A space mirroring IOChunk without mmap-backed storage.
+pub struct ByteSpace {
+    pub content: Vec<u8>,  // Line content without newline
     pub has_newline: bool, // True if \n-terminated
 }
 
@@ -100,9 +110,9 @@ pub enum Address {
 #[derive(Debug)]
 /// A single part of an RE replacement
 pub enum ReplacementPart {
-    Literal(String), // Normal text
-    WholeMatch,      // &
-    Group(u32),      // \1 to \9
+    Literal(Vec<u8>), // Normal text
+    WholeMatch,       // &
+    Group(u32),       // \1 to \9
 }
 
 // The maximum value allowed in regex quantifier
@@ -152,8 +162,8 @@ impl ReplacementTemplate {
     /// let result = regex.replace_all(input, |caps: &Captures| {
     ///    template.apply_captures(&command, caps) });
     /// Returns an error if a backreference in the template was not matched by the RE.
-    pub fn apply_captures(&self, command: &Command, caps: &Captures) -> UResult<String> {
-        let mut result = String::new();
+    pub fn apply_captures(&self, command: &Command, caps: &Captures) -> UResult<Vec<u8>> {
+        let mut result = Vec::new();
 
         // Invalid group numbers may end here through (unkown at compile time)
         // reused REs.
@@ -169,15 +179,17 @@ impl ReplacementTemplate {
 
         for part in &self.parts {
             match part {
-                ReplacementPart::Literal(s) => result.push_str(s),
+                ReplacementPart::Literal(s) => result.extend_from_slice(s),
 
                 ReplacementPart::WholeMatch => {
-                    result.push_str(caps.get(0)?.map(|m| m.as_str()).unwrap_or_default());
+                    result
+                        .extend_from_slice(caps.get(0)?.map(|m| m.as_bytes()).unwrap_or_default());
                 }
 
                 ReplacementPart::Group(n) => {
                     let i: usize = (*n).try_into().unwrap();
-                    result.push_str(caps.get(i)?.map(|m| m.as_str()).unwrap_or_default());
+                    result
+                        .extend_from_slice(caps.get(i)?.map(|m| m.as_bytes()).unwrap_or_default());
                 }
             }
         }
@@ -186,14 +198,14 @@ impl ReplacementTemplate {
     }
 
     /// Apply the template to the given RE single match.
-    pub fn apply_match(&self, m: &Match) -> String {
-        let mut result = String::new();
+    pub fn apply_match(&self, m: &Match) -> Vec<u8> {
+        let mut result = Vec::new();
 
         for part in &self.parts {
             match part {
-                ReplacementPart::Literal(s) => result.push_str(s),
+                ReplacementPart::Literal(s) => result.extend_from_slice(s),
 
-                ReplacementPart::WholeMatch => result.push_str(m.as_str()),
+                ReplacementPart::WholeMatch => result.extend_from_slice(m.as_bytes()),
 
                 ReplacementPart::Group(_) => {
                     panic!("unexpected Regex group replacement")
@@ -211,12 +223,20 @@ pub struct Substitution {
     pub replacement: ReplacementTemplate,             // Specified broken-down replacement
     pub occurrence: usize,                            // Which occurrence to substitute
     pub print_flag: bool,                             // True if 'p' flag
+    pub p_before_e: bool,                             // True if 'p' appears before 'e'
     pub ignore_case: bool,                            // True if 'I' flag
     pub execute: bool,                                // True if 'e' flag (GNU extension)
     pub multiline: bool,                              // True if 'm' or 'M' flag (GNU extension)
     pub write_file: Option<Rc<RefCell<NamedWriter>>>, // Writer to file if 'w' flag is used
 }
 
+#[derive(Debug, PartialEq, Eq)]
+/// Result of parsing a transliteration string in byte or character mode.
+pub enum ParsedTransliteration {
+    Bytes(Vec<u8>),
+    Text(String),
+}
+
 /// The block of the first and most common Unicode characters:
 /// ASCII, Latin Extended, Greek, Curillic, Coptic, Arabic, etc.
 /// It comprises all UCS-2 characters.  We use a fast lookup array for these.
@@ -225,20 +245,28 @@ const COMMON_UNICODE: usize = 2048;
 #[derive(Debug)]
 /// Transliteration command (y)
 pub struct Transliteration {
-    fast: [char; COMMON_UNICODE],
-    slow: HashMap<char, char>,
+    byte_fast: [u8; 256],
+    unicode_fast: [char; COMMON_UNICODE],
+    unicode_slow: HashMap<char, char>,
+    pub(crate) is_byte_identity: bool,
 }
 
 impl Default for Transliteration {
     /// Create a new Transliteration with identity mapping for the fast-path.
     fn default() -> Self {
-        let mut fast = ['\0'; COMMON_UNICODE];
+        let mut fast = [0u8; 256];
         for (i, slot) in fast.iter_mut().enumerate() {
+            *slot = i as u8;
+        }
+        let mut unicode_fast = ['\0'; COMMON_UNICODE];
+        for (i, slot) in unicode_fast.iter_mut().enumerate() {
             *slot = char::from_u32(i as u32).unwrap_or('\0');
         }
         Self {
-            fast,
-            slow: HashMap::new(),
+            byte_fast: fast,
+            unicode_fast,
+            unicode_slow: HashMap::new(),
+            is_byte_identity: true,
         }
     }
 }
@@ -253,25 +281,44 @@ impl Transliteration {
         result
     }
 
+    /// Create through byte mappings from `source` to `target`.
+    pub fn from_bytes(source: &[u8], target: &[u8]) -> Self {
+        let mut result = Self::default();
+        for (&from, &to) in source.iter().zip(target) {
+            result.byte_fast[from as usize] = to;
+        }
+        result
+    }
+
     /// Set a transliteration mapping from one character to another.
     fn insert(&mut self, from: char, to: char) {
         let cp = from as usize;
         if cp < COMMON_UNICODE {
-            self.fast[cp] = to;
+            self.unicode_fast[cp] = to;
         } else {
-            self.slow.insert(from, to);
+            self.unicode_slow.insert(from, to);
+        }
+        if from.is_ascii() && to.is_ascii() {
+            self.byte_fast[from as usize] = to as u8;
+        } else {
+            self.is_byte_identity = false;
         }
     }
 
     /// Look up a character transliteration.
-    pub fn lookup(&self, ch: char) -> char {
+    pub fn lookup_char(&self, ch: char) -> char {
         let cp = ch as usize;
         if cp < COMMON_UNICODE {
-            self.fast[cp]
+            self.unicode_fast[cp]
         } else {
-            self.slow.get(&ch).copied().unwrap_or(ch)
+            self.unicode_slow.get(&ch).copied().unwrap_or(ch)
         }
     }
+
+    /// Fast byte lookup for pure ASCII transliterations.
+    pub fn lookup_byte(&self, byte: u8) -> u8 {
+        self.byte_fast[byte as usize]
+    }
 }
 
 #[derive(Debug)]
@@ -314,16 +361,16 @@ impl Command {
 
 #[derive(Debug)]
 /// Command-specific data
-/// After parsing, t, b Label elements are converted into BranchTarget ones.
+/// After parsing, t, T, b Label elements are converted into BranchTarget ones.
 pub enum CommandData {
     None,
-    BranchTarget(Option<Rc<RefCell<Command>>>), // Commands for 'b', 't', '{'
-    Label(Option<String>),                      // Label name for 'b', 't', ':'
+    BranchTarget(Option<Rc<RefCell<Command>>>), // Commands for 'b', 't', 'T', '{'
+    Label(Option<String>),                      // Label name for 'b', 't', 'T', ':'
     Path(PathBuf),                              // File path for 'r'
     NamedWriter(Rc<RefCell<NamedWriter>>),      // File output for 'w'
     Number(usize),                              // Number for 'l', 'q', 'Q' (GNU)
     Substitution(Box<Substitution>),            // Substitute command 's'
-    Text(Rc<str>),                              // Text for 'a', 'c', 'i'
+    Text(Rc<[u8]>),                             // Text for 'a', 'c', 'i'
     Transliteration(Box<Transliteration>),      // Transliteration command 'y'
 }
 
@@ -340,7 +387,7 @@ pub struct InputAction {
     /// Next command to execute (rather than commands from start)
     pub next_command: Option<Rc<RefCell<Command>>>,
     /// Data to prepend to the read contents
-    pub prepend: String,
+    pub prepend: Vec<u8>,
 }
 
 #[cfg(test)]
@@ -350,7 +397,7 @@ mod tests {
 
     // Return the captures for the RE applied to the specified string
     fn caps_for<'a>(re: &str, chunk: &'a mut IOChunk) -> Captures<'a> {
-        Regex::new(re)
+        Regex::new(re, CharacterMode::Utf8)
             .unwrap()
             .captures(chunk)
             .unwrap()
@@ -366,26 +413,26 @@ mod tests {
         let cmd = Command::default();
 
         let result = template.apply_captures(&cmd, &caps).unwrap();
-        assert_eq!(result, "");
+        assert_eq!(result, b"");
     }
 
     #[test]
     // s/abc/hello/
     fn test_literal_only() {
-        let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]);
+        let template = ReplacementTemplate::new(vec![ReplacementPart::Literal(b"hello".to_vec())]);
         let input = &mut IOChunk::new_from_str("abc");
         let caps = caps_for("abc", input);
         let cmd = Command::default();
 
         let result = template.apply_captures(&cmd, &caps).unwrap();
-        assert_eq!(result, "hello");
+        assert_eq!(result, b"hello");
     }
 
     #[test]
     // s/foo\d+/got: &/
     fn test_whole_match() {
         let template = ReplacementTemplate::new(vec![
-            ReplacementPart::Literal("got: ".into()),
+            ReplacementPart::Literal(b"got: ".to_vec()),
             ReplacementPart::WholeMatch,
         ]);
         let input = &mut IOChunk::new_from_str("foo42");
@@ -393,14 +440,26 @@ mod tests {
         let cmd = Command::default();
 
         let result = template.apply_captures(&cmd, &caps).unwrap();
-        assert_eq!(result, "got: foo42");
+        assert_eq!(result, b"got: foo42");
+    }
+
+    #[test]
+    fn test_apply_match_uses_matched_bytes() {
+        let template = ReplacementTemplate::new(vec![
+            ReplacementPart::Literal(b"<".to_vec()),
+            ReplacementPart::WholeMatch,
+            ReplacementPart::Literal(b">".to_vec()),
+        ]);
+        let m = Match::from_bytes(1, 3, b"\xE9x");
+
+        assert_eq!(template.apply_match(&m), b"<\xE9x>");
     }
 
     #[test]
     // s/foo(\d+)/number: \1/
     fn test_backreference() {
         let template = ReplacementTemplate::new(vec![
-            ReplacementPart::Literal("number: ".into()),
+            ReplacementPart::Literal(b"number: ".to_vec()),
             ReplacementPart::Group(1),
         ]);
         let input = &mut IOChunk::new_from_str("foo42");
@@ -408,16 +467,16 @@ mod tests {
         let cmd = Command::default();
 
         let result = template.apply_captures(&cmd, &caps).unwrap();
-        assert_eq!(result, "number: 42");
+        assert_eq!(result, b"number: 42");
     }
 
     #[test]
     // s/(\w+):(\d+)/key: \1, value: \2/
     fn test_multiple_parts() {
         let template = ReplacementTemplate::new(vec![
-            ReplacementPart::Literal("key: ".into()),
+            ReplacementPart::Literal(b"key: ".to_vec()),
             ReplacementPart::Group(1),
-            ReplacementPart::Literal(", value: ".into()),
+            ReplacementPart::Literal(b", value: ".to_vec()),
             ReplacementPart::Group(2),
         ]);
         let input = &mut IOChunk::new_from_str("x:123");
@@ -425,16 +484,16 @@ mod tests {
         let cmd = Command::default();
 
         let result = template.apply_captures(&cmd, &caps).unwrap();
-        assert_eq!(result, "key: x, value: 123");
+        assert_eq!(result, b"key: x, value: 123");
     }
 
     #[test]
     // s/(\w+):(\d+)/key: \1, value: \3/
     fn test_invalid_group() {
         let template = ReplacementTemplate::new(vec![
-            ReplacementPart::Literal("key: ".into()),
+            ReplacementPart::Literal(b"key: ".to_vec()),
             ReplacementPart::Group(1),
-            ReplacementPart::Literal(", value: ".into()),
+            ReplacementPart::Literal(b", value: ".to_vec()),
             ReplacementPart::Group(3),
         ]);
         let input = &mut IOChunk::new_from_str("x:123");
@@ -452,11 +511,11 @@ mod tests {
     #[test]
     fn test_max_group_number_with_groups() {
         let template = ReplacementTemplate::new(vec![
-            ReplacementPart::Literal("a".into()),
+            ReplacementPart::Literal(b"a".to_vec()),
             ReplacementPart::Group(2),
             ReplacementPart::WholeMatch,
             ReplacementPart::Group(5),
-            ReplacementPart::Literal("z".into()),
+            ReplacementPart::Literal(b"z".to_vec()),
         ]);
         assert_eq!(template.max_group_number, 5);
     }
@@ -464,9 +523,9 @@ mod tests {
     #[test]
     fn test_max_group_number_without_groups() {
         let template = ReplacementTemplate::new(vec![
-            ReplacementPart::Literal("no".into()),
+            ReplacementPart::Literal(b"no".to_vec()),
             ReplacementPart::WholeMatch,
-            ReplacementPart::Literal("groups".into()),
+            ReplacementPart::Literal(b"groups".to_vec()),
         ]);
         assert_eq!(template.max_group_number, 0);
     }
@@ -476,16 +535,31 @@ mod tests {
     #[test]
     fn test_identity_lookup_fast_path() {
         let t = Transliteration::default();
-        assert_eq!(t.lookup('A'), 'A');
-        assert_eq!(t.lookup('z'), 'z');
-        assert_eq!(t.lookup('\u{07FF}'), '\u{07FF}'); // highest 2-byte UTF-8 char
+        assert_eq!(t.lookup_char('A'), 'A');
+        assert_eq!(t.lookup_char('z'), 'z');
+        assert_eq!(t.lookup_char('\u{07FF}'), '\u{07FF}'); // highest 2-byte UTF-8 char
     }
 
     #[test]
     fn test_identity_lookup_slow_path() {
         let t = Transliteration::default();
-        assert_eq!(t.lookup('\u{0800}'), '\u{0800}'); // just outside fast path
-        assert_eq!(t.lookup('\u{1F600}'), '\u{1F600}'); // 😀
+        assert_eq!(t.lookup_char('\u{0800}'), '\u{0800}'); // just outside fast path
+        assert_eq!(t.lookup_char('\u{1F600}'), '\u{1F600}'); // 😀
+    }
+
+    #[test]
+    fn test_from_bytes_and_lookup_byte() {
+        let t = Transliteration::from_bytes(b"a\xE9", b"Z!");
+        assert_eq!(t.lookup_byte(b'a'), b'Z');
+        assert_eq!(t.lookup_byte(0xE9), b'!');
+        assert_eq!(t.lookup_byte(b'b'), b'b');
+    }
+
+    #[test]
+    fn test_is_byte_identity_tracks_non_ascii_character_mappings() {
+        assert!(Transliteration::from_strings("ab", "xy").is_byte_identity);
+        assert!(!Transliteration::from_strings("aé", "xy").is_byte_identity);
+        assert!(!Transliteration::from_strings("ab", "xé").is_byte_identity);
     }
 
     #[test]
@@ -493,26 +567,26 @@ mod tests {
         let mut t = Transliteration::default();
         t.insert('a', 'α');
         t.insert('b', 'β');
-        assert_eq!(t.lookup('a'), 'α');
-        assert_eq!(t.lookup('b'), 'β');
-        assert_eq!(t.lookup('c'), 'c'); // unchanged
+        assert_eq!(t.lookup_char('a'), 'α');
+        assert_eq!(t.lookup_char('b'), 'β');
+        assert_eq!(t.lookup_char('c'), 'c'); // unchanged
     }
 
     #[test]
     fn test_insert_and_lookup_slow_path() {
         let mut t = Transliteration::default();
         t.insert('🦀', 'c'); // U+1F980 Crab emoji -> 'c'
-        assert_eq!(t.lookup('🦀'), 'c');
-        assert_eq!(t.lookup('🦁'), '🦁'); // unchanged
+        assert_eq!(t.lookup_char('🦀'), 'c');
+        assert_eq!(t.lookup_char('🦁'), '🦁'); // unchanged
     }
 
     #[test]
     fn test_overwrite_mapping() {
         let mut t = Transliteration::default();
         t.insert('x', '1');
-        assert_eq!(t.lookup('x'), '1');
+        assert_eq!(t.lookup_char('x'), '1');
         t.insert('x', '2');
-        assert_eq!(t.lookup('x'), '2');
+        assert_eq!(t.lookup_char('x'), '2');
     }
 
     #[test]
@@ -523,8 +597,8 @@ mod tests {
                 t.insert(ch, ' ');
             }
         }
-        assert_eq!(t.lookup('A'), ' ');
-        assert_eq!(t.lookup('\u{07FF}'), ' ');
+        assert_eq!(t.lookup_char('A'), ' ');
+        assert_eq!(t.lookup_char('\u{07FF}'), ' ');
     }
 
     // from_strings
@@ -532,11 +606,11 @@ mod tests {
     fn test_basic_transliteration() {
         let t = Transliteration::from_strings("abcδ", "1234");
 
-        assert_eq!(t.lookup('a'), '1');
-        assert_eq!(t.lookup('b'), '2');
-        assert_eq!(t.lookup('c'), '3');
-        assert_eq!(t.lookup('δ'), '4');
-        assert_eq!(t.lookup('e'), 'e'); // not mapped, fallback
+        assert_eq!(t.lookup_char('a'), '1');
+        assert_eq!(t.lookup_char('b'), '2');
+        assert_eq!(t.lookup_char('c'), '3');
+        assert_eq!(t.lookup_char('δ'), '4');
+        assert_eq!(t.lookup_char('e'), 'e'); // not mapped, fallback
     }
 
     #[test]
@@ -545,16 +619,16 @@ mod tests {
         let target = "e文c";
         let t = Transliteration::from_strings(source, target);
 
-        assert_eq!(t.lookup('é'), 'e');
-        assert_eq!(t.lookup('漢'), '文');
-        assert_eq!(t.lookup('🦀'), 'c');
-        assert_eq!(t.lookup('x'), 'x'); // fast fallback
-        assert_eq!(t.lookup('文'), '文'); // slow fallback
+        assert_eq!(t.lookup_char('é'), 'e');
+        assert_eq!(t.lookup_char('漢'), '文');
+        assert_eq!(t.lookup_char('🦀'), 'c');
+        assert_eq!(t.lookup_char('x'), 'x'); // fast fallback
+        assert_eq!(t.lookup_char('文'), '文'); // slow fallback
     }
 
     #[test]
     fn test_overwrite_fast_path() {
         let t = Transliteration::from_strings("aa", "12");
-        assert_eq!(t.lookup('a'), '2'); // last mapping wins
+        assert_eq!(t.lookup_char('a'), '2'); // last mapping wins
     }
 }

src/sed/fast_io.rs

@@ -37,7 +37,6 @@ use std::str;
 use std::path::PathBuf;
 use uucore::error::UError;
 
-#[cfg(unix)]
 use uucore::error::USimpleError;
 
 #[cfg(unix)]
@@ -112,7 +111,7 @@ impl<'a> MmapLineCursor<'a> {
 /// Buffered line reader from any BufRead input.
 pub struct ReadLineCursor {
     reader: Box<dyn BufRead>,
-    buffer: String,
+    buffer: Vec<u8>,
 }
 
 impl ReadLineCursor {
@@ -121,20 +120,20 @@ impl ReadLineCursor {
         let buf = BufReader::new(r);
         Self {
             reader: Box::new(buf),
-            buffer: String::new(),
+            buffer: Vec::new(),
         }
     }
 
     /// If a line is available, return it and its \n termination.
-    fn get_line(&mut self) -> io::Result<Option<(String, bool)>> {
+    fn get_line(&mut self) -> io::Result<Option<(Vec<u8>, bool)>> {
         self.buffer.clear();
         // read_line *includes* the '\n' if present
-        let bytes_read = self.reader.read_line(&mut self.buffer)?;
+        let bytes_read = self.reader.read_until(b'\n', &mut self.buffer)?;
         if bytes_read == 0 {
             return Ok(None);
         }
         // O(1) check whether it ended in '\n'
-        let has_newline = self.buffer.ends_with('\n');
+        let has_newline = self.buffer.ends_with(b"\n");
         // strip it if you don’t want to expose it to the caller
         if has_newline {
             self.buffer.pop();
@@ -181,7 +180,7 @@ impl<'a> IOChunk<'a> {
             }
             #[cfg(unix)]
             _ => {
-                self.content = IOChunkContent::new_owned(String::new(), false);
+                self.content = IOChunkContent::new_owned(Vec::new(), false);
             }
         }
     }
@@ -210,7 +209,7 @@ impl<'a> IOChunk<'a> {
     /// Create an Owned newline-terminated IOChunk from a string.
     pub fn new_from_str(s: &str) -> Self {
         IOChunk {
-            content: IOChunkContent::new_owned(s.to_string(), true),
+            content: IOChunkContent::new_owned(s.as_bytes().to_vec(), true),
             utf8_verified: Cell::new(false),
         }
     }
@@ -218,7 +217,14 @@ impl<'a> IOChunk<'a> {
     /// Set the object's contents to the specified string.
     /// Convert it into Owned if needed.
     pub fn set_to_string(&mut self, new_content: String, add_newline: bool) {
+        self.set_to_bytes(new_content.into_bytes(), add_newline);
         self.utf8_verified.set(true);
+    }
+
+    /// Set the object's contents to the specified bytes.
+    /// Convert it into Owned if needed.
+    pub fn set_to_bytes(&mut self, new_content: Vec<u8>, add_newline: bool) {
+        self.utf8_verified.set(false);
         match &mut self.content {
             IOChunkContent::Owned {
                 content,
@@ -242,14 +248,23 @@ impl<'a> IOChunk<'a> {
             IOChunkContent::MmapInput { content, .. } => {
                 if self.utf8_verified.get() {
                     // Use cached result
+                    // SAFETY: `utf8_verified` is set only if `content`
+                    // is derived from a String or if it has been
+                    // successfully verified.
                     Ok(unsafe { self.content.as_str_unchecked() })
                 } else {
-                    let result = str::from_utf8(content);
-                    self.utf8_verified.set(true);
-                    result.map_err(|e| USimpleError::new(2, e.to_string()))
+                    match str::from_utf8(content) {
+                        Ok(s) => {
+                            self.utf8_verified.set(true);
+                            Ok(s)
+                        }
+                        Err(e) => Err(USimpleError::new(2, e.to_string())),
+                    }
                 }
             }
-            IOChunkContent::Owned { content, .. } => Ok(content),
+            IOChunkContent::Owned { content, .. } => {
+                str::from_utf8(content).map_err(|e| USimpleError::new(2, e.to_string()))
+            }
         }
     }
 
@@ -258,32 +273,28 @@ impl<'a> IOChunk<'a> {
         match &self.content {
             #[cfg(unix)]
             IOChunkContent::MmapInput { content, .. } => content,
-            IOChunkContent::Owned { content, .. } => content.as_bytes(),
+            IOChunkContent::Owned { content, .. } => content,
         }
     }
 
     /// Convert content to the Owned variant if it's not already.
-    /// Fails if the conversion to UTF-8 fails.
     pub fn ensure_owned(&mut self) -> Result<(), Box<dyn UError>> {
         match &self.content {
             IOChunkContent::Owned { .. } => Ok(()), // already owned
             #[cfg(unix)]
             IOChunkContent::MmapInput {
                 content, full_span, ..
-            } => match std::str::from_utf8(content) {
-                Ok(valid_str) => {
-                    let has_newline = full_span.last().copied() == Some(b'\n');
-                    self.content = IOChunkContent::new_owned(valid_str.to_string(), has_newline);
-                    self.utf8_verified.set(true);
-                    Ok(())
-                }
-                Err(e) => Err(USimpleError::new(2, e.to_string())),
-            },
+            } => {
+                let has_newline = full_span.last().copied() == Some(b'\n');
+                self.content = IOChunkContent::new_owned(content.to_vec(), has_newline);
+                self.utf8_verified.set(false);
+                Ok(())
+            }
         }
     }
 
     /// Return mutable access to the content and has_newline fields.
-    pub fn fields_mut(&mut self) -> Result<(&mut String, &mut bool), Box<dyn UError>> {
+    pub fn fields_mut(&mut self) -> Result<(&mut Vec<u8>, &mut bool), Box<dyn UError>> {
         self.ensure_owned()?;
 
         match &mut self.content {
@@ -312,7 +323,7 @@ enum IOChunkContent<'a> {
         full_span: &'a [u8], // Line including original newline, if any
     },
     Owned {
-        content: String,   // Line content without newline
+        content: Vec<u8>,  // Line content without newline
         has_newline: bool, // True if \n-terminated
         #[cfg(not(unix))]
         _phantom: PhantomData<&'a ()>, // Silence E0392 warning
@@ -321,7 +332,7 @@ enum IOChunkContent<'a> {
 
 impl IOChunkContent<'_> {
     /// Construct a new Owned chunk.
-    pub fn new_owned(content: String, has_newline: bool) -> Self {
+    pub fn new_owned(content: Vec<u8>, has_newline: bool) -> Self {
         #[cfg(unix)]
         return IOChunkContent::Owned {
             content,
@@ -343,7 +354,9 @@ impl IOChunkContent<'_> {
             IOChunkContent::MmapInput { content, .. } => unsafe {
                 std::str::from_utf8_unchecked(content)
             },
-            IOChunkContent::Owned { content, .. } => content,
+            IOChunkContent::Owned { content, .. } => unsafe {
+                std::str::from_utf8_unchecked(content)
+            },
         }
     }
 
@@ -627,7 +640,20 @@ impl OutputBuffer {
             s.truncate(s.len() - 1);
         }
         self.write_chunk(&IOChunk::from_content(IOChunkContent::new_owned(
-            s,
+            s.into_bytes(),
+            has_newline,
+        )))
+    }
+
+    /// Schedule the specified bytes for eventual output.
+    pub fn write_bytes(&mut self, bytes: &[u8]) -> io::Result<()> {
+        let (content, has_newline) = if bytes.ends_with(b"\n") {
+            (&bytes[..bytes.len() - 1], true)
+        } else {
+            (bytes, false)
+        };
+        self.write_chunk(&IOChunk::from_content(IOChunkContent::new_owned(
+            content.to_vec(),
             has_newline,
         )))
     }
@@ -654,9 +680,7 @@ impl OutputBuffer {
 /// Implementation of the std::io::Write trait
 impl Write for OutputBuffer {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        let s =
-            std::str::from_utf8(buf).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
-        self.write_str(s)?;
+        self.write_bytes(buf)?;
         Ok(buf.len())
     }
 
@@ -741,7 +765,7 @@ impl OutputBuffer {
                 ..
             } => {
                 self.flush_mmap(WriteRange::Complete)?;
-                self.out.write_all(content.as_bytes())?;
+                self.out.write_all(content)?;
                 if *has_newline {
                     self.out.write_all(b"\n")?;
                 }
@@ -830,7 +854,7 @@ impl OutputBuffer {
                 has_newline,
                 ..
             } => {
-                self.out.write_all(content.as_bytes())?;
+                self.out.write_all(content)?;
                 if *has_newline {
                     self.out.write_all(b"\n")?;
                 }
@@ -1388,7 +1412,7 @@ mod tests {
             ..
         }) = reader.get_line()?
         {
-            assert_eq!(content, "first line");
+            assert_eq!(content, b"first line");
             assert_eq!(content.len(), 10);
             assert!(has_newline);
             assert!(!utf8_verified.get());
@@ -1407,7 +1431,7 @@ mod tests {
             ..
         }) = reader.get_line()?
         {
-            assert_eq!(content, "second line");
+            assert_eq!(content, b"second line");
             assert!(has_newline);
             assert!(!reader.last_line().unwrap());
         } else {
@@ -1492,21 +1516,21 @@ mod tests {
     // is_newline_terminated, is_empty
     #[test]
     fn test_owned_newline_terminated_non_empty() {
-        let chunk = IOChunk::from_content(IOChunkContent::new_owned("line".to_string(), true));
+        let chunk = IOChunk::from_content(IOChunkContent::new_owned(b"line".to_vec(), true));
         assert!(chunk.is_newline_terminated());
         assert!(!chunk.is_empty());
     }
 
     #[test]
     fn test_owned_newline_terminated_empty() {
-        let chunk = IOChunk::from_content(IOChunkContent::new_owned(String::new(), true));
+        let chunk = IOChunk::from_content(IOChunkContent::new_owned(Vec::new(), true));
         assert!(chunk.is_newline_terminated());
         assert!(chunk.is_empty());
     }
 
     #[test]
     fn test_owned_not_newline_terminated() {
-        let chunk = IOChunk::from_content(IOChunkContent::new_owned("line".to_string(), false));
+        let chunk = IOChunk::from_content(IOChunkContent::new_owned(b"line".to_vec(), false));
         assert!(!chunk.is_newline_terminated());
     }
 
@@ -1541,7 +1565,7 @@ mod tests {
     #[test]
     fn test_ensure_owned_on_owned() {
         let mut chunk =
-            IOChunk::from_content(IOChunkContent::new_owned("already owned".to_string(), true));
+            IOChunk::from_content(IOChunkContent::new_owned(b"already owned".to_vec(), true));
 
         let result = chunk.ensure_owned();
         assert!(result.is_ok());
@@ -1553,7 +1577,7 @@ mod tests {
                 has_newline,
                 ..
             } => {
-                assert_eq!(content, "already owned");
+                assert_eq!(content, b"already owned");
                 assert!(*has_newline);
             }
             #[cfg(unix)]
@@ -1578,7 +1602,7 @@ mod tests {
                 has_newline,
                 ..
             } => {
-                assert_eq!(content, "mmap string");
+                assert_eq!(content, b"mmap string");
                 assert!(*has_newline);
             }
             _ => panic!("Expected Owned variant after ensure_owned"),
@@ -1602,7 +1626,7 @@ mod tests {
                 has_newline,
                 ..
             } => {
-                assert_eq!(content, "no newline");
+                assert_eq!(content, b"no newline");
                 assert!(!*has_newline);
             }
             _ => panic!("Expected Owned variant after ensure_owned"),
@@ -1618,23 +1642,18 @@ mod tests {
         let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span));
 
         let result = chunk.ensure_owned();
-        assert!(result.is_err());
-        let err_msg = format!("{}", result.unwrap_err());
-        assert!(
-            err_msg.contains("invalid utf-8"),
-            "Unexpected error message: {}",
-            err_msg
-        );
+        assert!(result.is_ok());
+        assert_eq!(chunk.as_bytes(), b"bad\xFFutf8");
+        assert!(chunk.as_str().is_err());
     }
 
     // fields_mut
     #[test]
     fn test_fields_mut_on_owned() {
-        let mut chunk =
-            IOChunk::from_content(IOChunkContent::new_owned("hello".to_string(), false));
+        let mut chunk = IOChunk::from_content(IOChunkContent::new_owned(b"hello".to_vec(), false));
 
         let (s, _) = chunk.fields_mut().unwrap();
-        s.push_str(" world");
+        s.extend_from_slice(b" world");
 
         assert_eq!(chunk.as_str().unwrap(), "hello world");
     }
@@ -1648,7 +1667,7 @@ mod tests {
 
         {
             let (s, _) = chunk.fields_mut().unwrap();
-            s.push_str("bar");
+            s.extend_from_slice(b"bar");
         }
 
         assert_eq!(chunk.as_str().unwrap(), "foobar");
@@ -1662,7 +1681,7 @@ mod tests {
         let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span));
 
         let (s, _) = chunk.fields_mut().unwrap();
-        s.push_str(" Δεδομένα");
+        s.extend_from_slice(" Δεδομένα".as_bytes());
 
         assert_eq!(chunk.as_str().unwrap(), "Ζωντανά! Δεδομένα");
     }
@@ -1675,8 +1694,9 @@ mod tests {
         let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span));
 
         let result = chunk.fields_mut();
-        assert!(result.is_err());
-        assert!(format!("{}", result.unwrap_err()).contains("invalid utf-8"));
+        assert!(result.is_ok());
+        assert_eq!(chunk.as_bytes(), b"abc\xFF");
+        assert!(chunk.as_str().is_err());
     }
 
     #[cfg(unix)]
@@ -1873,7 +1893,7 @@ mod tests {
         IOChunk {
             utf8_verified: Cell::new(true),
             content: IOChunkContent::Owned {
-                content: s.to_string(),
+                content: s.as_bytes().to_vec(),
                 has_newline: has_nl,
                 #[cfg(not(unix))]
                 _phantom: std::marker::PhantomData,

src/sed/mod.rs

@@ -20,14 +20,15 @@ pub mod processor;
 pub mod script_char_provider;
 pub mod script_line_provider;
 
-use crate::sed::command::{ProcessingContext, StringSpace};
+use crate::sed::command::{ByteSpace, CharacterMode, ProcessingContext};
 use crate::sed::compiler::compile;
 use crate::sed::processor::process_all_files;
 use crate::sed::script_line_provider::ScriptValue;
 use clap::{Arg, ArgMatches, Command, arg, crate_version};
 use std::collections::HashMap;
+use std::env;
 use std::path::PathBuf;
-use uucore::error::{UResult, UUsageError};
+use uucore::error::{UResult, USimpleError, UUsageError};
 use uucore::format_usage;
 
 const ABOUT: &str = "Stream editor for filtering and transforming text";
@@ -46,7 +47,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
     }
 
     let (scripts, files) = get_scripts_files(&matches)?;
-    let mut context = build_context(&matches);
+    let mut context = build_context(&matches)?;
 
     let executable = compile(scripts, &mut context)?;
     process_all_files(executable, files, &mut context)?;
@@ -192,9 +193,40 @@ fn get_scripts_files(matches: &ArgMatches) -> UResult<(Vec<ScriptValue>, Vec<Pat
     Ok((scripts, files))
 }
 
-// Parse CLI flag arguments and return a ProcessingContext struct based on them
-fn build_context(matches: &ArgMatches) -> ProcessingContext {
-    ProcessingContext {
+/// Return the character interpretation mode implied by the process locale.
+///
+/// The mode is determined from the first non-empty of LC_ALL, LC_CTYPE, and
+/// LANG. The C and POSIX locales select byte mode. UTF-8 locales (including
+/// C.UTF-8) select UTF-8 mode. All other locales result in an error.
+pub fn character_mode_for_locale(locale: &str) -> UResult<CharacterMode> {
+    if locale == "C" || locale == "POSIX" {
+        Ok(CharacterMode::Byte)
+    } else if locale.eq_ignore_ascii_case("C.UTF-8")
+        || locale.to_ascii_lowercase().ends_with(".utf-8")
+        || locale.to_ascii_lowercase().ends_with(".utf8")
+    {
+        Ok(CharacterMode::Utf8)
+    } else {
+        Err(USimpleError::new(
+            1,
+            format!("unsupported locale: {locale}"),
+        ))
+    }
+}
+
+/// Return a ProcessingContext based on parsed CLI flags and environment.
+fn build_context(matches: &ArgMatches) -> UResult<ProcessingContext> {
+    let locale = ["LC_ALL", "LC_CTYPE", "LANG"]
+        .into_iter()
+        .find_map(|name| {
+            let value = env::var(name).ok()?;
+            (!value.is_empty()).then_some(value)
+        })
+        // Same default as GNU sed
+        .unwrap_or_else(|| "C".to_string());
+
+    Ok(ProcessingContext {
+        // CLI arguments
         all_output_files: matches.get_flag("all-output-files"),
         debug: matches.get_flag("debug"),
         regex_extended: matches.get_flag("regexp-extended"),
@@ -211,6 +243,9 @@ fn build_context(matches: &ArgMatches) -> ProcessingContext {
         unbuffered: matches.get_flag("unbuffered"),
         null_data: matches.get_flag("null-data"),
 
+        // Environment
+        character_mode: character_mode_for_locale(&locale)?,
+
         // Other context
         input_name: "<stdin>".to_string(),
         line_number: 0,
@@ -220,8 +255,8 @@ fn build_context(matches: &ArgMatches) -> ProcessingContext {
         stop_processing: false,
         saved_regex: None,
         input_action: None,
-        hold: StringSpace {
-            content: String::new(),
+        hold: ByteSpace {
+            content: Vec::new(),
             has_newline: true,
         },
         parsed_block_nesting: 0,
@@ -229,7 +264,7 @@ fn build_context(matches: &ArgMatches) -> ProcessingContext {
         range_commands: Vec::new(),
         substitution_made: false,
         append_elements: Vec::new(),
-    }
+    })
 }
 
 #[cfg(test)]
@@ -338,7 +373,7 @@ mod tests {
     #[test]
     fn test_defaults() {
         let matches = test_matches(&[]);
-        let ctx = build_context(&matches);
+        let ctx = build_context(&matches).unwrap();
 
         assert!(!ctx.all_output_files);
         assert!(!ctx.debug);
@@ -373,7 +408,7 @@ mod tests {
             "-z",
         ]);
 
-        let ctx = build_context(&matches);
+        let ctx = build_context(&matches).unwrap();
 
         assert!(ctx.all_output_files);
         assert!(ctx.debug);
@@ -393,7 +428,7 @@ mod tests {
     #[test]
     fn test_multiple_same_arguments() {
         let matches = test_matches(&["-E", "-r"]);
-        let ctx = build_context(&matches);
+        let ctx = build_context(&matches).unwrap();
 
         assert!(ctx.regex_extended);
     }
@@ -401,7 +436,7 @@ mod tests {
     #[test]
     fn test_in_place_with_suffix() {
         let matches = test_matches(&["-i", ".bak"]);
-        let ctx = build_context(&matches);
+        let ctx = build_context(&matches).unwrap();
 
         assert!(ctx.in_place);
         assert_eq!(ctx.in_place_suffix, Some(".bak".to_string()));
@@ -412,10 +447,59 @@ mod tests {
         let matches_default = test_matches(&[]);
         let matches_custom = test_matches(&["-l", "120"]);
 
-        let ctx_default = build_context(&matches_default);
-        let ctx_custom = build_context(&matches_custom);
+        let ctx_default = build_context(&matches_default).unwrap();
+        let ctx_custom = build_context(&matches_custom).unwrap();
 
         assert_eq!(ctx_default.length, 70);
         assert_eq!(ctx_custom.length, 120);
     }
+
+    #[test]
+    fn c_locale_selects_byte_mode() {
+        assert_eq!(character_mode_for_locale("C").unwrap(), CharacterMode::Byte);
+    }
+
+    #[test]
+    fn posix_locale_selects_byte_mode() {
+        assert_eq!(
+            character_mode_for_locale("POSIX").unwrap(),
+            CharacterMode::Byte
+        );
+    }
+
+    #[test]
+    fn c_utf8_locale_selects_utf8_mode() {
+        assert_eq!(
+            character_mode_for_locale("C.UTF-8").unwrap(),
+            CharacterMode::Utf8
+        );
+    }
+
+    #[test]
+    fn dot_utf8_locale_selects_utf8_mode() {
+        assert_eq!(
+            character_mode_for_locale("en_US.UTF-8").unwrap(),
+            CharacterMode::Utf8
+        );
+    }
+
+    #[test]
+    fn dot_utf8_locale_is_case_insensitive() {
+        assert_eq!(
+            character_mode_for_locale("el_GR.utf8").unwrap(),
+            CharacterMode::Utf8
+        );
+    }
+
+    #[test]
+    fn unsupported_locale_reports_locale_name() {
+        let err = character_mode_for_locale("el_GR.ISO-8859-7").unwrap_err();
+
+        assert_eq!(err.code(), 1);
+        assert!(
+            err.to_string()
+                .contains("unsupported locale: el_GR.ISO-8859-7"),
+            "{err}"
+        );
+    }
 }

src/sed/named_writer.rs

@@ -57,13 +57,21 @@ impl NamedWriter {
 
     /// Write a line to the file with a newline, returning descriptive errors.
     pub fn write_line(&mut self, line: &str) -> UResult<()> {
-        writeln!(self.writer, "{line}").map_err(|e| {
-            runtime_error::<()>(
-                &self.location,
-                format!("writing to file {}: {e}", self.path.quote()),
-            )
-            .unwrap_err()
-        })
+        self.write_line_bytes(line.as_bytes())
+    }
+
+    /// Write bytes to the file with a newline, returning descriptive errors.
+    pub fn write_line_bytes(&mut self, line: &[u8]) -> UResult<()> {
+        self.writer
+            .write_all(line)
+            .and_then(|()| self.writer.write_all(b"\n"))
+            .map_err(|e| {
+                runtime_error::<()>(
+                    &self.location,
+                    format!("writing to file {}: {e}", self.path.quote()),
+                )
+                .unwrap_err()
+            })
     }
 
     /// Flush the writer, returning a descriptive error.
@@ -88,3 +96,22 @@ pub fn flush_all() -> UResult<()> {
         Ok(())
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn test_write_line_bytes_appends_newline() {
+        let file = NamedTempFile::new().unwrap();
+        let path = file.path().to_path_buf();
+        let writer = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
+
+        writer.borrow_mut().write_line_bytes(b"a\xE9").unwrap();
+        writer.borrow_mut().flush().unwrap();
+
+        assert_eq!(fs::read(path).unwrap(), b"a\xE9\n");
+    }
+}

src/sed/script_char_provider.rs

@@ -1,4 +1,4 @@
-// Provide the script contents character by character
+// Provide the script contents byte by byte
 //
 // SPDX-License-Identifier: MIT
 // Copyright (c) 2025 Diomidis Spinellis
@@ -10,19 +10,20 @@
 
 #[derive(Debug)]
 pub struct ScriptCharProvider {
-    line: Vec<char>,
+    line: Vec<u8>,
     pos: usize,
 }
 
 impl ScriptCharProvider {
-    pub fn new(line_string: &str) -> Self {
+    /// Build from raw script bytes.
+    pub fn new(line: impl AsRef<[u8]>) -> Self {
         Self {
-            line: line_string.chars().collect(),
+            line: line.as_ref().to_vec(),
             pos: 0,
         }
     }
 
-    /// Advances to the next character, if not at end of line.
+    /// Advances to the next byte, if not at end of line.
     pub fn advance(&mut self) {
         if self.pos < self.line.len() {
             self.pos += 1;
@@ -39,8 +40,13 @@ impl ScriptCharProvider {
         self.pos = pos;
     }
 
-    /// Returns the current character. Panics if out of bounds.
+    /// Returns the current byte as a character. Panics if out of bounds.
     pub fn current(&self) -> char {
+        char::from(self.line[self.pos])
+    }
+
+    /// Returns the current script byte. Panics if out of bounds.
+    pub fn current_byte(&self) -> u8 {
         self.line[self.pos]
     }
 
@@ -51,8 +57,8 @@ impl ScriptCharProvider {
 
     /// Advances the position past any whitespace characters.
     pub fn eat_spaces(&mut self) {
-        while self.pos < self.line.len() && self.line[self.pos].is_whitespace() {
-            self.pos += 1;
+        while self.pos < self.line.len() && self.current().is_whitespace() {
+            self.advance();
         }
     }
 
@@ -139,4 +145,28 @@ mod tests {
         assert_eq!(chars.get_pos(), 2);
         assert_eq!(chars.current(), 'c');
     }
+
+    #[test]
+    fn test_current_for_utf8_bytes() {
+        let mut chars = ScriptCharProvider::new("αb");
+
+        assert_eq!(chars.current(), char::from(0xCE));
+        assert_eq!(chars.current_byte(), 0xCE);
+        chars.advance();
+        assert_eq!(chars.current(), char::from(0xB1));
+        assert_eq!(chars.current_byte(), 0xB1);
+        chars.advance();
+        assert_eq!(chars.current(), 'b');
+    }
+
+    #[test]
+    fn test_current_for_invalid_utf8_byte() {
+        let mut chars = ScriptCharProvider::new(vec![0xC2, b'a']);
+
+        assert_eq!(chars.current(), char::from(0xC2));
+        assert_eq!(chars.current_byte(), 0xC2);
+        chars.advance();
+        assert_eq!(chars.current(), 'a');
+        assert_eq!(chars.current_byte(), b'a');
+    }
 }

src/sed/script_line_provider.rs

@@ -69,8 +69,8 @@ impl ScriptLineProvider {
     }
 
     /// Return the next script line to process across all scripts.
-    pub fn next_line(&mut self) -> UResult<Option<String>> {
-        let mut line = String::new();
+    pub fn next_line(&mut self) -> UResult<Option<Vec<u8>>> {
+        let mut line = Vec::new();
 
         loop {
             let advance = match &mut self.state {
@@ -82,13 +82,13 @@ impl ScriptLineProvider {
                     ..
                 } => {
                     line.clear();
-                    let bytes = reader.read_line(&mut line)?;
+                    let bytes = reader.read_until(b'\n', &mut line)?;
                     if bytes == 0 {
                         Some(*index + 1) // finished reading this source
                     } else {
                         *line_number += 1;
                         // Remove trailing newline
-                        if line.ends_with('\n') {
+                        if line.ends_with(b"\n") {
                             line.pop();
                         }
                         return Ok(Some(line));
@@ -114,7 +114,7 @@ impl ScriptLineProvider {
 
         match &self.sources[next_index] {
             ScriptValue::StringVal(s) => {
-                let cursor = std::io::Cursor::new(s.clone());
+                let cursor = std::io::Cursor::new(s.as_bytes().to_vec());
                 self.state = State::Active {
                     index: next_index,
                     reader: Box::new(BufReader::new(cursor)),
@@ -199,7 +199,7 @@ mod tests {
 
         let mut lines = Vec::new();
         while let Some(line) = provider.next_line().unwrap() {
-            lines.push(line.trim_end().to_string());
+            lines.push(String::from_utf8(line).unwrap().trim_end().to_string());
         }
 
         assert_eq!(lines, vec!["line one", "line two", "line three"]);
@@ -216,7 +216,7 @@ mod tests {
 
         let mut lines = Vec::new();
         while let Some(line) = provider.next_line().unwrap() {
-            lines.push(line.trim_end().to_string());
+            lines.push(String::from_utf8(line).unwrap().trim_end().to_string());
         }
 
         assert_eq!(lines, vec!["file line 1", "file line 2"]);
@@ -241,7 +241,7 @@ mod tests {
 
         let mut lines = Vec::new();
         while let Some(line) = provider.next_line().unwrap() {
-            lines.push(line.trim_end().to_string());
+            lines.push(String::from_utf8(line).unwrap().trim_end().to_string());
         }
 
         assert_eq!(
@@ -266,7 +266,7 @@ mod tests {
         let mut provider = ScriptLineProvider::new(input);
 
         if let Some(line) = provider.next_line().unwrap() {
-            assert_eq!(line.trim(), "l1");
+            assert_eq!(String::from_utf8(line).unwrap().trim(), "l1");
             assert_eq!(provider.get_line_number(), 1);
             assert_eq!(provider.get_input_name(), "<script argument 1>");
         } else {
@@ -274,7 +274,7 @@ mod tests {
         }
 
         if let Some(line) = provider.next_line().unwrap() {
-            assert_eq!(line.trim(), "l2");
+            assert_eq!(String::from_utf8(line).unwrap().trim(), "l2");
             assert_eq!(provider.get_line_number(), 2);
             assert_eq!(provider.get_input_name(), "<script argument 1>");
         } else {
@@ -282,11 +282,22 @@ mod tests {
         }
 
         if let Some(line) = provider.next_line().unwrap() {
-            assert_eq!(line.trim(), "l3");
+            assert_eq!(String::from_utf8(line).unwrap().trim(), "l3");
             assert_eq!(provider.get_line_number(), 1);
             assert_eq!(provider.get_input_name(), "<script argument 2>");
         } else {
             panic!("Expected a line");
         }
     }
+
+    #[test]
+    fn test_file_source_preserves_invalid_utf8_bytes() {
+        let mut temp_file = NamedTempFile::new().unwrap();
+        temp_file.write_all(b"s/\xC2\xE7/X/\n").unwrap();
+
+        let input = vec![ScriptValue::PathVal(temp_file.path().to_path_buf())];
+        let mut provider = ScriptLineProvider::new(input);
+
+        assert_eq!(provider.next_line().unwrap().unwrap(), b"s/\xC2\xE7/X/");
+    }
 }

tests/fixtures/sed/output/branch_no_sub_clears

@@ -0,0 +1,14 @@
+CLEARED ^l1_1
+CLEARED ^l1_2
+CLEARED ^l1_3
+CLEARED ^l1_4
+CLEARED ^l1_5
+CLEARED ^l1_6
+CLEARED ^l1_7
+CLEARED ^l1_8
+CLEARED ^l1_9
+CLEARED ^l1_10
+CLEARED ^l1_11
+CLEARED ^l1_12
+CLEARED ^l1_13
+CLEARED ^l1_14

tests/fixtures/sed/output/branch_no_sub_simple

@@ -0,0 +1,14 @@
+l1_1
+SUB l1_X
+l1_3
+l1_4
+l1_5
+l1_6
+l1_7
+l1_8
+l1_9
+l1_10
+l1_11
+SUB l1_1X
+l1_13
+l1_14

tests/tests.rs

@@ -16,6 +16,9 @@ fn init() {
         std::env::remove_var("UUTESTS_UTIL_NAME");
         std::env::set_var("UUTESTS_UTIL_NAME", "");
         std::env::set_var("UUTILS_MULTICALL", "0");
+        // Keep fixture-based integration tests in UTF-8 mode; individual
+        // locale tests override LC_ALL to exercise byte mode.
+        std::env::set_var("LC_ALL", "C.UTF-8");
     }
 }