Author SHA1 Message Date
Diomidis Spinellis 95cbd302fe Fix handling of lines ending without a newline 2026-08-20 21:05:37 +02:00
Diomidis Spinellis 1e390a63c0 Revert d7a713e fast_io.rs pending_newline addition
It violated layering by having it track higher-level newline behavior
and caused R of non-newline terminated files to fail; (see
read_one_no_newline test).
2026-08-20 21:05:37 +02:00
Diomidis SpinellisandMukunda Katta 452e249296 Implement GNU R (read next line) command
Co-authored-by: Mukunda Katta <mukunda.vjcs6@gmail.com>
2026-08-20 21:05:37 +02:00
Diomidis Spinellis c8b5dc53e3 Fix multiple write commands to same file
Maintining different writers for the same path can fail due to buffering
or underlying OS restrictions. In particular, this fails on Windows.
Fix it through a map of canonicalized write paths.
2026-08-20 10:20:38 +03:00
Diomidis SpinellisandMukunda Katta 714ec8a7bf Implement GNU W (write pattern first line) command
While at it, also fix the w command on patterns missing a newline.

Co-authored-by: Mukunda Katta <mukunda.vjcs6@gmail.com>
2026-08-19 20:22:54 +03:00
21 changed files with 858 additions and 246 deletions

No files matched your search

+5
View File
@@ -102,8 +102,13 @@ cargo test
* The `a`, `i`, `=`, `l`, `q` and `r` commands support address range as an extension to POSIX.
* The substitution command replacement group `\0` is a synonym for &.
* An `F` command outputs the name of the file currently being processed.
* The `p` command adds a missing newline.
(POSIX is silent; FreeBSD sed ignores the newline; original sed ignores
the line).
* A `Q` command (optionally followed by an exit code) quits immediately.
* The `q` command can be optionally followed by an exit code.
* An `R` schedules reading the next line from the specified file.
* A `W` command writes to a file the pattern's first line.
* The `l` command can be optionally followed by the output width.
* The `--follow-symlinks` option for in-place editing.
* The `--sandbox` option that limits potentially destructive commands.
+9 -4
View File
@@ -10,7 +10,8 @@
use crate::sed::error_handling::{ScriptLocation, runtime_error};
use crate::sed::fast_regex::{Captures, Match, Regex};
use crate::sed::named_writer::NamedWriter;
use crate::sed::named_io::{NamedReader, NamedWriter};
use crate::sed::processor::RecordSeparatorState;
use crate::sed::script_char_provider::ScriptCharProvider;
use crate::sed::script_line_provider::ScriptLineProvider;
@@ -72,13 +73,16 @@ pub struct ProcessingContext {
pub substitution_made: bool,
/// Elements to append at the end of each command processing cycle
pub append_elements: Vec<AppendElement>,
/// State used to separate consecutive output records.
pub rss: RecordSeparatorState,
}
#[derive(Clone, Debug)]
/// Elements that shall be appended at the end of each command processing cycle
pub enum AppendElement {
Text(Rc<[u8]>), // The specified text bytes
Path(PathBuf), // The contents of the specified file path
Text(Rc<[u8]>), // Text from the a command
ReaderText(Rc<[u8]>), // One line from the R command
Path(PathBuf), // The contents of the specified file path
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
@@ -367,7 +371,8 @@ pub enum CommandData {
BranchTarget(Option<Rc<RefCell<Command>>>), // Commands for 'b', 't', 'T', '{'
Label(Option<String>), // Label name for 'b', 't', 'T', ':'
Path(PathBuf), // File path for 'r'
NamedWriter(Rc<RefCell<NamedWriter>>), // File output for 'w'
NamedReader(Rc<RefCell<NamedReader>>), // File input for 'R'
NamedWriter(Rc<RefCell<NamedWriter>>), // File output for 'w', 'W'
Number(usize), // Number for 'l', 'q', 'Q' (GNU)
Substitution(Box<Substitution>), // Substitute command 's'
Text(Rc<[u8]>), // Text for 'a', 'c', 'i'
+18 -3
View File
@@ -18,7 +18,7 @@ use crate::sed::delimited_parser::{
};
use crate::sed::error_handling::{ScriptLocation, compilation_error, semantic_error};
use crate::sed::fast_regex::Regex;
use crate::sed::named_writer::NamedWriter;
use crate::sed::named_io::{NamedReader, NamedWriter};
use crate::sed::script_char_provider::ScriptCharProvider;
use crate::sed::script_line_provider::{ScriptLineProvider, ScriptValue};
@@ -1107,7 +1107,7 @@ fn compile_empty_command(
Ok(CommandHandling::Continue)
}
// Handles r
// Handles r, R
fn compile_read_file_command(
lines: &mut ScriptLineProvider,
line: &mut ScriptCharProvider,
@@ -1118,7 +1118,11 @@ fn compile_read_file_command(
return compilation_error(lines, line, ERR_SANDBOX);
}
let path = read_file_path(lines, line)?;
cmd.data = CommandData::Path(path);
cmd.data = if cmd.code == 'R' {
CommandData::NamedReader(NamedReader::new(path))
} else {
CommandData::Path(path)
};
Ok(CommandHandling::Continue)
}
@@ -1632,10 +1636,16 @@ fn get_cmd_spec(
n_addr: 2,
handler: compile_execute_command,
}),
// F is a GNU extension
'F' if !posix => Ok(CommandSpec {
n_addr: 2,
handler: compile_empty_command,
}),
// R is a GNU extension
'R' if !posix => Ok(CommandSpec {
n_addr: 2,
handler: compile_read_file_command,
}),
'r' => Ok(CommandSpec {
n_addr: if posix { 1 } else { 2 },
handler: compile_read_file_command,
@@ -1648,6 +1658,11 @@ fn get_cmd_spec(
n_addr: 2,
handler: compile_label_command,
}),
// W is a GNU extension
'W' if !posix => Ok(CommandSpec {
n_addr: 2,
handler: compile_write_file_command,
}),
'w' => Ok(CommandSpec {
n_addr: 2,
handler: compile_write_file_command,
+1 -94
View File
@@ -592,11 +592,8 @@ pub struct OutputBuffer {
max_pending_write: usize, // Max bytes to keep before flushing
#[cfg(unix)]
mmap_chunk: Option<MmapOutput>, // Chunk to write
// True when the last write didn't end with \n; the \n is deferred so
// that commands like `p` don't emit a spurious newline under -n.
pending_newline: bool,
#[cfg(test)]
low_level_flushes: usize, // Number of system call flushes
low_level_flushes: usize, // Number of system call flushes
}
/// Threshold to use buffered writes for output
@@ -627,7 +624,6 @@ impl OutputBuffer {
pub fn new(w: Box<dyn OutputWrite + 'static>) -> Self {
Self {
out: BufWriter::new(w),
pending_newline: false,
#[cfg(test)]
low_level_flushes: 0,
}
@@ -646,7 +642,6 @@ impl OutputBuffer {
fast_copy,
max_pending_write,
mmap_chunk: None,
pending_newline: false,
#[cfg(test)]
low_level_flushes: 0,
}
@@ -725,12 +720,6 @@ impl OutputBuffer {
return Ok(());
}
if self.pending_newline {
self.flush_mmap(WriteRange::Complete)?;
self.out.write_all(b"\n")?;
self.pending_newline = false;
}
match &new_chunk.content {
IOChunkContent::MmapInput {
full_span,
@@ -776,7 +765,6 @@ impl OutputBuffer {
len: new_len,
});
}
self.pending_newline = !new_chunk.is_newline_terminated();
}
IOChunkContent::Owned {
@@ -789,7 +777,6 @@ impl OutputBuffer {
if *has_newline {
self.out.write_all(b"\n")?;
}
self.pending_newline = !has_newline;
}
}
Ok(())
@@ -838,16 +825,6 @@ impl OutputBuffer {
Ok(())
}
/// Write a deferred newline if the last output didn't end with one.
pub fn flush_pending_newline(&mut self) -> io::Result<()> {
if self.pending_newline {
self.flush_mmap(WriteRange::Complete)?;
self.out.write_all(b"\n")?;
self.pending_newline = false;
}
Ok(())
}
/// Flush everything: pending mmap and buffered data.
pub fn flush(&mut self) -> io::Result<()> {
self.flush_mmap(WriteRange::Complete)?; // flush mmap if any
@@ -863,11 +840,6 @@ impl OutputBuffer {
return Ok(());
}
if self.pending_newline {
self.out.write_all(b"\n")?;
self.pending_newline = false;
}
match &chunk.content {
IOChunkContent::Owned {
content,
@@ -878,21 +850,11 @@ impl OutputBuffer {
if *has_newline {
self.out.write_all(b"\n")?;
}
self.pending_newline = !has_newline;
Ok(())
}
}
}
/// Write a deferred newline if the last output didn't end with one.
pub fn flush_pending_newline(&mut self) -> io::Result<()> {
if self.pending_newline {
self.out.write_all(b"\n")?;
self.pending_newline = false;
}
Ok(())
}
/// Flush everything: pending mmap and buffered data.
pub fn flush(&mut self) -> io::Result<()> {
self.out.flush() // then flush buffered data
@@ -1976,7 +1938,6 @@ mod tests {
max_pending_write: 8,
#[cfg(unix)]
mmap_chunk: None,
pending_newline: false,
low_level_flushes: 0,
};
(buf, file)
@@ -2097,58 +2058,4 @@ mod tests {
assert_eq!(out, "world\n");
}
// pending_newline is injected between two no-newline chunks
#[test]
fn pending_newline_injected_between_chunks() {
let (mut buf, mut file) = new_for_test();
buf.write_chunk(&make_owned_chunk("first", false)).unwrap();
buf.write_chunk(&make_owned_chunk("second", true)).unwrap();
buf.out.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "first\nsecond\n");
}
// flush_pending_newline emits the deferred newline
#[test]
fn flush_pending_newline_emits_newline() {
let (mut buf, mut file) = new_for_test();
buf.write_chunk(&make_owned_chunk("foo", false)).unwrap();
assert!(buf.pending_newline);
buf.flush_pending_newline().unwrap();
assert!(!buf.pending_newline);
buf.out.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "foo\n");
}
// write_str strips trailing newline and sets pending_newline correctly
#[test]
fn write_str_with_trailing_newline() {
let (mut buf, mut file) = new_for_test();
buf.write_str("bar\n").unwrap();
assert!(!buf.pending_newline);
buf.out.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "bar\n");
}
#[test]
fn write_str_without_trailing_newline() {
let (mut buf, mut file) = new_for_test();
buf.write_str("baz").unwrap();
assert!(buf.pending_newline);
buf.flush_pending_newline().unwrap();
buf.out.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "baz\n");
}
}
+2 -1
View File
@@ -15,7 +15,7 @@ pub mod error_handling;
pub mod fast_io;
pub mod fast_regex;
pub mod in_place;
pub mod named_writer;
pub mod named_io;
pub mod processor;
pub mod script_char_provider;
pub mod script_line_provider;
@@ -265,6 +265,7 @@ fn build_context(matches: &ArgMatches) -> UResult<ProcessingContext> {
range_commands: Vec::new(),
substitution_made: false,
append_elements: Vec::new(),
rss: processor::RecordSeparatorState::default(),
})
}
+324
View File
@@ -0,0 +1,324 @@
// An abstraction for output files created on entry and flushed on exit
//
// SPDX-License-Identifier: MIT
// Copyright (c) 2025 Diomidis Spinellis
//
// This file is part of the uutils sed package.
// It is licensed under the MIT License.
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
use crate::sed::error_handling::{ScriptLocation, runtime_error};
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::{self, File, OpenOptions};
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};
use std::rc::Rc;
use uucore::display::Quotable;
use uucore::error::UResult;
thread_local! {
/// Readers indexed by canonical input path, used to share state
/// between same-named path reads.
static READERS: RefCell<HashMap<PathBuf, Rc<RefCell<NamedReader>>>> = RefCell::new(HashMap::new());
/// Writers indexed by canonical output path, used to share same-named
/// paths and to flush writes.
static WRITERS: RefCell<HashMap<PathBuf, Rc<RefCell<NamedWriter>>>> = RefCell::new(HashMap::new());
}
#[derive(Debug)]
/// Reader that shares line-by-line state for GNU sed's R command.
pub struct NamedReader {
path: PathBuf,
reader: Option<BufReader<File>>,
done: bool,
}
impl NamedReader {
/// Create or retrieve the reader associated with `path`.
pub fn new(path: PathBuf) -> Rc<RefCell<Self>> {
let canonical_path = fs::canonicalize(&path).unwrap_or(path);
READERS.with(|readers| {
readers
.borrow_mut()
.entry(canonical_path.clone())
.or_insert_with(|| {
Rc::new(RefCell::new(Self {
path: canonical_path,
reader: None,
done: false,
}))
})
.clone()
})
}
/// Return the path associated with this reader.
pub fn original_path(&self) -> &Path {
&self.path
}
/// Read the next line, including its newline. Missing files and read errors
/// are treated as end-of-file, as required by the R command.
pub fn read_line(&mut self) -> Option<Vec<u8>> {
if self.done {
return None;
}
if self.reader.is_none() {
if let Ok(file) = File::open(&self.path) {
self.reader = Some(BufReader::new(file));
} else {
self.done = true;
return None;
}
}
let mut line = Vec::new();
match self.reader.as_mut().unwrap().read_until(b'\n', &mut line) {
Ok(0) | Err(_) => {
self.done = true;
None
}
Ok(_) => Some(line),
}
}
}
#[derive(Debug)]
/// Writer that tracks its file name for better error messages
pub struct NamedWriter {
pub path: PathBuf,
writer: BufWriter<File>,
location: ScriptLocation,
}
impl NamedWriter {
/// Create a new writer, truncate the file, and register it for flushing.
pub fn new(path: PathBuf, location: ScriptLocation) -> UResult<Rc<RefCell<Self>>> {
let canonical_path = canonicalize_output_path(&path, &location)?;
if let Some(writer) = WRITERS.with(|writers| writers.borrow().get(&canonical_path).cloned())
{
return Ok(writer);
}
let file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&path)
.map_err(|e| {
runtime_error::<()>(&location, format!("creating file {}: {}", path.quote(), e))
.unwrap_err()
})?;
let writer = Rc::new(RefCell::new(NamedWriter {
path,
writer: BufWriter::new(file),
location,
}));
WRITERS.with(|writers| {
writers
.borrow_mut()
.insert(canonical_path, Rc::clone(&writer));
});
Ok(writer)
}
/// Return the path used when this writer was first opened.
pub fn original_path(&self) -> &Path {
&self.path
}
/// Write String to the file, possibly with a newline, returning errors.
pub fn write_line(&mut self, line: &str, newline: bool) -> UResult<()> {
self.write_line_bytes(line.as_bytes(), newline)
}
/// Write bytes to the file, possibly with a newline, returning errors.
pub fn write_line_bytes(&mut self, line: &[u8], newline: bool) -> UResult<()> {
self.writer
.write_all(line)
.and_then(|()| {
if newline {
self.writer.write_all(b"\n")
} else {
Ok(())
}
})
.map_err(|e| {
runtime_error::<()>(
&self.location,
format!("writing to file {}: {e}", self.path.quote()),
)
.unwrap_err()
})
}
/// Flush the writer, returning a descriptive error.
pub fn flush(&mut self) -> UResult<()> {
self.writer.flush().map_err(|e| {
runtime_error::<()>(
&self.location,
format!("writing to file {}: {}", self.path.quote(), e),
)
.unwrap_err()
})
}
}
fn canonicalize_output_path(path: &Path, location: &ScriptLocation) -> UResult<PathBuf> {
let parent = path
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
let canonical_parent = fs::canonicalize(parent).map_err(|e| {
runtime_error::<()>(location, format!("creating file {}: {}", path.quote(), e)).unwrap_err()
})?;
Ok(match path.file_name() {
Some(file_name) => canonical_parent.join(file_name),
None => canonical_parent,
})
}
/// Flush buffered content to all open files, returning descriptive errors.
pub fn flush_all() -> UResult<()> {
WRITERS.with(|writers| {
for writer in writers.borrow().values() {
writer.borrow_mut().flush()?;
}
Ok(())
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::{NamedTempFile, tempdir};
#[test]
fn test_reader_reads_lines_as_bytes() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
fs::write(&path, b"first\nsecond\xE9").unwrap();
let reader = NamedReader::new(path);
assert_eq!(reader.borrow_mut().read_line(), Some(b"first\n".to_vec()));
assert_eq!(
reader.borrow_mut().read_line(),
Some(b"second\xE9".to_vec())
);
assert_eq!(reader.borrow_mut().read_line(), None);
assert_eq!(reader.borrow_mut().read_line(), None);
}
#[test]
fn test_new_reuses_reader_and_shared_position_for_same_path() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
fs::write(&path, b"first\nsecond\n").unwrap();
let first = NamedReader::new(path.clone());
let second = NamedReader::new(path);
assert!(Rc::ptr_eq(&first, &second));
assert_eq!(first.borrow_mut().read_line(), Some(b"first\n".to_vec()));
assert_eq!(second.borrow_mut().read_line(), Some(b"second\n".to_vec()));
}
#[test]
fn test_new_reuses_reader_for_canonical_duplicate_path() {
let dir = tempdir().unwrap();
let path = dir.path().join("input");
fs::write(&path, b"first\nsecond\n").unwrap();
let duplicate_path = dir.path().join(".").join("input");
let first = NamedReader::new(path.clone());
let second = NamedReader::new(duplicate_path);
assert!(Rc::ptr_eq(&first, &second));
assert_eq!(
first.borrow().original_path(),
fs::canonicalize(path).unwrap()
);
assert_eq!(first.borrow_mut().read_line(), Some(b"first\n".to_vec()));
assert_eq!(second.borrow_mut().read_line(), Some(b"second\n".to_vec()));
}
#[test]
fn test_reader_silently_ignores_missing_file() {
let dir = tempdir().unwrap();
let reader = NamedReader::new(dir.path().join("missing"));
assert_eq!(reader.borrow_mut().read_line(), None);
assert_eq!(reader.borrow_mut().read_line(), None);
}
#[test]
fn test_write_line_bytes_appends_newline() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
let writer = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
writer
.borrow_mut()
.write_line_bytes(b"a\xE9", true)
.unwrap();
writer.borrow_mut().flush().unwrap();
assert_eq!(fs::read(path).unwrap(), b"a\xE9\n");
}
#[test]
fn test_write_line_bytes_appends_no_newline() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
let writer = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
writer
.borrow_mut()
.write_line_bytes(b"a\xE9", false)
.unwrap();
writer.borrow_mut().flush().unwrap();
assert_eq!(fs::read(path).unwrap(), b"a\xE9");
}
#[test]
fn test_new_reuses_writer_for_same_path() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
let first = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
let second = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
assert!(Rc::ptr_eq(&first, &second));
assert_eq!(first.borrow().original_path(), path.as_path());
first.borrow_mut().write_line("first", true).unwrap();
second.borrow_mut().write_line("second", true).unwrap();
first.borrow_mut().flush().unwrap();
assert_eq!(fs::read_to_string(path).unwrap(), "first\nsecond\n");
}
#[test]
fn test_new_reuses_writer_for_canonical_duplicate_path() {
let dir = tempdir().unwrap();
let subdir = dir.path().join("subdir");
fs::create_dir(&subdir).unwrap();
let path = subdir.join("output");
let duplicate_path = subdir.join(".").join("output");
let first = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
let second = NamedWriter::new(duplicate_path, ScriptLocation::default()).unwrap();
assert!(Rc::ptr_eq(&first, &second));
assert_eq!(first.borrow().original_path(), path.as_path());
}
}
-117
View File
@@ -1,117 +0,0 @@
// An abstraction for output files created on entry and flushed on exit
//
// SPDX-License-Identifier: MIT
// Copyright (c) 2025 Diomidis Spinellis
//
// This file is part of the uutils sed package.
// It is licensed under the MIT License.
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
use crate::sed::error_handling::{ScriptLocation, runtime_error};
use std::cell::RefCell;
use std::fs::{File, OpenOptions};
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use std::rc::Rc;
use uucore::display::Quotable;
use uucore::error::UResult;
thread_local! {
/// Global list of all writers that should be flushed at shutdown
static FLUSH_LIST: RefCell<Vec<Rc<RefCell<NamedWriter>>>> = const { RefCell::new(Vec::new()) };
}
#[derive(Debug)]
/// Writer that tracks its file name for better error messages
pub struct NamedWriter {
pub path: PathBuf,
writer: BufWriter<File>,
location: ScriptLocation,
}
impl NamedWriter {
/// Create a new writer, truncate the file, and register it for flushing.
pub fn new(path: PathBuf, location: ScriptLocation) -> UResult<Rc<RefCell<Self>>> {
let file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&path)
.map_err(|e| {
runtime_error::<()>(&location, format!("creating file {}: {}", path.quote(), e))
.unwrap_err()
})?;
let writer = Rc::new(RefCell::new(NamedWriter {
path,
writer: BufWriter::new(file),
location,
}));
FLUSH_LIST.with(|list| list.borrow_mut().push(Rc::clone(&writer)));
Ok(writer)
}
/// Write a line to the file with a newline, returning descriptive errors.
pub fn write_line(&mut self, line: &str) -> UResult<()> {
self.write_line_bytes(line.as_bytes())
}
/// Write bytes to the file with a newline, returning descriptive errors.
pub fn write_line_bytes(&mut self, line: &[u8]) -> UResult<()> {
self.writer
.write_all(line)
.and_then(|()| self.writer.write_all(b"\n"))
.map_err(|e| {
runtime_error::<()>(
&self.location,
format!("writing to file {}: {e}", self.path.quote()),
)
.unwrap_err()
})
}
/// Flush the writer, returning a descriptive error.
pub fn flush(&mut self) -> UResult<()> {
self.writer.flush().map_err(|e| {
runtime_error::<()>(
&self.location,
format!("writing to file {}: {}", self.path.quote(), e),
)
.unwrap_err()
})
}
}
/// Flush buffered content to the file, returning descriptive errors.
pub fn flush_all() -> UResult<()> {
FLUSH_LIST.with(|cell| {
for handle in cell.borrow().iter() {
handle.borrow_mut().flush()?;
}
Ok(())
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::NamedTempFile;
#[test]
fn test_write_line_bytes_appends_newline() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
let writer = NamedWriter::new(path.clone(), ScriptLocation::default()).unwrap();
writer.borrow_mut().write_line_bytes(b"a\xE9").unwrap();
writer.borrow_mut().flush().unwrap();
assert_eq!(fs::read(path).unwrap(), b"a\xE9\n");
}
}
+193 -25
View File
@@ -17,7 +17,7 @@ use crate::sed::error_handling::{ScriptLocation, input_runtime_error};
use crate::sed::fast_io::{IOChunk, LineReader, OutputBuffer};
use crate::sed::fast_regex::Regex;
use crate::sed::in_place::InPlace;
use crate::sed::named_writer;
use crate::sed::named_io;
use memchr::memchr;
use std::borrow::Cow;
@@ -29,6 +29,40 @@ use std::rc::Rc;
use uucore::display::Quotable;
use uucore::error::{FromIo, UResult, set_exit_code};
#[derive(Clone, Copy, Debug)]
/// Track whether a separator is required before the next output record.
// This implements the following documented GNU sed behavior:
// "If sed prints a line without the terminating newline, it will
// nevertheless print the missing newline as soon as more text is
// sent to the same output stream."
pub struct RecordSeparatorState {
previous_had_newline: bool,
}
impl Default for RecordSeparatorState {
fn default() -> Self {
Self {
previous_had_newline: true,
}
}
}
impl RecordSeparatorState {
/// Begin an output record, terminating the previous one when necessary.
pub fn new_record(&mut self, output: &mut OutputBuffer) -> io::Result<()> {
if !self.previous_had_newline {
output.write_bytes(b"\n")?;
self.previous_had_newline = true;
}
Ok(())
}
/// Record whether the output record ended with a newline.
pub fn has_newline(&mut self, has_newline: bool) {
self.previous_had_newline = has_newline;
}
}
/// Return the specified command variant or panic.
// Example: let path = extract_variant!(command, Path);
macro_rules! extract_variant {
@@ -164,13 +198,41 @@ fn applies(
}
}
/// Write the specified chunk to the output for a given processing context.
fn write_chunk(
/// Output the specified record as a chunk.
fn write_chunk_record(
output: &mut OutputBuffer,
context: &ProcessingContext,
context: &mut ProcessingContext,
chunk: &IOChunk,
) -> std::io::Result<()> {
// Completely empty chunks are no-ops regarding state.
if chunk.is_empty() && !chunk.is_newline_terminated() {
return Ok(());
}
context.rss.new_record(output)?;
output.write_chunk(chunk)?;
context.rss.has_newline(chunk.is_newline_terminated());
if context.unbuffered {
output.flush()?;
}
Ok(())
}
/// Write one owned output record.
fn write_buffer_record(
output: &mut OutputBuffer,
context: &mut ProcessingContext,
bytes: &[u8],
has_newline: bool,
) -> UResult<()> {
if bytes.is_empty() {
return Ok(());
}
context.rss.new_record(output)?;
output.write_bytes(bytes)?;
context.rss.has_newline(has_newline);
if context.unbuffered {
output.flush()?;
@@ -386,18 +448,20 @@ fn substitute(
// prints the pre-execution text then executes, while 'ep' executes
// then prints the result.
if sub.print_flag && sub.p_before_e {
write_chunk(output, context, pattern)?;
write_chunk_record(output, context, pattern)?;
}
if sub.execute {
execute_pattern_as_shell_command(pattern, command, context)?;
}
if sub.print_flag && !sub.p_before_e {
write_chunk(output, context, pattern)?;
write_chunk_record(output, context, pattern)?;
}
// Write to file if needed.
if let Some(ref writer) = sub.write_file {
writer.borrow_mut().write_line_bytes(pattern.as_bytes())?;
writer
.borrow_mut()
.write_line_bytes(pattern.as_bytes(), pattern.is_newline_terminated())?;
}
context.substitution_made = true;
}
@@ -465,10 +529,21 @@ fn flush_appends(output: &mut OutputBuffer, context: &mut ProcessingContext) ->
for elem in &context.append_elements {
match elem {
AppendElement::Text(text) => {
context.rss.new_record(output)?;
output.write_bytes(text.as_ref())?;
context.rss.has_newline(text.ends_with(b"\n"));
}
AppendElement::ReaderText(text) => {
context.rss.new_record(output)?;
output.write_bytes(text.as_ref())?;
// GNU sed doesn't care about read file \n ending.
context.rss.has_newline(true);
}
AppendElement::Path(path) => {
context.rss.new_record(output)?;
output.copy_file(path)?;
// GNU sed doesn't care about read file \n ending.
context.rss.has_newline(true);
}
}
}
@@ -608,6 +683,7 @@ fn list(
fn process_address_0(
commands: Option<Rc<RefCell<Command>>>,
output: &mut OutputBuffer,
context: &mut ProcessingContext,
) -> UResult<()> {
// Prescan for zero-address which must produce output
// before any input line is read.
@@ -622,7 +698,10 @@ fn process_address_0(
&& cmd.addr2.is_none()
{
let path = extract_variant!(cmd, Path);
context.rss.new_record(output)?;
output.copy_file(path)?;
// GNU sed doesn't care about read file \n ending.
context.rss.has_newline(true);
}
cmd.next.clone()
@@ -641,7 +720,7 @@ fn process_file(
output: &mut OutputBuffer,
context: &mut ProcessingContext,
) -> UResult<()> {
process_address_0(commands.clone(), output)?;
process_address_0(commands.clone(), output, context)?;
// Loop over the input lines as pattern space.
'lines: while let Some(mut pattern) = reader.get_line()? {
@@ -706,7 +785,7 @@ fn process_file(
pattern.clear();
if command.addr2.is_none() || context.last_address || reader.last_line()? {
let text = extract_variant!(command, Text);
output.write_bytes(text.as_ref())?;
write_buffer_record(output, context, text, true)?;
}
break;
}
@@ -733,7 +812,7 @@ fn process_file(
}
CommandData::Text(cmd_bytes) => {
let shell_out = shell_stdout(cmd_bytes.to_vec(), &command, context)?;
output.write_bytes(&shell_out)?;
write_buffer_record(output, context, &shell_out, true)?;
}
_ => panic!("invalid 'e' command data"),
},
@@ -741,7 +820,7 @@ fn process_file(
// Output current input file name.
let mut bytes = context.input_name.as_os_str().as_encoded_bytes().to_vec();
bytes.push(b'\n');
output.write_bytes(&bytes)?;
write_buffer_record(output, context, &bytes, true)?;
}
'g' => {
// Replace pattern with the contents of the hold space.
@@ -768,11 +847,13 @@ fn process_file(
'i' => {
// Write text to standard output.
let text = extract_variant!(command, Text);
output.write_bytes(text.as_ref())?;
write_buffer_record(output, context, text, true)?;
}
'l' => {
let width = *extract_variant!(command, Number);
context.rss.new_record(output)?;
list(output, &pattern, width, &command.location, context)?;
context.rss.has_newline(true);
}
'n' => {
break;
@@ -790,14 +871,14 @@ fn process_file(
continue 'lines;
}
'p' => {
write_chunk(output, context, &pattern)?;
write_chunk_record(output, context, &pattern)?;
}
'P' => {
let line = pattern.as_bytes();
if let Some(pos) = memchr(b'\n', line) {
output.write_bytes(&line[..=pos])?;
write_buffer_record(output, context, &line[..=pos], true)?;
} else {
write_chunk(output, context, &pattern)?;
write_chunk_record(output, context, &pattern)?;
}
}
'q' => {
@@ -824,6 +905,15 @@ fn process_file(
.append_elements
.push(AppendElement::Path(path.clone()));
}
'R' => {
// Copy one line from the file to standard output later.
let reader = extract_variant!(command, NamedReader);
if let Some(line) = reader.borrow_mut().read_line() {
context
.append_elements
.push(AppendElement::ReaderText(Rc::from(line)));
}
}
's' => {
substitute(&mut pattern, &command, context, output)?;
}
@@ -861,7 +951,24 @@ fn process_file(
'w' => {
// Append the pattern space to the specified file.
let writer = extract_variant!(command, NamedWriter);
writer.borrow_mut().write_line_bytes(pattern.as_bytes())?;
writer
.borrow_mut()
.write_line_bytes(pattern.as_bytes(), pattern.is_newline_terminated())?;
}
'W' => {
// Append only the first line of the pattern space.
let writer = extract_variant!(command, NamedWriter);
let pattern_bytes = pattern.as_bytes();
let (first_line, found_newline) =
match pattern_bytes.iter().position(|&b| b == b'\n') {
// A slice including the newline
Some(pos) => (&pattern_bytes[..=pos], true),
None => (pattern_bytes, false),
};
writer.borrow_mut().write_line_bytes(
first_line,
!found_newline && pattern.is_newline_terminated(),
)?;
}
'x' => {
// Exchange the contents of the pattern and hold spaces.
@@ -888,7 +995,12 @@ fn process_file(
}
'=' => {
// Output current line number.
output.write_str(format!("{}\n", context.line_number))?;
write_buffer_record(
output,
context,
format!("{}\n", context.line_number).as_bytes(),
true,
)?;
}
// The compilation should supply only valid codes.
_ => panic!("invalid command code"),
@@ -898,13 +1010,13 @@ fn process_file(
}
if !context.quiet {
write_chunk(output, context, &pattern)?;
write_chunk_record(output, context, &pattern)?;
}
flush_appends(output, context)?;
if context.stop_processing {
output.flush_pending_newline()?;
context.rss.new_record(output)?;
break;
}
}
@@ -916,10 +1028,7 @@ fn process_file(
{
let mut pending = action.prepend;
pending.push(b'\n');
output.write_bytes(&pending)?;
if context.unbuffered {
output.flush()?;
}
write_buffer_record(output, context, &pending, true)?;
}
Ok(())
@@ -977,7 +1086,7 @@ pub fn process_all_files(
{
let mut pending = action.prepend;
pending.push(b'\n');
output.write_bytes(&pending)?;
write_buffer_record(output, context, &pending, true)?;
}
in_place.end()?;
@@ -988,7 +1097,7 @@ pub fn process_all_files(
}
// Flush all output files
named_writer::flush_all()?;
named_io::flush_all()?;
Ok(())
}
@@ -999,6 +1108,65 @@ mod tests {
use std::io::{Read, Seek, SeekFrom};
use tempfile::tempfile;
fn record_separator_output(
state: &mut RecordSeparatorState,
configure: impl FnOnce(&mut RecordSeparatorState),
) -> String {
let mut file = tempfile().unwrap();
let mut output = OutputBuffer::new(Box::new(file.try_clone().unwrap()));
configure(state);
state.new_record(&mut output).unwrap();
output.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut written = String::new();
file.read_to_string(&mut written).unwrap();
written
}
#[test]
fn record_separator_default_does_not_emit_newline() {
let mut state = RecordSeparatorState::default();
assert_eq!(record_separator_output(&mut state, |_| {}), "");
}
#[test]
fn record_separator_terminates_unterminated_record() {
let mut state = RecordSeparatorState::default();
assert_eq!(
record_separator_output(&mut state, |state| state.has_newline(false)),
"\n"
);
}
#[test]
fn record_separator_emits_missing_newline_only_once() {
let mut state = RecordSeparatorState::default();
state.has_newline(false);
let mut file = tempfile().unwrap();
let mut output = OutputBuffer::new(Box::new(file.try_clone().unwrap()));
state.new_record(&mut output).unwrap();
state.new_record(&mut output).unwrap();
output.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut written = String::new();
file.read_to_string(&mut written).unwrap();
assert_eq!(written, "\n");
}
#[test]
fn record_separator_newline_terminated_record_needs_no_separator() {
let mut state = RecordSeparatorState::default();
assert_eq!(
record_separator_output(&mut state, |state| state.has_newline(true)),
""
);
}
#[test]
fn test_readable_ascii_byte_named_escapes() {
assert_eq!(readable_ascii_byte(b'\n'), r"\012");
+161 -1
View File
@@ -1088,6 +1088,15 @@ fn test_e_command_with_arg_non_utf8_output_passthrough() {
.stdout_is_bytes(b"\xff\na\n");
}
#[cfg(unix)]
#[test]
fn test_e_command_no_newline_ignored() {
new_ucmd!()
.args(&["3e printf hi", LINES1])
.succeeds()
.stdout_is_fixture_bytes("output/e_command_no_newline_ignored");
}
////////////////////////////////////////////////////////////
// Transliteration: y
check_output!(trans_simple, ["-e", r"y/0123456789/9876543210/", LINES1]);
@@ -1478,6 +1487,16 @@ fn test_branch_no_sub_non_posix() {
.stderr_contains("invalid command code");
}
check_output!(
multiple_no_newline,
[
"-n",
"p",
"input/no-new-line.txt",
"input/no-new-line.txt",
"input/no-new-line.txt",
]
);
////////////////////////////////////////////////////////////
// Text: a, c, i
@@ -1646,8 +1665,32 @@ check_output!(
);
////////////////////////////////////////////////////////////
// r, w commands
// r, R, w, W commands
check_output!(read_ok, [format!("4r {LINES2}"), LINES1.to_string()]);
check_output!(read_no_newline, ["4r input/no-new-line.txt", LINES1]);
// r Doesn't doesn't record lacking newline
check_output!(
read_no_newline_update,
[
"-e",
"4r input/no-new-line.txt",
"-e",
"4r input/no-new-line.txt",
LINES1
]
);
// r respects a lacking newline and adds it and also resets the state
// to no newline needed.
check_output!(
read_no_newline_respect,
[
"-e",
"s/^/i: /;r input/no-new-line.txt",
"-e",
"r input/no-new-line.txt",
"input/no-new-line.txt"
]
);
check_output!(read_missing, ["5r /xyzzyxyzy42", LINES1]);
check_output!(read_empty, ["6r input/empty", LINES1]);
check_output!(
@@ -1667,6 +1710,25 @@ fn sandbox_rejects_read_command() {
.stderr_contains("command not allowed with --sandbox");
}
check_output!(read_one, ["6R input/lines2", LINES1]);
check_output!(
read_one_twice,
["-e", "5R input/lines2", "-e", "7R input/lines2", LINES1]
);
check_output!(read_one_many, ["R input/lines2", LINES1]);
check_output!(read_one_empty, ["R input/empty", LINES1]);
check_output!(read_one_missing, ["R input/xyzzy42", LINES1]);
check_output!(read_one_no_newline, ["R input/no-new-line.txt", LINES1]);
#[test]
fn read_one_line_rejected_in_posix_mode() {
new_ucmd!()
.args(&["--posix", "R /tmp/read-one-line"])
.fails()
.code_is(1)
.stderr_is("sed: <script argument 1>:1:1: error: invalid command code `R'\n");
}
#[test]
fn sandbox_rejects_subst_write_flag() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
@@ -1701,6 +1763,23 @@ fn sandbox_rejects_write_command() -> std::io::Result<()> {
Ok(())
}
#[test]
fn sandbox_rejects_first_line_write_command() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
let cmd = format!("W {}", temp.path().display());
new_ucmd!()
.args(&["--sandbox", &cmd, LINES1])
.fails()
.stderr_contains("command not allowed with --sandbox");
let mut actual = String::new();
temp.reopen()?.read_to_string(&mut actual)?;
assert!(actual.is_empty());
Ok(())
}
#[test]
fn write_single_file() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
@@ -1717,6 +1796,44 @@ fn write_single_file() -> std::io::Result<()> {
Ok(())
}
#[test]
fn write_single_file_twice() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
// Write odd and even lines to the same file through different commands.
// Unless the writers are unified, this fails on Windows.
let cmd1 = format!("3~2w {}", temp.path().display());
let cmd2 = format!("4~2w {}", temp.path().display());
new_ucmd!()
.args(&["-e", &cmd1, "-e", &cmd2, "-e", "12q", LINES1])
.succeeds();
let mut actual = String::new();
temp.reopen()?.read_to_string(&mut actual)?;
let expected = fs::read_to_string("tests/fixtures/sed/output/write_single_file")?;
assert_eq!(actual, expected, "Output did not match fixture");
Ok(())
}
#[test]
fn write_single_file_no_newline() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
let cmd = format!("w {}", temp.path().display());
new_ucmd!()
.args(&[cmd.as_str(), "input/no-new-line.txt"])
.succeeds();
let mut actual = String::new();
temp.reopen()?.read_to_string(&mut actual)?;
assert_eq!(actual, "Hello", "Output did not match expected");
Ok(())
}
#[test]
fn write_two_files() -> std::io::Result<()> {
let temp1 = NamedTempFile::new()?;
@@ -1744,6 +1861,49 @@ fn write_two_files() -> std::io::Result<()> {
Ok(())
}
#[test]
fn write_first_line_newline() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
let cmd = format!("N;W {}", temp.path().display());
new_ucmd!()
.args(&["-n", "-e", &cmd])
.pipe_in("abc\ndef\n")
.succeeds();
let mut actual = String::new();
temp.reopen()?.read_to_string(&mut actual)?;
assert_eq!(actual, "abc\n");
Ok(())
}
#[test]
fn write_first_line_no_newline() -> std::io::Result<()> {
let temp = NamedTempFile::new()?;
let cmd = format!("W {}", temp.path().display());
new_ucmd!()
.args(&["-n", "-e", &cmd])
.pipe_in("abc")
.succeeds();
let mut actual = String::new();
temp.reopen()?.read_to_string(&mut actual)?;
assert_eq!(actual, "abc");
Ok(())
}
#[test]
fn write_first_line_with_w_command_is_non_posix() {
new_ucmd!()
.args(&["--posix", "W /tmp/out"])
.fails()
.code_is(1)
.stderr_is("sed: <script argument 1>:1:1: error: invalid command code `W'\n");
}
////////////////////////////////////////////////////////////
// =, l, F commands
check_output!(number_continuous, ["/l2_/=", LINES1, LINES2]);
+14
View File
@@ -0,0 +1,14 @@
l1_1
l1_2
hil1_3
l1_4
l1_5
l1_6
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
+2 -1
View File
@@ -14,7 +14,8 @@
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
Hello..........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
Hello
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Line truncated
+3
View File
@@ -0,0 +1,3 @@
Hello
Hello
Hello
+14
View File
@@ -0,0 +1,14 @@
l1_1
l1_2
l1_3
l1_4
Hellol1_5
l1_6
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
+2
View File
@@ -0,0 +1,2 @@
i: Hello
HelloHello
+14
View File
@@ -0,0 +1,14 @@
l1_1
l1_2
l1_3
l1_4
HelloHellol1_5
l1_6
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
+15
View File
@@ -0,0 +1,15 @@
l1_1
l1_2
l1_3
l1_4
l1_5
l1_6
l2_1
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
+14
View File
@@ -0,0 +1,14 @@
l1_1
l1_2
l1_3
l1_4
l1_5
l1_6
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
+23
View File
@@ -0,0 +1,23 @@
l1_1
l2_1
l1_2
l2_2
l1_3
l2_3
l1_4
l2_4
l1_5
l2_5
l1_6
l2_6
l1_7
l2_7
l1_8
l2_8
l1_9
l2_9
l1_10
l1_11
l1_12
l1_13
l1_14
+14
View File
@@ -0,0 +1,14 @@
l1_1
l1_2
l1_3
l1_4
l1_5
l1_6
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
+14
View File
@@ -0,0 +1,14 @@
l1_1
Hellol1_2
l1_3
l1_4
l1_5
l1_6
l1_7
l1_8
l1_9
l1_10
l1_11
l1_12
l1_13
l1_14
Loaded 20 of 21 files, more files were not shown because too many files have changed in this diff. Show more