From 8d65c2baddac1226cc6d172acf3998b24d47e722 Mon Sep 17 00:00:00 2001 From: Olivier Tilloy Date: Mon, 4 Mar 2024 21:41:27 +0100 Subject: [PATCH] Implement -t/--expand-tabs option --- Cargo.lock | 7 +++ Cargo.toml | 1 + src/context_diff.rs | 29 +++++++++--- src/ed_diff.rs | 23 +++++---- src/lib.rs | 1 + src/main.rs | 16 +++++-- src/normal_diff.rs | 26 +++++----- src/params.rs | 53 +++++++++++++++++++++ src/unified_diff.rs | 21 +++++++-- src/utils.rs | 112 ++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 255 insertions(+), 34 deletions(-) create mode 100644 src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 92ad218..9dc8153 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,6 +83,7 @@ dependencies = [ "pretty_assertions", "same-file", "tempfile", + "unicode-width", ] [[package]] @@ -313,6 +314,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 3804a81..7eeb35b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] diff = "0.1.10" same-file = "1.0.6" +unicode-width = "0.1.11" [dev-dependencies] pretty_assertions = "1" diff --git a/src/context_diff.rs b/src/context_diff.rs index af262a3..408821f 100644 --- a/src/context_diff.rs +++ b/src/context_diff.rs @@ -6,6 +6,8 @@ use std::collections::VecDeque; use std::io::Write; +use crate::utils::do_write_line; + #[derive(Debug, PartialEq)] pub enum DiffLine { Context(Vec), @@ -270,6 +272,7 @@ pub fn diff( actual_filename: &str, context_size: usize, stop_early: bool, + expand_tabs: bool, ) -> Vec { let mut output = format!("*** {expected_filename}\t\n--- {actual_filename}\t\n").into_bytes(); let diff_results = make_diff(expected, actual, context_size, stop_early); @@ -314,17 +317,20 @@ pub fn diff( match line { DiffLine::Context(e) => { write!(output, " ").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::Change(e) => { write!(output, "! ").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::Add(e) => { write!(output, "- ").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } } @@ -341,17 +347,20 @@ pub fn diff( match line { DiffLine::Context(e) => { write!(output, " ").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::Change(e) => { write!(output, "! ").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::Add(e) => { write!(output, "+ ").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } } @@ -424,6 +433,7 @@ mod tests { &format!("{target}/alef"), 2, false, + false, ); File::create(&format!("{target}/ab.diff")) .unwrap() @@ -503,6 +513,7 @@ mod tests { &format!("{target}/alef_"), 2, false, + false, ); File::create(&format!("{target}/ab_.diff")) .unwrap() @@ -585,6 +596,7 @@ mod tests { &format!("{target}/alefx"), 2, false, + false, ); File::create(&format!("{target}/abx.diff")) .unwrap() @@ -670,6 +682,7 @@ mod tests { &format!("{target}/alefr"), 2, false, + false, ); File::create(&format!("{target}/abr.diff")) .unwrap() @@ -715,6 +728,7 @@ mod tests { to_filename, context_size, false, + false, ); let expected_full = [ "*** foo\t", @@ -740,6 +754,7 @@ mod tests { to_filename, context_size, true, + false, ); let expected_brief = ["*** foo\t", "--- bar\t", ""].join("\n"); assert_eq!(diff_brief, expected_brief.as_bytes()); @@ -751,6 +766,7 @@ mod tests { to_filename, context_size, false, + false, ); assert!(nodiff_full.is_empty()); @@ -761,6 +777,7 @@ mod tests { to_filename, context_size, true, + false, ); assert!(nodiff_brief.is_empty()); } diff --git a/src/ed_diff.rs b/src/ed_diff.rs index 7613b22..6d47b9f 100644 --- a/src/ed_diff.rs +++ b/src/ed_diff.rs @@ -5,6 +5,8 @@ use std::io::Write; +use crate::utils::do_write_line; + #[derive(Debug, PartialEq)] struct Mismatch { pub line_number_expected: usize, @@ -107,7 +109,12 @@ fn make_diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Result Result, DiffError> { +pub fn diff( + expected: &[u8], + actual: &[u8], + stop_early: bool, + expand_tabs: bool, +) -> Result, DiffError> { let mut output = Vec::new(); let diff_results = make_diff(expected, actual, stop_early)?; if stop_early && !diff_results.is_empty() { @@ -145,7 +152,7 @@ pub fn diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Result, if actual == b"." { writeln!(&mut output, "..\n.\ns/.//\na").unwrap(); } else { - output.write_all(actual).unwrap(); + do_write_line(&mut output, actual, expand_tabs).unwrap(); writeln!(&mut output).unwrap(); } } @@ -160,7 +167,7 @@ mod tests { use super::*; use pretty_assertions::assert_eq; pub fn diff_w(expected: &[u8], actual: &[u8], filename: &str) -> Result, DiffError> { - let mut output = diff(expected, actual, false)?; + let mut output = diff(expected, actual, false, false)?; writeln!(&mut output, "w {filename}").unwrap(); Ok(output) } @@ -169,7 +176,7 @@ mod tests { fn test_basic() { let from = b"a\n"; let to = b"b\n"; - let diff = diff(from, to, false).unwrap(); + let diff = diff(from, to, false, false).unwrap(); let expected = ["1c", "b", ".", ""].join("\n"); assert_eq!(diff, expected.as_bytes()); } @@ -404,18 +411,18 @@ mod tests { let from = ["a", "b", "c", ""].join("\n"); let to = ["a", "d", "c", ""].join("\n"); - let diff_full = diff(from.as_bytes(), to.as_bytes(), false).unwrap(); + let diff_full = diff(from.as_bytes(), to.as_bytes(), false, false).unwrap(); let expected_full = ["2c", "d", ".", ""].join("\n"); assert_eq!(diff_full, expected_full.as_bytes()); - let diff_brief = diff(from.as_bytes(), to.as_bytes(), true).unwrap(); + let diff_brief = diff(from.as_bytes(), to.as_bytes(), true, false).unwrap(); let expected_brief = "\0".as_bytes(); assert_eq!(diff_brief, expected_brief); - let nodiff_full = diff(from.as_bytes(), from.as_bytes(), false).unwrap(); + let nodiff_full = diff(from.as_bytes(), from.as_bytes(), false, false).unwrap(); assert!(nodiff_full.is_empty()); - let nodiff_brief = diff(from.as_bytes(), from.as_bytes(), true).unwrap(); + let nodiff_brief = diff(from.as_bytes(), from.as_bytes(), true, false).unwrap(); assert!(nodiff_brief.is_empty()); } } diff --git a/src/lib.rs b/src/lib.rs index a78b64d..faf5df2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub mod context_diff; pub mod ed_diff; pub mod normal_diff; pub mod unified_diff; +pub mod utils; // Re-export the public functions/types you need pub use context_diff::diff as context_diff; diff --git a/src/main.rs b/src/main.rs index 6ff2a0f..f074cb1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod ed_diff; mod normal_diff; mod params; mod unified_diff; +mod utils; // Exit codes are documented at // https://www.gnu.org/software/diffutils/manual/html_node/Invoking-diff.html. @@ -30,6 +31,7 @@ fn main() -> ExitCode { format, report_identical_files, brief, + expand_tabs, } = parse_params(opts).unwrap_or_else(|error| { eprintln!("{error}"); exit(2); @@ -65,7 +67,7 @@ fn main() -> ExitCode { }; // run diff let result: Vec = match format { - Format::Normal => normal_diff::diff(&from_content, &to_content, brief), + Format::Normal => normal_diff::diff(&from_content, &to_content, brief, expand_tabs), Format::Unified => unified_diff::diff( &from_content, &from.to_string_lossy(), @@ -73,6 +75,7 @@ fn main() -> ExitCode { &to.to_string_lossy(), context_count, brief, + expand_tabs, ), Format::Context => context_diff::diff( &from_content, @@ -81,11 +84,14 @@ fn main() -> ExitCode { &to.to_string_lossy(), context_count, brief, + expand_tabs, ), - Format::Ed => ed_diff::diff(&from_content, &to_content, brief).unwrap_or_else(|error| { - eprintln!("{error}"); - exit(2); - }), + Format::Ed => { + ed_diff::diff(&from_content, &to_content, brief, expand_tabs).unwrap_or_else(|error| { + eprintln!("{error}"); + exit(2); + }) + } }; if brief && !result.is_empty() { println!( diff --git a/src/normal_diff.rs b/src/normal_diff.rs index aeef145..e25a6c6 100644 --- a/src/normal_diff.rs +++ b/src/normal_diff.rs @@ -5,6 +5,8 @@ use std::io::Write; +use crate::utils::do_write_line; + #[derive(Debug, PartialEq)] struct Mismatch { pub line_number_expected: usize, @@ -114,7 +116,7 @@ fn make_diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Vec } #[must_use] -pub fn diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Vec { +pub fn diff(expected: &[u8], actual: &[u8], stop_early: bool, expand_tabs: bool) -> Vec { // See https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Normal.html // for details on the syntax of the normal format. let mut output = Vec::new(); @@ -188,7 +190,7 @@ pub fn diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Vec { } for expected in &result.expected { write!(&mut output, "< ").unwrap(); - output.write_all(expected).unwrap(); + do_write_line(&mut output, expected, expand_tabs).unwrap(); writeln!(&mut output).unwrap(); } if result.expected_missing_nl { @@ -199,7 +201,7 @@ pub fn diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Vec { } for actual in &result.actual { write!(&mut output, "> ").unwrap(); - output.write_all(actual).unwrap(); + do_write_line(&mut output, actual, expand_tabs).unwrap(); writeln!(&mut output).unwrap(); } if result.actual_missing_nl { @@ -220,7 +222,7 @@ mod tests { a.write_all(b"a\n").unwrap(); let mut b = Vec::new(); b.write_all(b"b\n").unwrap(); - let diff = diff(&a, &b, false); + let diff = diff(&a, &b, false, false); let expected = b"1c1\n< a\n---\n> b\n".to_vec(); assert_eq!(diff, expected); } @@ -273,7 +275,7 @@ mod tests { } // This test diff is intentionally reversed. // We want it to turn the alef into bet. - let diff = diff(&alef, &bet, false); + let diff = diff(&alef, &bet, false, false); File::create(&format!("{target}/ab.diff")) .unwrap() .write_all(&diff) @@ -365,7 +367,7 @@ mod tests { } // This test diff is intentionally reversed. // We want it to turn the alef into bet. - let diff = diff(&alef, &bet, false); + let diff = diff(&alef, &bet, false, false); File::create(&format!("{target}/abn.diff")) .unwrap() .write_all(&diff) @@ -439,7 +441,7 @@ mod tests { } // This test diff is intentionally reversed. // We want it to turn the alef into bet. - let diff = diff(&alef, &bet, false); + let diff = diff(&alef, &bet, false, false); File::create(&format!("{target}/ab_.diff")) .unwrap() .write_all(&diff) @@ -517,7 +519,7 @@ mod tests { } // This test diff is intentionally reversed. // We want it to turn the alef into bet. - let diff = diff(&alef, &bet, false); + let diff = diff(&alef, &bet, false, false); File::create(&format!("{target}/abr.diff")) .unwrap() .write_all(&diff) @@ -552,18 +554,18 @@ mod tests { let from = ["a", "b", "c"].join("\n"); let to = ["a", "d", "c"].join("\n"); - let diff_full = diff(from.as_bytes(), to.as_bytes(), false); + let diff_full = diff(from.as_bytes(), to.as_bytes(), false, false); let expected_full = ["2c2", "< b", "---", "> d", ""].join("\n"); assert_eq!(diff_full, expected_full.as_bytes()); - let diff_brief = diff(from.as_bytes(), to.as_bytes(), true); + let diff_brief = diff(from.as_bytes(), to.as_bytes(), true, false); let expected_brief = "\0".as_bytes(); assert_eq!(diff_brief, expected_brief); - let nodiff_full = diff(from.as_bytes(), from.as_bytes(), false); + let nodiff_full = diff(from.as_bytes(), from.as_bytes(), false, false); assert!(nodiff_full.is_empty()); - let nodiff_brief = diff(from.as_bytes(), from.as_bytes(), true); + let nodiff_brief = diff(from.as_bytes(), from.as_bytes(), true, false); assert!(nodiff_brief.is_empty()); } } diff --git a/src/params.rs b/src/params.rs index 661cd37..a576f3d 100644 --- a/src/params.rs +++ b/src/params.rs @@ -27,6 +27,7 @@ pub struct Params { pub context_count: usize, pub report_identical_files: bool, pub brief: bool, + pub expand_tabs: bool, } pub fn parse_params>(opts: I) -> Result { @@ -42,6 +43,7 @@ pub fn parse_params>(opts: I) -> Result>(opts: I) -> Result>(opts: I) -> Result), @@ -241,6 +243,7 @@ pub fn diff( actual_filename: &str, context_size: usize, stop_early: bool, + expand_tabs: bool, ) -> Vec { let mut output = format!("--- {expected_filename}\t\n+++ {actual_filename}\t\n").into_bytes(); let diff_results = make_diff(expected, actual, context_size, stop_early); @@ -371,17 +374,20 @@ pub fn diff( match line { DiffLine::Expected(e) => { write!(output, "-").expect("write to Vec is infallible"); - output.write_all(&e).expect("write to Vec is infallible"); + do_write_line(&mut output, &e, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::Context(c) => { write!(output, " ").expect("write to Vec is infallible"); - output.write_all(&c).expect("write to Vec is infallible"); + do_write_line(&mut output, &c, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::Actual(r) => { write!(output, "+",).expect("write to Vec is infallible"); - output.write_all(&r).expect("write to Vec is infallible"); + do_write_line(&mut output, &r, expand_tabs) + .expect("write to Vec is infallible"); writeln!(output).unwrap(); } DiffLine::MissingNL => { @@ -454,6 +460,7 @@ mod tests { &format!("{target}/alef"), 2, false, + false, ); File::create(&format!("{target}/ab.diff")) .unwrap() @@ -568,6 +575,7 @@ mod tests { &format!("{target}/alefn"), 2, false, + false, ); File::create(&format!("{target}/abn.diff")) .unwrap() @@ -662,6 +670,7 @@ mod tests { &format!("{target}/alef_"), 2, false, + false, ); File::create(&format!("{target}/ab_.diff")) .unwrap() @@ -741,6 +750,7 @@ mod tests { &format!("{target}/alefx"), 2, false, + false, ); File::create(&format!("{target}/abx.diff")) .unwrap() @@ -825,6 +835,7 @@ mod tests { &format!("{target}/alefr"), 2, false, + false, ); File::create(&format!("{target}/abr.diff")) .unwrap() @@ -869,6 +880,7 @@ mod tests { to_filename, context_size, false, + false, ); let expected_full = [ "--- foo\t", @@ -890,6 +902,7 @@ mod tests { to_filename, context_size, true, + false, ); let expected_brief = ["--- foo\t", "+++ bar\t", ""].join("\n"); assert_eq!(diff_brief, expected_brief.as_bytes()); @@ -901,6 +914,7 @@ mod tests { to_filename, context_size, false, + false, ); assert!(nodiff_full.is_empty()); @@ -911,6 +925,7 @@ mod tests { to_filename, context_size, true, + false, ); assert!(nodiff_brief.is_empty()); } diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..1d13682 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,112 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use std::io::Write; + +use unicode_width::UnicodeWidthStr; + +/// Replace tabs by spaces in the input line. +/// Correctly handle multi-bytes characters. +/// This assumes that line does not contain any line breaks (if it does, the result is undefined). +pub fn do_expand_tabs(line: &[u8], tabsize: usize) -> Vec { + let tab = b'\t'; + let ntabs = line.iter().filter(|c| **c == tab).count(); + if ntabs == 0 { + return line.to_vec(); + } + let mut result = Vec::with_capacity(line.len() + ntabs * (tabsize - 1)); + let mut offset = 0; + + let mut iter = line.split(|c| *c == tab).peekable(); + while let Some(chunk) = iter.next() { + match String::from_utf8(chunk.to_vec()) { + Ok(s) => offset += UnicodeWidthStr::width(s.as_str()), + Err(_) => offset += chunk.len(), + } + result.extend_from_slice(chunk); + if iter.peek().is_some() { + result.resize(result.len() + tabsize - offset % tabsize, b' '); + offset = 0; + } + } + + result +} + +/// Write a single line to an output stream, expanding tabs to space if necessary. +/// This assumes that line does not contain any line breaks +/// (if it does and tabs are to be expanded to spaces, the result is undefined). +pub fn do_write_line(output: &mut Vec, line: &[u8], expand_tabs: bool) -> std::io::Result<()> { + if expand_tabs { + output.write_all(do_expand_tabs(line, 8).as_slice()) + } else { + output.write_all(line) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + mod expand_tabs { + use super::*; + use pretty_assertions::assert_eq; + + fn assert_tab_expansion(line: &str, tabsize: usize, expected: &str) { + assert_eq!( + do_expand_tabs(line.as_bytes(), tabsize), + expected.as_bytes() + ); + } + + #[test] + fn basics() { + assert_tab_expansion("foo barr baz", 8, "foo barr baz"); + assert_tab_expansion("foo\tbarr\tbaz", 8, "foo barr baz"); + assert_tab_expansion("foo\tbarr\tbaz", 5, "foo barr baz"); + assert_tab_expansion("foo\tbarr\tbaz", 2, "foo barr baz"); + } + + #[test] + fn multibyte_chars() { + assert_tab_expansion("foo\tépée\tbaz", 8, "foo épée baz"); + assert_tab_expansion("foo\t😉\tbaz", 5, "foo 😉 baz"); + + // Note: The Woman Scientist emoji (👩‍🔬) is a ZWJ sequence combining + // the Woman emoji (👩) and the Microscope emoji (🔬). On supported platforms + // it is displayed as a single emoji and should have a print size of 2 columns, + // but terminal emulators tend to not support this, and display the two emojis + // side by side, thus accounting for a print size of 4 columns. + assert_tab_expansion("foo\t👩‍🔬\tbaz", 6, "foo 👩‍🔬 baz"); + } + + #[test] + fn invalid_utf8() { + // [240, 240, 152, 137] is an invalid UTF-8 sequence, so it is handled as 4 bytes + assert_eq!( + do_expand_tabs(&[240, 240, 152, 137, 9, 102, 111, 111], 8), + &[240, 240, 152, 137, 32, 32, 32, 32, 102, 111, 111] + ); + } + } + + mod write_line { + use super::*; + use pretty_assertions::assert_eq; + + fn assert_line_written(line: &str, expand_tabs: bool, expected: &str) { + let mut output: Vec = Vec::new(); + assert!(do_write_line(&mut output, line.as_bytes(), expand_tabs).is_ok()); + assert_eq!(output, expected.as_bytes()); + } + + #[test] + fn basics() { + assert_line_written("foo bar baz", false, "foo bar baz"); + assert_line_written("foo bar\tbaz", false, "foo bar\tbaz"); + assert_line_written("foo bar\tbaz", true, "foo bar baz"); + } + } +}