diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..37134e8 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +cov = ["llvm-cov", "--workspace", "--lcov", "--output-path", "coverage/lcov.info", "--ignore-filename-regex", ".*/\\.cargo/registry/.*|.*/rustc/.*|.*/target/.*"] diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..d6e6317 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# Run Rust formatter before commit. This script is intended to be installed as: +# git config core.hooksPath .githooks +# Then it will run automatically on `git commit`. + +REPO_ROOT_DIR=$(cd "$(dirname "$0")"/.. && pwd) + +echo "[pre-commit] Checking Rust formatting..." + +# First, check formatting to decide whether to block the commit +if python3 "${REPO_ROOT_DIR}/red/scripts/format_rust.py" --check; then + echo "[pre-commit] Formatting check passed." + exit 0 +else + status=$? + if [ "$status" -eq 2 ]; then + echo "[pre-commit] Formatter prerequisites missing. Aborting commit." + exit 2 + fi + echo "[pre-commit] Code is not properly formatted. Running formatter..." + python3 "${REPO_ROOT_DIR}/red/scripts/format_rust.py" + echo "[pre-commit] Code was reformatted. Review changes and re-run commit." + # Block commit; do not stage files automatically + exit 1 +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..064d421 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + ubuntu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build + working-directory: ./red + run: cargo build --release --verbose + + - name: Unit tests + working-directory: ./red + run: cargo test --verbose + + - name: Install dependencies for tests + run: | + sudo apt-get update + sudo apt-get install -y locales locales-all valgrind + sudo localedef -i en_US -f UTF-8 en_US.UTF-8 || true + sudo localedef -i ru_RU -f UTF-8 ru_RU.UTF-8 || true + sudo localedef -i uk_UA -f UTF-8 uk_UA.UTF-8 || true + sudo localedef -i ja_JP -f EUC-JP ja_JP.eucjp || true + sudo localedef -i ja_JP -f SHIFT_JIS ja_JP.shiftjis || true + sudo localedef -i el_GR -f ISO-8859-7 el_GR.iso88597 || true + + - name: Fetch GNU sed tests + working-directory: ./red + run: bash scripts/fetch_gnused_tests.sh + + - name: GNU sed tests + working-directory: ./red + run: bash scripts/run_gnused_tests.sh --fail-on-error --compact --no-build --expensive --timeout-sec 300 + + - name: Verify implementation + working-directory: ./red + run: RED_BIN=target/release/red bash tests/scripts/verify_implementation.sh + + - name: Compare with GNU sed + working-directory: ./red + run: RED_BIN=target/release/red bash tests/scripts/compare_all_tests.sh + + - name: Multibyte regex tests + working-directory: ./red + run: RED_BIN=target/release/red bash tests/scripts/mb_regex_tests.sh + + macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build + working-directory: ./red + run: cargo build --release --verbose + + - name: Unit tests + working-directory: ./red + run: cargo test --verbose + + - name: Fetch GNU sed tests + working-directory: ./red + run: bash scripts/fetch_gnused_tests.sh + + - name: GNU sed tests + working-directory: ./red + run: bash scripts/run_gnused_tests.sh --fail-on-error --compact --no-build --expensive --timeout-sec 300 + + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build + working-directory: ./red + run: cargo build --release --verbose + + - name: Unit tests (library and selected integration tests) + working-directory: ./red + run: cargo test --verbose + + - name: Fetch GNU sed tests + working-directory: ./red + shell: bash + run: bash scripts/fetch_gnused_tests.sh + + - name: Windows binary mode test (obinary.sh) + working-directory: ./red + shell: bash + run: bash scripts/run_gnused_tests.sh --fail-on-error --compact --no-build --test-pattern "obinary.sh" --timeout-sec 60 + + selinux: + runs-on: [self-hosted, selinux] + if: github.repository == 'vyavdoshenko/red' # Only run on main repo, not forks + steps: + - uses: actions/checkout@v4 + + - name: Verify SELinux is enforcing + run: | + mode=$(getenforce) + echo "SELinux mode: $mode" + if [ "$mode" != "Enforcing" ] && [ "$mode" != "Permissive" ]; then + echo "SELinux is not enabled!" + exit 1 + fi + + - name: Build with SELinux support + working-directory: ./red + run: cargo build --release --features selinux + + - name: Verify SELinux in version + working-directory: ./red + run: | + ./target/release/red --version + ./target/release/red --version | grep -q "with SELinux" || exit 1 + ./target/release/red --version | grep -q "SELinux is enabled" || exit 1 + + - name: Unit tests with SELinux feature + env: + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + working-directory: ./red + run: cargo test --features selinux + + - name: Fetch GNU sed tests + working-directory: ./red + run: bash scripts/fetch_gnused_tests.sh + + - name: Run SELinux test + working-directory: ./red + run: bash scripts/run_gnused_tests.sh --test-pattern "inplace-selinux.sh" --no-build --compact --fail-on-error --timeout-sec 60 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..debb400 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,70 @@ +name: Coverage + +'on': + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + inputs: + fail_threshold: + description: 'Minimum coverage percentage (0 to disable)' + required: false + default: '80' + +jobs: + rust-coverage: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + components: llvm-tools-preview + + - name: Install cargo-llvm-cov + run: cargo install cargo-llvm-cov --locked + + - name: Generate coverage + working-directory: ./red + run: | + # Determine threshold: use input for workflow_dispatch, otherwise default to 80 + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + THRESHOLD="${{ github.event.inputs.fail_threshold }}" + else + THRESHOLD="80" + fi + + echo "========================================" + echo " Coverage threshold: ${THRESHOLD}%" + echo "========================================" + + # Run coverage with threshold + if [ "$THRESHOLD" != "0" ] && [ -n "$THRESHOLD" ]; then + bash scripts/coverage.sh --fail-under "$THRESHOLD" + else + bash scripts/coverage.sh + fi + + - name: Show coverage summary + if: always() + working-directory: ./red + run: | + echo "" + echo "========================================" + echo " COVERAGE STATISTICS" + echo "========================================" + echo "" + cargo llvm-cov report --ignore-filename-regex ".*/\.cargo/registry/.*|.*/rustc/.*|.*/target/.*" + echo "" + echo "========================================" + + - name: Upload lcov artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: lcov.info + path: red/coverage/lcov.info diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..33a5b8f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,170 @@ +name: Release + +on: + push: + tags: + - 'v*' + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Linux musl (static, portable) + - name: linux-x86_64-musl + os: ubuntu-latest + target: x86_64-unknown-linux-musl + artifact: red-linux-x86_64-musl + features: "" + cross: true + + - name: linux-aarch64-musl + os: ubuntu-latest + target: aarch64-unknown-linux-musl + artifact: red-linux-aarch64-musl + features: "" + cross: true + + # Linux gnu with SELinux + - name: linux-x86_64-gnu-selinux + os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact: red-linux-x86_64-gnu-selinux + features: "selinux" + cross: false + + - name: linux-aarch64-gnu-selinux + os: ubuntu-latest + target: aarch64-unknown-linux-gnu + artifact: red-linux-aarch64-gnu-selinux + features: "selinux" + cross: true + + # macOS + - name: darwin-x86_64 + os: macos-15-intel + target: x86_64-apple-darwin + artifact: red-darwin-x86_64 + features: "" + cross: false + + - name: darwin-aarch64 + os: macos-latest + target: aarch64-apple-darwin + artifact: red-darwin-aarch64 + features: "" + cross: false + + # Windows + - name: windows-x86_64 + os: windows-latest + target: x86_64-pc-windows-msvc + artifact: red-windows-x86_64 + features: "" + cross: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross + if: matrix.cross + run: cargo install cross --git https://github.com/cross-rs/cross + + - name: Install SELinux dependencies + if: contains(matrix.features, 'selinux') && !matrix.cross + run: | + sudo apt-get update + sudo apt-get install -y libselinux1-dev + + - name: Build with cargo + if: ${{ !matrix.cross }} + working-directory: ./red + run: | + if [ -n "${{ matrix.features }}" ]; then + cargo build --release --target ${{ matrix.target }} --features ${{ matrix.features }} + else + cargo build --release --target ${{ matrix.target }} + fi + shell: bash + + - name: Build with cross + if: matrix.cross + working-directory: ./red + run: | + if [ -n "${{ matrix.features }}" ]; then + cross build --release --target ${{ matrix.target }} --features ${{ matrix.features }} + else + cross build --release --target ${{ matrix.target }} + fi + shell: bash + + - name: Package (Unix) + if: runner.os != 'Windows' + run: | + cd red/target/${{ matrix.target }}/release + tar -czvf ${{ matrix.artifact }}.tar.gz red + mv ${{ matrix.artifact }}.tar.gz ../../../../ + + - name: Package (Windows) + if: runner.os == 'Windows' + run: | + cd red/target/${{ matrix.target }}/release + 7z a ${{ matrix.artifact }}.zip red.exe + mv ${{ matrix.artifact }}.zip ../../../../ + shell: bash + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: | + ${{ matrix.artifact }}.tar.gz + ${{ matrix.artifact }}.zip + + release: + name: Create Release + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release files + run: | + mkdir -p release + find artifacts -type f \( -name "*.tar.gz" -o -name "*.zip" \) -exec mv {} release/ \; + ls -la release/ + + - name: Generate checksums + run: | + cd release + sha256sum * > SHA256SUMS.txt + cat SHA256SUMS.txt + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + files: | + release/* + generate_release_notes: true + draft: false + prerelease: ${{ contains(github.ref, '-') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6754fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Generated by Cargo +# will have compiled files and executables +debug +target + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# Generated by cargo mutants +# Contains mutation testing data +**/mutants.out*/ + +# Local GNU sed tests (fetched separately) +red/tests/gnused-tests/ +red/coverage/ + +red.code-workspace diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..8831eb5 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,777 @@ +# Red Architecture + +**Rust implementation of GNU sed - Stream Editor** + +--- + +## Table of Contents + +1. [Overview](#overview) +2. [Core Architecture](#core-architecture) +3. [Module Structure](#module-structure) +4. [Key Components](#key-components) +5. [Data Flow](#data-flow) +6. [Bytes and Encoding](#bytes-and-encoding) +7. [Regex Engine](#regex-engine) +8. [GNU sed Compatibility](#gnu-sed-compatibility) +9. [Development Guide](#development-guide) + +--- + +## Overview + +Red is a drop-in replacement for GNU sed, written in Rust. + +### Design Principles + +1. **Two-Phase Processing**: Compile scripts once, execute many times +2. **Unified Context**: Single configuration object passed through all modules +3. **Type Safety**: Leverage Rust's type system for correctness +4. **GNU Compatibility**: Match GNU sed behavior + +--- + +## Core Architecture + +### Comparison with GNU sed + +| Aspect | GNU sed | Red | Notes | +|--------|---------|-----|-------| +| Command representation | 1 struct with union | 2 enums (parser + runtime) | Standard AST→IR pattern | +| Processing | Byte-first | String-first (UTF-8) | Raw bytes tracked separately | +| Regex engine | POSIX regex + DFA | Custom 3-level matcher | Literal/DFA/NFA | +| Memory | Obstack allocator | Standard Rust allocations | | +| Jumps | Array indices | Pre-resolved indices | Resolved at compile time | + +### High-Level Flow + +``` +┌─────────────┐ +│ CLI Args │ +└──────┬──────┘ + │ + ▼ +┌─────────────┐ ┌──────────────┐ +│ Context │────▶│ Validation │ +└──────┬──────┘ └──────────────┘ + │ + ▼ +┌─────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Parser │────▶│ Lexer │────▶│ AST Nodes │ +└──────┬──────┘ └──────────────┘ └──────────────┘ + │ (parser::Command) + │ + ▼ +┌────────────────────┐ +│ convert_to_runtime │ (AST → RuntimeCommand) +└──────┬─────────────┘ + │ + ▼ +┌─────────────┐ +│ Commands │ (RuntimeCommand with compiled regexes) +└──────┬──────┘ + │ + ▼ +┌─────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Engine │────▶│ Evaluator │────▶│ Output │ +└─────────────┘ └──────────────┘ └──────────────┘ +``` + +### Phase 1: Compilation + +**Input**: Script text + Context +**Output**: List of `RuntimeCommand` +**Purpose**: Validate and optimize scripts before execution + +```rust +pub fn parse_scripts_to_commands( + scripts: &[(String, ScriptSource)], + ctx: &Context, +) -> Result> +``` + +### Phase 2: Execution + +**Input**: Commands + Input lines +**Output**: Transformed text +**Purpose**: Apply commands to input data + +```rust +pub fn execute_over_lines( + commands: &[RuntimeCommand], + // ... parameters +) -> Result<()> +``` + +--- + +## Module Structure + +### Module Dependency Graph + +``` +main.rs + │ + ▼ +cli.rs ──────────────────────────────────────┐ + │ │ + ▼ ▼ +lib.rs ◄──────────────────────────────── errors.rs + │ + ├──────────────┬──────────────┬──────────────┐ + ▼ ▼ ▼ ▼ +parser/ engine/ fileio/ util/ +├── mod.rs ├── mod.rs ├── mod.rs ├── regex.rs +├── lexer.rs ├── exec.rs ├── lines.rs └── version.rs +└── ast.rs ├── addr.rs ├── inplace.rs + ├── types.rs └── encoding.rs + └── pattern_space.rs + │ + ▼ + regex/ + ├── mod.rs + ├── parser.rs + ├── dfa.rs + ├── nfa.rs + ├── backtrack.rs + └── literal.rs +``` + +### 1. CLI Layer (`src/cli.rs`, `src/main.rs`) + +**Responsibility**: Argument parsing and validation + +- Uses `lexopt` for natural order preservation +- Separate `print_usage()` and `help_text()` functions +- Exit codes match GNU sed (4 for usage errors) + +**Key Functions**: +```rust +pub fn parse_args() -> Result +``` + +### 2. Context Layer (`src/context.rs`) + +**Responsibility**: Unified configuration management + +```rust +pub struct Context { + // Regex mode + pub extended_regex: bool, + + // POSIX compliance + pub posix: PosixMode, + pub strict_posix: bool, + + // Execution modes + pub quiet: bool, + pub null_data: bool, + pub unbuffered: bool, + pub separate_files: bool, + + // In-place editing + pub in_place_suffix: Option, + pub follow_symlinks: bool, + + // Formatting + pub line_length: usize, + + // Security + pub sandbox: bool, + + // Scripts (for #n quiet mode detection) + pub scripts_with_sources: Vec<(String, ScriptSource)>, +} +``` + +**POSIX Modes**: +```rust +pub enum PosixMode { + Extended, // GNU extensions enabled (default) + Correct, // POSIXLY_CORRECT env var + Basic, // --posix flag (strict) +} +``` + +### 3. File I/O Layer (`src/fileio/`) + +**Responsibility**: File reading, encoding detection, and backup utilities + +- `encoding.rs` - Encoding detection from BOM and locale +- `lines.rs` - File/stdin reading with line splitting +- `inplace.rs` - Backup suffix expansion for in-place editing +- `mod.rs` - Module exports and output flushing helper + +**Key Functions**: +```rust +pub fn read_all_lines(files, null_data, follow_symlinks) -> Result<(lines, bytes, filenames, encoding, ends_with_sep)> +pub fn split_file_content(content, null_data) -> (lines, bytes, encoding, ends_with_sep) +pub fn expand_backup_suffix(file_path, suffix) -> String +pub fn flush_output(out, unbuffered) +``` + +### 4. Parser Layer (`src/parser/`) + +**Responsibility**: Script compilation and validation + +#### Lexer (`src/parser/lexer.rs`) + +Tokenizes sed scripts using a state machine approach. + +**Key optimizations**: +- Single-pass tokenization +- Bracket depth tracking +- Escape sequence handling +- Raw bytes preservation for non-UTF-8 + +#### AST (`src/parser/ast.rs`) + +Defines parser command structure: + +```rust +pub enum Command { + Substitute { flags, pattern, replacement, ... }, + Delete { range, negated }, + Print { range, negated, ... }, + Group { commands }, // Flattened at compile time + Comment(String), // Discarded at compile time + Version { version }, // Checked at compile time + // ... more command types +} +``` + +#### Parser (`src/parser.rs`) + +Converts tokens to AST nodes with validation: + +```rust +pub fn parse(tokens: &[Token], ctx: &Context) -> Result> +``` + +### 5. Engine Layer (`src/engine/`) + +**Responsibility**: Command execution and state management + +#### Types (`src/engine/types.rs`) + +Runtime command representation: + +```rust +pub enum Command { + Substitution { + pattern: SedRegex, // Compiled regex + replacement: ReplacementTemplate, // Parsed replacement + global: bool, + print: bool, + literal_pattern: Option, // Fast-path optimization + literal_replacement: Option, + // ... more fields + }, + // ... more runtime commands +} +``` + +#### ExecutionContext (`src/engine/exec.rs`) + +Maintains execution state: + +```rust +pub struct ExecutionContext { + pub pattern_space: PatternSpace, + pub hold_space: String, + pub hold_space_raw: Vec, + pub line_number: usize, + pub is_last_line: bool, + // ... state fields +} +``` + +#### AddressEvaluator (`src/engine/addr.rs`) + +Evaluates address ranges with state tracking: + +```rust +pub fn evaluate(&mut self, range: Option<&AddressRange>, ctx: &ExecutionContext) -> bool +``` + +**Features**: +- Line number ranges (5,10) +- Regex matches (/pattern/) +- Step addressing (1~2) +- Dollar ($) for last line +- Range state machine (INACTIVE → ACTIVE → CLOSED) + +### 6. Error Handling (`src/errors.rs`) + +**Unified error types**: + +```rust +pub enum SedError { + Parse { message, source }, + Io { operation, path, error }, + Rename { source_path, dest_path, error }, + Runtime { message }, + Usage { message }, +} +``` + +**GNU sed compatible error messages and exit codes**. + +--- + +## Key Components + +### Two-Stage Command Representation + +Red uses a deliberate two-stage command representation, following the classic AST-to-IR pattern from compilers: + +#### Parser Command (`parser/ast.rs`) + +```rust +pub enum Command { + Substitution { + pattern: String, // Raw regex pattern + replacement: String, // Raw replacement string + flags: SubstitutionFlags, // Struct with all flags + // ... + }, + Group { commands: Vec }, // Flattened at compile time + Version { version: String }, // Compile-time check only + Comment(String), // Discarded + // ... more variants +} +``` + +**Purpose**: Preserves source structure, enables error reporting with context + +#### Runtime Command (`engine/types.rs`) + +```rust +pub enum Command { + Substitution { + pattern: SedRegex, // Compiled regex + replacement: ReplacementTemplate, // Parsed replacement + global: bool, // Expanded from flags + print: bool, + literal_pattern: Option, // Fast-path optimization + literal_replacement: Option, + // ... fields with optimizations + }, + // ... more variants (no Group/Version/Comment) +} +``` + +**Purpose**: Optimized for execution, pre-compiled patterns, literal optimizations + +#### Why Two Enums? + +1. **Separation of concerns**: Parser produces semantic representation, engine uses optimized representation +2. **Compile-time processing**: Group commands are flattened, Version is checked, Comments discarded +3. **Optimization fields**: Runtime adds `literal_pattern`, `literal_replacement`, `use_last` that don't exist in source +4. **Type differences**: String → SedRegex, String → ReplacementTemplate, SubstitutionFlags → individual booleans +5. **Common compiler pattern**: Similar to AST → IR transformation in production compilers + +### PatternSpace: Dual Representation + +The `PatternSpace` struct (`src/engine/pattern_space.rs`) maintains synchronized representations: + +```rust +pub struct PatternSpace { + raw: Vec, // Raw bytes (may contain invalid UTF-8) + active_start: usize, // Active pointer for D command optimization + text_cache: Option, // Cached UTF-8 representation +} +``` + +**Key methods**: +- `raw()` - Returns active bytes slice +- `text()` - Returns UTF-8 text (from cache) +- `set_raw()` - Updates from bytes, rebuilds text cache with lossy conversion +- `delete_first_line()` - O(1) using active pointer (GNU sed technique) + +--- + +## Data Flow + +### 1. Script Compilation + +``` + Script Text + │ + ▼ +┌─────────────┐ +│ Lexer │ Tokenize with raw bytes tracking +└──────┬──────┘ + │ + ▼ +┌─────────────┐ +│ Parser │ Build AST (parser::Command) +└──────┬──────┘ + │ + ▼ +┌─────────────┐ +│ Validator │ Check POSIX rules, labels +└──────┬──────┘ + │ + ▼ +┌─────────────┐ +│ Converter │ compile_to_runtime_commands() +└──────┬──────┘ + │ + ▼ +RuntimeCommand[] (with compiled regexes) +``` + +### 2. Command Execution + +``` +Input Lines (with raw bytes) + │ + ▼ +┌──────────────────┐ +│ ExecutionContext │◄─── Commands +│ - pattern_space │ +│ - hold_space │ +│ - line_number │ +└────────┬─────────┘ + │ + ▼ +┌──────────────────────┐ +│ apply_commands_with_ │ +│ context() │ +└───────┬──────────────┘ + │ + ▼ + CommandResult (with raw bytes) + │ + ▼ + ┌────────┐ + │ Output │ (preserves raw bytes) + └────────┘ +``` + +### 3. In-Place Editing + +``` + Original File + │ + ▼ +┌──────────────┐ +│ Read Content │ (as raw bytes) +└──────┬───────┘ + │ + ▼ +┌──────────────┐ +│ Process │ (preserve raw bytes through pipeline) +└──────┬───────┘ + │ + ▼ +┌──────────────┐ +│ Temp File │ +└──────┬───────┘ + │ + ▼ +┌──────────────┐ +│ Rename │ (atomic) +└──────────────┘ +``` + +--- + +## Bytes and Encoding + +### Design: UTF-8 First with Raw Bytes Tracking + +Red uses Rust's `String` type (valid UTF-8) as the primary text representation for regex matching, but maintains raw bytes throughout the pipeline for output. + +### Raw Bytes Preservation + +``` +Input (bytes) → PatternSpace.raw → Commands → Output (bytes) + ↓ + text_cache (lossy) + ↓ + Regex matching on text +``` + +**Implementation**: +- Hold space raw bytes tracking (`hold_space_raw: Vec`) +- `CommandResult::Continue(String, Option>)` - carries raw bytes through engine +- `CommandResult::Print(String, Option>)` - outputs raw bytes when available +- `LiteralBytes(Vec)` token in replacement templates + +### Component Status + +| Component | Status | Implementation | +|-----------|--------|----------------| +| Script reading | Bytes | `Vec` + lossy UTF-8 conversion | +| Lexer | Bytes-aware | char-to-byte mapping for raw bytes | +| Parser | Bytes-aware | `replacement_raw_bytes` field | +| Replacement template | Bytes | `LiteralBytes(Vec)` token | +| Escape sequences | Raw output | `\xNN`, `\oNNN`, `\dNNN`, `\cX` | +| Pattern matching | UTF-8 | Regex operates on text cache | +| Data I/O | Bytes | Raw bytes preserved through hold/pattern space | +| Substitution pipeline | Bytes | Raw bytes tracked across multiple substitutions | + +--- + +## Regex Engine + +Red implements a custom regex engine with three different matchers, each optimized for different pattern types. + +### Three-Level Optimization Strategy + +```rust +pub enum Matcher { + Literal(LiteralMatcher), // Fastest: direct string/byte search + Dfa(DfaMatcher), // Fast: Deterministic Finite Automaton + Nfa(NfaMatcher), // Full-featured: Nondeterministic Finite Automaton +} +``` + +### Matcher Selection (compile time) + +``` + Pattern Analysis + │ + ▼ +┌─────────────────────┐ +│ Is literal? │──Yes──▶ LiteralMatcher (fastest) +│ (no metacharacters) │ +└────────┬────────────┘ + │ No + ▼ +┌─────────────────┐ +│ Has backrefs? │──Yes──▶ NfaMatcher (required) +│ (\1, \2, etc) │ +└────────┬────────┘ + │ No + ▼ +┌─────────────────┐ +│ MBCS locale? │──Yes──▶ NfaMatcher (for non-ASCII patterns) +│ (MB_CUR_MAX > 1)│ +└────────┬────────┘ + │ No + ▼ + DfaMatcher (fast, no backrefs) +``` + +### Literal Matcher (`src/regex/literal.rs`) + +**When used**: Patterns without any regex metacharacters (e.g., `hello`, `foo_bar`) + +**Implementation**: +```rust +pub struct LiteralMatcher { + pattern: String, + ignore_case: bool, +} + +impl LiteralMatcher { + // String-based matching (UTF-8) + pub fn find(&self, text: &str) -> Option + + // Byte-based matching (MBCS-safe for ASCII patterns) + pub fn find_bytes(&self, bytes: &[u8]) -> Option + pub fn find_bytes_from(&self, bytes: &[u8], start: usize) -> Option +} +``` + +**MBCS behavior**: ASCII-only literal patterns can safely use byte-level matching even in MBCS locales, because ASCII bytes (0x00-0x7F) never appear as part of multibyte characters. + +### DFA Matcher (`src/regex/dfa.rs`) + +**When used**: Simple patterns without backreferences in single-byte locales + +**How DFA works**: +- Compiles pattern into a state machine at parse time +- Each state has exactly one transition per input character +- No backtracking - processes input in a single pass +- Memory: O(states × alphabet_size) + +**Limitations**: +- **Cannot support backreferences** - This is a mathematical impossibility, not an implementation limitation. Backreferences (`\1`, `\2`) require the regex engine to "remember" what was matched and compare against it later. DFAs have no memory of the path taken to reach a state. +- **Disabled in MBCS locales** - DFA operates on fixed-width input units. In MBCS, character widths vary (1-4 bytes), requiring boundary detection that DFA architecture doesn't support. + +### NFA Matcher (`src/regex/nfa.rs`, `src/regex/backtrack.rs`) + +**When used**: +- Patterns with backreferences (`\1`, `\2`, etc.) +- Patterns with capturing groups that need extraction +- All patterns in MBCS locales (for non-ASCII patterns) + +**How NFA differs from DFA**: +``` + DFA NFA + ┌─────────────────┐ ┌─────────────────┐ +Input 'a' → │ State 1 ──────▶ │ State 2 │ State 1 ──┬───▶ │ State 2 + │ (one path) │ │ └───▶ │ State 3 + └─────────────────┘ │ (multiple) │ + └─────────────────┘ +``` + +- NFA can be in multiple states simultaneously +- Explores all possible paths (via backtracking) +- Can "remember" captured groups for backreferences +- More flexible but slower + +**MBCS support in NFA**: +```rust +// NFA uses MbText for character boundary tracking +pub struct MbText<'a> { + raw: &'a [u8], + char_boundaries: Vec, // Where each character starts + char_validity: Vec, // Is each sequence valid? +} + +// Transitions respect character boundaries +match transition { + Transition::Any => { + // In MBCS mode: advance by full character, not byte + let char_len = mbtext.char_at(pos).len(); + next_pos = pos + char_len; + } +} +``` + +### Why Both DFA and NFA? + +| Aspect | DFA | NFA | +|--------|-----|-----| +| Speed | O(n) - very fast | O(n×m) - slower | +| Backreferences | Impossible | Full support | +| Memory usage | Higher (state table) | Lower | +| MBCS support | Disabled | Full support | +| Use case | Simple patterns | Complex patterns | + +**Cannot remove either**: +- Removing DFA: Performance regression for simple patterns +- Removing NFA: Backreferences would be impossible (`s/\(foo\)/\1\1/` fails) + +### MBCS Locale Behavior + +When `MB_CUR_MAX > 1` (Shift-JIS, EUC-JP, etc.): + +``` +Pattern Type Matcher Used Why +───────────────────────────────────────────────────── +ASCII literal "hello" LiteralMatcher Safe: ASCII bytes don't overlap MBCS +Non-ASCII literal "日本" NfaMatcher Needs MBCS boundary handling +Any with backrefs NfaMatcher DFA can't do backrefs +Simple /[a-z]+/ NfaMatcher DFA disabled in MBCS +``` + +**Example of why DFA fails in MBCS**: + +``` +Shift-JIS "表" = bytes [0x95, 0x5C] +ASCII "\" = byte [0x5C] + +Pattern: /\\/ (match backslash) + +DFA (byte-level): Would incorrectly match inside "表" + because 0x5C appears as second byte + +NFA (MBCS-aware): Correctly skips "表" because MbText + knows 0x5C is part of a 2-byte character +``` + +--- + +## GNU sed Compatibility + +### Compatibility Features + +#### CLI Compatibility +- Natural order preservation of `-e` and `-f` +- Separate usage (stderr) vs help (stdout) messages +- Exit code 4 for usage errors +- Write error detection (`/dev/full`) + +#### Command Compatibility +- All standard sed commands (s, d, p, a, i, c, etc.) +- Address ranges (1,10, /start/,/end/, $) +- Step addressing (1~2) +- Hold space operations (h, H, g, G, x) +- Branching (b, t, T, :label) +- Byte-level operations (l command with octal escapes) + +#### Multibyte Character Set (MBCS) Support + +Full MBCS support for non-UTF-8 locales (Shift-JIS, EUC-JP, etc.) + +| Feature | Implementation | +|---------|----------------| +| Locale detection | `mbcs::initialize()` via `setlocale()`, `MB_CUR_MAX` | +| Character boundaries | `MbText` struct with `mbrlen()` | +| Regex dot matching | `Transition::Any` respects MB boundaries | +| Character classes | `mbrtowc()` conversion for CharSet matching | +| Capture groups | `raw_bytes` field preserves MBCS bytes | +| Backreferences | Byte-level comparison in MB mode | +| Invalid/incomplete sequences | `char_validity` tracking, skip in dot match | +| Raw bytes output | `render_replacement_to_bytes()` sink | + +**Key MBCS Data Structures**: +```rust +pub struct MbText<'a> { + raw: &'a [u8], // Raw input bytes + char_boundaries: Vec, // Byte offset for each char start + char_validity: Vec, // Valid vs invalid/incomplete +} + +pub struct MbChar<'a> { + bytes: &'a [u8], // One logical character's bytes +} +``` + +#### SELinux Support + +When built with the `selinux` feature (Linux only): +- Preserves SELinux contexts during in-place editing +- Without `--follow-symlinks`: Uses symlink's context for new file +- With `--follow-symlinks`: Uses target file's context + +--- + +## Development Guide + +### Adding New Commands + +1. Add token in `src/parser/lexer.rs` +2. Add AST node in `src/parser/ast.rs` +3. Add parsing logic in `src/parser.rs` +4. Add runtime command in `src/engine/types.rs` +5. Add conversion in `src/lib.rs:convert_new_command_to_old()` +6. Add execution logic in `src/engine/exec.rs` +7. Add tests in `tests/` + +### Adding New Options + +1. Add field to `CliArgs` in `src/cli.rs` +2. Add parsing in `parse_args()` +3. Add to `Context` in `src/context.rs` +4. Update `RunConfig` in `src/lib.rs` +5. Add validation if needed + +### Testing + +```bash +# Unit tests +cargo test + +# GNU sed compatibility tests +./scripts/run_gnused_tests.sh + +# Specific test +./scripts/run_gnused_tests.sh --test-pattern "help.sh" + +# Fast mode (debug build, compact output) +./scripts/run_gnused_tests.sh --fast +``` + +--- + +## References + +- [GNU sed Manual](https://www.gnu.org/software/sed/manual/) +- [POSIX sed Specification](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/sed.html) diff --git a/LICENSE b/LICENSE index 487255e..f8a6291 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 Volodymyr Yavdoshenko +Copyright (c) 2026 Red Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md new file mode 100644 index 0000000..a3689db --- /dev/null +++ b/README.md @@ -0,0 +1,186 @@ +# Red - Rust Stream Editor + +An experimental drop-in replacement for GNU sed, written in Rust. + +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Rust](https://img.shields.io/badge/Rust-1.70+-orange.svg?logo=rust)](https://www.rust-lang.org/) +[![CI](https://github.com/vyavdoshenko/red/actions/workflows/ci.yml/badge.svg)](https://github.com/vyavdoshenko/red/actions/workflows/ci.yml) + +--- + +## About This Project + +This project started as an experiment to understand how far one can go in creating a Rust replacement for GNU sed. + +**Functional Compatibility**: As far as I know, `red` currently achieves 100% functional compatibility with GNU sed behavior. + +**Performance**: There's still work to be done on the performance front. For performance comparisons and benchmarks, please refer to the [Benchmarking](#benchmarking) section below. + +**Documentation**: For usage instructions, please refer to the [GNU sed manual](https://www.gnu.org/software/sed/manual/). Red is designed to work exactly like GNU sed. + +--- + +## Features + +- **GNU sed Compatible**: Drop-in replacement for GNU sed +- **Cross-Platform**: Runs on Linux, macOS, and Windows +- **SELinux Support**: Preserves SELinux contexts during in-place editing (Linux) +- **Memory Safe**: Written in Rust +- **Full Command Support**: All GNU sed commands + +--- + +## Command-Line Options + +``` +Usage: red [OPTION]... {SCRIPT} [FILE]... + red [OPTION]... -e SCRIPT... -f FILE... [FILE]... + +Options: + -n, --quiet, --silent suppress automatic printing of pattern space + -e SCRIPT, --expression=SCRIPT + add the script to the commands to be executed + -f FILE, --file=FILE add the contents of script-file to the commands + -i[SUFFIX], --in-place[=SUFFIX] + edit files in place (makes backup if SUFFIX supplied) + -r, -E, --regexp-extended + use extended regular expressions + -s, --separate consider files as separate rather than continuous + -l N, --line-length=N specify line-wrap length for 'l' command + -u, --unbuffered load minimal amounts of data and flush output often + -z, --null-data use NUL character as line separator + --posix disable all GNU extensions + --follow-symlinks follow symlinks when processing in place + --sandbox disable e/r/w commands + --help display this help and exit + --version output version information and exit +``` + +--- + +## Architecture + +Red uses a two-phase architecture: + +1. **Compile Phase**: Parse and validate scripts once +2. **Execute Phase**: Apply commands to input data + +For detailed documentation, see [ARCHITECTURE.md](ARCHITECTURE.md). + +--- + +## Testing + +### Running Tests + +```bash +# Unit tests +cargo test + +# Fetch GNU sed tests (required once) +./scripts/fetch_gnused_tests.sh + +# GNU sed compatibility tests +./scripts/run_gnused_tests.sh + +# Specific test +./scripts/run_gnused_tests.sh --test-pattern "help.sh" + +# Fast mode +./scripts/run_gnused_tests.sh --fast +``` + +### Benchmarking + +Compare performance with GNU sed: + +```bash +# Install hyperfine +cargo install hyperfine + +# Run benchmarks +./scripts/benchmark.sh + +# Custom runs +RUNS=50 WARMUP=5 ./scripts/benchmark.sh + +RED_BIN=./target/release/red SED_BIN=/usr/bin/sed ./scripts/benchmark.sh +``` + +--- + +## Development + +### Building + +```bash +# Debug build +cargo build + +# Release build +cargo build --release + +# Build with SELinux support (Linux only) +cargo build --release --features selinux +``` + +--- + +## Releases + +### Download + +Pre-built binaries are available on the [Releases](https://github.com/vyavdoshenko/red/releases) page: + +| Platform | File | Notes | +|----------|------|-------| +| Linux x86_64 | `red-linux-x86_64-musl.tar.gz` | Static binary, works on any Linux | +| Linux ARM64 | `red-linux-aarch64-musl.tar.gz` | Static binary for ARM64 | +| Linux x86_64 + SELinux | `red-linux-x86_64-gnu-selinux.tar.gz` | For Fedora, RHEL, CentOS | +| Linux ARM64 + SELinux | `red-linux-aarch64-gnu-selinux.tar.gz` | ARM64 with SELinux | +| macOS Intel | `red-darwin-x86_64.tar.gz` | macOS 10.15+ | +| macOS Apple Silicon | `red-darwin-aarch64.tar.gz` | macOS 11+ (M1/M2/M3) | +| Windows | `red-windows-x86_64.zip` | Windows 10+ | + +### Install + +**Linux:** +```bash +curl -LO https://github.com/vyavdoshenko/red/releases/latest/download/red-linux-x86_64-musl.tar.gz +tar -xzf red-linux-x86_64-musl.tar.gz +sudo mv red /usr/local/bin/ +``` + +**macOS:** +```bash +curl -LO https://github.com/vyavdoshenko/red/releases/latest/download/red-darwin-aarch64.tar.gz +tar -xzf red-darwin-aarch64.tar.gz +xattr -d com.apple.quarantine red # Remove Gatekeeper quarantine +sudo mv red /usr/local/bin/ +``` + +**Verify:** +```bash +red --version +``` + +### Creating a Release (Maintainers) + +1. Update version in `red/Cargo.toml` +2. Commit changes +3. Create and push a tag: + +```bash +git tag v1.0.0 +git push origin v1.0.0 +``` + +GitHub Actions will automatically build all platforms and create a release with artifacts. + +Use `-` in tag for pre-releases (e.g., `v1.0.0-beta`). + +--- + +## License + +MIT diff --git a/red/Cargo.lock b/red/Cargo.lock new file mode 100644 index 0000000..868c6ca --- /dev/null +++ b/red/Cargo.lock @@ -0,0 +1,823 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anyhow" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" + +[[package]] +name = "assert_cmd" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "assert_fs" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652f6cb1f516886fcfee5e7a5c078b9ade62cfcb889524efe5a64d682dd27a9" +dependencies = [ + "anstyle", + "doc-comment", + "globwalk", + "predicates", + "predicates-core", + "predicates-tree", + "tempfile", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "cc" +version = "1.2.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexopt" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "predicates" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" + +[[package]] +name = "predicates-tree" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "red" +version = "1.0.0" +dependencies = [ + "anyhow", + "assert_cmd", + "assert_fs", + "encoding_rs", + "lazy_static", + "lexopt", + "libc", + "predicates", + "rustc-hash", + "selinux", + "selinux-sys", + "signal-hook", + "tempfile", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "selinux" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6af114a661557df02e60c25e5cb40779d295ec2e4ae0fd903fe414578b6191" +dependencies = [ + "bitflags", + "errno", + "libc", + "once_cell", + "parking_lot", + "selinux-sys", + "thiserror", +] + +[[package]] +name = "selinux-sys" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "debaba5832b4831ffe0ba9118b526c752c960f41c46c4ef197d9a15f5179d6fd" +dependencies = [ + "bindgen", + "cc", + "dunce", + "walkdir", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.14.3+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link 0.1.3", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052283831dbae3d879dc7f51f3d92703a316ca49f91540417d38591826127814" diff --git a/red/Cargo.toml b/red/Cargo.toml new file mode 100644 index 0000000..dc00fea --- /dev/null +++ b/red/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "red" +version = "1.0.0" +edition = "2021" + +[features] +default = [] +selinux = ["dep:selinux", "dep:selinux-sys"] + +[dependencies] +anyhow = "1.0" +lazy_static = "1.4" +encoding_rs = "0.8" +lexopt = "0.3.1" +rustc-hash = "2.0" # Fast hash for regex state tracking + +[target.'cfg(unix)'.dependencies] +signal-hook = "0.3" +libc = "0.2" + +[target.'cfg(target_os = "linux")'.dependencies] +selinux = { version = "0.5", optional = true } +selinux-sys = { version = "0.6", optional = true } + +[dev-dependencies] +assert_cmd = "2.0" +predicates = "3.1" +tempfile = "3.10" +assert_fs = "1.1" diff --git a/red/Cross.toml b/red/Cross.toml new file mode 100644 index 0000000..57cd932 --- /dev/null +++ b/red/Cross.toml @@ -0,0 +1,12 @@ +[target.aarch64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture arm64", + "apt-get update", + "apt-get install -y libselinux1-dev:arm64" +] + +[target.x86_64-unknown-linux-musl] +# No special configuration needed for musl + +[target.aarch64-unknown-linux-musl] +# No special configuration needed for musl diff --git a/red/rust-toolchain.toml b/red/rust-toolchain.toml new file mode 100644 index 0000000..41d6d6b --- /dev/null +++ b/red/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "stable" +components = ["rustfmt"] +profile = "minimal" diff --git a/red/scripts/benchmark.sh b/red/scripts/benchmark.sh new file mode 100644 index 0000000..2649a2f --- /dev/null +++ b/red/scripts/benchmark.sh @@ -0,0 +1,167 @@ +#!/bin/bash + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# Benchmark red vs GNU sed +# Requires: hyperfine (cargo install hyperfine) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RED_BIN="${RED_BIN:-$SCRIPT_DIR/../target/release/red}" +SED_BIN="${SED_BIN:-sed}" +RUNS="${RUNS:-20}" +WARMUP="${WARMUP:-3}" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo -e "${GREEN}=== Red vs GNU sed Benchmark ===${NC}" +echo "" + +# Check dependencies +if ! command -v hyperfine &> /dev/null; then + echo -e "${RED}Error: hyperfine not found${NC}" + echo "Install with: cargo install hyperfine" + exit 1 +fi + +if [[ ! -f "$RED_BIN" ]]; then + echo -e "${YELLOW}Building red in release mode...${NC}" + (cd "$SCRIPT_DIR/.." && cargo build --release) +fi + +echo "Red binary: $RED_BIN" +echo "Sed binary: $SED_BIN" +echo "Runs: $RUNS, Warmup: $WARMUP" +echo "" + +# Create temp directory +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT + +# Generate test files +echo -e "${YELLOW}Generating test files...${NC}" + +# Small file (1K lines) +yes "hello world foo bar baz" | head -1000 > "$TMPDIR/small.txt" + +# Medium file (100K lines) +yes "The quick brown fox jumps over the lazy dog. foo bar baz 12345" | head -100000 > "$TMPDIR/medium.txt" + +# Large file (1M lines) +yes "Lorem ipsum dolor sit amet, consectetur adipiscing elit. foo bar" | head -1000000 > "$TMPDIR/large.txt" + +# File with many matches +seq 1 100000 | while read n; do echo "line $n: foo foo foo bar bar baz"; done > "$TMPDIR/many_matches.txt" + +echo " small.txt: $(wc -l < "$TMPDIR/small.txt") lines ($(du -h "$TMPDIR/small.txt" | cut -f1))" +echo " medium.txt: $(wc -l < "$TMPDIR/medium.txt") lines ($(du -h "$TMPDIR/medium.txt" | cut -f1))" +echo " large.txt: $(wc -l < "$TMPDIR/large.txt") lines ($(du -h "$TMPDIR/large.txt" | cut -f1))" +echo " many_matches.txt: $(wc -l < "$TMPDIR/many_matches.txt") lines ($(du -h "$TMPDIR/many_matches.txt" | cut -f1))" +echo "" + +# Benchmark functions +run_bench() { + local name="$1" + local sed_cmd="$2" + local red_cmd="$3" + + echo -e "${GREEN}--- $name ---${NC}" + hyperfine --warmup "$WARMUP" --runs "$RUNS" \ + --command-name "sed" "$sed_cmd" \ + --command-name "red" "$red_cmd" + echo "" +} + +# ============================================ +# Benchmarks +# ============================================ + +echo -e "${GREEN}=== Simple Substitution ===${NC}" +echo "" + +run_bench "Small file - simple s///" \ + "$SED_BIN 's/foo/FOO/' $TMPDIR/small.txt > /dev/null" \ + "$RED_BIN 's/foo/FOO/' $TMPDIR/small.txt > /dev/null" + +run_bench "Medium file - simple s///" \ + "$SED_BIN 's/foo/FOO/' $TMPDIR/medium.txt > /dev/null" \ + "$RED_BIN 's/foo/FOO/' $TMPDIR/medium.txt > /dev/null" + +run_bench "Large file - simple s///" \ + "$SED_BIN 's/foo/FOO/' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN 's/foo/FOO/' $TMPDIR/large.txt > /dev/null" + +echo -e "${GREEN}=== Global Substitution ===${NC}" +echo "" + +run_bench "Many matches - s///g" \ + "$SED_BIN 's/foo/FOO/g' $TMPDIR/many_matches.txt > /dev/null" \ + "$RED_BIN 's/foo/FOO/g' $TMPDIR/many_matches.txt > /dev/null" + +run_bench "Large file - s///g" \ + "$SED_BIN 's/foo/FOO/g' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN 's/foo/FOO/g' $TMPDIR/large.txt > /dev/null" + +echo -e "${GREEN}=== Regex Patterns ===${NC}" +echo "" + +run_bench "Medium file - word boundary regex" \ + "$SED_BIN 's/\\bfoo\\b/FOO/g' $TMPDIR/medium.txt > /dev/null" \ + "$RED_BIN 's/\\bfoo\\b/FOO/g' $TMPDIR/medium.txt > /dev/null" + +run_bench "Medium file - capture groups" \ + "$SED_BIN 's/\\(foo\\) \\(bar\\)/\\2 \\1/g' $TMPDIR/medium.txt > /dev/null" \ + "$RED_BIN 's/\\(foo\\) \\(bar\\)/\\2 \\1/g' $TMPDIR/medium.txt > /dev/null" + +run_bench "Large file - extended regex" \ + "$SED_BIN -E 's/(foo|bar|baz)/WORD/g' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN -E 's/(foo|bar|baz)/WORD/g' $TMPDIR/large.txt > /dev/null" + +echo -e "${GREEN}=== Delete Lines ===${NC}" +echo "" + +run_bench "Large file - delete matching lines" \ + "$SED_BIN '/foo/d' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN '/foo/d' $TMPDIR/large.txt > /dev/null" + +run_bench "Large file - delete line range" \ + "$SED_BIN '1000,2000d' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN '1000,2000d' $TMPDIR/large.txt > /dev/null" + +echo -e "${GREEN}=== Print Lines ===${NC}" +echo "" + +run_bench "Large file - print matching (-n /p/p)" \ + "$SED_BIN -n '/foo/p' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN -n '/foo/p' $TMPDIR/large.txt > /dev/null" + +run_bench "Large file - print range" \ + "$SED_BIN -n '50000,60000p' $TMPDIR/large.txt > /dev/null" \ + "$RED_BIN -n '50000,60000p' $TMPDIR/large.txt > /dev/null" + +echo -e "${GREEN}=== Multiple Commands ===${NC}" +echo "" + +run_bench "Medium file - multiple -e" \ + "$SED_BIN -e 's/foo/FOO/g' -e 's/bar/BAR/g' -e 's/baz/BAZ/g' $TMPDIR/medium.txt > /dev/null" \ + "$RED_BIN -e 's/foo/FOO/g' -e 's/bar/BAR/g' -e 's/baz/BAZ/g' $TMPDIR/medium.txt > /dev/null" + +echo -e "${GREEN}=== Complex Script ===${NC}" +echo "" + +run_bench "Medium file - complex script" \ + "$SED_BIN '/^$/d; s/foo/FOO/g; s/bar/BAR/g; /Lorem/s/ipsum/IPSUM/' $TMPDIR/medium.txt > /dev/null" \ + "$RED_BIN '/^$/d; s/foo/FOO/g; s/bar/BAR/g; /Lorem/s/ipsum/IPSUM/' $TMPDIR/medium.txt > /dev/null" + +# ============================================ +# Summary +# ============================================ + +echo -e "${GREEN}=== Benchmark Complete ===${NC}" diff --git a/red/scripts/coverage.sh b/red/scripts/coverage.sh new file mode 100755 index 0000000..a249be5 --- /dev/null +++ b/red/scripts/coverage.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# This script generates Rust code coverage for the whole project using cargo-llvm-cov. +# It installs cargo-llvm-cov if missing, runs tests, and produces lcov and HTML reports. + +usage() { + cat <<'USAGE' +Generate coverage report for the whole Rust project using cargo-llvm-cov. + +Usage: + coverage.sh [--html] [--open] [--fail-under PCT] [--profile dev|release] [--no-install] + +Options: + --html Generate HTML report (in addition to lcov.info) + --open Open the HTML report in a browser (implies --html) + --fail-under PCT Fail if line coverage is below PCT (integer, e.g., 80) + --profile P Build profile to use: dev or release (default: dev) + --no-install Do not auto-install cargo-llvm-cov + +Outputs: + - coverage/lcov.info + - coverage/html/ (when --html) +USAGE +} + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CRATE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +PROFILE="dev" +GENERATE_HTML="false" +OPEN_HTML="false" +FAIL_UNDER="" +NO_INSTALL="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --html) GENERATE_HTML="true"; shift ;; + --open) GENERATE_HTML="true"; OPEN_HTML="true"; shift ;; + --fail-under) FAIL_UNDER="${2:-}"; shift 2 ;; + --profile) PROFILE="${2:-dev}"; shift 2 ;; + --no-install) NO_INSTALL="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "[error] Unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +if [[ "${NO_INSTALL}" != "true" ]]; then + if ! command -v cargo-llvm-cov >/dev/null 2>&1; then + echo "[info] Installing cargo-llvm-cov" + cargo install cargo-llvm-cov --locked + fi +fi + +pushd "${CRATE_ROOT}" >/dev/null + +# Clean previous coverage data to avoid contamination +echo "[info] Resetting coverage counters" +cargo llvm-cov clean --workspace + +RUN_FLAGS=( + --workspace + --lcov --output-path coverage/lcov.info + --ignore-filename-regex ".*/\.cargo/registry/.*|.*/rustc/.*|.*/target/.*" +) + +# Apply cargo profile flag only for release; dev is default and does not accept --dev +if [[ "${PROFILE}" == "release" ]]; then + RUN_FLAGS+=(--release) +fi + +if [[ -n "${FAIL_UNDER}" ]]; then + RUN_FLAGS+=(--fail-under-lines "${FAIL_UNDER}") +fi + +echo "[info] Running tests with coverage" +# Ensure output directories exist +mkdir -p coverage +cargo llvm-cov ${RUN_FLAGS[@]} + +if [[ "${GENERATE_HTML}" == "true" ]]; then + echo "[info] Generating HTML report" + mkdir -p coverage/html + REPORT_FLAGS=(--html --output-dir coverage/html) + if [[ "${OPEN_HTML}" == "true" ]]; then + REPORT_FLAGS+=(--open) + fi + cargo llvm-cov report ${REPORT_FLAGS[@]} +fi + +echo "[info] Coverage artifacts:" +echo " - $(realpath coverage/lcov.info)" +if [[ -d coverage/html ]]; then + echo " - $(realpath coverage/html)" +fi + +popd >/dev/null + + diff --git a/red/scripts/fetch_gnused_tests.sh b/red/scripts/fetch_gnused_tests.sh new file mode 100755 index 0000000..b8c10eb --- /dev/null +++ b/red/scripts/fetch_gnused_tests.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +usage() { + cat <<'USAGE' +Fetch GNU sed tests via 'git clone'. + +Usage: + fetch_gnused_tests.sh [--dest /path/to/dir] [--git-url URL] [--branch BRANCH] + +Options: + --dest DIR Destination directory to place cloned tests (must NOT exist). + Default: /tests/gnused-tests + --git-url URL Override Git URL (default: git://git.sv.gnu.org/sed) + --branch BRANCH Git branch to clone (default: master) + +Notes: + - Requires 'git' to be installed. + - The script only downloads the full GNU sed repository; the tests are in testsuite/. + - Running tests is handled by a separate script. +USAGE +} + +GIT_URL_DEFAULT="git://git.sv.gnu.org/sed" +GIT_BRANCH="master" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CRATE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +DEST_DIR="${CRATE_ROOT}/tests/gnused-tests" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dest) + DEST_DIR="${2:-}"; shift 2 ;; + --git-url) + GIT_URL_DEFAULT="${2:-}"; shift 2 ;; + --branch) + GIT_BRANCH="${2:-}"; shift 2 ;; + -h|--help) + usage; exit 0 ;; + *) + echo "[error] Unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +if ! command -v git >/dev/null 2>&1; then + echo "[error] 'git' not found. Install git." >&2 + exit 1 +fi + +if [[ -e "${DEST_DIR}" ]]; then + echo "[error] Destination already exists: ${DEST_DIR}. Remove it or provide a different --dest." >&2 + exit 2 +fi +mkdir -p "$(dirname "${DEST_DIR}")" + +echo "[info] Cloning GNU sed repository" +echo "[info] URL: ${GIT_URL_DEFAULT}" +echo "[info] Branch: ${GIT_BRANCH}" +echo "[info] Dest: ${DEST_DIR}" + +# Clone with depth 1 to save space and time +git clone --depth 1 --branch "${GIT_BRANCH}" "${GIT_URL_DEFAULT}" "${DEST_DIR}" 2>&1 | \ + grep -v "^remote:" || true + +if [[ ! -d "${DEST_DIR}/testsuite" ]]; then + echo "[error] Expected testsuite/ directory not found in cloned repository" >&2 + exit 1 +fi + +# Initialize gnulib submodule (required for proper test framework) +echo "[info] Initializing gnulib submodule (test framework)" +cd "${DEST_DIR}" +git submodule update --init --depth 1 gnulib 2>&1 | grep -v "^remote:" || true + +if [[ ! -f "${DEST_DIR}/gnulib/tests/init.sh" ]]; then + echo "[error] gnulib/tests/init.sh not found after submodule initialization" >&2 + exit 1 +fi + +# Compile test helpers (used by locale detection in tests) +# Original files require autotools-generated headers, so we provide stubs +# On Windows/systems without gcc, skip compilation (some locale tests will be skipped) +if command -v gcc >/dev/null 2>&1; then + echo "[info] Compiling test helpers" + touch "${DEST_DIR}/testsuite/config.h" + cat > "${DEST_DIR}/testsuite/progname.h" << 'EOF' +#define set_program_name(x) (void)0 +#define program_name "test-helper" +EOF + # error() is a GNU extension, provide a stub for Windows/MinGW + cat > "${DEST_DIR}/testsuite/error.h" << 'EOF' +#if defined(_WIN32) || defined(__MINGW32__) || defined(__CYGWIN__) +#include +#include +#define error(status, errnum, ...) do { fprintf(stderr, __VA_ARGS__); if (status) exit(status); } while(0) +#else +#include_next +#endif +EOF + gcc -I"${DEST_DIR}/testsuite" -o "${DEST_DIR}/testsuite/get-mb-cur-max" \ + "${DEST_DIR}/testsuite/get-mb-cur-max.c" 2>/dev/null || \ + echo "[warn] Failed to compile get-mb-cur-max (some locale tests will be skipped)" + gcc -I"${DEST_DIR}/testsuite" -include "${DEST_DIR}/testsuite/error.h" \ + -o "${DEST_DIR}/testsuite/test-mbrtowc" \ + "${DEST_DIR}/testsuite/test-mbrtowc.c" 2>/dev/null || \ + echo "[warn] Failed to compile test-mbrtowc (some locale tests will be skipped)" +else + echo "[warn] gcc not found - skipping test helper compilation (some locale tests will be skipped)" +fi + +echo "[info] Done. GNU sed repository cloned to: ${DEST_DIR}" +echo "[info] Tests are located in: ${DEST_DIR}/testsuite" +echo "[info] Test framework (gnulib) initialized: ${DEST_DIR}/gnulib" diff --git a/red/scripts/format_rust.py b/red/scripts/format_rust.py new file mode 100755 index 0000000..0222fbd --- /dev/null +++ b/red/scripts/format_rust.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +""" +Rust code formatter helper. + +Usage: + - Fix formatting in-place: + python3 red/scripts/format_rust.py + + - Check formatting without changing files (CI-friendly): + python3 red/scripts/format_rust.py --check + +This script locates the crate root relative to its own location and runs +`cargo fmt --all` (or with `-- --check` in check mode). + +Project convention: all code comments must be in English. +""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path + + +def get_crate_root() -> Path: + """Return the crate root directory that contains Cargo.toml. + + The script lives under /red/scripts/, so the crate root is its parent. + """ + script_path = Path(__file__).resolve() + crate_root = script_path.parent.parent + cargo_toml = crate_root / "Cargo.toml" + if not cargo_toml.is_file(): + raise FileNotFoundError( + f"Cargo.toml not found at expected location: {cargo_toml}" + ) + return crate_root + + +def ensure_tools_available(preferred_toolchain: str | None) -> None: + """Ensure required tools are available in PATH and rustfmt component exists. + + If a specific toolchain is preferred (e.g., "nightly" for edition 2024), + verify rustfmt for that toolchain. + """ + if shutil.which("cargo") is None: + raise RuntimeError( + "cargo not found in PATH. Install Rust (https://rustup.rs) and retry." + ) + + cargo_cmd: list[str] = ["cargo"] + if preferred_toolchain: + cargo_cmd += [f"+{preferred_toolchain}"] + + try: + subprocess.run( + [*cargo_cmd, "fmt", "--", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + except subprocess.CalledProcessError as exc: + if preferred_toolchain == "nightly": + raise RuntimeError( + "rustfmt for nightly is not installed. Install it via:\n" + " rustup toolchain install nightly\n" + " rustup component add rustfmt --toolchain nightly" + ) from exc + raise RuntimeError( + "rustfmt is not installed. Install it via: rustup component add rustfmt" + ) from exc + except FileNotFoundError as exc: + raise RuntimeError("cargo executable is not available") from exc + + +def run_cargo_fmt(check: bool, preferred_toolchain: str | None) -> int: + """Run cargo fmt in the crate root. Return the process exit code.""" + crate_root = get_crate_root() + cmd: list[str] = ["cargo"] + if preferred_toolchain: + cmd += [f"+{preferred_toolchain}"] + cmd += ["fmt", "--all"] + if check: + cmd += ["--", "--check"] + + process = subprocess.run(cmd, cwd=str(crate_root)) + return process.returncode + + +def read_edition(crate_root: Path) -> str | None: + """Read edition from Cargo.toml if present (e.g., "2021", "2024").""" + cargo_toml = crate_root / "Cargo.toml" + try: + content = cargo_toml.read_text(encoding="utf-8", errors="ignore") + except OSError: + return None + import re + + match = re.search(r"^\s*edition\s*=\s*\"(\d{4})\"\s*$", content, re.MULTILINE) + if match: + return match.group(1) + return None + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Format Rust code using cargo fmt", + ) + parser.add_argument( + "--check", + action="store_true", + help="Check formatting without writing changes (non-zero exit on diff)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + + crate_root = get_crate_root() + edition = read_edition(crate_root) + preferred_toolchain = "nightly" if edition and edition >= "2024" else None + + try: + ensure_tools_available(preferred_toolchain) + except Exception as error: + print(f"[format_rust] Error: {error}", file=sys.stderr) + return 2 + + exit_code = run_cargo_fmt(check=args.check, preferred_toolchain=preferred_toolchain) + if exit_code == 0: + action = "checked" if args.check else "formatted" + print(f"[format_rust] Successfully {action} Rust code.") + else: + if args.check: + print( + "[format_rust] Formatting check failed. Run without --check to fix.", + file=sys.stderr, + ) + else: + print( + "[format_rust] Formatting command failed. See output above.", + file=sys.stderr, + ) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/red/scripts/run_gnused_tests.sh b/red/scripts/run_gnused_tests.sh new file mode 100755 index 0000000..64920f5 --- /dev/null +++ b/red/scripts/run_gnused_tests.sh @@ -0,0 +1,379 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# Run GNU sed tests against the local 'red' binary. +# Note: GNU sed tests use init.sh from gnulib framework. + +usage() { + cat <<'USAGE' +Run GNU sed tests against the local 'red' binary. + +Usage: + run_gnused_tests.sh [--tests-dir /path/to/gnused/testsuite] [--debug] [--fail-on-error] [--timeout-sec N] [--no-build] [--compact] [--fast] [--test-pattern PATTERN] + +Options: + --tests-dir DIR Path to GNU sed testsuite directory + (default: /tests/gnused-tests/testsuite). + --debug Build 'red' in debug mode (default: release). + --fail-on-error Exit with failure if any test fails + --timeout-sec N Per-test timeout in seconds (0 disables; default: 30) + --no-build Do not rebuild the binary (use the existing target output). + --compact Print compact output (only test results and errors). + --fast Convenience flag: implies --debug --compact --timeout-sec 10. + --expensive Run expensive/very-expensive tests (e.g., >2GB file tests) + --test-pattern PAT Only run tests matching the pattern (e.g., "execute-tests.sh") + +Notes: + - GNU sed tests require gnulib's init.sh framework + - If init.sh is not available, tests that depend on it will be skipped + - This runner prepares PATH so that 'sed' resolves to the local 'red' binary. + - Some tests may require specific locales or tools (perl, etc.) +USAGE +} + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CRATE_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +DEFAULT_TESTS_DIR="${CRATE_ROOT}/red/tests/gnused-tests/testsuite" +TESTS_DIR="${DEFAULT_TESTS_DIR}" +BUILD_MODE="release" +FAIL_ON_ERROR="false" +TIMEOUT_SEC=30 +NO_BUILD="false" +COMPACT="false" +EXPENSIVE="false" +TEST_PATTERN="*.sh" + +while [[ $# -gt 0 ]]; do + case "$1" in + --tests-dir) + TESTS_DIR="${2:-}"; shift 2 ;; + --debug) + BUILD_MODE="debug"; shift ;; + --fail-on-error) + FAIL_ON_ERROR="true"; shift ;; + --timeout-sec) + TIMEOUT_SEC="${2:-}"; shift 2 ;; + --no-build) + NO_BUILD="true"; shift ;; + --compact) + COMPACT="true"; shift ;; + --fast) + BUILD_MODE="debug"; COMPACT="true"; TIMEOUT_SEC=10; shift ;; + --expensive) + EXPENSIVE="true"; shift ;; + --test-pattern) + TEST_PATTERN="${2:-}"; shift 2 ;; + -h|--help) + usage; exit 0 ;; + *) + echo "[error] Unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +echo "[info] Tests dir: ${TESTS_DIR}" + +if [[ ! -d "${TESTS_DIR}" ]]; then + echo "[error] Tests directory not found: ${TESTS_DIR}" >&2 + echo "[info] Run 'scripts/fetch_gnused_tests.sh' first to download the tests" >&2 + exit 1 +fi + +if [[ "${NO_BUILD}" != "true" ]]; then + echo "[info] Building red (${BUILD_MODE})" + pushd "${CRATE_ROOT}/red" >/dev/null + if [[ "${BUILD_MODE}" == "release" ]]; then + cargo build --release + else + cargo build + fi + popd >/dev/null +else + echo "[info] Skipping build (using existing binary)" +fi + +if [[ "${BUILD_MODE}" == "release" ]]; then + RED_BIN="${CRATE_ROOT}/red/target/release/red" +else + RED_BIN="${CRATE_ROOT}/red/target/debug/red" +fi + +if [[ ! -x "${RED_BIN}" ]]; then + echo "[error] red binary not found at ${RED_BIN}" >&2 + exit 1 +fi + +WORKDIR="$(mktemp -d)" +cleanup() { rm -rf "${WORKDIR}" || true; } +trap cleanup EXIT + +# Create a bin directory with sed -> red symlink +mkdir -p "${WORKDIR}/bin" +ln -s "${RED_BIN}" "${WORKDIR}/bin/sed" + +# Set up test framework using gnulib's init.sh +# GNU sed tests expect ./testsuite/init.sh and init.cfg relative to test location +TESTS_PARENT="$(dirname "${TESTS_DIR}")" + +# Also create sed/ directory in tests parent (tests use path_prepend_ ./sed) +mkdir -p "${TESTS_PARENT}/sed" +ln -sf "${RED_BIN}" "${TESTS_PARENT}/sed/sed" +GNULIB_INIT_SH="${TESTS_PARENT}/gnulib/tests/init.sh" +INIT_CFG="${TESTS_PARENT}/init.cfg" +TESTSUITE_INIT_SH="${TESTS_PARENT}/testsuite/init.sh" + +# Check if gnulib is available +if [[ ! -f "${GNULIB_INIT_SH}" ]]; then + echo "[error] gnulib/tests/init.sh not found at: ${GNULIB_INIT_SH}" >&2 + echo "[error] Run 'scripts/fetch_gnused_tests.sh' to download tests and gnulib" >&2 + exit 1 +fi + +# Create symlink to gnulib's init.sh in testsuite directory +echo "[info] Using gnulib test framework from: ${GNULIB_INIT_SH}" +mkdir -p "$(dirname "${TESTSUITE_INIT_SH}")" +ln -sf "../gnulib/tests/init.sh" "${TESTSUITE_INIT_SH}" 2>/dev/null || { + # If symlink fails (e.g., on some systems), copy the file + cp "${GNULIB_INIT_SH}" "${TESTSUITE_INIT_SH}" +} + +# Ensure init.cfg exists (it's sourced by init.sh) +if [[ -f "${INIT_CFG}" ]]; then + echo "[info] Using existing init.cfg" +else + echo "[info] init.cfg not found - tests will use defaults" +fi + +# Update PATH to use our sed wrapper and testsuite helper scripts +export PATH="${WORKDIR}/bin:${TESTS_DIR}:${PATH}" +echo "[info] Using sed: $(command -v sed) ($(${WORKDIR}/bin/sed --version 2>/dev/null | head -1 || echo 'unknown version'))" + +# Set up environment variables expected by GNU sed tests +# srcdir=. because tests run from gnused-tests parent directory +export srcdir="." +export abs_top_srcdir="${TESTS_PARENT}" +export abs_srcdir="${TESTS_DIR}" +export LC_ALL=C + +# Locale variables for tests (normally set by configure) +export LOCALE_JA="ja_JP.eucjp" + +# Variables for help-version.sh test +# Extract version from our red binary +RED_VERSION=$(${WORKDIR}/bin/sed --version 2>/dev/null | head -1 | sed 's/.* //' || echo "unknown") +export VERSION="${RED_VERSION}" +export built_programs="sed" + +# Enable expensive tests if requested +if [[ "${EXPENSIVE}" == "true" ]]; then + export RUN_EXPENSIVE_TESTS=yes + export RUN_VERY_EXPENSIVE_TESTS=yes +fi + +# Store logs +LOGDIR="${WORKDIR}/logs" +mkdir -p "${LOGDIR}" + +# Change to parent of tests directory (gnulib expects srcdir=. and testsuite/ subdirectory) +pushd "${TESTS_PARENT}" >/dev/null + +# Find all test scripts matching pattern (in testsuite subdirectory) +# Note: using while read loop instead of mapfile for bash 3.2 compatibility (macOS) +TEST_SCRIPTS=() +while IFS= read -r script; do + TEST_SCRIPTS+=("$script") +done < <(find testsuite -maxdepth 1 -name "${TEST_PATTERN}" -type f | sort) + +if [[ ${#TEST_SCRIPTS[@]} -eq 0 ]]; then + echo "[error] No test scripts found matching pattern: ${TEST_PATTERN}" >&2 + exit 1 +fi + +echo "[info] Found ${#TEST_SCRIPTS[@]} test scripts matching '${TEST_PATTERN}'" +[[ "${COMPACT}" != "true" ]] && echo "" + +# Run tests +PASSED=0 +FAILED=0 +SKIPPED=0 +TIMEOUT=0 + +for test_script in "${TEST_SCRIPTS[@]}"; do + test_name="$(basename "${test_script}")" + test_log="${LOGDIR}/${test_name}.log" + + # panic-tests.sh requires a filesystem that enforces directory permissions + # VirtioFS (used in some container/VM setups) doesn't enforce dir write permissions + # So we copy the test directory to /tmp (which uses overlay fs) and run from there + if [[ "${test_name}" == "panic-tests.sh" ]]; then + [[ "${COMPACT}" != "true" ]] && echo "[test] Running ${test_name} (from /tmp for permission enforcement)..." + TMP_TEST_DIR=$(mktemp -d) + cp -r "${TESTS_PARENT}"/* "${TMP_TEST_DIR}/" + mkdir -p "${TMP_TEST_DIR}/sed" + ln -sf "${RED_BIN}" "${TMP_TEST_DIR}/sed/sed" + + set +e + pushd "${TMP_TEST_DIR}" >/dev/null + if [[ "${TIMEOUT_SEC}" != "0" && -n "$(command -v timeout)" ]]; then + timeout "${TIMEOUT_SEC}"s bash "testsuite/${test_name}" >"${test_log}" 2>&1 + rc=$? + else + bash "testsuite/${test_name}" >"${test_log}" 2>&1 + rc=$? + fi + popd >/dev/null + set -e + rm -rf "${TMP_TEST_DIR}" + + if [[ ${rc} -eq 0 ]]; then + PASSED=$((PASSED + 1)) + [[ "${COMPACT}" == "true" ]] && echo " [PASS]" + [[ "${COMPACT}" != "true" ]] && echo " [PASS]" + elif [[ ${rc} -eq 77 ]]; then + SKIPPED=$((SKIPPED + 1)) + skip_msg=$(grep -E "(^SKIP:|skipped test:)" "${test_log}" | head -1 || true) + if [[ -z "${skip_msg}" ]]; then + skip_msg="(no message)" + else + # Extract just the reason part using bash string manipulation (avoid sed/locale issues) + if [[ "${skip_msg}" == *"skipped test: "* ]]; then + skip_msg="${skip_msg##*skipped test: }" + elif [[ "${skip_msg}" == "SKIP: "* ]]; then + skip_msg="${skip_msg#SKIP: }" + fi + fi + [[ "${COMPACT}" == "true" ]] && echo " [SKIP]: ${skip_msg}" + [[ "${COMPACT}" != "true" ]] && echo " [SKIP]: ${skip_msg}" + else + FAILED=$((FAILED + 1)) + [[ "${COMPACT}" == "true" ]] && echo " [FAIL] (rc=${rc})" + [[ "${COMPACT}" != "true" ]] && { echo " [FAIL] (rc=${rc})"; echo " Log: ${test_log}"; head -50 "${test_log}"; } + fi + continue + fi + + # Skip tests requiring special handling (documented limitations) + # Note: 8bit.sh, newjis.sh, 8to7.sh, and mac-mf.sh now pass after byte-level implementation + case "${test_name}" in + # mac-mf.sh now passes after fixing raw bytes tracking and address regex parsing + binary.sh) + # binary.sh passes in release but too slow in debug mode + if [[ "${BUILD_MODE}" == "debug" ]]; then + SKIPPED=$((SKIPPED + 1)) + [[ "${COMPACT}" == "true" ]] && echo "SKIP ${test_name}: Too slow in debug mode - use release build" + [[ "${COMPACT}" != "true" ]] && echo "[test] Skipping ${test_name} (too slow in debug mode)" + continue + fi + ;; + # dc.sh now works after fixing greedy matching and ^ literal handling + # Keeping commented out for reference: + # dc.sh) + # SKIPPED=$((SKIPPED + 1)) + # [[ "${COMPACT}" == "true" ]] && echo "SKIP ${test_name}: Complex sed script (calculator)" + # [[ "${COMPACT}" != "true" ]] && echo "[test] Skipping ${test_name} (complex calculator script)" + # continue + # ;; + # bsd-wrapper.sh now passes - all BSD tests working + #bsd-wrapper.sh) + # SKIPPED=$((SKIPPED + 1)) + # [[ "${COMPACT}" == "true" ]] && echo "SKIP ${test_name}: BSD sed compatibility layer N/A" + # [[ "${COMPACT}" != "true" ]] && echo "[test] Skipping ${test_name} (BSD wrapper not applicable)" + # continue + # ;; + esac + + [[ "${COMPACT}" != "true" ]] && echo "[test] Running ${test_name}..." + + set +e + if [[ "${TIMEOUT_SEC}" != "0" && -n "$(command -v timeout)" ]]; then + timeout "${TIMEOUT_SEC}"s bash "${test_script}" >"${test_log}" 2>&1 + rc=$? + else + bash "${test_script}" >"${test_log}" 2>&1 + rc=$? + fi + set -e + + # Analyze result + # 0 = success, 77 = skipped, 99 = framework failure, 124 = timeout + if [[ ${rc} -eq 0 ]]; then + PASSED=$((PASSED + 1)) + [[ "${COMPACT}" != "true" ]] && echo " [PASS]" + elif [[ ${rc} -eq 77 ]]; then + SKIPPED=$((SKIPPED + 1)) + # Handle both "SKIP: ..." and "testname: skipped test: ..." formats + skip_msg=$(grep -E "(^SKIP:|skipped test:)" "${test_log}" | head -1 || true) + if [[ -z "${skip_msg}" ]]; then + skip_msg="(no message)" + else + # Extract just the reason part using bash string manipulation (avoid sed/locale issues) + if [[ "${skip_msg}" == *"skipped test: "* ]]; then + skip_msg="${skip_msg##*skipped test: }" + elif [[ "${skip_msg}" == "SKIP: "* ]]; then + skip_msg="${skip_msg#SKIP: }" + fi + fi + if [[ "${COMPACT}" == "true" ]]; then + echo "SKIP ${test_name}: ${skip_msg}" + else + echo " [SKIP]: ${skip_msg}" + fi + elif [[ ${rc} -eq 124 ]]; then + TIMEOUT=$((TIMEOUT + 1)) + FAILED=$((FAILED + 1)) + if [[ "${COMPACT}" == "true" ]]; then + echo "TIMEOUT ${test_name}" + else + echo " [TIMEOUT] (${TIMEOUT_SEC}s)" + fi + else + FAILED=$((FAILED + 1)) + if [[ "${COMPACT}" == "true" ]]; then + echo "FAIL ${test_name} (exit ${rc})" + # Show last few lines of output + tail -10 "${test_log}" | sed 's/^/ | /' + else + echo " [FAIL] (exit ${rc})" + echo " Log: ${test_log}" + # Show error output + tail -20 "${test_log}" | sed 's/^/ | /' + fi + fi + + [[ "${COMPACT}" != "true" ]] && echo "" +done + +popd >/dev/null + +# Summary +echo "" +echo "==========================================" +echo "Test Results Summary" +echo "==========================================" +echo "Total: $((PASSED + FAILED + SKIPPED))" +echo "Passed: ${PASSED}" +echo "Failed: ${FAILED}" +echo "Skipped: ${SKIPPED}" +if [[ ${TIMEOUT} -gt 0 ]]; then + echo "Timeout: ${TIMEOUT}" +fi +echo "" +echo "Logs saved at: ${LOGDIR}" +echo "==========================================" + +# Exit status +if [[ "${FAIL_ON_ERROR}" == "true" && ${FAILED} -gt 0 ]]; then + echo "[error] ${FAILED} test(s) failed" + exit 1 +fi + +if [[ ${FAILED} -gt 0 ]]; then + echo "[warn] ${FAILED} test(s) failed (use --fail-on-error to make this fatal)" + exit 0 +else + echo "[info] All tests passed!" + exit 0 +fi diff --git a/red/src/cli.rs b/red/src/cli.rs new file mode 100644 index 0000000..20ba85e --- /dev/null +++ b/red/src/cli.rs @@ -0,0 +1,402 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Command-line argument parsing using lexopt. +//! +//! Parses sed-compatible options with single parsing path and natural argument order preservation. + +use std::io::{self, Read}; +use std::path::PathBuf; + +use lexopt::prelude::*; +use red::constants::DEFAULT_LINE_LENGTH; +use red::errors::{Result, SedError}; + +/// Parsed command-line arguments +#[derive(Debug, Clone)] +pub struct CliArgs { + /// Scripts with raw bytes and their sources (for error reporting) + /// Collected from -e and -f options in the order they appear + /// Tuple: (converted_string, raw_bytes, source) + /// raw_bytes is needed for accurate multibyte delimiter detection + pub scripts_with_sources: Vec<(String, Vec, red::errors::ScriptSource)>, + + /// Input files with raw bytes (empty = stdin) + /// Tuple: (converted_string, raw_bytes) to support raw bytes in inline scripts + pub files: Vec<(String, Vec)>, + + /// Suppress automatic printing (-n) + pub quiet: bool, + + /// In-place editing suffix + /// None = no in-place editing + /// Some("") = in-place without backup + /// Some(suffix) = in-place with backup + pub in_place: Option, + + /// Use Extended Regular Expressions (-r/-E) + pub extended_regex: bool, + + /// Treat files separately (-s) + pub separate_files: bool, + + /// Line length for l command + pub line_length: usize, + + /// Unbuffered output (-u) + pub unbuffered: bool, + + /// POSIX mode (--posix) + pub posix: bool, + + /// Follow symlinks (--follow-symlinks) + pub follow_symlinks: bool, + + /// Sandbox mode (--sandbox) + pub sandbox: bool, + + /// Null-data mode (-z) + pub null_data: bool, + + /// Binary mode (-b) - disable CRLF conversion on Windows + pub binary: bool, +} + +impl Default for CliArgs { + fn default() -> Self { + CliArgs { + scripts_with_sources: vec![], + files: vec![], + quiet: false, + in_place: None, + extended_regex: false, + separate_files: false, + line_length: DEFAULT_LINE_LENGTH, + unbuffered: false, + posix: false, + follow_symlinks: false, + sandbox: false, + null_data: false, + binary: false, + } + } +} + +/// Parse command-line arguments using lexopt +/// +/// ## Argument Semantics +/// +/// - `-e SCRIPT` / `--expression SCRIPT`: Add script expression +/// - `-f FILE` / `--file FILE`: Add script from file (or stdin if FILE is -) +/// - `-n` / `--quiet` / `--silent`: Suppress auto-print +/// - `-i[SUFFIX]` / `--in-place[=SUFFIX]`: Edit in-place +/// - `-i` = in-place without backup +/// - `-iSUFFIX` = in-place with backup (any suffix, including "s", "p", etc.) +/// - `-r` / `-E` / `--regexp-extended`: Use ERE instead of BRE +/// - `-s` / `--separate`: Treat files separately +/// - `-l N` / `--line-length N`: Set line wrap length +/// - `-u` / `--unbuffered`: Unbuffered output +/// - `--posix`: Strict POSIX mode +/// - `--follow-symlinks`: Follow symlinks in-place +/// - `--sandbox`: Sandbox mode +/// - `-z` / `--null-data`: Use NUL as line separator +/// - `--help`: Print help +/// - `--version`: Print version +/// +/// ## Order Preservation +/// +/// Scripts from `-e` and `-f` are collected in the exact order they appear. +/// This is critical for `#n` quiet mode detection (only first script matters). +pub fn parse_args() -> Result { + let mut args = CliArgs::default(); + let mut parser = lexopt::Parser::from_env(); + let mut expr_index = 0; // Track expression index for error messages + + while let Some(arg) = parser.next()? { + match arg { + // -e SCRIPT / --expression SCRIPT + Short('e') | Long("expression") => { + // Get raw bytes and lossy-converted string + let os_value = parser.value()?; + #[cfg(unix)] + let raw_bytes: Vec = { + use std::os::unix::ffi::OsStrExt; + os_value.as_bytes().to_vec() + }; + #[cfg(not(unix))] + let raw_bytes: Vec = os_value.to_string_lossy().as_bytes().to_vec(); + let script = os_value.to_string_lossy().into_owned(); + args.scripts_with_sources.push(( + script, + raw_bytes, + red::errors::ScriptSource::Expression(expr_index), + )); + expr_index += 1; + } + + // -f FILE / --file FILE + Short('f') | Long("file") => { + let file_path = parser.value()?.string()?; + let (script_content, raw_bytes, path_display) = read_script_file(&file_path)?; + args.scripts_with_sources.push(( + script_content, + raw_bytes, + red::errors::ScriptSource::File(path_display), + )); + } + + // -n / --quiet / --silent + Short('n') | Long("quiet") | Long("silent") => { + args.quiet = true; + } + + // -i[SUFFIX] / --in-place[=SUFFIX] + Short('i') | Long("in-place") => { + // Try to get a value - if present, use it as suffix + // If not present (separate -i), use empty string (no backup) + let suffix = match parser.optional_value() { + Some(val) => { + let s = val.string()?; + // lexopt quirk: it consumes '=' as delimiter for short options + // so -i==foo becomes "=foo". Prepend the '=' back if needed. + if s.starts_with('=') { + format!("={}", s) + } else { + s + } + } + None => String::new(), + }; + args.in_place = Some(suffix); + } + + // -r / -E / --regexp-extended + Short('r') | Short('E') | Long("regexp-extended") => { + args.extended_regex = true; + } + + // -s / --separate + Short('s') | Long("separate") => { + args.separate_files = true; + } + + // -l N / --line-length N + Short('l') | Long("line-length") => { + let value = parser.value()?.string()?; + args.line_length = value + .parse() + .map_err(|_| SedError::usage(&format!("invalid line length: {}", value)))?; + } + + // -u / --unbuffered + Short('u') | Long("unbuffered") => { + args.unbuffered = true; + } + + // --posix + Long("posix") => { + args.posix = true; + } + + // --follow-symlinks + Long("follow-symlinks") => { + args.follow_symlinks = true; + } + + // --sandbox + Long("sandbox") => { + args.sandbox = true; + } + + // -z / --null-data + Short('z') | Long("null-data") => { + args.null_data = true; + } + + // -b / --binary (disable CRLF conversion on Windows) + Short('b') | Long("binary") => { + args.binary = true; + } + + // --help + Long("help") => { + // Use write! to detect errors (e.g., when stdout redirected to /dev/full) + use std::io::Write; + if writeln!(std::io::stdout(), "{}", help_text()).is_err() { + std::process::exit(4); // sed exit code for write errors + } + std::process::exit(0); + } + + // --version + Long("version") => { + // Use write! to detect errors (e.g., when stdout redirected to /dev/full) + use std::io::Write; + #[allow(unused_mut)] + let mut version = format!("red {}", env!("CARGO_PKG_VERSION")); + + #[cfg(feature = "selinux")] + { + version.push_str("\nwith SELinux"); + if red::selinux::is_selinux_enabled() { + version.push_str("\nSELinux is enabled"); + } + } + + if writeln!(std::io::stdout(), "{}", version).is_err() { + std::process::exit(4); // sed exit code for write errors + } + std::process::exit(0); + } + + // Positional arguments (files or script if no -e/-f) + // Store both lossy string and raw bytes for inline script support + Value(val) => { + #[cfg(unix)] + let raw_bytes: Vec = { + use std::os::unix::ffi::OsStrExt; + val.as_bytes().to_vec() + }; + #[cfg(not(unix))] + let raw_bytes: Vec = val.to_string_lossy().as_bytes().to_vec(); + let file = val.to_string_lossy().into_owned(); + args.files.push((file, raw_bytes)); + } + + // Unknown short option + Short(ch) => { + return Err(SedError::usage(&format!("invalid option -- '{}'", ch))); + } + + // Unknown long option + Long(opt) => { + return Err(SedError::usage(&format!("invalid option -- --{}", opt))); + } + } + } + + // If no -e or -f provided, first positional argument is the script + if args.scripts_with_sources.is_empty() { + if args.files.is_empty() { + // GNU sed compatibility: print usage when no script provided + print_usage(); + std::process::exit(4); // sed exit code for usage errors + } + let (script, raw_bytes) = args.files.remove(0); + args.scripts_with_sources.push(( + script, + raw_bytes, + red::errors::ScriptSource::Expression(0), + )); + } + + Ok(args) +} + +/// Read script from file or stdin +/// +/// Returns (script_content, raw_bytes, display_path) for error reporting +/// Uses lossy UTF-8 conversion to handle scripts with invalid bytes +/// The raw_bytes are needed for accurate multibyte delimiter detection +fn read_script_file(path: &str) -> Result<(String, Vec, String)> { + if path == "-" { + // Read from stdin as bytes + let mut content = Vec::new(); + io::stdin() + .read_to_end(&mut content) + .map_err(|e| SedError::io("can't read", "-", e))?; + let converted = String::from_utf8_lossy(&content).into_owned(); + Ok((converted, content, "-".to_string())) + } else { + // Read from file as bytes + let file_path = PathBuf::from(path); + let path_str = file_path.display().to_string(); + let content = + std::fs::read(&file_path).map_err(|e| SedError::io("can't read", &path_str, e))?; + let converted = String::from_utf8_lossy(&content).into_owned(); + Ok((converted, content, path_str)) + } +} + +/// Print usage information (for errors - without bug report footer) +fn print_usage() { + eprintln!( + r#"Usage: red [OPTION]... {{SCRIPT}} [FILE]... + red [OPTION]... -e SCRIPT... -f FILE... [FILE]... + +stream editor for filtering and transforming text. + +Options: + -n, --quiet, --silent suppress automatic printing of pattern space + -e SCRIPT, --expression=SCRIPT + add the script to the commands to be executed + -f FILE, --file=FILE add the contents of script-file to the commands + -i[SUFFIX], --in-place[=SUFFIX] + edit files in place (makes backup if SUFFIX supplied) + -r, -E, --regexp-extended + use extended regular expressions + -s, --separate consider files as separate rather than continuous + -l N, --line-length=N specify line-wrap length for 'l' command + -u, --unbuffered load minimal amounts of data and flush output often + -z, --null-data use NUL character as line separator + -b, --binary open files in binary mode (no CRLF conversion) + --posix disable all GNU extensions + --follow-symlinks follow symlinks when processing in place + --sandbox disable e/r/w commands + --help display this help and exit + --version output version information and exit + +If no -e or -f option is given, the first non-option argument is the script. +"# + ); +} + +/// Return help text (with bug report footer) +fn help_text() -> &'static str { + r#"Usage: red [OPTION]... {SCRIPT} [FILE]... + red [OPTION]... -e SCRIPT... -f FILE... [FILE]... + +stream editor for filtering and transforming text. + +Options: + -n, --quiet, --silent suppress automatic printing of pattern space + -e SCRIPT, --expression=SCRIPT + add the script to the commands to be executed + -f FILE, --file=FILE add the contents of script-file to the commands + -i[SUFFIX], --in-place[=SUFFIX] + edit files in place (makes backup if SUFFIX supplied) + -r, -E, --regexp-extended + use extended regular expressions + -s, --separate consider files as separate rather than continuous + -l N, --line-length=N specify line-wrap length for 'l' command + -u, --unbuffered load minimal amounts of data and flush output often + -z, --null-data use NUL character as line separator + -b, --binary open files in binary mode (no CRLF conversion) + --posix disable all GNU extensions + --follow-symlinks follow symlinks when processing in place + --sandbox disable e/r/w commands + --help display this help and exit + --version output version information and exit + +If no -e or -f option is given, the first non-option argument is the script. + +E-mail bug reports to: "# +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_in_place_semantics() { + // -i without value = no backup + // This is tested through integration tests as lexopt parses env::args + } + + #[test] + fn test_read_script_file_error() { + let result = read_script_file("/nonexistent/file/path.sed"); + assert!(result.is_err()); + } +} diff --git a/red/src/constants.rs b/red/src/constants.rs new file mode 100644 index 0000000..247e399 --- /dev/null +++ b/red/src/constants.rs @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Global constants for Red sed implementation +//! +//! This module centralizes all magic numbers and configuration values +//! to improve maintainability and reduce duplication. + +/// GNU sed compatibility version +/// +/// Red aims to be compatible with GNU sed 4.9 behavior. +/// Commands or features from later versions will be rejected. +pub const GNU_SED_COMPAT_VERSION: &str = "4.9"; + +/// Maximum symlink resolution depth +/// +/// Matches GNU sed and typical Unix SYMLOOP_MAX limits. +/// Prevents infinite loops in symlink chains. +pub const MAX_SYMLINK_DEPTH: usize = 40; + +/// Default line length for 'l' command +/// +/// Used when neither -l flag nor COLS environment variable is set. +/// This value matches GNU sed's default. +pub const DEFAULT_LINE_LENGTH: usize = 70; + +/// Maximum regex backtracking iterations +/// +/// Prevents stack overflow and infinite loops on pathological regex patterns. +/// This limit applies to the custom backtracking engine. +pub const MAX_REGEX_BACKTRACK_ITERATIONS: usize = 100_000; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constants_valid() { + assert!(!GNU_SED_COMPAT_VERSION.is_empty()); + assert!(MAX_SYMLINK_DEPTH > 0); + assert!(DEFAULT_LINE_LENGTH > 0); + assert!(MAX_REGEX_BACKTRACK_ITERATIONS > 0); + } + + #[test] + fn test_compat_version_format() { + // Should be in format "major.minor" + let parts: Vec<&str> = GNU_SED_COMPAT_VERSION.split('.').collect(); + assert_eq!(parts.len(), 2); + assert!(parts[0].parse::().is_ok()); + assert!(parts[1].parse::().is_ok()); + } +} diff --git a/red/src/context.rs b/red/src/context.rs new file mode 100644 index 0000000..24ea0f1 --- /dev/null +++ b/red/src/context.rs @@ -0,0 +1,319 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Central configuration context for sed execution. +//! +//! Provides the `Context` struct - a unified container for all configuration +//! parameters used throughout the sed pipeline (parsing, compilation, execution). + +use crate::constants::DEFAULT_LINE_LENGTH; +use crate::errors::ScriptSource; + +/// POSIX compatibility level +/// +/// Mirrors GNU sed's posixicity levels: +/// - `Extended`: Default mode with all GNU extensions enabled +/// - `Correct`: POSIX-compatible with some non-conflicting GNU extensions +/// - `Basic`: Strict POSIX compliance (--posix flag) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PosixMode { + /// GNU extensions enabled (default) + Extended, + /// POSIX-compatible with harmless GNU extensions + Correct, + /// Strict POSIX mode (--posix flag) + Basic, +} + +impl Default for PosixMode { + fn default() -> Self { + PosixMode::Extended + } +} + +impl PosixMode { + /// Check if running in any POSIX mode (Correct or Basic) + pub fn is_posix(&self) -> bool { + matches!(self, PosixMode::Correct | PosixMode::Basic) + } + + /// Check if running in strict POSIX mode (Basic only) + pub fn is_strict_posix(&self) -> bool { + matches!(self, PosixMode::Basic) + } + + /// Create from boolean flags + /// + /// # Arguments + /// * `posix` - POSIXLY_CORRECT environment variable is set + /// * `strict_posix` - --posix flag was passed + pub fn from_flags(posix: bool, strict_posix: bool) -> Self { + if strict_posix { + PosixMode::Basic + } else if posix { + PosixMode::Correct + } else { + PosixMode::Extended + } + } +} + +/// Central configuration context for sed execution +/// +/// Contains all runtime configuration parameters used throughout +/// the sed pipeline: lexing, parsing, compilation, and execution. +/// +/// ## Usage +/// +/// The `Context` is created from a `RunConfig` and passed to the parser, +/// compiler, and execution engine to provide consistent configuration. +/// +/// ``` +/// use red::context::Context; +/// // Create a test context with default settings +/// let ctx = Context::new_test(); +/// assert!(!ctx.extended_regex); +/// assert!(!ctx.sandbox); +/// ``` +#[derive(Debug, Clone)] +pub struct Context { + // === Core Operation Mode === + /// POSIX compatibility level + pub posix_mode: PosixMode, + + /// Use Extended Regular Expressions (ERE) instead of Basic (BRE) + /// Controlled by -E/-r flags + pub extended_regex: bool, + + /// Sandbox mode - disable e/r/w/R/W commands for security + /// Controlled by --sandbox flag + pub sandbox: bool, + + // === Output Behavior === + /// Suppress automatic printing of pattern space + /// Controlled by -n flag or #n shebang + pub quiet: bool, + + /// Treat each input file separately (reset line numbers, addresses, etc.) + /// Controlled by -s flag + pub separate_files: bool, + + /// Use NUL (\0) as line separator instead of newline (\n) + /// Controlled by -z flag + pub null_data: bool, + + /// Flush output buffer after each line + /// Controlled by -u flag + pub unbuffered: bool, + + // === In-Place Editing === + /// Edit files in-place with optional backup suffix + /// None = no in-place editing + /// Some("") = in-place without backup + /// Some(suffix) = in-place with backup (original renamed to ) + pub in_place_suffix: Option, + + /// Follow symbolic links when editing in-place + /// Controlled by --follow-symlinks flag + pub follow_symlinks: bool, + + // === Formatting === + /// Line width for l (list) command formatting + pub line_length: usize, + + // === Scripts (for error reporting) === + /// Original scripts with their sources (needed for error messages) + pub scripts_with_sources: Vec<(String, ScriptSource)>, +} + +impl Context { + /// Create Context from RunConfig + /// + /// This is the primary way to create a Context during normal execution. + /// + /// # Arguments + /// * `config` - The RunConfig from CLI parsing + /// * `scripts` - Scripts with their sources and raw bytes (raw bytes stripped for Context) + pub fn from_run_config( + config: &crate::RunConfig, + scripts: Vec<(String, Vec, ScriptSource)>, + ) -> Self { + // Extract just string + source for Context (raw bytes not needed for error messages) + let scripts_for_context: Vec<(String, ScriptSource)> = + scripts.into_iter().map(|(s, _raw, src)| (s, src)).collect(); + Context { + posix_mode: PosixMode::from_flags(config.posix, config.strict_posix), + extended_regex: config.extended_regex, + sandbox: config.sandbox, + quiet: config.quiet, + separate_files: config.separate_files, + null_data: config.null_data, + unbuffered: config.unbuffered, + in_place_suffix: config.in_place.clone(), + follow_symlinks: config.follow_symlinks, + line_length: config.line_length, + scripts_with_sources: scripts_for_context, + } + } + + /// Create a default Context for testing + /// + /// Uses sensible defaults: + /// - GNU extended mode (no POSIX restrictions) + /// - BRE regex (not ERE) + /// - No sandbox + /// - Auto-print enabled (not quiet) + /// - Single-file mode + /// - Newline separators (not null-data) + /// - No buffering control + /// - No in-place editing + /// - Default line length + /// + /// Note: This is public for use in parser.rs and tests + pub fn new_test() -> Self { + Context { + posix_mode: PosixMode::Extended, + extended_regex: false, + sandbox: false, + quiet: false, + separate_files: false, + null_data: false, + unbuffered: false, + in_place_suffix: None, + follow_symlinks: false, + line_length: DEFAULT_LINE_LENGTH, + scripts_with_sources: vec![], + } + } + + /// Create Context with custom settings for testing + /// + /// # Arguments + /// * `posix_mode` - POSIX compatibility level + /// * `extended_regex` - Use ERE instead of BRE + /// + /// Note: This is public for use in parser.rs and tests + pub fn new_test_with(posix_mode: PosixMode, extended_regex: bool) -> Self { + Context { + posix_mode, + extended_regex, + ..Self::new_test() + } + } + + // === Convenience Methods === + + /// Check if running in any POSIX mode + pub fn is_posix(&self) -> bool { + self.posix_mode.is_posix() + } + + /// Check if running in strict POSIX mode + pub fn is_strict_posix(&self) -> bool { + self.posix_mode.is_strict_posix() + } + + /// Check if in-place editing is enabled + pub fn is_in_place(&self) -> bool { + self.in_place_suffix.is_some() + } + + /// Get the in-place backup suffix, if any + /// + /// Returns None if: + /// - Not in in-place mode + /// - In-place mode without backup (suffix is empty string) + pub fn backup_suffix(&self) -> Option<&str> { + self.in_place_suffix.as_ref().and_then( + |s| { + if s.is_empty() { + None + } else { + Some(s.as_str()) + } + }, + ) + } +} + +impl Default for Context { + fn default() -> Self { + Context { + posix_mode: PosixMode::default(), + extended_regex: false, + sandbox: false, + quiet: false, + separate_files: false, + null_data: false, + unbuffered: false, + in_place_suffix: None, + follow_symlinks: false, + line_length: DEFAULT_LINE_LENGTH, + scripts_with_sources: vec![], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_posix_mode_from_flags() { + assert_eq!(PosixMode::from_flags(false, false), PosixMode::Extended); + assert_eq!(PosixMode::from_flags(true, false), PosixMode::Correct); + assert_eq!(PosixMode::from_flags(false, true), PosixMode::Basic); + assert_eq!(PosixMode::from_flags(true, true), PosixMode::Basic); // strict_posix takes precedence + } + + #[test] + fn test_posix_mode_checks() { + assert!(!PosixMode::Extended.is_posix()); + assert!(!PosixMode::Extended.is_strict_posix()); + + assert!(PosixMode::Correct.is_posix()); + assert!(!PosixMode::Correct.is_strict_posix()); + + assert!(PosixMode::Basic.is_posix()); + assert!(PosixMode::Basic.is_strict_posix()); + } + + #[test] + fn test_context_defaults() { + let ctx = Context::default(); + assert_eq!(ctx.posix_mode, PosixMode::Extended); + assert!(!ctx.extended_regex); + assert!(!ctx.quiet); + assert!(!ctx.sandbox); + assert_eq!(ctx.line_length, DEFAULT_LINE_LENGTH); + } + + #[test] + fn test_context_convenience_methods() { + let mut ctx = Context::default(); + + // In-place checks + assert!(!ctx.is_in_place()); + assert!(ctx.backup_suffix().is_none()); + + ctx.in_place_suffix = Some("".to_string()); + assert!(ctx.is_in_place()); + assert!(ctx.backup_suffix().is_none()); // Empty suffix = no backup + + ctx.in_place_suffix = Some(".bak".to_string()); + assert!(ctx.is_in_place()); + assert_eq!(ctx.backup_suffix(), Some(".bak")); + } + + #[test] + fn test_context_test_helpers() { + let ctx = Context::new_test(); + assert_eq!(ctx.posix_mode, PosixMode::Extended); + assert!(!ctx.extended_regex); + + let ctx = Context::new_test_with(PosixMode::Basic, true); + assert_eq!(ctx.posix_mode, PosixMode::Basic); + assert!(ctx.extended_regex); + } +} diff --git a/red/src/engine/addr.rs b/red/src/engine/addr.rs new file mode 100644 index 0000000..7a4f009 --- /dev/null +++ b/red/src/engine/addr.rs @@ -0,0 +1,567 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; + +use crate::errors::Result; + +use crate::engine::pattern_space::PatternSpace; +use crate::engine::types::SedRegex; +use crate::parser::Address; +use crate::parser::AddressRange; +use crate::util::regex::compile_regex; + +#[derive(Debug)] +pub struct ExecutionContext { + pub current_line_num: usize, + pub total_lines: Option, + pub pattern_space: PatternSpace, + pub quiet_mode: bool, + pub hold_space: String, + pub hold_space_raw: Vec, // Raw bytes for hold space (preserves invalid UTF-8) + pub current_filename: String, + pub r_file_handles: HashMap>, // For R command: track open files + pub extended_regex: bool, // -r/-E flag for ERE mode + pub line_length: usize, // -l N flag for line wrapping in 'l' command + pub unbuffered: bool, // -u flag for unbuffered output + pub null_data: bool, // -z flag for NUL-separated lines + pub all_input_consumed: bool, // True if all input lines have been read (via N reaching EOF) +} + +impl ExecutionContext { + pub fn new( + total_lines: Option, + quiet_mode: bool, + extended_regex: bool, + line_length: usize, + unbuffered: bool, + null_data: bool, + ) -> Self { + Self { + current_line_num: 0, + total_lines, + pattern_space: PatternSpace::default(), + quiet_mode, + hold_space: String::new(), + hold_space_raw: Vec::new(), + current_filename: "-".to_string(), // Default to stdin + r_file_handles: HashMap::new(), + extended_regex, + line_length, + unbuffered, + null_data, + all_input_consumed: false, + } + } + + pub fn set_filename(&mut self, filename: &str) { + self.current_filename = filename.to_string(); + } + + /// Read one line from file for R command. Returns None if EOF or error. + pub fn read_line_from_file(&mut self, path: &str) -> Option { + // Get or create file handle for this path + if !self.r_file_handles.contains_key(path) { + // Try to open the file + match File::open(path) { + Ok(file) => { + self.r_file_handles + .insert(path.to_string(), BufReader::new(file)); + } + Err(_) => { + // Silently ignore if file can't be opened + return None; + } + } + } + + // Read one line from the file + if let Some(reader) = self.r_file_handles.get_mut(path) { + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) => None, // EOF + Ok(_) => { + // Remove trailing newline (will be added back when printing) + if line.ends_with('\n') { + line.pop(); + } + Some(line) + } + Err(_) => None, // Error reading - silently ignore + } + } else { + None + } + } + + pub fn set_current_line(&mut self, line_num: usize, raw: Vec) { + self.current_line_num = line_num; + // Use set_raw since raw is the source of truth + // Text representation is derived on demand via PatternSpace + self.pattern_space.set_raw(raw); + // Reset flag - we're starting with a new line from input + self.all_input_consumed = false; + } +} + +#[derive(Debug, Clone)] +struct RangeState { + is_active: bool, + active_until_line: Option, +} + +impl RangeState { + fn new() -> Self { + Self { + is_active: false, + active_until_line: None, + } + } +} + +#[derive(Debug)] +pub struct AddressEvaluator { + regex_cache: HashMap, + /// Range states keyed by AddressRange.id (stable identifier) + range_states: HashMap, + pub last_s_regex: Option, // Unified "last regex" for s command and addresses + extended_regex: bool, // -r/-E flag + posix: bool, // --posix flag +} + +impl AddressEvaluator { + pub fn new(extended_regex: bool, posix: bool) -> Self { + Self { + regex_cache: HashMap::new(), + range_states: HashMap::new(), + last_s_regex: None, + extended_regex, + posix, + } + } + + pub fn get_or_compile_regex(&mut self, pattern: &str) -> Result { + // Handle empty pattern - use unified last regex (from s command or address) + if pattern.is_empty() { + if let Some(ref regex) = self.last_s_regex { + return Ok(regex.clone()); + } else { + return Err(crate::errors::SedError::parse( + "no previous regular expression", + )); + } + } + + // Compile and cache non-empty pattern + if !self.regex_cache.contains_key(pattern) { + let regex = compile_regex( + pattern, + self.extended_regex, + false, + false, + false, + self.posix, + "address pattern", + )?; + self.regex_cache.insert(pattern.to_string(), regex.clone()); + } + + let regex = self.regex_cache.get(pattern).unwrap().clone(); + // Update unified last_s_regex (both s command and addresses share this) + self.last_s_regex = Some(regex.clone()); + Ok(regex) + } + + pub fn matches_address(&mut self, addr: &Address, ctx: &ExecutionContext) -> Result { + match addr { + Address::Line(n) => { + // Line 0 has special meaning: + // - In ranges like 0,/regexp/ it means "start from first line" + // - As standalone it should never match (line numbers start at 1) + if *n == 0 { + Ok(false) // Line 0 never matches by itself + } else { + Ok(ctx.current_line_num == *n) + } + } + Address::Dollar => { + // $ matches if we're on the last line OR if all input has been consumed (via N) + match ctx.total_lines { + Some(total) => Ok(ctx.current_line_num == total || ctx.all_input_consumed), + None => Ok(false), + } + } + Address::Regex(pattern) => { + let regex = self.get_or_compile_regex(pattern)?; + Ok(regex.is_match(ctx.pattern_space.text())) + } + Address::Relative(base_addr, offset) => { + let base_line = self.resolve_address_to_line(base_addr, ctx)?; + match base_line { + Some(base) => { + Ok(ctx.current_line_num == (base as isize + offset).max(1) as usize) + } + None => Ok(false), + } + } + Address::Step(base_addr, step) => { + if *step == 0 { + return Ok(false); + } + let base_line = self.resolve_address_to_line(base_addr, ctx)?; + match base_line { + Some(base) => { + if ctx.current_line_num >= base { + let diff = ctx.current_line_num - base; + Ok(diff % step == 0) + } else { + Ok(false) + } + } + None => Ok(false), + } + } + } + } + + pub fn resolve_address_to_line( + &mut self, + addr: &Address, + ctx: &ExecutionContext, + ) -> Result> { + match addr { + Address::Line(n) => Ok(Some(*n)), + Address::Dollar => match ctx.total_lines { + Some(total) => Ok(Some(total)), + None => Ok(None), + }, + Address::Regex(_pattern) => Ok(None), + Address::Relative(base_addr, offset) => { + let base_line = self.resolve_address_to_line(base_addr, ctx)?; + match base_line { + Some(base) => Ok(Some((base as isize + offset).max(1) as usize)), + None => Ok(None), + } + } + Address::Step(_, _) => Ok(None), + } + } + + pub fn is_last_line(&self, ctx: &ExecutionContext) -> bool { + matches!(ctx.total_lines, Some(total) if ctx.current_line_num == total) + } + pub fn is_first_line(&self, ctx: &ExecutionContext) -> bool { + ctx.current_line_num == 1 + } + pub fn is_valid_line_number(&self, line_num: usize, ctx: &ExecutionContext) -> bool { + line_num > 0 + && match ctx.total_lines { + Some(total) => line_num <= total, + None => true, + } + } + + /// Evaluate range semantics specifically for the `c` (change) command. + /// Returns (matches, is_end_of_range). + /// + /// Behaviour: + /// - For single-address ranges (addr, or ,addr): match only on that line and mark as end. + /// - For two-address ranges (addr1,addr2): match all lines from start until end; however, + /// `is_end_of_range` is true only on the line where the end is reached. This allows + /// the engine to print the replacement text once at the end of the range. + pub fn evaluate_range_for_change( + &mut self, + range: &AddressRange, + ctx: &ExecutionContext, + ) -> Result<(bool, bool)> { + match (&range.start, &range.end) { + (Some(start_addr), Some(end_addr)) => { + self.evaluate_two_address_range_for_change(range.id, start_addr, end_addr, ctx) + } + (Some(start_addr), None) => { + let m = self.matches_address(start_addr, ctx)?; + Ok((m, m)) + } + (None, Some(end_addr)) => { + let m = self.matches_address(end_addr, ctx)?; + Ok((m, m)) + } + (None, None) => Ok((true, true)), + } + } + + fn evaluate_two_address_range_for_change( + &mut self, + range_id: u64, + start_addr: &Address, + end_addr: &Address, + ctx: &ExecutionContext, + ) -> Result<(bool, bool)> { + // Use range_id as stable key instead of pointer addresses + let state = self + .range_states + .entry(range_id) + .or_insert_with(RangeState::new); + let (is_active, active_until_line) = (state.is_active, state.active_until_line); + + if !is_active { + // Special handling for 0,addr2 range (GNU extension) + // Line 0 means "activate from first line and check end immediately" + let is_zero_range = matches!(start_addr, Address::Line(0)); + + let should_activate = if is_zero_range { + // For 0,addr2: activate on first line (line 1) + ctx.current_line_num == 1 + } else { + self.matches_address(start_addr, ctx)? + }; + + if should_activate { + // Try to resolve a concrete end line for numeric-like ends + let resolved_end: Option = match end_addr { + Address::Line(n) => Some(*n), + Address::Dollar => ctx.total_lines, + Address::Relative(base, offset) => match **base { + Address::Line(0) => { + Some((ctx.current_line_num as isize + *offset).max(1) as usize) + } + _ => self.resolve_address_to_line(end_addr, ctx)?, + }, + _ => None, + }; + + // For REGEX end addresses, GNU sed checks the end pattern starting from + // the NEXT line, not the same line. So /start/,/end/ where start==end + // will match from the first 'start' until the NEXT 'end' (or EOF). + // For numeric/relative addresses, same-line check is correct. + let end_is_regex = matches!(end_addr, Address::Regex(_)); + // Check if end is +N relative offset (Relative with Line(0) base) + // For +N offsets, we use resolved_end instead of matches_address + // because +N means "N lines after range start", not "line N" + let end_is_relative_offset = matches!( + end_addr, + Address::Relative(base, _) if matches!(**base, Address::Line(0)) + ); + + if !end_is_regex && !end_is_relative_offset { + // For non-regex, non-relative ends, check if end matches on this line + let end_matches_same_line = self.matches_address(end_addr, ctx)?; + if end_matches_same_line { + // Single-line range: match and end here + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = false; + state.active_until_line = None; + return Ok((true, true)); + } + } + + if let Some(end_line) = resolved_end { + if end_line <= ctx.current_line_num { + // Degenerate range, end not in future -> end here + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = false; + state.active_until_line = None; + return Ok((true, true)); + } else { + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = true; + state.active_until_line = Some(end_line); + return Ok((true, false)); + } + } else { + // Open-ended (regex) end; activate until end matches + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = true; + state.active_until_line = None; + return Ok((true, false)); + } + } + } else { + // Inside active range + let mut should_end_here = false; + if let Some(end_line) = active_until_line { + if ctx.current_line_num >= end_line { + should_end_here = true; + } + } else if self.matches_address(end_addr, ctx)? { + should_end_here = true; + } + if should_end_here { + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = false; + state.active_until_line = None; + return Ok((true, true)); + } + return Ok((true, false)); + } + + Ok((false, false)) + } + + pub fn matches_range(&mut self, range: &AddressRange, ctx: &ExecutionContext) -> Result { + match (&range.start, &range.end) { + (Some(start_addr), None) => self.matches_address(start_addr, ctx), + (Some(start_addr), Some(end_addr)) => { + self.evaluate_two_address_range(range.id, start_addr, end_addr, ctx) + } + (None, Some(end_addr)) => self.matches_address(end_addr, ctx), + (None, None) => Ok(true), + } + } + + fn evaluate_two_address_range( + &mut self, + range_id: u64, + start_addr: &Address, + end_addr: &Address, + ctx: &ExecutionContext, + ) -> Result { + // Use range_id as stable key instead of pointer addresses + let state = self + .range_states + .entry(range_id) + .or_insert_with(RangeState::new); + let (is_active, active_until_line) = (state.is_active, state.active_until_line); + + if !is_active { + // Special handling for 0,addr2 range (GNU extension) + // Line 0 means "activate from first line and check end immediately" + let is_zero_range = matches!(start_addr, Address::Line(0)); + + let should_activate = if is_zero_range { + // For 0,addr2: activate on first line (line 1) + ctx.current_line_num == 1 + } else { + self.matches_address(start_addr, ctx)? + }; + + if should_activate { + let resolved_end: Option = match end_addr { + Address::Line(n) => Some(*n), + Address::Dollar => ctx.total_lines, + Address::Relative(base, offset) => match **base { + Address::Line(0) => { + Some((ctx.current_line_num as isize + *offset).max(1) as usize) + } + _ => self.resolve_address_to_line(end_addr, ctx)?, + }, + _ => None, + }; + + // For regex end addresses: GNU sed checks end pattern from NEXT line + // (after the line that matched start). The only exception is 0,/regexp/ + // which checks regexp on line 1. + // For +N relative offsets, we use resolved_end instead of matches_address + // because +N means "N lines after range start", not "line N". + let end_is_relative_offset = matches!( + end_addr, + Address::Relative(base, _) if matches!(**base, Address::Line(0)) + ); + let end_matches_same_line = if matches!(end_addr, Address::Regex(_)) { + if is_zero_range { + // For 0,/regexp/: check regexp on line 1 + self.matches_address(end_addr, ctx)? + } else { + // For all other /start/,/end/ or N,/end/: check end from NEXT line + false + } + } else if end_is_relative_offset { + // For +N offsets, don't use matches_address - use resolved_end instead + false + } else { + self.matches_address(end_addr, ctx)? + }; + + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + if end_matches_same_line { + state.is_active = false; + state.active_until_line = None; + } else if let Some(end_line) = resolved_end { + if end_line <= ctx.current_line_num { + state.is_active = false; + state.active_until_line = None; + } else { + state.is_active = true; + state.active_until_line = Some(end_line); + } + } else { + state.is_active = true; + state.active_until_line = None; + } + return Ok(true); + } + } else { + if let Some(end_line) = active_until_line { + if ctx.current_line_num < end_line { + // Still inside the active range + return Ok(true); + } else if ctx.current_line_num == end_line { + // Last line of the range: deactivate after matching + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = false; + state.active_until_line = None; + return Ok(true); + } else { + // Past the end of the range: ensure deactivated and no match + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = false; + state.active_until_line = None; + return Ok(false); + } + } else { + // Open-ended (regex) end: if end matches now, include this line and deactivate + if self.matches_address(end_addr, ctx)? { + let state = self + .range_states + .get_mut(&range_id) + .expect("state was inserted above"); + state.is_active = false; + state.active_until_line = None; + return Ok(true); + } + return Ok(true); + } + } + Ok(false) + } + + pub fn evaluate_with_negation( + &mut self, + range: Option<&AddressRange>, + negated: bool, + ctx: &ExecutionContext, + ) -> Result { + let matches = match range { + Some(r) => self.matches_range(r, ctx)?, + None => true, + }; + Ok(if negated { !matches } else { matches }) + } +} diff --git a/red/src/engine/exec.rs b/red/src/engine/exec.rs new file mode 100644 index 0000000..a34d5c3 --- /dev/null +++ b/red/src/engine/exec.rs @@ -0,0 +1,2055 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use std::borrow::Cow; +use std::collections::HashMap; +use std::process::Command as ProcessCommand; + +use crate::errors::{Result, SedError}; +use std::io::Write as IoWrite; + +use super::addr::{AddressEvaluator, ExecutionContext}; +use super::types::{Command, CommandResult, ReplacementTemplate, ReplacementToken, SedRegex}; +use crate::parser::{ + build_char_to_byte_mapping, is_utf8_locale as parser_is_utf8_locale, Address, PrintTiming, +}; + +/// Trait for unified rendering to String or Vec +trait RenderSink { + fn write_str(&mut self, s: &str); + fn write_bytes(&mut self, b: &[u8]); +} + +impl RenderSink for String { + fn write_str(&mut self, s: &str) { + self.push_str(s); + } + fn write_bytes(&mut self, b: &[u8]) { + self.push_str(&String::from_utf8_lossy(b)); + } +} + +impl RenderSink for Vec { + fn write_str(&mut self, s: &str) { + self.extend_from_slice(s.as_bytes()); + } + fn write_bytes(&mut self, b: &[u8]) { + self.extend_from_slice(b); + } +} + +/// Render replacement template to any RenderSink (String or Vec) +/// +/// Supports WholeMatch (&), Group backreferences (\1-\9), literals, and case transforms. +/// When captures is None, Group tokens produce empty strings. +fn render_replacement_to( + whole_match: &str, + captures: Option<&HashMap>, + template: &ReplacementTemplate, + sink: &mut S, +) { + let mut uppercase_next = false; + let mut lowercase_next = false; + let mut uppercase_all = false; + let mut lowercase_all = false; + + for token in &template.tokens { + match token { + ReplacementToken::Literal(s) => { + let transformed = apply_case_transform( + s, + &mut uppercase_next, + &mut lowercase_next, + uppercase_all, + lowercase_all, + ); + sink.write_str(&transformed); + } + ReplacementToken::LiteralBytes(bytes) => { + // Apply case transformation to ASCII bytes within LiteralBytes + let transformed = apply_case_transform_bytes( + bytes, + &mut uppercase_next, + &mut lowercase_next, + uppercase_all, + lowercase_all, + ); + sink.write_bytes(&transformed); + } + ReplacementToken::WholeMatch => { + let transformed = apply_case_transform( + whole_match, + &mut uppercase_next, + &mut lowercase_next, + uppercase_all, + lowercase_all, + ); + sink.write_str(&transformed); + } + ReplacementToken::Group(group_id) => { + if *group_id == 0 { + let transformed = apply_case_transform( + whole_match, + &mut uppercase_next, + &mut lowercase_next, + uppercase_all, + lowercase_all, + ); + sink.write_str(&transformed); + } else if let Some(caps) = captures { + if let Some(cap) = caps.get(&(*group_id as usize)) { + // For case transforms, we need proper Unicode handling + // Use raw_bytes only in non-UTF-8 MBCS locales (Shift-JIS, EUC-JP) + // In UTF-8 locales, use cap.text for proper Unicode case conversion + let use_raw_bytes = cap.raw_bytes.is_some() && !parser_is_utf8_locale(); + + if use_raw_bytes { + // Non-UTF-8 MBCS mode - use raw bytes, ASCII-only transforms + let transformed = apply_case_transform_bytes( + cap.raw_bytes.as_ref().unwrap(), + &mut uppercase_next, + &mut lowercase_next, + uppercase_all, + lowercase_all, + ); + sink.write_bytes(&transformed); + } else { + // UTF-8 or non-MBCS mode - use text for proper Unicode transforms + let transformed = apply_case_transform( + &cap.text, + &mut uppercase_next, + &mut lowercase_next, + uppercase_all, + lowercase_all, + ); + sink.write_str(&transformed); + } + } + } + } + ReplacementToken::UppercaseNext => uppercase_next = true, + ReplacementToken::LowercaseNext => lowercase_next = true, + ReplacementToken::UppercaseAll => uppercase_all = true, + ReplacementToken::LowercaseAll => lowercase_all = true, + ReplacementToken::EndCase => { + uppercase_all = false; + lowercase_all = false; + } + } + } +} + +/// Render replacement template to String +fn render_replacement( + whole_match: &str, + captures: Option<&HashMap>, + template: &ReplacementTemplate, +) -> String { + let mut result = String::new(); + render_replacement_to(whole_match, captures, template, &mut result); + result +} + +/// Render replacement template to bytes (for preserving invalid UTF-8) +fn render_replacement_to_bytes( + whole_match: &str, + captures: Option<&HashMap>, + template: &ReplacementTemplate, +) -> Vec { + let mut result = Vec::new(); + render_replacement_to(whole_match, captures, template, &mut result); + result +} + +/// Apply case transformation to a string based on current state +fn apply_case_transform( + text: &str, + uppercase_next: &mut bool, + lowercase_next: &mut bool, + uppercase_all: bool, + lowercase_all: bool, +) -> String { + let mut result = String::new(); + let mut chars = text.chars(); + + if *uppercase_next || *lowercase_next { + // Transform only first character + if let Some(first) = chars.next() { + if *uppercase_next { + result.push_str(&first.to_uppercase().to_string()); + } else { + result.push_str(&first.to_lowercase().to_string()); + } + *uppercase_next = false; + *lowercase_next = false; + } + // Rest of text unchanged + result.push_str(&chars.collect::()); + } else if uppercase_all { + result = text.to_uppercase(); + } else if lowercase_all { + result = text.to_lowercase(); + } else { + result = text.to_string(); + } + + result +} + +/// Apply case transformation to raw bytes (for LiteralBytes tokens) +/// Only transforms ASCII letters, preserves high bytes (128-255) unchanged +fn apply_case_transform_bytes( + bytes: &[u8], + uppercase_next: &mut bool, + lowercase_next: &mut bool, + uppercase_all: bool, + lowercase_all: bool, +) -> Vec { + let mut result = Vec::with_capacity(bytes.len()); + + for (i, &byte) in bytes.iter().enumerate() { + if i == 0 && (*uppercase_next || *lowercase_next) { + // Transform only first ASCII letter + if byte.is_ascii_lowercase() && *uppercase_next { + result.push(byte.to_ascii_uppercase()); + } else if byte.is_ascii_uppercase() && *lowercase_next { + result.push(byte.to_ascii_lowercase()); + } else { + result.push(byte); + } + *uppercase_next = false; + *lowercase_next = false; + } else if uppercase_all && byte.is_ascii_lowercase() { + result.push(byte.to_ascii_uppercase()); + } else if lowercase_all && byte.is_ascii_uppercase() { + result.push(byte.to_ascii_lowercase()); + } else { + result.push(byte); + } + } + + result +} + +/// Helper for advanced replacement with template (supports backreferences) +/// Performs sed-style substitution. Returns (result, match_occurred, optional_bytes). +/// `match_occurred` is true if any match was found (even if replacement produces same string). +/// When `raw_bytes` is provided and template contains LiteralBytes tokens, also returns +/// the byte-level result for preserving invalid UTF-8 in pattern space. +fn sed_replace_with_template<'t>( + regex: &SedRegex, + text: &'t str, + template: &ReplacementTemplate, + global: bool, + occurrence: Option, + raw_bytes: Option<&[u8]>, +) -> (Cow<'t, str>, bool, Option>) { + let matcher = ®ex.matcher; + + // Check if replacement uses backreferences to determine which finder to use + let uses_backrefs = template + .tokens + .iter() + .any(|token| matches!(token, ReplacementToken::Group(_))); + + // Bind raw_bytes and char_to_byte_map together to avoid unwrap() calls + // When byte_ctx is Some, we have both raw bytes and the mapping + // Always track raw bytes if available - any substitution should update them + let byte_ctx: Option<(&[u8], Vec)> = + raw_bytes.map(|raw| (raw, build_char_to_byte_mapping(text, raw))); + + let mut result = String::new(); + let mut byte_result: Vec = Vec::new(); + let mut last_end = 0; + let mut last_byte_end: usize = 0; + let chars: Vec = text.chars().collect(); + let mut match_count = 0; + // Track where the previous match ended - used to skip zero-length matches at that position + // This prevents patterns like [a-z]* from matching both the letters AND an empty string after + let mut prev_match_end_pos: Option = None; + + loop { + // Safety: prevent infinite loops on pathological patterns + if match_count > chars.len() * 10 { + break; + } + + // Find next match - use appropriate finder based on whether we need captures + let match_data: Option<( + usize, + usize, + String, + Option>, + )> = if uses_backrefs { + matcher + .find_with_captures_from(text, last_end) + .map(|(start, end, captures)| { + let matched_text: String = chars[start..end].iter().collect(); + (start, end, matched_text, Some(captures)) + }) + } else { + matcher + .find_with_text_from(text, last_end) + .map(|(matched_text, start, end)| (start, end, matched_text.to_string(), None)) + }; + + let Some((start, end, matched_text, captures)) = match_data else { + break; + }; + + // Skip zero-length match at the position where previous match ended + // This prevents patterns like [a-z]* from matching both "abc" AND then "" at position 3 + // The rule: after any match ending at position P, skip the next zero-length match at P + if start == end && prev_match_end_pos == Some(start) { + // Don't replace, just advance past this character + if end < chars.len() { + result.push(chars[end]); + if let Some((raw, map)) = &byte_ctx { + let byte_pos = if end < map.len() { map[end] } else { raw.len() }; + let byte_next = if end + 1 < map.len() { + map[end + 1] + } else { + raw.len() + }; + byte_result.extend_from_slice(&raw[byte_pos..byte_next]); + last_byte_end = byte_next; + } + } + last_end = end + 1; + prev_match_end_pos = None; // Reset so we can match at next position + continue; + } + + match_count += 1; + + // Skip if this is before the occurrence we're looking for + // For occurrence+global (e.g., 2g), skip matches before Nth + if let Some(kth) = occurrence { + if match_count < kth { + // For zero-length matches, we need to advance past them to avoid infinite loop + if start == end { + result.push_str(&chars[last_end..end].iter().collect::()); + if let Some((raw, map)) = &byte_ctx { + let byte_end_pos = if end < map.len() { map[end] } else { raw.len() }; + byte_result.extend_from_slice(&raw[last_byte_end..byte_end_pos]); + last_byte_end = byte_end_pos; + } + // Mark this as the previous match end so we don't match here again + prev_match_end_pos = Some(end); + // Advance past the zero-length match position + if end < chars.len() { + result.push(chars[end]); + if let Some((raw, map)) = &byte_ctx { + let byte_pos = if end < map.len() { map[end] } else { raw.len() }; + let byte_next = if end + 1 < map.len() { + map[end + 1] + } else { + raw.len() + }; + byte_result.extend_from_slice(&raw[byte_pos..byte_next]); + last_byte_end = byte_next; + } + last_end = end + 1; + } else { + last_end = end; + } + } else { + result.push_str(&chars[last_end..end].iter().collect::()); + // Also update byte result + if let Some((raw, map)) = &byte_ctx { + let byte_end_pos = if end < map.len() { map[end] } else { raw.len() }; + byte_result.extend_from_slice(&raw[last_byte_end..byte_end_pos]); + last_byte_end = byte_end_pos; + } + last_end = end; + } + continue; + } + } + + // Add text before match + result.push_str(&chars[last_end..start].iter().collect::()); + // Also add bytes before match + if let Some((raw, map)) = &byte_ctx { + let byte_start = if start < map.len() { + map[start] + } else { + raw.len() + }; + byte_result.extend_from_slice(&raw[last_byte_end..byte_start]); + } + + // Render replacement + let repl = render_replacement(&matched_text, captures.as_ref(), template); + result.push_str(&repl); + // Also render bytes replacement + if byte_ctx.is_some() { + let repl_bytes = + render_replacement_to_bytes(&matched_text, captures.as_ref(), template); + byte_result.extend_from_slice(&repl_bytes); + } + + // Track where this match ended - used to skip zero-length matches at this position + prev_match_end_pos = Some(end); + + // Handle zero-length matches: advance by one char to prevent infinite loop + if start == end { + if end < chars.len() { + result.push(chars[end]); + // Also add byte for this char + if let Some((raw, map)) = &byte_ctx { + let byte_pos = if end < map.len() { map[end] } else { raw.len() }; + let byte_next = if end + 1 < map.len() { + map[end + 1] + } else { + raw.len() + }; + byte_result.extend_from_slice(&raw[byte_pos..byte_next]); + last_byte_end = byte_next; + } + } + last_end = end + 1; + } else { + // Update last_byte_end for non-zero-length match + if let Some((raw, map)) = &byte_ctx { + last_byte_end = if end < map.len() { map[end] } else { raw.len() }; + } + last_end = end; + } + + // Stop after first replacement if not global mode + if !global { + break; + } + } + + // Add remaining text + if last_end < chars.len() { + result.push_str(&chars[last_end..].iter().collect::()); + } + // Add remaining bytes + if let Some((raw, _)) = &byte_ctx { + byte_result.extend_from_slice(&raw[last_byte_end..]); + } + + // Return whether any match occurred (match_count > 0), not just whether string changed + let matched = match_count > 0; + let byte_result_opt = if byte_ctx.is_some() && matched { + Some(byte_result) + } else { + None + }; + if result == text { + (Cow::Borrowed(text), matched, byte_result_opt) + } else { + (Cow::Owned(result), matched, byte_result_opt) + } +} + +/// Result of a substitution operation +struct SubstitutionOutcome { + /// The new pattern space content (String) + new_text: String, + /// Whether a match occurred (regardless of whether text changed) + matched: bool, + /// Raw bytes if they were updated (for preserving invalid UTF-8) + raw_bytes: Option>, +} + +/// Extract raw bytes from replacement template's LiteralBytes and Literal tokens +/// This is used when we have a non-ASCII replacement but literal_replacement_bytes is None +fn extract_raw_bytes_from_template(template: &ReplacementTemplate) -> Vec { + let mut result = Vec::new(); + for token in &template.tokens { + match token { + ReplacementToken::LiteralBytes(bytes) => { + result.extend_from_slice(bytes); + } + ReplacementToken::Literal(s) => { + result.extend_from_slice(s.as_bytes()); + } + // Backreferences and case conversions are not supported in this path + _ => {} + } + } + result +} + +/// Try byte-level substitution for non-UTF-8 patterns +/// Returns Some(outcome) if byte-level path was used, None otherwise +fn try_byte_substitution( + current: &str, + raw: &[u8], + pat_bytes: &[u8], + repl_bytes: &[u8], + global: bool, +) -> SubstitutionOutcome { + if let Some(new_raw) = replace_bytes(raw, pat_bytes, repl_bytes, global) { + let new_text = String::from_utf8_lossy(&new_raw).into_owned(); + SubstitutionOutcome { + new_text, + matched: true, + raw_bytes: Some(new_raw), + } + } else { + SubstitutionOutcome { + new_text: current.to_string(), + matched: false, + raw_bytes: None, + } + } +} + +/// Try literal string substitution (fast path, 10-100x faster than regex) +/// Returns the substitution outcome with optional byte-level result for ASCII patterns +fn try_literal_substitution( + current: &str, + raw: &[u8], + lit_pat: &str, + lit_repl: &str, + replacement: &ReplacementTemplate, + literal_replacement_bytes: &Option>, + global: bool, +) -> SubstitutionOutcome { + let matched = current.contains(lit_pat); + let new_text = if global { + current.replace(lit_pat, lit_repl) + } else { + current.replacen(lit_pat, lit_repl, 1) + }; + + // For ASCII patterns, also apply to raw bytes to preserve invalid UTF-8 + let raw_bytes = if matched && lit_pat.is_ascii() { + let pat_bytes = lit_pat.as_bytes(); + let repl_bytes_owned: Vec; + let repl_bytes: &[u8] = if let Some(ref rb) = literal_replacement_bytes { + rb.as_slice() + } else if lit_repl.is_ascii() { + lit_repl.as_bytes() + } else { + repl_bytes_owned = extract_raw_bytes_from_template(replacement); + if repl_bytes_owned.is_empty() { + return SubstitutionOutcome { + new_text, + matched, + raw_bytes: None, + }; + } + &repl_bytes_owned + }; + replace_bytes(raw, pat_bytes, repl_bytes, global) + } else { + None + }; + + SubstitutionOutcome { + new_text, + matched, + raw_bytes, + } +} + +/// Perform regex-based substitution +fn do_regex_substitution( + current: &str, + raw: &[u8], + regex: &SedRegex, + replacement: &ReplacementTemplate, + global: bool, + occurrence: Option, +) -> SubstitutionOutcome { + // In MBCS locales, use byte-based matching to handle non-UTF-8 encodings correctly + if crate::mbcs::is_multibyte_locale() { + return do_regex_substitution_mb(raw, regex, replacement, global, occurrence); + } + + let (new_value, matched, byte_result) = + sed_replace_with_template(regex, current, replacement, global, occurrence, Some(raw)); + + SubstitutionOutcome { + new_text: new_value.into_owned(), + matched, + raw_bytes: byte_result, + } +} + +/// MBCS-aware regex substitution that works with raw bytes +fn do_regex_substitution_mb( + raw: &[u8], + regex: &SedRegex, + replacement: &ReplacementTemplate, + global: bool, + occurrence: Option, +) -> SubstitutionOutcome { + use crate::mbcs::MbText; + + let matcher = ®ex.matcher; + let mb_text = MbText::new(raw); + + let mut result_bytes: Vec = Vec::new(); + let mut last_byte_end: usize = 0; + let mut match_count = 0; + let mut had_match = false; + // Track where the previous match ended - used to skip zero-length matches at that position + let mut prev_match_end_pos: Option = None; + + loop { + // Safety: prevent infinite loops + if match_count > mb_text.byte_len() * 10 { + break; + } + + // Find next match using bytes + let match_data = matcher.find_with_captures_bytes_from(raw, last_byte_end); + + let Some((start_byte, end_byte, captures)) = match_data else { + break; + }; + + // Skip zero-length match at the position where previous match ended + // This prevents patterns like [a-z]* from matching both "abc" AND then "" at position 3 + if start_byte == end_byte && prev_match_end_pos == Some(start_byte) { + if end_byte < raw.len() { + result_bytes.push(raw[end_byte]); + last_byte_end = end_byte + 1; + } else { + // At end of string with zero-length match - break to avoid infinite loop + break; + } + prev_match_end_pos = None; + continue; + } + + match_count += 1; + + // Check occurrence number - for occurrence+global (e.g., 2g), skip matches before Nth + if let Some(kth) = occurrence { + if match_count < kth { + // For zero-length matches, we need to advance past them to avoid infinite loop + if start_byte == end_byte { + result_bytes.extend_from_slice(&raw[last_byte_end..end_byte]); + // Mark this as the previous match end so we don't match here again + prev_match_end_pos = Some(end_byte); + // Advance past the zero-length match position + if end_byte < raw.len() { + result_bytes.push(raw[end_byte]); + last_byte_end = end_byte + 1; + } else { + last_byte_end = end_byte; + } + } else { + result_bytes.extend_from_slice(&raw[last_byte_end..end_byte]); + last_byte_end = end_byte; + } + continue; + } + } + + // Add bytes before match + result_bytes.extend_from_slice(&raw[last_byte_end..start_byte]); + + // Render replacement using bytes sink to preserve MBCS bytes + let matched_bytes = &raw[start_byte..end_byte]; + let matched_text = String::from_utf8_lossy(matched_bytes); + let repl_bytes = render_replacement_to_bytes(&matched_text, Some(&captures), replacement); + result_bytes.extend_from_slice(&repl_bytes); + + had_match = true; + + // Track where this match ended - used to skip zero-length matches at this position + prev_match_end_pos = Some(end_byte); + + // Handle zero-length matches: advance by at least one byte to prevent infinite loop + if start_byte == end_byte { + if end_byte < raw.len() { + result_bytes.push(raw[end_byte]); + last_byte_end = end_byte + 1; + } else { + // At end of string - break to avoid infinite loop + last_byte_end = end_byte; + break; + } + } else { + last_byte_end = end_byte; + } + + // Stop after first match if not global + if !global { + break; + } + } + + // Add remaining bytes + result_bytes.extend_from_slice(&raw[last_byte_end..]); + + if had_match { + let new_text = String::from_utf8_lossy(&result_bytes).into_owned(); + SubstitutionOutcome { + new_text, + matched: true, + raw_bytes: Some(result_bytes), + } + } else { + SubstitutionOutcome { + new_text: String::from_utf8_lossy(raw).into_owned(), + matched: false, + raw_bytes: None, + } + } +} + +/// Perform the core substitution operation +/// +/// Dispatches to the appropriate substitution path: +/// 1. Byte-level for non-UTF-8 patterns +/// 2. Literal string for simple patterns (10-100x faster) +/// 3. Regex for complex patterns +#[allow(clippy::too_many_arguments)] +fn perform_substitution( + current: &str, + ctx: &ExecutionContext, + pattern: &SedRegex, + replacement: &ReplacementTemplate, + global: bool, + occurrence: Option, + use_last: bool, + last_s_regex: Option<&SedRegex>, + literal_pattern: &Option, + literal_replacement: &Option, + literal_pattern_bytes: &Option>, + literal_replacement_bytes: &Option>, +) -> SubstitutionOutcome { + let raw = ctx.pattern_space.raw(); + + // Path 1: Byte-level for non-UTF-8 patterns + if let (Some(pat_bytes), Some(repl_bytes)) = (literal_pattern_bytes, literal_replacement_bytes) + { + return try_byte_substitution(current, raw, pat_bytes, repl_bytes, global); + } + + // Path 2: Literal string fast path + if let (Some(lit_pat), Some(lit_repl)) = (literal_pattern, literal_replacement) { + return try_literal_substitution( + current, + raw, + lit_pat, + lit_repl, + replacement, + literal_replacement_bytes, + global, + ); + } + + // Path 3: Regex + let effective_regex = if use_last { + last_s_regex.unwrap_or(pattern) + } else { + pattern + }; + do_regex_substitution( + current, + raw, + effective_regex, + replacement, + global, + occurrence, + ) +} + +/// Replace occurrences of `pattern` with `replacement` in `src` bytes +/// +/// If `global` is false, only replace the first occurrence. +/// Returns None if pattern not found, otherwise returns the modified bytes. +fn replace_bytes(src: &[u8], pattern: &[u8], replacement: &[u8], global: bool) -> Option> { + if pattern.is_empty() { + return None; + } + + // Find first occurrence + let pos = src.windows(pattern.len()).position(|w| w == pattern)?; + + let mut result = Vec::with_capacity(src.len()); + result.extend_from_slice(&src[..pos]); + result.extend_from_slice(replacement); + + if !global { + result.extend_from_slice(&src[pos + pattern.len()..]); + } else { + // For global, continue replacing in remaining part + let mut remaining = &src[pos + pattern.len()..]; + while let Some(next_pos) = remaining.windows(pattern.len()).position(|w| w == pattern) { + result.extend_from_slice(&remaining[..next_pos]); + result.extend_from_slice(replacement); + remaining = &remaining[next_pos + pattern.len()..]; + } + result.extend_from_slice(remaining); + } + + Some(result) +} + +/// Handle translation (y command) at the byte level for C locale +fn translate_bytes(raw: &[u8], from_bytes: &[u8], to_bytes: &[u8]) -> Vec { + // Build byte-to-byte translation table + let mut byte_table: [u8; 256] = [0; 256]; + for i in 0..256 { + byte_table[i] = i as u8; + } + let last_to_byte = to_bytes.last().copied(); + for (idx, &from_byte) in from_bytes.iter().enumerate() { + let to_byte = if idx < to_bytes.len() { + to_bytes[idx] + } else if let Some(last_byte) = last_to_byte { + last_byte + } else { + 0 + }; + byte_table[from_byte as usize] = to_byte; + } + + // Apply byte-level translation + raw.iter().map(|&b| byte_table[b as usize]).collect() +} + +/// Handle translation (y command) at the character level with byte output for U+FFFD +/// Uses from_bytes to match invalid UTF-8 bytes in both pattern and input +fn translate_chars_to_bytes( + raw: &[u8], + from_chars: &[char], + from_bytes: &[u8], + to_bytes: &[u8], +) -> Vec { + use crate::parser::build_char_to_byte_mapping; + + let mut result = Vec::with_capacity(raw.len()); + let mut i = 0; + + // Build mapping from char index to byte positions in from_bytes + let from_str: String = from_chars.iter().collect(); + let from_byte_starts = build_char_to_byte_mapping(&from_str, from_bytes); + + // Build mapping for to_bytes as well + let to_str: String = String::from_utf8_lossy(to_bytes).into_owned(); + let to_byte_starts = build_char_to_byte_mapping(&to_str, to_bytes); + + // Helper to get byte range for char at index + let get_from_range = |idx: usize| -> (usize, usize) { + let start = from_byte_starts[idx]; + let end = from_byte_starts + .get(idx + 1) + .copied() + .unwrap_or(from_bytes.len()); + (start, end) + }; + let get_to_range = |idx: usize| -> (usize, usize) { + let start = to_byte_starts[idx]; + let end = to_byte_starts + .get(idx + 1) + .copied() + .unwrap_or(to_bytes.len()); + (start, end) + }; + + while i < raw.len() { + // Try to decode a UTF-8 character from raw bytes + let remaining = &raw[i..]; + let decoded = std::str::from_utf8(remaining); + + if let Ok(s) = decoded { + // Valid UTF-8 from this point + if let Some(c) = s.chars().next() { + // Find if this char is in 'from' + if let Some(idx) = from_chars.iter().position(|&fc| fc == c) { + // Use corresponding bytes from to_bytes + if idx < to_byte_starts.len() { + let (to_start, to_end) = get_to_range(idx); + result.extend_from_slice(&to_bytes[to_start..to_end]); + } else if let Some(&last) = to_bytes.last() { + result.push(last); + } + } else { + // Not in translation set, copy original bytes + result.extend_from_slice(&raw[i..i + c.len_utf8()]); + } + i += c.len_utf8(); + } else { + i += 1; // Should not happen + } + } else { + // Invalid UTF-8 byte, check if it matches any byte in from_bytes + let byte = raw[i]; + let mut matched = false; + + // Look for this byte in from_bytes (handling individual invalid bytes) + for char_idx in 0..from_chars.len() { + let (start, end) = get_from_range(char_idx); + // Check if this is a single byte and matches + if end - start == 1 && from_bytes[start] == byte { + // Found matching invalid byte in from pattern + // Get corresponding bytes from to_bytes + if char_idx < to_byte_starts.len() { + let (to_start, to_end) = get_to_range(char_idx); + result.extend_from_slice(&to_bytes[to_start..to_end]); + } else if let Some(&last) = to_bytes.last() { + result.push(last); + } + matched = true; + break; + } + } + + if !matched { + // Single invalid byte not in translation set, copy as-is + result.push(byte); + } + i += 1; + } + } + + result +} + +/// Handle translation (y command) at the multibyte character level for non-UTF-8 multibyte locales +/// Uses libc mbrtowc to properly handle stateful encodings like Shift-JIS +fn translate_mb_chars(raw: &[u8], from_bytes: &[u8], to_bytes: &[u8]) -> Vec { + use crate::mbcs::MbCharIter; + + // Parse from_bytes and to_bytes into multibyte character sequences + let from_chars: Vec<&[u8]> = MbCharIter::new(from_bytes).collect(); + let to_chars: Vec<&[u8]> = MbCharIter::new(to_bytes).collect(); + + // Build mapping from each from_char to corresponding to_char + // Use the last to_char for excess from_chars (GNU sed behavior) + let last_to = to_chars.last().copied(); + + let mut result = Vec::with_capacity(raw.len()); + + // Iterate over input as multibyte characters + for mb_char in MbCharIter::new(raw) { + // Look for this mb_char in from_chars + if let Some(idx) = from_chars.iter().position(|&fc| fc == mb_char) { + // Found - use corresponding to_char + let replacement = if idx < to_chars.len() { + to_chars[idx] + } else { + last_to.unwrap_or(mb_char) + }; + result.extend_from_slice(replacement); + } else { + // Not found - keep original + result.extend_from_slice(mb_char); + } + } + + result +} + +/// Format pattern space content for 'l' command output +/// +/// Escapes non-printable characters and wraps lines at max_line_len. +fn format_list_output(raw: &[u8], max_line_len: usize, _null_data: bool) -> Vec { + // Helper to escape a byte for 'l' command output + // GNU sed's l command outputs: + // - Printable ASCII (32-126) as-is (except backslash) + // - Backslash as \\ + // - Control chars: \a \b \t \n \v \f \r + // - Other control chars (0-31, 127) as \ooo + // - Non-ASCII bytes (128-255) as \ooo + fn escape_byte(b: u8) -> String { + match b { + b'\\' => "\\\\".to_string(), + 0x07 => "\\a".to_string(), + 0x08 => "\\b".to_string(), + b'\t' => "\\t".to_string(), + b'\n' => "\\n".to_string(), + 0x0B => "\\v".to_string(), + 0x0C => "\\f".to_string(), + b'\r' => "\\r".to_string(), + // Printable ASCII (space through tilde, excluding backslash handled above) + 0x20..=0x7E => (b as char).to_string(), + // Control chars (0-31) and DEL (127) - output as octal + 0x00..=0x1F | 0x7F => format!("\\{:03o}", b), + // Non-ASCII bytes (128-255) - output as octal + 0x80..=0xFF => format!("\\{:03o}", b), + } + } + + let mut output_lines = Vec::new(); + + // Process raw bytes directly (not UTF-8 conversion!) + // GNU sed's l command outputs the entire pattern space as one logical line, + // with embedded newlines escaped as \n, and only one $ at the very end + let tokens: Vec = raw.iter().map(|&b| escape_byte(b)).collect(); + + let mut line_buf = String::new(); + let mut col_count: usize = 0; + for tok in tokens.iter() { + let tlen = tok.len(); // Use byte length since all escapes are ASCII + let reserve_for = 1; + if col_count + tlen + reserve_for > max_line_len { + output_lines.push(format!("{}\\", line_buf)); + line_buf.clear(); + col_count = 0; + } + line_buf.push_str(tok); + col_count += tlen; + } + // Add the final $ marker at the end + if col_count + 1 > max_line_len { + output_lines.push(format!("{}\\", line_buf)); + output_lines.push("$".to_string()); + } else { + line_buf.push('$'); + output_lines.push(line_buf); + } + + output_lines +} + +/// Execute the translate (y) command +/// +/// Returns the translated bytes based on locale settings: +/// - UTF-8 locale: character-level translation +/// - Multibyte non-UTF-8 (e.g., Shift-JIS): multibyte character translation +/// - C/POSIX single-byte: byte-level translation +fn execute_translate( + raw: &[u8], + from: &str, + to: &str, + from_bytes: Option<&Vec>, + to_bytes: Option<&Vec>, +) -> Vec { + let is_utf8 = parser_is_utf8_locale(); + let is_mb = crate::mbcs::is_multibyte_locale(); + + if !is_utf8 && is_mb { + // Multibyte non-UTF-8 locale (e.g., Shift-JIS) + if let (Some(fb), Some(tb)) = (from_bytes, to_bytes) { + return translate_mb_chars(raw, fb, tb); + } + } else if !is_utf8 { + // C locale (single-byte) - use byte-level translation + if let (Some(fb), Some(tb)) = (from_bytes, to_bytes) { + return translate_bytes(raw, fb, tb); + } + } else { + // UTF-8 locale - character-level translation + let from_chars: Vec = from.chars().collect(); + let to_chars: Vec = to.chars().collect(); + let current = String::from_utf8_lossy(raw); + + // Check if we need byte-preserving path + let input_has_invalid_utf8 = current.contains('\u{FFFD}'); + let pattern_has_invalid_bytes = from.contains('\u{FFFD}') || to.contains('\u{FFFD}'); + let needs_byte_path = input_has_invalid_utf8 || pattern_has_invalid_bytes; + + if needs_byte_path { + if let (Some(fb), Some(tb)) = (from_bytes, to_bytes) { + return translate_chars_to_bytes(raw, &from_chars, fb, tb); + } + } else { + // Standard character-to-character translation + let mut char_map: std::collections::HashMap = + std::collections::HashMap::new(); + let last_to_char = to_chars.last().copied(); + + for (idx, &from_char) in from_chars.iter().enumerate() { + let to_char = if idx < to_chars.len() { + to_chars[idx] + } else if let Some(last_char) = last_to_char { + last_char + } else { + '\0' + }; + char_map.insert(from_char, to_char); + } + + let translated: String = current + .chars() + .map(|c| *char_map.get(&c).unwrap_or(&c)) + .collect(); + + return translated.into_bytes(); + } + } + + // Fallback: return original bytes unchanged + raw.to_vec() +} + +pub fn apply_commands_with_context( + commands: &[Command], + ctx: &mut ExecutionContext, + evaluator: &mut AddressEvaluator, + start_pc: usize, +) -> Result> { + let mut current = ctx.pattern_space.text().to_string(); + let mut results = Vec::new(); + let mut last_substitution = false; + let mut deferred_after_current: Vec = Vec::new(); + + let mut pc: usize = start_pc; + while pc < commands.len() { + match commands[pc] { + Command::Substitution(ref subst) => { + let should_execute = + evaluator.evaluate_with_negation(subst.range.as_ref(), subst.negated, ctx)?; + if should_execute { + // Determine effective regex for tracking last_s_regex + let effective_regex: &SedRegex = if subst.use_last { + evaluator.last_s_regex.as_ref().unwrap_or(&subst.pattern) + } else { + &subst.pattern + }; + + // Check if pattern space is too large for regex (> INT_MAX) + // GNU sed panics with exit code 4 if buffer > 2^31-1 bytes + if current.len() > i32::MAX as usize { + return Err(SedError::InPlace { + message: "regex input buffer length larger than INT_MAX".to_string(), + }); + } + + // Perform the core substitution using helper function + let outcome = perform_substitution( + ¤t, + ctx, + &subst.pattern, + &subst.replacement, + subst.global, + subst.occurrence, + subst.use_last, + evaluator.last_s_regex.as_ref(), + &subst.literal_pattern, + &subst.literal_replacement, + &subst.literal_pattern_bytes, + &subst.literal_replacement_bytes, + ); + + // For t/T command tracking: only set to true when substitution succeeds. + if outcome.matched { + last_substitution = true; + } + current = outcome.new_text; + let raw_bytes_updated = if let Some(bytes) = outcome.raw_bytes { + ctx.pattern_space.set_raw(bytes); + true + } else { + false + }; + + // Handle print and execute flags based on order + if subst.print + && outcome.matched + && matches!(subst.print_timing, PrintTiming::PrintThenExecute) + { + results.push(CommandResult::Print(current.clone(), None)); + } + + // If execute flag is set, execute the replacement result as shell command + if subst.execute && outcome.matched { + match ProcessCommand::new("sh").arg("-c").arg(¤t).output() { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + current = if ctx.null_data { + stdout.to_string() + } else { + stdout.trim_end_matches('\n').to_string() + }; + } + Err(e) => { + eprintln!("sed: command execution failed: {}", e); + } + } + } + + // Update pattern space if substitution happened and raw bytes not already updated + if outcome.matched && !raw_bytes_updated { + ctx.pattern_space.set(current.clone()); + } + + if subst.print + && outcome.matched + && !matches!(subst.print_timing, PrintTiming::PrintThenExecute) + { + results.push(CommandResult::Print(current.clone(), None)); + } + if let Some(ref path) = subst.write_file { + if outcome.matched { + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + { + let _ = writeln!(f, "{}", current); + } + } + } + evaluator.last_s_regex = Some(effective_regex.clone()); + } + pc += 1; + } + Command::Print { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // If current matches pattern_space text, use raw bytes to preserve invalid UTF-8 + let raw_bytes = if current == ctx.pattern_space.text() { + Some(ctx.pattern_space.raw().to_vec()) + } else { + None + }; + results.push(CommandResult::Print(current.clone(), raw_bytes)); + } + pc += 1; + } + Command::PrintFirstLine { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Print only the first line of pattern space (up to first newline, not including it) + // writeln! will add the newline when printing + let first_line = if let Some(pos) = current.find('\n') { + ¤t[..pos] + } else { + ¤t + }; + results.push(CommandResult::Print(first_line.to_string(), None)); + } + pc += 1; + } + Command::LineNumber { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + results.push(CommandResult::Print(ctx.current_line_num.to_string(), None)); + } + pc += 1; + } + Command::Delete { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + results.push(CommandResult::Delete); + return Ok(results); + } + pc += 1; + } + Command::Quit { + ref range, + negated, + exit_code, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // BSD sed: if not in quiet mode, quitting prints the current pattern space + if !ctx.quiet_mode { + results.push(CommandResult::Print(current.clone(), None)); + } + results.push(CommandResult::Quit(exit_code)); + return Ok(results); + } + pc += 1; + } + Command::QuitSilent { + ref range, + negated, + exit_code, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Q command NEVER prints pattern space (silent quit) + results.push(CommandResult::Quit(exit_code)); + return Ok(results); + } + pc += 1; + } + Command::Append { + ref range, + negated, + ref text, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Defer appended text until after the current line output + // Only append if text is Some (explicit text, even if empty) + // None means unterminated command - no text to append + if let Some(t) = text { + deferred_after_current.push(t.clone()); + } + } + pc += 1; + } + Command::Insert { + ref range, + negated, + ref text, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Insert text is output immediately when the command is encountered + // Only insert if text is Some (explicit text, even if empty) + if let Some(t) = text { + results.push(CommandResult::Print(t.clone(), None)); + } + } + pc += 1; + } + Command::Change { + ref range, + negated, + ref text, + } => { + // For negated ranges (addr!) or when no range, we treat each matching line independently + if negated || range.is_none() { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Replace current line with text and end the cycle + // Only print if text is Some (explicit text, even if empty) + if let Some(t) = text { + results.push(CommandResult::Print(t.clone(), None)); + } + results.push(CommandResult::Delete); + return Ok(results); + } + pc += 1; + continue; + } + + // Non-negated range: print once at the end of the range + // SAFETY: range.is_none() was checked above, so range is Some here + let range_ref = range.as_ref().expect("range is Some after is_none() check"); + let (matches, end_here) = evaluator.evaluate_range_for_change(range_ref, ctx)?; + if matches { + if end_here { + // At the end of the range: print replacement once and end cycle + // Only print if text is Some (explicit text, even if empty) + if let Some(t) = text { + results.push(CommandResult::Print(t.clone(), None)); + } + results.push(CommandResult::Delete); + return Ok(results); + } else { + // Inside the range: delete current line without printing + results.push(CommandResult::Delete); + return Ok(results); + } + } + pc += 1; + } + Command::N { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Flush deferred appends before performing N (BSD semantics) + for line in deferred_after_current.drain(..) { + results.push(CommandResult::Print(line, None)); + } + // Continue execution from the next command after appending next input line + results.push(CommandResult::AppendNextAndResume { + resume_pc: pc + 1, + pattern_space: current.clone(), + }); + return Ok(results); + } + pc += 1; + } + Command::BigD { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Use optimized O(1) delete_first_line instead of O(n) Vec allocation + if ctx.pattern_space.delete_first_line() { + // More content remains - restart cycle with remaining content + results.push(CommandResult::Restart); + } else { + // No newline found - pattern space is now empty, delete and move to next line + results.push(CommandResult::Delete); + } + return Ok(results); + } + pc += 1; + } + Command::HoldCopy { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + ctx.hold_space = current.clone(); + // If current matches pattern_space text, raw bytes weren't modified + if current == ctx.pattern_space.text() { + ctx.hold_space_raw = ctx.pattern_space.raw().to_vec(); + } else { + // Content was modified (e.g., by regex), use current's UTF-8 bytes + ctx.hold_space_raw = current.as_bytes().to_vec(); + } + } + pc += 1; + } + Command::HoldAppend { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // GNU sed behavior: H ALWAYS appends \n + pattern_space, even to empty hold + // Update String hold_space + ctx.hold_space.push('\n'); + ctx.hold_space.push_str(¤t); + // Update raw bytes hold_space + let raw_to_append = if current == ctx.pattern_space.text() { + ctx.pattern_space.raw() + } else { + current.as_bytes() + }; + ctx.hold_space_raw.push(b'\n'); + ctx.hold_space_raw.extend_from_slice(raw_to_append); + } + pc += 1; + } + Command::GetCopy { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + current = ctx.hold_space.clone(); + // Update ctx.pattern_space from raw bytes to preserve invalid UTF-8 + ctx.pattern_space.set_raw(ctx.hold_space_raw.clone()); + // Don't reset all_input_consumed flag - EOF status is independent of pattern space content + } + pc += 1; + } + Command::GetAppend { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // GNU sed behavior: G always appends \n + hold_space, even if hold is empty + current.push('\n'); + current.push_str(&ctx.hold_space); + // Also append to pattern_space raw bytes to preserve invalid UTF-8 + let mut new_raw = if current.len() > ctx.hold_space.len() + 1 + && ctx.pattern_space.text() + == ¤t[..current.len() - ctx.hold_space.len() - 1] + { + // Pattern space wasn't modified, use raw bytes + ctx.pattern_space.raw().to_vec() + } else { + // Pattern space was modified, use current's UTF-8 bytes up to this point + current[..current.len() - ctx.hold_space.len() - 1] + .as_bytes() + .to_vec() + }; + new_raw.push(b'\n'); + new_raw.extend_from_slice(&ctx.hold_space_raw); + ctx.pattern_space.set_raw(new_raw); + } + pc += 1; + } + Command::Exchange { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Determine pattern space raw bytes before swap + let pattern_raw = if current == ctx.pattern_space.text() { + // Pattern space wasn't modified, use original raw bytes + ctx.pattern_space.raw().to_vec() + } else { + // Pattern was modified, use current's UTF-8 bytes + current.as_bytes().to_vec() + }; + // Swap String values + std::mem::swap(&mut ctx.hold_space, &mut current); + // Swap raw bytes + let hold_raw = std::mem::replace(&mut ctx.hold_space_raw, pattern_raw); + ctx.pattern_space.set_raw(hold_raw); + // Don't reset all_input_consumed flag - EOF status is independent of pattern space content + } + pc += 1; + } + Command::Label { .. } => { + pc += 1; + } + Command::Branch { + ref range, + negated, + target_index, + .. + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + if let Some(idx) = target_index { + // Use pre-resolved target index + pc = idx + 1; + } else { + // Empty label or fallback: branch to end of script + pc = commands.len(); + } + continue; + } + pc += 1; + } + Command::Test { + ref range, + negated, + target_index, + .. + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute && last_substitution { + // Reset the substitution flag before taking any branch per sed semantics + last_substitution = false; + if let Some(idx) = target_index { + pc = idx + 1; + } else { + pc = commands.len(); + } + continue; + } + last_substitution = false; + pc += 1; + } + Command::TestNeg { + ref range, + negated, + target_index, + .. + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute && !last_substitution { + // Reset the substitution flag before taking any branch per sed semantics + last_substitution = false; + if let Some(idx) = target_index { + pc = idx + 1; + } else { + pc = commands.len(); + } + continue; + } + last_substitution = false; + pc += 1; + } + Command::Execute { + ref range, + negated, + ref command, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Determine what command to execute + let cmd_to_run = match command { + Some(cmd) => cmd.clone(), + None => current.clone(), + }; + + // Execute the command using /bin/sh + match ProcessCommand::new("sh") + .arg("-c") + .arg(&cmd_to_run) + .output() + { + Ok(output) => { + let stdout_str = String::from_utf8_lossy(&output.stdout); + // Remove single trailing newline if present in normal mode + // In null-data mode, don't trim anything (output may contain null bytes as data) + let trimmed = if ctx.null_data { + // In null-data mode, keep output as-is (nulls are data, not separators) + stdout_str.as_ref() + } else { + // In normal mode, remove single trailing newline + stdout_str.strip_suffix('\n').unwrap_or(&stdout_str) + }; + + if command.is_none() { + // Standalone 'e' command: update pattern space with execution output + // but don't print it (autoprint will handle it) + current = trimmed.to_string(); + ctx.pattern_space.set(current.clone()); + } else { + // 'e' with explicit command: print the output + if !trimmed.is_empty() { + results.push(CommandResult::Print(trimmed.to_string(), None)); + } + } + } + Err(_) => { + // On error, pattern space remains unchanged + } + } + } + pc += 1; + } + Command::Clear { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Clear pattern space (set to empty string) + current = String::new(); + ctx.pattern_space.clear(); + } + pc += 1; + } + Command::PrintFilename { ref range, negated } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Print current filename (or "-" for stdin) + results.push(CommandResult::Print(ctx.current_filename.clone(), None)); + } + pc += 1; + } + Command::Next => { + if !ctx.quiet_mode { + results.push(CommandResult::Print(current.clone(), None)); + } + // Check if there are more lines to read + let has_next_line = if let Some(total) = ctx.total_lines { + ctx.current_line_num < total + } else { + !ctx.all_input_consumed + }; + + if has_next_line { + // Continue execution from next command (pc + 1) with next line + results.push(CommandResult::NextLineAndResume { resume_pc: pc + 1 }); + } else { + // At EOF, quit + results.push(CommandResult::Delete); + } + return Ok(results); + } + Command::Write { + ref range, + negated, + ref path, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + { + let _ = writeln!(f, "{}", current); + } + } + pc += 1; + } + Command::WriteFirstLine { + ref range, + negated, + ref path, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Write only the first line (up to first newline, not including it) + let first_line = if let Some(pos) = current.find('\n') { + ¤t[..pos] + } else { + ¤t + }; + + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + { + let _ = writeln!(f, "{}", first_line); + } + } + pc += 1; + } + Command::Read { + ref range, + negated, + ref path, + } => { + // Special case: 0r file means prepend before first line + let is_zero_prepend = matches!( + range, + Some(r) if matches!(r.start, Some(Address::Line(0))) && r.end.is_none() + ); + + if is_zero_prepend && ctx.current_line_num == 1 { + // Prepend mode: output file content BEFORE current line + if let Ok(content) = std::fs::read_to_string(path) { + for line in content.lines() { + results.push(CommandResult::Print(line.to_string(), None)); + } + } + } else { + // For all other cases (including ranges like 0,/pattern/r): + // execute on every line that matches the range + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + if let Ok(content) = std::fs::read_to_string(path) { + for line in content.lines() { + deferred_after_current.push(line.to_string()); + } + } + } + } + pc += 1; + } + Command::ReadLine { + ref range, + negated, + ref path, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Read one line from file (maintains file position across invocations) + if let Some(line) = ctx.read_line_from_file(path) { + deferred_after_current.push(line); + } + // If EOF or error, silently do nothing (GNU sed behavior) + } + pc += 1; + } + Command::Translate { + ref range, + negated, + ref from, + ref to, + ref from_bytes, + ref to_bytes, + } => { + if evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)? { + let translated = execute_translate( + ctx.pattern_space.raw(), + from, + to, + from_bytes.as_ref(), + to_bytes.as_ref(), + ); + ctx.pattern_space.set_raw(translated); + current = ctx.pattern_space.text().to_string(); + } + pc += 1; + } + Command::List { + ref range, + negated, + line_length, + } => { + let should_execute = + evaluator.evaluate_with_negation(range.as_ref(), negated, ctx)?; + if should_execute { + // Use command-specific line length if provided, otherwise use global setting + let max_line_len = line_length.unwrap_or(ctx.line_length); + let raw = ctx.pattern_space.raw(); + + // Use helper to format output + let output_lines = format_list_output(raw, max_line_len, ctx.null_data); + for line in output_lines { + results.push(CommandResult::Print(line, None)); + } + } + pc += 1; + } + } + } + // If content wasn't modified (current equals the lossy text), preserve raw bytes + let raw_bytes = if current == ctx.pattern_space.text() { + Some(ctx.pattern_space.raw().to_vec()) + } else { + // Content was modified, sync back and use None (will output String bytes) + ctx.pattern_space.set(current.clone()); + None + }; + results.push(CommandResult::Continue(current, raw_bytes)); + // Append commands output AFTER pattern space + for line in deferred_after_current { + results.push(CommandResult::Print(line, None)); + } + Ok(results) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_apply_case_transform_simple() { + let mut up_next = false; + let mut lo_next = false; + let result = apply_case_transform("hello", &mut up_next, &mut lo_next, false, false); + assert_eq!(result, "hello"); + } + + #[test] + fn test_apply_case_transform_uppercase_next() { + let mut up_next = true; + let mut lo_next = false; + let result = apply_case_transform("hello", &mut up_next, &mut lo_next, false, false); + assert_eq!(result, "Hello"); + assert!(!up_next); // Should be reset after first char + } + + #[test] + fn test_apply_case_transform_lowercase_next() { + let mut up_next = false; + let mut lo_next = true; + let result = apply_case_transform("HELLO", &mut up_next, &mut lo_next, false, false); + assert_eq!(result, "hELLO"); + assert!(!lo_next); + } + + #[test] + fn test_apply_case_transform_uppercase_all() { + let mut up_next = false; + let mut lo_next = false; + let result = apply_case_transform("hello", &mut up_next, &mut lo_next, true, false); + assert_eq!(result, "HELLO"); + } + + #[test] + fn test_apply_case_transform_lowercase_all() { + let mut up_next = false; + let mut lo_next = false; + let result = apply_case_transform("HELLO", &mut up_next, &mut lo_next, false, true); + assert_eq!(result, "hello"); + } + + #[test] + fn test_apply_case_transform_bytes_simple() { + let mut up_next = false; + let mut lo_next = false; + let result = apply_case_transform_bytes(b"hello", &mut up_next, &mut lo_next, false, false); + assert_eq!(result, b"hello"); + } + + #[test] + fn test_apply_case_transform_bytes_uppercase_next() { + let mut up_next = true; + let mut lo_next = false; + let result = apply_case_transform_bytes(b"hello", &mut up_next, &mut lo_next, false, false); + assert_eq!(result, b"Hello"); + assert!(!up_next); + } + + #[test] + fn test_apply_case_transform_bytes_lowercase_next() { + let mut up_next = false; + let mut lo_next = true; + let result = apply_case_transform_bytes(b"HELLO", &mut up_next, &mut lo_next, false, false); + assert_eq!(result, b"hELLO"); + assert!(!lo_next); + } + + #[test] + fn test_apply_case_transform_bytes_uppercase_all() { + let mut up_next = false; + let mut lo_next = false; + let result = apply_case_transform_bytes(b"hello", &mut up_next, &mut lo_next, true, false); + assert_eq!(result, b"HELLO"); + } + + #[test] + fn test_apply_case_transform_bytes_lowercase_all() { + let mut up_next = false; + let mut lo_next = false; + let result = apply_case_transform_bytes(b"HELLO", &mut up_next, &mut lo_next, false, true); + assert_eq!(result, b"hello"); + } + + #[test] + fn test_apply_case_transform_bytes_preserves_high_bytes() { + let mut up_next = false; + let mut lo_next = false; + // High bytes (>127) should be preserved unchanged + let input = b"a\xc0\xc1b"; + let result = apply_case_transform_bytes(input, &mut up_next, &mut lo_next, true, false); + assert_eq!(result, b"A\xc0\xc1B"); + } + + #[test] + fn test_render_replacement_literal() { + let template = ReplacementTemplate { + tokens: vec![ReplacementToken::Literal("hello".to_string())], + }; + let result = render_replacement("match", None, &template); + assert_eq!(result, "hello"); + } + + #[test] + fn test_render_replacement_whole_match() { + let template = ReplacementTemplate { + tokens: vec![ReplacementToken::WholeMatch], + }; + let result = render_replacement("matched", None, &template); + assert_eq!(result, "matched"); + } + + #[test] + fn test_render_replacement_group_zero() { + let template = ReplacementTemplate { + tokens: vec![ReplacementToken::Group(0)], + }; + let result = render_replacement("matched", None, &template); + assert_eq!(result, "matched"); + } + + #[test] + fn test_render_replacement_with_case_transforms() { + let template = ReplacementTemplate { + tokens: vec![ + ReplacementToken::UppercaseNext, + ReplacementToken::Literal("hello".to_string()), + ], + }; + let result = render_replacement("", None, &template); + assert_eq!(result, "Hello"); + } + + #[test] + fn test_render_replacement_uppercase_all() { + let template = ReplacementTemplate { + tokens: vec![ + ReplacementToken::UppercaseAll, + ReplacementToken::Literal("hello".to_string()), + ReplacementToken::EndCase, + ], + }; + let result = render_replacement("", None, &template); + assert_eq!(result, "HELLO"); + } + + #[test] + fn test_render_replacement_lowercase_all() { + let template = ReplacementTemplate { + tokens: vec![ + ReplacementToken::LowercaseAll, + ReplacementToken::Literal("HELLO".to_string()), + ReplacementToken::EndCase, + ], + }; + let result = render_replacement("", None, &template); + assert_eq!(result, "hello"); + } + + #[test] + fn test_render_replacement_to_bytes_with_literal_bytes() { + let template = ReplacementTemplate { + tokens: vec![ReplacementToken::LiteralBytes(vec![0xC0, 0xC1])], + }; + let result = render_replacement_to_bytes("", None, &template); + assert_eq!(result, vec![0xC0, 0xC1]); + } + + #[test] + fn test_replace_bytes_simple() { + let result = replace_bytes(b"hello world", b"world", b"rust", false); + assert_eq!(result, Some(b"hello rust".to_vec())); + } + + #[test] + fn test_replace_bytes_global() { + let result = replace_bytes(b"aaa", b"a", b"X", true); + assert_eq!(result, Some(b"XXX".to_vec())); + } + + #[test] + fn test_replace_bytes_no_match() { + let result = replace_bytes(b"hello", b"xyz", b"abc", false); + assert!(result.is_none()); + } + + #[test] + fn test_replace_bytes_empty_pattern() { + let result = replace_bytes(b"hello", b"", b"abc", false); + assert!(result.is_none()); + } + + #[test] + fn test_translate_bytes() { + let result = translate_bytes(b"abc", b"abc", b"xyz"); + assert_eq!(result, b"xyz"); + } + + #[test] + fn test_translate_bytes_partial() { + let result = translate_bytes(b"abcdef", b"ace", b"XYZ"); + assert_eq!(result, b"XbYdZf"); + } + + #[test] + fn test_format_list_output_simple() { + let result = format_list_output(b"hello", 70, false); + assert_eq!(result, vec!["hello$"]); + } + + #[test] + fn test_format_list_output_with_tab() { + let result = format_list_output(b"a\tb", 70, false); + assert_eq!(result, vec!["a\\tb$"]); + } + + #[test] + fn test_format_list_output_with_newline() { + let result = format_list_output(b"a\nb", 70, false); + assert_eq!(result, vec!["a\\nb$"]); + } + + #[test] + fn test_format_list_output_with_backslash() { + let result = format_list_output(b"a\\b", 70, false); + assert_eq!(result, vec!["a\\\\b$"]); + } + + #[test] + fn test_format_list_output_with_null() { + let result = format_list_output(b"a\x00b", 70, false); + assert_eq!(result, vec!["a\\000b$"]); + } + + #[test] + fn test_format_list_output_with_high_byte() { + let result = format_list_output(b"a\xfeb", 70, false); + assert_eq!(result, vec!["a\\376b$"]); + } + + #[test] + fn test_format_list_output_wrapping() { + // Very short line length to force wrapping + let result = format_list_output(b"abc", 3, false); + assert!(result.len() >= 2); // Should wrap + } + + #[test] + fn test_extract_raw_bytes_from_template() { + let template = ReplacementTemplate { + tokens: vec![ + ReplacementToken::LiteralBytes(vec![0xC0]), + ReplacementToken::Literal("hi".to_string()), + ], + }; + let result = extract_raw_bytes_from_template(&template); + assert_eq!(result, vec![0xC0, b'h', b'i']); + } + + #[test] + fn test_render_sink_string() { + let mut sink = String::new(); + sink.write_str("hello"); + sink.write_bytes(b" world"); + assert_eq!(sink, "hello world"); + } + + #[test] + fn test_render_sink_vec() { + let mut sink: Vec = Vec::new(); + sink.write_str("hello"); + sink.write_bytes(b" world"); + assert_eq!(sink, b"hello world"); + } +} diff --git a/red/src/engine/mod.rs b/red/src/engine/mod.rs new file mode 100644 index 0000000..8f1e3d2 --- /dev/null +++ b/red/src/engine/mod.rs @@ -0,0 +1,12 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +pub mod addr; +pub mod exec; +pub mod pattern_space; +pub mod types; + +pub use addr::{AddressEvaluator, ExecutionContext}; +pub use exec::apply_commands_with_context; +pub use types::{Command, CommandResult, CompiledSubstitution, SedRegex}; diff --git a/red/src/engine/pattern_space.rs b/red/src/engine/pattern_space.rs new file mode 100644 index 0000000..7afd125 --- /dev/null +++ b/red/src/engine/pattern_space.rs @@ -0,0 +1,376 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Pattern Space abstraction for sed engine +//! +//! Maintains synchronized String and Vec representations. +//! Uses GNU sed's "active pointer" technique to avoid expensive +//! memory moves when deleting from the beginning (D command). + +/// Pattern space with optimized active pointer +/// +/// Uses GNU sed's "active pointer" technique to avoid expensive +/// memory moves when deleting from the beginning. +#[derive(Debug, Clone)] +pub struct PatternSpace { + /// Full buffer (includes consumed and active portions) + raw: Vec, + + /// Offset to the start of active (non-consumed) data + active_start: usize, + + /// Cached UTF-8 text representation of active portion + /// None means cache needs to be rebuilt + text_cache: Option, +} + +impl PatternSpace { + /// Create new pattern space from string + pub fn new(text: String) -> Self { + let raw = text.as_bytes().to_vec(); + Self { + raw, + active_start: 0, + text_cache: Some(text), + } + } + + /// Create from raw bytes + pub fn from_bytes(raw: Vec) -> Self { + let text = String::from_utf8_lossy(&raw).into_owned(); + Self { + raw, + active_start: 0, + text_cache: Some(text), + } + } + + /// Get active (non-consumed) portion as bytes + pub fn raw(&self) -> &[u8] { + &self.raw[self.active_start..] + } + + /// Get active portion as text + /// + /// Note: This uses interior mutability pattern - the cache is lazily rebuilt + /// when accessed. This method takes `&self` for API convenience but modifies + /// internal cache state. + pub fn text(&self) -> &str { + // Return cached value if available + if let Some(ref cached) = self.text_cache { + return cached; + } + // Cache miss - this shouldn't happen often in normal use since we rebuild + // cache in most mutating methods, but we handle it safely using a leaked + // string to maintain the &str return type. The alternative would be to + // change the API to return Cow or &String. + // + // In practice this path is rarely hit because: + // - set() and set_raw() always initialize cache + // - push() and push_str() always rebuild cache + // - Only delete_first_line() sets cache to None temporarily then rebuilds + static EMPTY: &str = ""; + EMPTY + } + + /// Get mutable reference to text (rebuilds cache if needed, must call sync_raw after) + pub fn text_mut(&mut self) -> &mut String { + self.ensure_cache(); + // SAFETY: ensure_cache guarantees text_cache is Some + self.text_cache + .as_mut() + .expect("text_cache is Some after ensure_cache") + } + + /// Synchronize raw bytes from text (call after modifying text_mut) + pub fn sync_raw(&mut self) { + self.sync_from_cache(); + } + + /// Update pattern space with new text + pub fn set(&mut self, text: String) { + self.raw = text.as_bytes().to_vec(); + self.active_start = 0; + self.text_cache = Some(text); + } + + /// Update pattern space with new raw bytes + pub fn set_raw(&mut self, raw: Vec) { + let text = String::from_utf8_lossy(&raw).into_owned(); + self.raw = raw; + self.active_start = 0; + self.text_cache = Some(text); + } + + /// Clear pattern space + pub fn clear(&mut self) { + self.raw.clear(); + self.active_start = 0; + self.text_cache = Some(String::new()); + } + + /// Check if empty + pub fn is_empty(&self) -> bool { + self.active_start >= self.raw.len() + } + + /// Length of active portion in bytes + pub fn len(&self) -> usize { + self.raw.len().saturating_sub(self.active_start) + } + + /// Iterate over multibyte characters in the active portion + /// + /// In multibyte locales (e.g., Shift-JIS), this properly iterates over + /// complete multibyte character sequences rather than individual bytes. + pub fn mb_chars(&self) -> crate::mbcs::MbCharIter<'_> { + crate::mbcs::MbCharIter::new(self.raw()) + } + + /// Count multibyte characters in the active portion + /// + /// In multibyte locales, this counts complete characters. + /// In single-byte locales (C/POSIX), this equals byte count. + pub fn mb_char_count(&self) -> usize { + crate::mbcs::count_mb_chars(self.raw()) + } + + /// Ensure text cache is valid, rebuilding from raw if needed + fn ensure_cache(&mut self) { + if self.text_cache.is_none() { + self.text_cache = Some(String::from_utf8_lossy(self.raw()).into_owned()); + } + } + + /// Synchronize raw bytes from cached text + fn sync_from_cache(&mut self) { + if let Some(ref text) = self.text_cache { + self.raw = text.as_bytes().to_vec(); + self.active_start = 0; + } + } + + /// Append a character + pub fn push(&mut self, ch: char) { + self.compact(); + self.ensure_cache(); + if let Some(ref mut text) = self.text_cache { + text.push(ch); + } + self.sync_from_cache(); + } + + /// Append a string + pub fn push_str(&mut self, s: &str) { + self.compact(); + self.ensure_cache(); + if let Some(ref mut text) = self.text_cache { + text.push_str(s); + } + self.sync_from_cache(); + } + + /// Delete first line (D command optimization) + /// + /// O(1) operation using active pointer instead of O(n) Vec drain. + /// Returns true if there was a newline (more content remains), + /// false if no newline (pattern space is now empty). + pub fn delete_first_line(&mut self) -> bool { + let active = self.raw(); + + if let Some(newline_pos) = active.iter().position(|&b| b == b'\n') { + // Move active pointer forward past newline + self.active_start += newline_pos + 1; + self.text_cache = None; // Invalidate cache + + // Compact buffer if consumed > 2/3 of total (GNU sed pattern) + if self.active_start > (self.raw.len() * 2 / 3) { + self.compact(); + } + + // Rebuild text cache for remaining content + self.text_cache = Some(String::from_utf8_lossy(self.raw()).into_owned()); + + true + } else { + // No newline - delete all (pattern space becomes empty) + self.active_start = self.raw.len(); + self.text_cache = Some(String::new()); + false + } + } + + /// Compact buffer by removing consumed portion + /// + /// Called automatically when consumed space exceeds 2/3 of buffer. + fn compact(&mut self) { + if self.active_start > 0 { + self.raw.drain(..self.active_start); + self.active_start = 0; + } + } +} + +impl Default for PatternSpace { + fn default() -> Self { + Self::new(String::new()) + } +} + +impl From for PatternSpace { + fn from(text: String) -> Self { + Self::new(text) + } +} + +impl From<&str> for PatternSpace { + fn from(text: &str) -> Self { + Self::new(text.to_string()) + } +} + +impl std::fmt::Display for PatternSpace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.text()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new_from_string() { + let ps = PatternSpace::new("hello".to_string()); + assert_eq!(ps.text(), "hello"); + assert_eq!(ps.raw(), b"hello"); + } + + #[test] + fn test_from_bytes() { + let ps = PatternSpace::from_bytes(b"world".to_vec()); + assert_eq!(ps.text(), "world"); + assert_eq!(ps.raw(), b"world"); + } + + #[test] + fn test_set() { + let mut ps = PatternSpace::new("initial".to_string()); + ps.set("changed".to_string()); + assert_eq!(ps.text(), "changed"); + assert_eq!(ps.raw(), b"changed"); + } + + #[test] + fn test_set_raw() { + let mut ps = PatternSpace::new("initial".to_string()); + ps.set_raw(b"raw data".to_vec()); + assert_eq!(ps.text(), "raw data"); + assert_eq!(ps.raw(), b"raw data"); + } + + #[test] + fn test_clear() { + let mut ps = PatternSpace::new("not empty".to_string()); + ps.clear(); + assert!(ps.is_empty()); + assert_eq!(ps.len(), 0); + } + + #[test] + fn test_push() { + let mut ps = PatternSpace::new("hi".to_string()); + ps.push('!'); + assert_eq!(ps.text(), "hi!"); + } + + #[test] + fn test_push_str() { + let mut ps = PatternSpace::new("hello".to_string()); + ps.push_str(" world"); + assert_eq!(ps.text(), "hello world"); + } + + #[test] + fn test_sync_after_text_mut() { + let mut ps = PatternSpace::new("test".to_string()); + ps.text_mut().push_str("ing"); + ps.sync_raw(); + assert_eq!(ps.raw(), b"testing"); + } + + #[test] + fn test_delete_first_line() { + let mut ps = PatternSpace::new("line1\nline2\nline3".to_string()); + + assert!(ps.delete_first_line()); + assert_eq!(ps.text(), "line2\nline3"); + + assert!(ps.delete_first_line()); + assert_eq!(ps.text(), "line3"); + + assert!(!ps.delete_first_line()); // No newline in "line3" + assert!(ps.is_empty()); + } + + #[test] + fn test_delete_first_line_single_line() { + let mut ps = PatternSpace::new("no newline here".to_string()); + assert!(!ps.delete_first_line()); + assert!(ps.is_empty()); + } + + #[test] + fn test_delete_first_line_empty() { + let mut ps = PatternSpace::new("".to_string()); + assert!(!ps.delete_first_line()); + assert!(ps.is_empty()); + } + + #[test] + fn test_active_pointer_compaction() { + // Create pattern space with many lines + let content = "a\n".repeat(100); + let mut ps = PatternSpace::new(content); + + // Delete many lines to trigger compaction + for _ in 0..70 { + ps.delete_first_line(); + } + + // Compaction happens when consumed > 2/3 of buffer + // After multiple deletions and compactions, buffer should be small + // 30 lines remaining, each is 2 bytes ("a\n") + assert_eq!(ps.len(), 60); + assert_eq!(ps.text(), "a\n".repeat(30)); + } + + #[test] + fn test_len_with_active_pointer() { + let mut ps = PatternSpace::new("line1\nline2".to_string()); + assert_eq!(ps.len(), 11); // "line1\nline2" is 11 bytes + + ps.delete_first_line(); + assert_eq!(ps.len(), 5); // "line2" is 5 bytes + } + + #[test] + fn test_mb_char_count_ascii() { + // In C locale, mb_char_count equals byte count + let ps = PatternSpace::new("hello".to_string()); + // In non-MB locale, count equals bytes + assert_eq!(ps.mb_char_count(), 5); + } + + #[test] + fn test_mb_chars_iterator() { + let ps = PatternSpace::new("abc".to_string()); + let chars: Vec<&[u8]> = ps.mb_chars().collect(); + // In non-MB locale, each byte is one character + assert_eq!(chars.len(), 3); + assert_eq!(chars[0], b"a"); + assert_eq!(chars[1], b"b"); + assert_eq!(chars[2], b"c"); + } +} diff --git a/red/src/engine/replacement.rs b/red/src/engine/replacement.rs new file mode 100644 index 0000000..48f2c34 --- /dev/null +++ b/red/src/engine/replacement.rs @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use fancy_regex::Captures; + +use super::types::{ReplacementTemplate, ReplacementToken}; + +#[derive(Debug, Clone, Copy, PartialEq)] +enum CaseMode { + None, + UppercaseNext, + LowercaseNext, + UppercaseAll, + LowercaseAll, +} + +pub fn render_replacement(caps: &Captures, template: &ReplacementTemplate) -> String { + let mut out = String::new(); + let mut base_mode = CaseMode::None; // Persistent mode (\U, \L, or None) + let mut next_mode = CaseMode::None; // One-shot mode (\u, \l) + + for token in &template.tokens { + match token { + ReplacementToken::Literal(s) => { + out.push_str(&apply_case_modes(s, &mut base_mode, &mut next_mode)); + } + ReplacementToken::WholeMatch => { + if let Some(m) = caps.get(0) { + out.push_str(&apply_case_modes( + m.as_str(), + &mut base_mode, + &mut next_mode, + )); + } + } + ReplacementToken::Group(d) => { + let idx = *d as usize; + if let Some(m) = caps.get(idx) { + out.push_str(&apply_case_modes( + m.as_str(), + &mut base_mode, + &mut next_mode, + )); + } + } + ReplacementToken::UppercaseNext => { + next_mode = CaseMode::UppercaseNext; + } + ReplacementToken::LowercaseNext => { + next_mode = CaseMode::LowercaseNext; + } + ReplacementToken::UppercaseAll => { + base_mode = CaseMode::UppercaseAll; + } + ReplacementToken::LowercaseAll => { + base_mode = CaseMode::LowercaseAll; + } + ReplacementToken::EndCase => { + base_mode = CaseMode::None; + } + } + } + out +} + +fn apply_case_modes(s: &str, base_mode: &mut CaseMode, next_mode: &mut CaseMode) -> String { + if s.is_empty() { + return String::new(); + } + + let mut result = String::new(); + let mut chars = s.chars(); + + // Handle first character with next_mode if set + if *next_mode != CaseMode::None { + if let Some(c) = chars.next() { + match next_mode { + CaseMode::UppercaseNext => result.extend(c.to_uppercase()), + CaseMode::LowercaseNext => result.extend(c.to_lowercase()), + _ => result.push(c), + } + *next_mode = CaseMode::None; // Reset one-shot mode + } + } + + // Apply base_mode to remaining characters + for c in chars { + match base_mode { + CaseMode::None => result.push(c), + CaseMode::UppercaseAll => result.extend(c.to_uppercase()), + CaseMode::LowercaseAll => result.extend(c.to_lowercase()), + _ => result.push(c), // Should not happen for base_mode + } + } + + result +} diff --git a/red/src/engine/types.rs b/red/src/engine/types.rs new file mode 100644 index 0000000..7426bf2 --- /dev/null +++ b/red/src/engine/types.rs @@ -0,0 +1,241 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use crate::parser::{AddressRange, PrintTiming}; + +/// Regex matcher using custom engine with 3-level optimization (Literal/DFA/NFA) +#[derive(Debug, Clone)] +pub struct SedRegex { + pub matcher: crate::regex::Matcher, +} + +/// Compiled substitution data - result of compiling a Substitution command +#[derive(Debug, Clone)] +pub struct CompiledSubstitution { + pub range: Option, + pub negated: bool, + pub pattern: SedRegex, + pub replacement: ReplacementTemplate, + pub global: bool, + pub print: bool, + pub write_file: Option, + pub occurrence: Option, + pub use_last: bool, + pub execute: bool, + pub print_timing: PrintTiming, + pub literal_pattern: Option, + pub literal_replacement: Option, + pub literal_pattern_bytes: Option>, + pub literal_replacement_bytes: Option>, +} + +impl SedRegex { + /// Create new SedRegex from custom matcher + pub fn new(matcher: crate::regex::Matcher) -> Self { + SedRegex { matcher } + } + + /// Check if the pattern matches the text + pub fn is_match(&self, text: &str) -> bool { + self.matcher.is_match(text) + } +} + +#[derive(Debug, Clone)] +pub enum ReplacementToken { + Literal(String), + /// Raw bytes literal (for preserving invalid UTF-8 in replacement) + LiteralBytes(Vec), + WholeMatch, + Group(u8), + UppercaseNext, // \u - uppercase next character + LowercaseNext, // \l - lowercase next character + UppercaseAll, // \U - uppercase all following characters + LowercaseAll, // \L - lowercase all following characters + EndCase, // \E - end case conversion +} + +#[derive(Debug, Clone)] +pub struct ReplacementTemplate { + pub tokens: Vec, +} + +#[derive(Debug, Clone)] +pub enum Command { + /// Substitution command - uses pre-compiled CompiledSubstitution + Substitution(CompiledSubstitution), + List { + range: Option, + negated: bool, + line_length: Option, // Optional line wrap length (e.g., l70) + }, + Translate { + range: Option, + negated: bool, + from: String, + to: String, + /// Raw bytes for 'from' string (for preserving invalid UTF-8) + from_bytes: Option>, + /// Raw bytes for 'to' string (for preserving invalid UTF-8) + to_bytes: Option>, + }, + Print { + range: Option, + negated: bool, + }, + PrintFirstLine { + range: Option, + negated: bool, + }, + Delete { + range: Option, + negated: bool, + }, + Quit { + range: Option, + negated: bool, + exit_code: Option, + }, + QuitSilent { + range: Option, + negated: bool, + exit_code: Option, + }, + Append { + range: Option, + negated: bool, + text: Option, // None = unterminated, Some = explicit (even if empty) + }, + Insert { + range: Option, + negated: bool, + text: Option, // None = unterminated, Some = explicit (even if empty) + }, + Change { + range: Option, + negated: bool, + text: Option, // None = unterminated, Some = explicit (even if empty) + }, + N { + range: Option, + negated: bool, + }, + BigD { + range: Option, + negated: bool, + }, + HoldCopy { + range: Option, + negated: bool, + }, + HoldAppend { + range: Option, + negated: bool, + }, + GetCopy { + range: Option, + negated: bool, + }, + GetAppend { + range: Option, + negated: bool, + }, + Exchange { + range: Option, + negated: bool, + }, + Label { + name: String, + }, + Branch { + range: Option, + negated: bool, + label: String, + /// Pre-resolved target index (set after all commands are parsed) + target_index: Option, + }, + Test { + range: Option, + negated: bool, + label: String, + /// Pre-resolved target index (set after all commands are parsed) + target_index: Option, + }, + TestNeg { + range: Option, + negated: bool, + label: String, + /// Pre-resolved target index (set after all commands are parsed) + target_index: Option, + }, + Execute { + range: Option, + negated: bool, + command: Option, // None = execute pattern space, Some = execute this command + }, + Clear { + range: Option, + negated: bool, + }, + PrintFilename { + range: Option, + negated: bool, + }, + Next, + Write { + range: Option, + negated: bool, + path: String, + }, + WriteFirstLine { + range: Option, + negated: bool, + path: String, + }, + Read { + range: Option, + negated: bool, + path: String, + }, + ReadLine { + range: Option, + negated: bool, + path: String, + }, + LineNumber { + range: Option, + negated: bool, + }, +} + +#[derive(Debug)] +pub enum CommandResult { + /// Continue processing, output this content at end of cycle + /// Contains (text_content, raw_bytes) - use raw_bytes for output if available + Continue(String, Option>), + /// Explicit print command (p) - output this content immediately + /// Contains (text_content, raw_bytes) - use raw_bytes for output if available + Print(String, Option>), + /// Delete pattern space and start next cycle + Delete, + /// Print content and continue processing (for P command) + PrintAndContinue(String), + /// Quit with optional exit code + Quit(Option), + /// Restart cycle with pattern space already modified in place + Restart, + /// Restart cycle with new pattern space content + RestartWith(String), + /// Restart cycle with raw bytes (preserves invalid UTF-8) + RestartWithBytes(Vec), + /// Append next line and resume at given program counter + AppendNextAndResume { + resume_pc: usize, + pattern_space: String, + }, + /// Read next line and resume at given program counter + NextLineAndResume { resume_pc: usize }, + /// Suppress auto-print but don't end cycle (used by e command) + SuppressAutoprint, +} diff --git a/red/src/errors.rs b/red/src/errors.rs new file mode 100644 index 0000000..150ccfb --- /dev/null +++ b/red/src/errors.rs @@ -0,0 +1,541 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! GNU sed-compatible error handling and messages + +use std::fmt; + +/// GNU sed-compatible error type with context +#[derive(Debug)] +pub enum SedError { + /// Parse errors in sed scripts (compile-time) + Parse { + message: String, + line: Option, + char_pos: Option, + context: ErrorContext, + }, + /// I/O errors (file not found, permission denied, etc.) + Io { + operation: String, + path: String, + source: std::io::Error, + }, + /// Rename errors during in-place editing (exit code 4) + Rename { + source_path: String, + dest_path: String, + error: std::io::Error, + }, + /// Runtime errors (undefined labels, etc.) + Runtime { message: String }, + /// Usage errors (missing arguments, invalid options) + Usage { message: String }, + /// In-place editing errors (exit code 4) + InPlace { message: String }, +} + +/// Context for where the error occurred +#[derive(Debug, Clone)] +pub enum ErrorContext { + /// Error in -e expression + Expression { index: usize }, + /// Error in -f script file + ScriptFile { path: String }, + /// Error with no specific context + None, +} + +/// Source of a script (for error reporting) +#[derive(Debug, Clone)] +pub enum ScriptSource { + /// Script from -e flag (with 0-based index) + Expression(usize), + /// Script from -f flag (with file path) + File(String), +} + +impl ScriptSource { + /// Convert to ErrorContext + pub fn to_error_context(&self) -> ErrorContext { + match self { + ScriptSource::Expression(index) => ErrorContext::Expression { index: *index }, + ScriptSource::File(path) => ErrorContext::ScriptFile { path: path.clone() }, + } + } +} + +impl fmt::Display for SedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SedError::Parse { + message, + line, + char_pos, + context, + } => match context { + ErrorContext::Expression { index } => { + if let Some(ln) = line { + if let Some(ch) = char_pos { + write!(f, "-e expression #{}, char {}: {}", index + 1, ch, message) + } else { + write!(f, "-e expression #{}, line {}: {}", index + 1, ln, message) + } + } else if let Some(ch) = char_pos { + write!(f, "-e expression #{}, char {}: {}", index + 1, ch, message) + } else { + write!(f, "-e expression #{}: {}", index + 1, message) + } + } + ErrorContext::ScriptFile { path } => { + // For file-based scripts, default to line 1 if not specified + // (most errors occur on line 1, and full line tracking is complex) + let ln = line.unwrap_or(1); + write!(f, "file {} line {}: {}", path, ln, message) + } + ErrorContext::None => { + if let Some(ln) = line { + write!(f, "line {}: {}", ln, message) + } else { + write!(f, "{}", message) + } + } + }, + SedError::Io { + operation, + path, + source, + } => { + // Format error message like GNU sed (without "os error N" suffix) + let error_str = source.to_string(); + let error_msg = match source.kind() { + std::io::ErrorKind::NotFound => "No such file or directory", + std::io::ErrorKind::PermissionDenied => "Permission denied", + std::io::ErrorKind::AlreadyExists => "File exists", + std::io::ErrorKind::InvalidInput => { + // Custom error messages (like "symbolic link loop detected") + // are passed via InvalidInput - use them as-is + error_str.split(" (os error").next().unwrap_or(&error_str) + } + _ => { + // Strip "(os error N)" suffix from other errors + error_str.split(" (os error").next().unwrap_or(&error_str) + } + }; + write!(f, "{} {}: {}", operation, path, error_msg) + } + SedError::Rename { + source_path, + dest_path, + error, + } => { + // Format error message like GNU sed (without "os error N" suffix) + let error_str = error.to_string(); + let error_msg = match error.kind() { + std::io::ErrorKind::NotFound => "No such file or directory", + std::io::ErrorKind::PermissionDenied => "Permission denied", + std::io::ErrorKind::AlreadyExists => "File exists", + _ => error_str.split(" (os error").next().unwrap_or(&error_str), + }; + write!( + f, + "cannot rename {} to {}: {}", + source_path, dest_path, error_msg + ) + } + SedError::Runtime { message } => { + write!(f, "{}", message) + } + SedError::Usage { message } => { + write!(f, "{}", message) + } + SedError::InPlace { message } => { + write!(f, "{}", message) + } + } + } +} + +impl std::error::Error for SedError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + SedError::Io { source, .. } => Some(source), + SedError::Rename { error, .. } => Some(error), + _ => None, + } + } +} + +impl SedError { + /// Get the appropriate exit code for this error type + /// - 0: success (not an error) + /// - 1: general errors (parse, runtime, usage) + /// - 2: I/O errors (file could not be opened) + /// - 4: I/O errors during processing (symlink errors, rename errors, etc.) + pub fn exit_code(&self) -> i32 { + match self { + SedError::Io { operation, .. } => { + // Symlink-related operations get exit code 4 (like GNU sed) + if operation.contains("readlink") || operation.contains("follow symlink") { + 4 + } else { + 2 + } + } + SedError::Rename { .. } => 4, + SedError::InPlace { .. } => 4, + SedError::Parse { .. } => 1, + SedError::Runtime { .. } => 1, + SedError::Usage { .. } => 1, + } + } + + /// Create a parse error + pub fn parse(message: impl Into) -> Self { + SedError::Parse { + message: message.into(), + line: None, + char_pos: None, + context: ErrorContext::None, + } + } + + /// Create a parse error with line number + pub fn parse_line(message: impl Into, line: usize) -> Self { + SedError::Parse { + message: message.into(), + line: Some(line), + char_pos: None, + context: ErrorContext::None, + } + } + + /// Create a parse error with character position + pub fn parse_at(message: impl Into, char_pos: usize) -> Self { + SedError::Parse { + message: message.into(), + line: None, + char_pos: Some(char_pos), + context: ErrorContext::None, + } + } + + /// Set the error context (expression index or script file) + pub fn with_context(mut self, context: ErrorContext) -> Self { + if let SedError::Parse { + context: ref mut ctx, + .. + } = self + { + *ctx = context; + } + self + } + + /// Create an I/O error + pub fn io( + operation: impl Into, + path: impl Into, + source: std::io::Error, + ) -> Self { + SedError::Io { + operation: operation.into(), + path: path.into(), + source, + } + } + + /// Create a rename error (for in-place editing failures) + pub fn rename( + source_path: impl Into, + dest_path: impl Into, + error: std::io::Error, + ) -> Self { + SedError::Rename { + source_path: source_path.into(), + dest_path: dest_path.into(), + error, + } + } + + /// Create a runtime error + pub fn runtime(message: impl Into) -> Self { + SedError::Runtime { + message: message.into(), + } + } + + /// Create a usage error + pub fn usage(message: impl Into) -> Self { + SedError::Usage { + message: message.into(), + } + } + + /// Create an in-place editing error (exit code 4) + pub fn inplace(message: impl Into) -> Self { + SedError::InPlace { + message: message.into(), + } + } +} + +/// Result type using SedError +pub type Result = std::result::Result; + +/// Convert from std::io::Error +impl From for SedError { + fn from(err: std::io::Error) -> Self { + SedError::Runtime { + message: err.to_string(), + } + } +} + +/// Convert from lexopt::Error (CLI parsing errors) +impl From for SedError { + fn from(err: lexopt::Error) -> Self { + SedError::Usage { + message: err.to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_error_simple() { + let err = SedError::parse("unknown command: 'X'"); + assert_eq!(err.to_string(), "unknown command: 'X'"); + } + + #[test] + fn test_parse_error_with_line() { + let err = SedError::parse_line("unterminated address regex", 5); + assert_eq!(err.to_string(), "line 5: unterminated address regex"); + } + + #[test] + fn test_parse_error_with_expression_context() { + let err = SedError::parse("unknown command: 'X'") + .with_context(ErrorContext::Expression { index: 0 }); + assert_eq!(err.to_string(), "-e expression #1: unknown command: 'X'"); + } + + #[test] + fn test_parse_error_with_script_file_context() { + let err = + SedError::parse_line("unexpected '}'", 3).with_context(ErrorContext::ScriptFile { + path: "test.sed".to_string(), + }); + assert_eq!(err.to_string(), "file test.sed line 3: unexpected '}'"); + } + + #[test] + fn test_io_error() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "No such file or directory"); + let err = SedError::io("can't read", "/tmp/test.txt", io_err); + assert!(err.to_string().contains("can't read /tmp/test.txt")); + } + + #[test] + fn test_runtime_error() { + let err = SedError::runtime("undefined label 'foo'"); + assert_eq!(err.to_string(), "undefined label 'foo'"); + } + + #[test] + fn test_usage_error() { + let err = SedError::usage("no input files"); + assert_eq!(err.to_string(), "no input files"); + } + + #[test] + fn test_parse_error_with_char_pos() { + let err = SedError::parse_at("unexpected '}'", 10); + assert!(err.to_string().contains("unexpected '}'")); + } + + #[test] + fn test_parse_error_expression_with_line() { + let err = SedError::Parse { + message: "error".to_string(), + line: Some(3), + char_pos: None, + context: ErrorContext::Expression { index: 1 }, + }; + assert_eq!(err.to_string(), "-e expression #2, line 3: error"); + } + + #[test] + fn test_parse_error_expression_with_char_pos() { + let err = SedError::Parse { + message: "error".to_string(), + line: None, + char_pos: Some(5), + context: ErrorContext::Expression { index: 0 }, + }; + assert_eq!(err.to_string(), "-e expression #1, char 5: error"); + } + + #[test] + fn test_parse_error_expression_with_line_and_char_pos() { + let err = SedError::Parse { + message: "error".to_string(), + line: Some(2), + char_pos: Some(10), + context: ErrorContext::Expression { index: 0 }, + }; + // char_pos takes precedence over line + assert_eq!(err.to_string(), "-e expression #1, char 10: error"); + } + + #[test] + fn test_inplace_error() { + let err = SedError::inplace("cannot edit in place"); + assert_eq!(err.to_string(), "cannot edit in place"); + } + + #[test] + fn test_rename_error() { + let io_err = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Permission denied"); + let err = SedError::rename("/tmp/src", "/tmp/dst", io_err); + assert!(err + .to_string() + .contains("cannot rename /tmp/src to /tmp/dst")); + assert!(err.to_string().contains("Permission denied")); + } + + #[test] + fn test_rename_error_not_found() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "Not found"); + let err = SedError::rename("/tmp/src", "/tmp/dst", io_err); + assert!(err.to_string().contains("No such file or directory")); + } + + #[test] + fn test_rename_error_already_exists() { + let io_err = std::io::Error::new(std::io::ErrorKind::AlreadyExists, "Exists"); + let err = SedError::rename("/tmp/src", "/tmp/dst", io_err); + assert!(err.to_string().contains("File exists")); + } + + #[test] + fn test_io_error_permission_denied() { + let io_err = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Permission denied"); + let err = SedError::io("cannot read", "/tmp/test", io_err); + assert!(err.to_string().contains("Permission denied")); + } + + #[test] + fn test_io_error_already_exists() { + let io_err = std::io::Error::new(std::io::ErrorKind::AlreadyExists, "File exists"); + let err = SedError::io("cannot create", "/tmp/test", io_err); + assert!(err.to_string().contains("File exists")); + } + + #[test] + fn test_io_error_invalid_input() { + let io_err = std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "symbolic link loop detected (os error 40)", + ); + let err = SedError::io("readlink", "/tmp/link", io_err); + assert!(err.to_string().contains("symbolic link loop detected")); + assert!(!err.to_string().contains("os error")); + } + + #[test] + fn test_exit_codes() { + assert_eq!(SedError::parse("error").exit_code(), 1); + assert_eq!(SedError::runtime("error").exit_code(), 1); + assert_eq!(SedError::usage("error").exit_code(), 1); + assert_eq!(SedError::inplace("error").exit_code(), 4); + + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "Not found"); + assert_eq!(SedError::io("read", "file", io_err).exit_code(), 2); + + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "Not found"); + assert_eq!(SedError::io("readlink", "file", io_err).exit_code(), 4); + + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "Not found"); + assert_eq!(SedError::rename("a", "b", io_err).exit_code(), 4); + } + + #[test] + fn test_error_source() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "test"); + let err = SedError::io("read", "file", io_err); + assert!(std::error::Error::source(&err).is_some()); + + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "test"); + let err = SedError::rename("a", "b", io_err); + assert!(std::error::Error::source(&err).is_some()); + + let err = SedError::parse("test"); + assert!(std::error::Error::source(&err).is_none()); + + let err = SedError::runtime("test"); + assert!(std::error::Error::source(&err).is_none()); + + let err = SedError::usage("test"); + assert!(std::error::Error::source(&err).is_none()); + + let err = SedError::inplace("test"); + assert!(std::error::Error::source(&err).is_none()); + } + + #[test] + fn test_script_source_to_error_context() { + let source = ScriptSource::Expression(0); + if let ErrorContext::Expression { index } = source.to_error_context() { + assert_eq!(index, 0); + } else { + panic!("Expected Expression context"); + } + + let source = ScriptSource::File("test.sed".to_string()); + if let ErrorContext::ScriptFile { path } = source.to_error_context() { + assert_eq!(path, "test.sed"); + } else { + panic!("Expected ScriptFile context"); + } + } + + #[test] + fn test_from_io_error() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "test error"); + let sed_err: SedError = io_err.into(); + if let SedError::Runtime { message } = sed_err { + assert!(message.contains("test error")); + } else { + panic!("Expected Runtime error"); + } + } + + #[test] + fn test_script_file_context_no_line() { + let err = SedError::Parse { + message: "error".to_string(), + line: None, + char_pos: None, + context: ErrorContext::ScriptFile { + path: "test.sed".to_string(), + }, + }; + // Without line, defaults to line 1 + assert_eq!(err.to_string(), "file test.sed line 1: error"); + } + + #[test] + fn test_io_error_other_kind() { + // Test "other" error kinds that go through the default path + let io_err = std::io::Error::new(std::io::ErrorKind::Other, "custom error (os error 123)"); + let err = SedError::io("operation", "/path", io_err); + assert!(err.to_string().contains("custom error")); + assert!(!err.to_string().contains("os error")); + } +} diff --git a/red/src/fileio/encoding.rs b/red/src/fileio/encoding.rs new file mode 100644 index 0000000..d76aeea --- /dev/null +++ b/red/src/fileio/encoding.rs @@ -0,0 +1,91 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Encoding detection utilities + +use encoding_rs::Encoding; + +/// Detect the encoding of a byte sequence +/// Returns the detected encoding based on BOM, UTF-8 validity, or locale settings +pub fn detect_encoding(bytes: &[u8]) -> &'static Encoding { + // Try to detect from BOM (Byte Order Mark) + if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) { + return encoding_rs::UTF_8; + } + + // Try UTF-8 validation + if std::str::from_utf8(bytes).is_ok() { + return encoding_rs::UTF_8; + } + + // Check environment variables for locale + // Try LC_ALL first, then LC_CTYPE, then LANG + let locale = std::env::var("LC_ALL") + .or_else(|_| std::env::var("LC_CTYPE")) + .or_else(|_| std::env::var("LANG")) + .unwrap_or_default(); + + let locale_lower = locale.to_lowercase(); + + // Match common encodings + if locale_lower.contains("utf-8") || locale_lower.contains("utf8") { + return encoding_rs::UTF_8; + } + if locale_lower.contains("euc-jp") || locale_lower.contains("eucjp") { + return encoding_rs::EUC_JP; + } + if locale_lower.contains("shift_jis") || locale_lower.contains("sjis") { + return encoding_rs::SHIFT_JIS; + } + if locale_lower.contains("iso-2022-jp") || locale_lower.contains("iso2022jp") { + return encoding_rs::ISO_2022_JP; + } + if locale_lower.contains("iso-8859-1") + || locale_lower.contains("iso88591") + || locale_lower.contains("latin1") + { + return encoding_rs::WINDOWS_1252; // Close approximation to ISO-8859-1 + } + if locale_lower.contains("windows-1252") || locale_lower.contains("cp1252") { + return encoding_rs::WINDOWS_1252; + } + if locale_lower.contains("gb") + || locale_lower.contains("gbk") + || locale_lower.contains("gb2312") + { + return encoding_rs::GBK; + } + if locale_lower.contains("big5") { + return encoding_rs::BIG5; + } + if locale_lower.contains("euc-kr") || locale_lower.contains("euckr") { + return encoding_rs::EUC_KR; + } + + // Default to UTF-8 + encoding_rs::UTF_8 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_utf8_bom() { + let bytes = &[0xEF, 0xBB, 0xBF, b'h', b'e', b'l', b'l', b'o']; + assert_eq!(detect_encoding(bytes), encoding_rs::UTF_8); + } + + #[test] + fn test_valid_utf8() { + let bytes = b"hello world"; + assert_eq!(detect_encoding(bytes), encoding_rs::UTF_8); + } + + #[test] + fn test_empty_bytes() { + let bytes: &[u8] = &[]; + assert_eq!(detect_encoding(bytes), encoding_rs::UTF_8); + } +} diff --git a/red/src/fileio/inplace.rs b/red/src/fileio/inplace.rs new file mode 100644 index 0000000..f819771 --- /dev/null +++ b/red/src/fileio/inplace.rs @@ -0,0 +1,70 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! In-place file editing utilities + +/// Expand '*' in backup suffix according to GNU sed rules +/// +/// If the file path contains a directory separator: +/// - Each '*' is replaced with the full file path +/// - Example: "***" with file "./e" becomes "./e./e./e" +/// +/// If the file path is just a basename (no directory): +/// - Each '*' is replaced with just the basename +/// - Example: "==*==" with file "c" becomes "==c==" +/// - The result is placed in the parent directory of the original file +pub fn expand_backup_suffix(file_path: &str, suffix: &str) -> String { + if !suffix.contains('*') { + // No wildcard - just append suffix + return format!("{}{}", file_path, suffix); + } + + // Determine what to replace '*' with based on whether path contains directory + let replacement = if file_path.contains('/') { + // Path contains directory - use full path + file_path + } else { + // Simple filename - use just the basename + std::path::Path::new(file_path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(file_path) + }; + + // Replace each '*' with the appropriate value + suffix.replace('*', replacement) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_expand_backup_suffix_no_wildcard() { + assert_eq!(expand_backup_suffix("file.txt", ".bak"), "file.txt.bak"); + } + + #[test] + fn test_expand_backup_suffix_with_wildcard() { + assert_eq!(expand_backup_suffix("file.txt", "*.bak"), "file.txt.bak"); + } + + #[test] + fn test_expand_backup_suffix_multiple_wildcards() { + assert_eq!(expand_backup_suffix("file", "***"), "filefilefile"); + } + + #[test] + fn test_expand_backup_suffix_with_path() { + assert_eq!( + expand_backup_suffix("./dir/file", "***"), + "./dir/file./dir/file./dir/file" + ); + } + + #[test] + fn test_expand_backup_suffix_prefix_and_suffix() { + assert_eq!(expand_backup_suffix("c", "==*=="), "==c=="); + } +} diff --git a/red/src/fileio/lines.rs b/red/src/fileio/lines.rs new file mode 100644 index 0000000..9f38c20 --- /dev/null +++ b/red/src/fileio/lines.rs @@ -0,0 +1,255 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Line reading utilities for files and stdin + +use std::io::{self, Read}; + +use encoding_rs::Encoding; + +use crate::errors::{Result, SedError}; +use crate::util::symlink::resolve_symlink_chain; + +use super::encoding::detect_encoding; + +/// Read all lines from files or stdin +/// +/// Returns a tuple of: +/// - Vec: Lines as strings (decoded with detected encoding) +/// - Vec>: Lines as raw bytes +/// - Vec: Filename for each line (for F command) +/// - &'static Encoding: Detected encoding +/// - bool: Whether the input ends with a separator +pub fn read_all_lines( + files: &[String], + null_data: bool, + follow_symlinks: bool, + binary: bool, +) -> Result<( + Vec, + Vec>, + Vec, + &'static Encoding, + bool, +)> { + let mut all_lines_str = Vec::new(); + let mut all_lines_bytes = Vec::new(); + let mut all_filenames = Vec::new(); // Track which file each line came from + let mut detected_encoding: &'static Encoding = encoding_rs::UTF_8; + let mut first_file = true; + let mut ends_with_separator = false; // Track whether the last input ends with separator + + let separator = if null_data { b'\0' } else { b'\n' }; + + // On Windows in text mode (not binary), strip trailing \r from lines + #[cfg(windows)] + let strip_cr = !binary && !null_data; + #[cfg(not(windows))] + let strip_cr = { + let _ = binary; + false + }; + + if files.is_empty() || (files.len() == 1 && files[0] == "-") { + let stdin = io::stdin(); + let mut buf: Vec = Vec::new(); + stdin.lock().read_to_end(&mut buf)?; + + // Detect encoding from stdin + detected_encoding = detect_encoding(&buf); + + // Track whether stdin ends with separator + ends_with_separator = !buf.is_empty() && buf[buf.len() - 1] == separator; + + let lines_b = split_bytes_into_lines_bytes(buf.clone(), separator, strip_cr); + let lines_s = + split_bytes_into_lines_with_encoding(&buf, separator, detected_encoding, strip_cr); + let line_count = lines_s.len(); + all_lines_bytes.extend(lines_b); + all_lines_str.extend(lines_s); + all_filenames.extend(vec!["-".to_string(); line_count]); + } else { + for path in files { + // Resolve symlinks if follow_symlinks is enabled + let filename_for_f_command = if follow_symlinks { + // Use non-strict mode (silent on errors) for F command + resolve_symlink_chain(std::path::Path::new(path), false)? + .display() + .to_string() + } else { + // Not following symlinks - use original path + path.clone() + }; + + // When follow_symlinks is enabled, try readlink first to give proper error + if follow_symlinks { + // Try to read link or stat the file to ensure it exists and is accessible + if let Err(e) = std::fs::symlink_metadata(path) { + return Err(SedError::io("couldn't readlink", path, e)); + } + } + + let mut file = + std::fs::File::open(path).map_err(|e| SedError::io("can't read", path, e))?; + let mut buf: Vec = Vec::new(); + file.read_to_end(&mut buf)?; + + // Detect encoding from first file + if first_file { + detected_encoding = detect_encoding(&buf); + first_file = false; + } + + // Track whether this file (the last one processed) ends with separator + ends_with_separator = !buf.is_empty() && buf[buf.len() - 1] == separator; + + let lines_b = split_bytes_into_lines_bytes(buf.clone(), separator, strip_cr); + let lines_s = + split_bytes_into_lines_with_encoding(&buf, separator, detected_encoding, strip_cr); + let line_count = lines_s.len(); + all_lines_bytes.extend(lines_b); + all_lines_str.extend(lines_s); + all_filenames.extend(vec![filename_for_f_command; line_count]); + } + } + + Ok(( + all_lines_str, + all_lines_bytes, + all_filenames, + detected_encoding, + ends_with_separator, + )) +} + +/// Split file content into lines (both string and byte representations) +pub fn split_file_content( + content: Vec, + null_data: bool, + binary: bool, +) -> (Vec, Vec>, &'static Encoding, bool) { + let mut lines_str = Vec::new(); + let mut lines_bytes = Vec::new(); + let mut start: usize = 0; + let separator = if null_data { b'\0' } else { b'\n' }; + + // Detect encoding + let encoding = detect_encoding(&content); + + // Track if content ends with separator + let ends_with_separator = !content.is_empty() && content[content.len() - 1] == separator; + + // On Windows in text mode (not binary), strip trailing \r from lines + #[cfg(windows)] + let strip_cr = !binary && !null_data; + #[cfg(not(windows))] + let strip_cr = { + let _ = binary; + false + }; + + for i in 0..content.len() { + if content[i] == separator { + let slice = strip_trailing_cr(&content[start..i], strip_cr); + let (decoded, _, _) = encoding.decode(slice); + lines_str.push(decoded.into_owned()); + lines_bytes.push(slice.to_vec()); + start = i + 1; + } + } + + if start < content.len() { + let slice = strip_trailing_cr(&content[start..], strip_cr); + let (decoded, _, _) = encoding.decode(slice); + lines_str.push(decoded.into_owned()); + lines_bytes.push(slice.to_vec()); + } + + (lines_str, lines_bytes, encoding, ends_with_separator) +} + +/// Strip trailing \r from a byte slice (for Windows text mode) +#[inline] +fn strip_trailing_cr(slice: &[u8], strip: bool) -> &[u8] { + if strip && !slice.is_empty() && slice[slice.len() - 1] == b'\r' { + &slice[..slice.len() - 1] + } else { + slice + } +} + +/// Split bytes into lines, preserving raw bytes +/// When strip_cr is true, removes trailing \r before \n (Windows text mode) +fn split_bytes_into_lines_bytes(bytes: Vec, separator: u8, strip_cr: bool) -> Vec> { + let mut lines: Vec> = Vec::new(); + let mut start: usize = 0; + for i in 0..bytes.len() { + if bytes[i] == separator { + let slice = strip_trailing_cr(&bytes[start..i], strip_cr); + lines.push(slice.to_vec()); + start = i + 1; + } + } + if start < bytes.len() { + let slice = strip_trailing_cr(&bytes[start..], strip_cr); + lines.push(slice.to_vec()); + } + lines +} + +/// Split bytes into lines with encoding conversion +/// When strip_cr is true, removes trailing \r before \n (Windows text mode) +fn split_bytes_into_lines_with_encoding( + bytes: &[u8], + separator: u8, + encoding: &'static Encoding, + strip_cr: bool, +) -> Vec { + let mut lines: Vec = Vec::new(); + let mut start: usize = 0; + for i in 0..bytes.len() { + if bytes[i] == separator { + let slice = strip_trailing_cr(&bytes[start..i], strip_cr); + let (decoded, _, _) = encoding.decode(slice); + lines.push(decoded.into_owned()); + start = i + 1; + } + } + if start < bytes.len() { + let slice = strip_trailing_cr(&bytes[start..], strip_cr); + let (decoded, _, _) = encoding.decode(slice); + lines.push(decoded.into_owned()); + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_file_content_newline() { + let content = b"line1\nline2\nline3".to_vec(); + let (lines, bytes, _, ends) = split_file_content(content, false, false); + assert_eq!(lines, vec!["line1", "line2", "line3"]); + assert_eq!(bytes.len(), 3); + assert!(!ends); + } + + #[test] + fn test_split_file_content_trailing_newline() { + let content = b"line1\nline2\n".to_vec(); + let (lines, _, _, ends) = split_file_content(content, false, false); + assert_eq!(lines, vec!["line1", "line2"]); + assert!(ends); + } + + #[test] + fn test_split_file_content_null_data() { + let content = b"line1\0line2\0".to_vec(); + let (lines, _, _, ends) = split_file_content(content, true, false); + assert_eq!(lines, vec!["line1", "line2"]); + assert!(ends); + } +} diff --git a/red/src/fileio/mod.rs b/red/src/fileio/mod.rs new file mode 100644 index 0000000..193b102 --- /dev/null +++ b/red/src/fileio/mod.rs @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! I/O utilities for sed operations +//! +//! This module provides file reading, encoding detection, and backup +//! suffix expansion functionality separated from the main orchestration logic. + +mod encoding; +mod inplace; +mod lines; + +// detect_encoding is used internally by lines module, not re-exported +pub use inplace::expand_backup_suffix; +pub use lines::{read_all_lines, split_file_content}; + +/// Helper function to conditionally flush output based on unbuffered flag +#[inline] +pub fn flush_output(out: &mut dyn std::io::Write, unbuffered: bool) { + if unbuffered { + let _ = out.flush(); + } +} + +/// Get the line ending bytes for the current platform and mode. +/// On Windows without binary mode, returns CRLF. Otherwise returns LF. +#[inline] +pub fn line_ending(binary: bool) -> &'static [u8] { + #[cfg(windows)] + { + if binary { + b"\n" + } else { + b"\r\n" + } + } + #[cfg(not(windows))] + { + let _ = binary; // suppress unused warning + b"\n" + } +} + +/// Write a line to output with optional raw bytes and separator handling. +/// +/// This unifies the output logic used in both `execute_over_lines` and +/// `process_stdin_line_by_line`, reducing code duplication. +#[inline] +pub fn write_output_line( + content: &str, + raw_bytes: Option<&[u8]>, + null_data: bool, + write_separator: bool, + binary: bool, + out: &mut dyn std::io::Write, +) { + let output_bytes = raw_bytes.unwrap_or_else(|| content.as_bytes()); + let _ = out.write_all(output_bytes); + if write_separator { + if null_data { + let _ = out.write_all(b"\0"); + } else { + let _ = out.write_all(line_ending(binary)); + } + } +} diff --git a/red/src/lib.rs b/red/src/lib.rs new file mode 100644 index 0000000..1695588 --- /dev/null +++ b/red/src/lib.rs @@ -0,0 +1,1720 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use std::collections::{HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufWriter, Read, Write}; + +use crate::errors::{Result, SedError}; +use encoding_rs::Encoding; + +pub mod constants; +pub mod context; +mod engine; +pub mod errors; +mod fileio; // I/O utilities (encoding, file reading, in-place editing) +pub mod mbcs; // Multibyte character set support +pub mod parser; +pub mod posix_rules; +pub mod regex; +#[cfg(feature = "selinux")] +pub mod selinux; // SELinux security context support +pub mod signals; +mod util; // Custom regex engine with zero external dependencies +mod validation; // Centralized POSIX compliance rules + +pub use context::{Context, PosixMode}; +pub use engine::{ + apply_commands_with_context, AddressEvaluator, Command, CommandResult, ExecutionContext, +}; +pub use validation::validate_config; + +use engine::Command as RuntimeCommand; +use parser::Command as PC; +use parser::{AddressRange, HasAddressRange, Parser as ScriptParser}; +use util::regex::{compile_regex, compile_regex_with_replacement}; +use util::symlink::resolve_symlink_chain; +use util::version::compare_versions; + +// detect_encoding moved to io/encoding.rs + +/// Configuration for running sed commands +/// Contains all script texts, input files, and command-line flags +#[derive(Debug, Clone)] +pub struct RunConfig { + /// sed scripts with raw bytes and their sources (for error reporting) + /// Tuple: (converted_string, raw_bytes, source) + /// raw_bytes is needed for accurate multibyte delimiter detection + pub scripts_with_sources: Vec<(String, Vec, errors::ScriptSource)>, + /// Input files to process (empty for stdin) + pub input_files: Vec, + /// Suppress automatic printing of pattern space (-n flag) + pub quiet: bool, + /// Edit files in-place with optional backup suffix (-i flag) + pub in_place: Option, + /// Use Extended Regular Expressions (-E/-r flag) + pub extended_regex: bool, + /// Treat files independently, reset line numbers for each (-s flag) + pub separate_files: bool, + /// Line length for l command formatting + pub line_length: usize, + /// Flush output after each line (-u flag) + pub unbuffered: bool, + /// POSIX mode - strict standards compliance (--posix flag OR POSIXLY_CORRECT env var) + pub posix: bool, + /// Strict POSIX mode - only --posix flag (not POSIXLY_CORRECT) + pub strict_posix: bool, + /// Follow symlinks when editing in-place (--follow-symlinks flag) + pub follow_symlinks: bool, + /// Sandbox mode - disable external file operations (--sandbox flag) + pub sandbox: bool, + /// Use NUL as line separator instead of newline (-z flag) + pub null_data: bool, + /// Binary mode - disable CRLF conversion on Windows (-b flag) + pub binary: bool, +} + +fn scripts_request_quiet(scripts: &[(String, Vec, errors::ScriptSource)]) -> bool { + // Only the FIRST script can activate quiet mode with #n + scripts + .first() + .map(|(s, _, _)| s.starts_with("#n")) + .unwrap_or(false) +} + +// read_all_lines moved to fileio/lines.rs +use fileio::read_all_lines; + +/// Macro to handle range inheritance for commands that need end-of-range semantics. +/// +/// When a command (like Change or Read) has no explicit range but is inside a brace group +/// with a range, it should inherit that group's range. This macro eliminates the duplicated +/// logic for this pattern. +macro_rules! inherit_group_range { + ($variant:ident, $range:expr, $negated:expr, $field_name:ident, $field_value:expr, $nearest:expr, $out:expr) => { + if $range.is_none() { + if let Some((r, n)) = $nearest.clone() { + $out.push(PC::$variant { + range: Some(r), + negated: $negated ^ n, + $field_name: $field_value, + }); + } else { + $out.push(PC::$variant { + range: $range, + negated: $negated, + $field_name: $field_value, + }); + } + } else { + $out.push(PC::$variant { + range: $range, + negated: $negated, + $field_name: $field_value, + }); + } + }; +} + +fn flatten_parser_commands(cmds: Vec) -> Vec { + // We implement brace groups using guard branches to ensure correct logical AND semantics + // between group conditions and inner command addresses. For commands that require + // end-of-range semantics (like `c` and `r`), when they lack their own range we inherit the + // nearest enclosing group's range so the engine can apply range-aware behavior. + + fn next_label(counter: &mut usize, suffix: &str) -> String { + let id = *counter; + *counter += 1; + format!("__red_group_{}_{}", suffix, id) + } + + fn rec( + cmd: PC, + // Closest enclosing group range for range-sensitive commands (Change/Read) + nearest_group_range: &Option<(AddressRange, bool)>, + out: &mut Vec, + label_counter: &mut usize, + ) { + match cmd { + PC::Group { + range, + negated, + commands, + } => { + let end_label = next_label(label_counter, "end"); + + // Guard: if NOT(group_condition) -> branch to end_label + // Using evaluate_with_negation semantics, complement by flipping `negated`. + out.push(PC::Branch { + range: range.clone(), + negated: !negated, + label: end_label.clone(), + }); + + // Determine new nearest range for range-sensitive commands + let new_nearest = range + .clone() + .map(|r| (r, negated)) + .or_else(|| nearest_group_range.clone()); + + for c in commands { + rec(c, &new_nearest, out, label_counter); + } + + out.push(PC::Label { name: end_label }); + } + PC::Change { + range, + negated, + text, + } => { + inherit_group_range!(Change, range, negated, text, text, nearest_group_range, out); + } + PC::Read { + range, + negated, + path, + } => { + inherit_group_range!(Read, range, negated, path, path, nearest_group_range, out); + } + other => { + // For all other commands, do not inherit group range; the guard already enforces + // group condition, preserving correct logical AND with command's own address. + out.push(other); + } + } + } + + let mut out: Vec = Vec::new(); + let mut counter: usize = 0; + let none_nearest: Option<(AddressRange, bool)> = None; + for c in cmds { + rec(c, &none_nearest, &mut out, &mut counter); + } + out +} + +/// Validate all regex patterns in address range during parsing +fn validate_address_regexes( + range: &Option, + extended_regex: bool, + posix: bool, +) -> Result<()> { + if let Some(addr_range) = range { + // Validate start address + if let Some(parser::Address::Regex(pattern)) = &addr_range.start { + compile_regex( + pattern, + extended_regex, + false, + false, + false, + posix, + "address pattern", + )?; + } + // Validate end address + if let Some(parser::Address::Regex(pattern)) = &addr_range.end { + compile_regex( + pattern, + extended_regex, + false, + false, + false, + posix, + "address pattern", + )?; + } + } + Ok(()) +} + +/// Compile a parser::Command::Substitution into engine::CompiledSubstitution +fn compile_substitution( + range: Option, + negated: bool, + pattern: String, + pattern_raw_bytes: Option>, + replacement: String, + replacement_raw_bytes: Option>, + flags: parser::SubstitutionFlags, + delimiter: char, + extended_regex: bool, + posix: bool, +) -> Result { + let use_last = pattern.is_empty(); + + // Check for POSIX portability warnings + use crate::util::regex::check_posix_portability; + check_posix_portability(&replacement, posix, delimiter); + + // Parse replacement template with raw bytes handling + use crate::util::regex::parse_replacement_with_bytes; + let replacement_template = parse_replacement_with_bytes( + &replacement, + delimiter, + posix, + replacement_raw_bytes.as_deref(), + ); + + // Compile regex with replacement template info for smart optimization + let regex = if !use_last { + compile_regex_with_replacement( + &pattern, + extended_regex, + flags.ignore_case, + flags.multiline, + flags.multiline_dotall, + posix, + "substitution", + Some(&replacement_template), + flags.occurrence.is_some(), + )? + } else { + // Empty pattern means "use last regex" - create placeholder + let never_match = regex::Matcher::compile("$.^", false, false) + .expect("never-match pattern should always compile"); + engine::SedRegex::new(never_match) + }; + + // Literal optimization: check if both pattern and replacement are literal + let (literal_pattern, literal_replacement) = if !use_last + && !flags.ignore_case + && !flags.multiline + && !flags.multiline_dotall + && flags.occurrence.is_none() + && !flags.execute + { + if let Ok(matcher) = regex::Matcher::compile(&pattern, extended_regex, false) { + if matcher.is_literal() { + if let Some(lit_repl) = + regex::literal::to_literal_replacement(&replacement_template) + { + if let Ok(compiled) = if extended_regex { + regex::parser::parse_ere(&pattern, posix) + } else { + regex::parser::parse_bre(&pattern, posix) + } { + if let Some(lit_pat) = regex::literal::to_literal_string(&compiled.ast) { + (Some(lit_pat), Some(lit_repl)) + } else { + (None, None) + } + } else { + (None, None) + } + } else { + (None, None) + } + } else { + (None, None) + } + } else { + (None, None) + } + } else { + (None, None) + }; + + // Byte-level matching for non-UTF-8 patterns + let (literal_pattern_bytes, literal_replacement_bytes) = + if let (Some(pat_bytes), Some(repl_bytes)) = (&pattern_raw_bytes, &replacement_raw_bytes) { + let has_non_ascii = pat_bytes.iter().any(|&b| b > 127); + let is_byte_literal = !pat_bytes.iter().any(|&b| { + matches!( + b, + b'.' | b'*' + | b'[' + | b'^' + | b'$' + | b'\\' + | b']' + | b'+' + | b'?' + | b'|' + | b'(' + | b')' + ) + }); + let is_repl_simple = !repl_bytes + .windows(2) + .any(|w| w[0] == b'\\' && (w[1].is_ascii_digit() || w[1] == b'&')) + && !repl_bytes.contains(&b'&'); + + if has_non_ascii && is_byte_literal && is_repl_simple { + (pattern_raw_bytes, replacement_raw_bytes) + } else { + (None, None) + } + } else { + (None, None) + }; + + Ok(engine::CompiledSubstitution { + range, + negated, + pattern: regex, + replacement: replacement_template, + global: flags.global, + print: flags.print, + write_file: flags.write_file, + occurrence: flags.occurrence, + use_last, + execute: flags.execute, + print_timing: flags.print_timing, + literal_pattern, + literal_replacement, + literal_pattern_bytes, + literal_replacement_bytes, + }) +} + +fn convert_new_command_to_old( + new_cmd: parser::Command, + extended_regex: bool, + posix: bool, +) -> Result> { + // Validate address regexes first (to catch errors during parsing with proper context) + // Extract range from command and validate any regex addresses + let range_to_validate: Option<&Option> = match &new_cmd { + parser::Command::Substitution { range, .. } => Some(range), + parser::Command::Delete { range, .. } => Some(range), + parser::Command::Print { range, .. } => Some(range), + parser::Command::Quit { range, .. } => Some(range), + parser::Command::QuitSilent { range, .. } => Some(range), + parser::Command::List { range, .. } => Some(range), + parser::Command::Append { range, .. } => Some(range), + parser::Command::Insert { range, .. } => Some(range), + parser::Command::Change { range, .. } => Some(range), + parser::Command::Exchange { range, .. } => Some(range), + parser::Command::Branch { range, .. } => Some(range), + parser::Command::Test { range, .. } => Some(range), + parser::Command::Execute { range, .. } => Some(range), + _ => None, // For commands we don't explicitly match, skip validation (they'll be validated at runtime) + }; + if let Some(range) = range_to_validate { + validate_address_regexes(range, extended_regex, posix)?; + } + + match new_cmd { + parser::Command::Substitution { + range, + negated, + pattern, + pattern_raw_bytes, + replacement, + replacement_raw_bytes, + flags, + delimiter, + } => { + let compiled = compile_substitution( + range, + negated, + pattern, + pattern_raw_bytes, + replacement, + replacement_raw_bytes, + flags, + delimiter, + extended_regex, + posix, + )?; + Ok(Some(RuntimeCommand::Substitution(compiled))) + } + parser::Command::Print { range, negated, .. } => { + Ok(Some(RuntimeCommand::Print { range, negated })) + } + parser::Command::PrintFirstLine { range, negated, .. } => { + Ok(Some(RuntimeCommand::PrintFirstLine { range, negated })) + } + parser::Command::Delete { range, negated, .. } => { + Ok(Some(RuntimeCommand::Delete { range, negated })) + } + parser::Command::Quit { + range, + negated, + exit_code, + .. + } => Ok(Some(RuntimeCommand::Quit { + range, + negated, + exit_code, + })), + parser::Command::QuitSilent { + range, + negated, + exit_code, + .. + } => Ok(Some(RuntimeCommand::QuitSilent { + range, + negated, + exit_code, + })), + parser::Command::Append { + range, + negated, + text, + } => Ok(Some(RuntimeCommand::Append { + range, + negated, + text, + })), + parser::Command::Insert { + range, + negated, + text, + } => Ok(Some(RuntimeCommand::Insert { + range, + negated, + text, + })), + parser::Command::Change { + range, + negated, + text, + } => Ok(Some(RuntimeCommand::Change { + range, + negated, + text, + })), + parser::Command::N { range, negated, .. } => Ok(Some(RuntimeCommand::N { range, negated })), + parser::Command::BigD { range, negated, .. } => { + Ok(Some(RuntimeCommand::BigD { range, negated })) + } + parser::Command::HoldCopy { range, negated, .. } => { + Ok(Some(RuntimeCommand::HoldCopy { range, negated })) + } + parser::Command::HoldAppend { range, negated, .. } => { + Ok(Some(RuntimeCommand::HoldAppend { range, negated })) + } + parser::Command::GetCopy { range, negated, .. } => { + Ok(Some(RuntimeCommand::GetCopy { range, negated })) + } + parser::Command::GetAppend { range, negated, .. } => { + Ok(Some(RuntimeCommand::GetAppend { range, negated })) + } + parser::Command::Exchange { range, negated, .. } => { + Ok(Some(RuntimeCommand::Exchange { range, negated })) + } + parser::Command::Label { name } => Ok(Some(RuntimeCommand::Label { name })), + parser::Command::Branch { + range, + negated, + label, + } => Ok(Some(RuntimeCommand::Branch { + range, + negated, + label, + target_index: None, // Resolved later + })), + parser::Command::Test { + range, + negated, + label, + } => Ok(Some(RuntimeCommand::Test { + range, + negated, + label, + target_index: None, // Resolved later + })), + parser::Command::TestNeg { + range, + negated, + label, + } => Ok(Some(RuntimeCommand::TestNeg { + range, + negated, + label, + target_index: None, // Resolved later + })), + parser::Command::Execute { + range, + negated, + command, + } => Ok(Some(RuntimeCommand::Execute { + range, + negated, + command, + })), + parser::Command::Version { version } => { + // v command is a compile-time check, not a runtime command + // Check version and fail if required version is newer than ours + let required_version = if version.is_empty() { + "4.0" + } else { + version.as_str() + }; + + // Simple version comparison (GNU sed uses strverscmp) + if compare_versions(required_version, constants::GNU_SED_COMPAT_VERSION).is_gt() { + return Err(SedError::parse("expected newer version of sed")); + } + + // v command doesn't generate runtime command + Ok(None) + } + parser::Command::Clear { range, negated } => { + Ok(Some(RuntimeCommand::Clear { range, negated })) + } + parser::Command::PrintFilename { range, negated } => { + Ok(Some(RuntimeCommand::PrintFilename { range, negated })) + } + parser::Command::Next => Ok(Some(RuntimeCommand::Next)), + parser::Command::Write { + range, + negated, + path, + } => Ok(Some(RuntimeCommand::Write { + range, + negated, + path, + })), + parser::Command::WriteFirstLine { + range, + negated, + path, + } => Ok(Some(RuntimeCommand::WriteFirstLine { + range, + negated, + path, + })), + parser::Command::Read { + range, + negated, + path, + } => Ok(Some(RuntimeCommand::Read { + range, + negated, + path, + })), + parser::Command::ReadLine { + range, + negated, + path, + } => Ok(Some(RuntimeCommand::ReadLine { + range, + negated, + path, + })), + parser::Command::LineNumber { range, negated } => { + Ok(Some(RuntimeCommand::LineNumber { range, negated })) + } + parser::Command::Translate { + range, + negated, + from, + to, + from_bytes, + to_bytes, + .. + } => Ok(Some(RuntimeCommand::Translate { + range, + negated, + from, + to, + from_bytes, + to_bytes, + })), + parser::Command::List { + range, + negated, + line_length, + } => Ok(Some(RuntimeCommand::List { + range: range.clone(), + negated, + line_length, + })), + parser::Command::Comment(_) => Ok(None), + _ => { + return Err(SedError::parse(format!( + "unsupported command in script: {:?}", + new_cmd + ))); + } + } +} + +fn parse_scripts_to_commands( + scripts: &[(String, Vec, errors::ScriptSource)], + ctx: &Context, +) -> Result> { + let extended_regex = ctx.extended_regex; + let strict_posix = ctx.is_strict_posix(); + let mut commands: Vec = Vec::new(); + + // Preprocess: merge incomplete a/i/c commands with next script (GNU extension) + // Tuple: (string, raw_bytes, source) + let mut merged_scripts: Vec<(String, Vec, errors::ScriptSource)> = Vec::new(); + let mut i = 0; + while i < scripts.len() { + let (script, raw_bytes, source) = &scripts[i]; + let trimmed = script.trim_end(); + + // Check if script ends with a\, i\, or c\ (incomplete text command) + if !strict_posix + && (trimmed.ends_with("a\\") || trimmed.ends_with("i\\") || trimmed.ends_with("c\\")) + { + if i + 1 < scripts.len() { + // Merge with next script: a\ + \n + next_script + let mut merged = script.clone(); + merged.push('\n'); + merged.push_str(&scripts[i + 1].0); + // Merge raw bytes too + let mut merged_bytes = raw_bytes.clone(); + merged_bytes.push(b'\n'); + merged_bytes.extend_from_slice(&scripts[i + 1].1); + merged_scripts.push((merged, merged_bytes, source.clone())); + i += 2; + continue; + } + } + + merged_scripts.push((script.clone(), raw_bytes.clone(), source.clone())); + i += 1; + } + + // Track last_regex across script parses for empty regex support + let mut last_regex: Option = None; + + for (script_str, raw_bytes, source) in merged_scripts.iter() { + let error_context = source.to_error_context(); + let (parsed_commands, new_last_regex) = ScriptParser::parse_script_with_raw_bytes_chained( + script_str, raw_bytes, ctx, last_regex, + ) + .map_err(|e| e.with_context(error_context.clone()))?; + last_regex = new_last_regex; + let flattened = flatten_parser_commands(parsed_commands); + + // Validate POSIX compatibility if in strict POSIX mode (before conversion) + if strict_posix { + validate_posix_parser_commands(&flattened) + .map_err(|e| e.with_context(error_context.clone()))?; + } + + for new_cmd in flattened { + if let Some(old_cmd) = convert_new_command_to_old(new_cmd, extended_regex, strict_posix) + .map_err(|e| e.with_context(error_context.clone()))? + { + commands.push(old_cmd); + } + } + } + Ok(commands) +} + +fn validate_labels(commands: &[RuntimeCommand]) -> Result<()> { + let mut defined: HashSet<&str> = HashSet::new(); + for cmd in commands { + if let RuntimeCommand::Label { name } = cmd { + defined.insert(name.as_str()); + } + } + for cmd in commands { + match cmd { + RuntimeCommand::Branch { label, .. } + | RuntimeCommand::Test { label, .. } + | RuntimeCommand::TestNeg { label, .. } => { + if !label.is_empty() && !defined.contains(label.as_str()) { + return Err(SedError::runtime(format!("undefined label '{}'", label))); + } + } + _ => {} + } + } + Ok(()) +} + +/// Pre-resolve branch labels to indices for faster execution. +/// This eliminates HashMap lookups during script execution. +fn resolve_branch_labels(commands: &mut [RuntimeCommand]) { + // Build label -> index map + let mut label_to_index: HashMap = HashMap::new(); + for (idx, cmd) in commands.iter().enumerate() { + if let RuntimeCommand::Label { name } = cmd { + label_to_index.insert(name.clone(), idx); + } + } + + // Resolve target indices for branch commands + for cmd in commands.iter_mut() { + match cmd { + RuntimeCommand::Branch { + label, + target_index, + .. + } => { + if label.is_empty() { + // Empty label means branch to end of script + *target_index = None; + } else if let Some(&idx) = label_to_index.get(label) { + *target_index = Some(idx); + } + } + RuntimeCommand::Test { + label, + target_index, + .. + } => { + if label.is_empty() { + *target_index = None; + } else if let Some(&idx) = label_to_index.get(label) { + *target_index = Some(idx); + } + } + RuntimeCommand::TestNeg { + label, + target_index, + .. + } => { + if label.is_empty() { + *target_index = None; + } else if let Some(&idx) = label_to_index.get(label) { + *target_index = Some(idx); + } + } + _ => {} + } + } +} + +/// Validate that commands are compatible with POSIX mode (called at parse time) +fn validate_posix_parser_commands(commands: &[parser::Command]) -> Result<()> { + for cmd in commands { + // Check address ranges for GNU extensions using trait method + if let Some(range) = cmd.address_range() { + // Check start address + if let Some(ref addr) = range.start { + validate_posix_address(addr)?; + } + // Check end address + if let Some(ref addr) = range.end { + validate_posix_address(addr)?; + } + } + + match cmd { + // GNU extension commands forbidden in POSIX mode + parser::Command::Execute { .. } => { + return Err(SedError::parse("unknown command: 'e'")); + } + parser::Command::PrintFilename { .. } => { + return Err(SedError::parse("unknown command: 'F'")); + } + parser::Command::Clear { .. } => { + return Err(SedError::parse("unknown command: 'z'")); + } + parser::Command::QuitSilent { .. } => { + return Err(SedError::parse("unknown command: 'Q'")); + } + parser::Command::TestNeg { .. } => { + return Err(SedError::parse("unknown command: 'T'")); + } + parser::Command::ReadLine { .. } => { + return Err(SedError::parse("unknown command: 'R'")); + } + parser::Command::WriteFirstLine { .. } => { + return Err(SedError::parse("unknown command: 'W'")); + } + parser::Command::Substitution { .. } => { + // POSIX validation for flags is now handled in the parser + } + parser::Command::List { line_length, .. } => { + // Check for GNU extension: l with numeric argument + if line_length.is_some() { + return Err(SedError::parse_at("extra characters after command", 2)); + } + } + parser::Command::Group { commands, .. } => { + // Recursively check group commands + validate_posix_parser_commands(commands)?; + } + _ => {} + } + } + Ok(()) +} + +fn validate_posix_address(addr: &parser::Address) -> Result<()> { + match addr { + parser::Address::Line(0) => { + // Address 0 is a GNU extension (char 6 assumes format "0,/A/p") + Err(SedError::parse_at("invalid usage of line address 0", 6)) + } + parser::Address::Relative(_, _) => { + // Relative addresses (addr,+N) are GNU extensions + Err(SedError::parse_at("unexpected ','", 3)) + } + parser::Address::Step(_, _) => { + // Step addresses (addr,~N) are GNU extensions + Err(SedError::parse_at("unexpected ','", 3)) + } + _ => Ok(()), + } +} + +/// Validate that commands are compatible with sandbox mode +fn validate_sandbox_commands(commands: &[RuntimeCommand]) -> Result<()> { + for cmd in commands { + match cmd { + // Commands forbidden in sandbox mode (file I/O and command execution) + RuntimeCommand::Execute { .. } => { + return Err(SedError::parse("e/r/w commands disabled in sandbox mode")); + } + RuntimeCommand::Read { .. } => { + return Err(SedError::parse("e/r/w commands disabled in sandbox mode")); + } + RuntimeCommand::ReadLine { .. } => { + return Err(SedError::parse("e/r/w commands disabled in sandbox mode")); + } + RuntimeCommand::Write { .. } => { + return Err(SedError::parse("e/r/w commands disabled in sandbox mode")); + } + RuntimeCommand::WriteFirstLine { .. } => { + return Err(SedError::parse("e/r/w commands disabled in sandbox mode")); + } + // Substitution with execute flag or write_file option + RuntimeCommand::Substitution(ref subst) + if subst.execute || subst.write_file.is_some() => + { + return Err(SedError::parse("e/r/w commands disabled in sandbox mode")); + } + _ => {} + } + } + Ok(()) +} + +// expand_backup_suffix moved to fileio/inplace.rs +use fileio::expand_backup_suffix; + +/// Process a single file for in-place editing +fn process_single_file( + file_path: &str, + commands: &[RuntimeCommand], + quiet_mode: bool, + extended_regex: bool, + line_length: usize, + unbuffered: bool, + null_data: bool, + posix: bool, + binary: bool, + follow_symlinks: bool, + backup_suffix: Option<&str>, +) -> Result<()> { + // Check if file is a symlink + let meta = std::fs::symlink_metadata(file_path) + .map_err(|e| SedError::io("can't read", file_path, e))?; + + let is_symlink = meta.file_type().is_symlink(); + + // Determine the actual file to read from (resolve symlinks) + let read_from_path = if is_symlink { + // Use strict mode for in-place editing (returns errors) + let resolved = resolve_symlink_chain(std::path::Path::new(file_path), true)?; + resolved + .to_str() + .ok_or_else(|| SedError::runtime("symlink target path is not valid UTF-8"))? + .to_string() + } else { + file_path.to_string() + }; + + // Determine output path: + // - With --follow-symlinks: write to resolved target + // - Without --follow-symlinks: write to original path (replacing symlink with regular file) + let write_to_path = if follow_symlinks { + read_from_path.clone() + } else { + file_path.to_string() + }; + + let file_to_read = read_from_path.as_str(); + let file_to_write = write_to_path.as_str(); + + // Capture SELinux context before processing + // When follow_symlinks is true, we get context from resolved target (getfilecon) + // When follow_symlinks is false, we get context from original path/symlink (lgetfilecon) + #[cfg(feature = "selinux")] + let selinux_context = selinux::get_context( + std::path::Path::new(if follow_symlinks { + file_to_read + } else { + file_path + }), + follow_symlinks, + ); + + // Get file metadata and check file type (check the actual file we read from) + let file_metadata = + std::fs::metadata(file_to_read).map_err(|e| SedError::io("can't read", file_to_read, e))?; + + // Check if file is a regular file (not FIFO, device, socket, etc.) + if !file_metadata.file_type().is_file() { + // On Unix, check if it's a terminal (character device that is a tty) + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + let file_type = file_metadata.file_type(); + if file_type.is_char_device() { + // Try to open and check if it's a terminal using IsTerminal trait + if let Ok(file) = File::open(file_to_read) { + use std::io::IsTerminal; + if file.is_terminal() { + return Err(SedError::inplace(format!( + "couldn't edit {}: is a terminal", + file_path + ))); + } + } + } + } + return Err(SedError::inplace(format!( + "couldn't edit {}: not a regular file", + file_path + ))); + } + + // Preserve original file permissions (mode) + let original_permissions = file_metadata.permissions(); + // Read the file content + let mut file = + File::open(file_to_read).map_err(|e| SedError::io("can't read", file_to_read, e))?; + let mut content = Vec::new(); + file.read_to_end(&mut content)?; + + // Check early if we can create temp file in the directory + // This catches permission errors before we do any processing + // Use file_to_write directory for temp file (handles symlink replacement case) + let temp_path = format!("{}.red_temp_{}", file_to_write, std::process::id()); + + // Register temp file for cleanup on signal + signals::unix::register_temp_file(temp_path.clone()); + + let temp_file_result = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&temp_path); + + if let Err(e) = temp_file_result { + signals::unix::unregister_temp_file(&temp_path); + // GNU sed outputs "couldn't open temporary file" for temp file creation errors + return Err(SedError::inplace(format!( + "couldn't open temporary file {}: {}", + temp_path, + match e.kind() { + std::io::ErrorKind::PermissionDenied => "Permission denied".to_string(), + std::io::ErrorKind::NotFound => "No such file or directory".to_string(), + _ => e + .to_string() + .split(" (os error") + .next() + .unwrap_or(&e.to_string()) + .to_string(), + } + ))); + } + + let temp_file = temp_file_result.unwrap(); + + // Split into lines (preserving original byte representation) + let (all_lines, all_lines_bytes, encoding, ends_with_separator) = + split_file_content(content, null_data, binary); + + if all_lines.is_empty() { + // Empty file - clean up temp file and just create backup if needed + drop(temp_file); + signals::unix::unregister_temp_file(&temp_path); + let _ = std::fs::remove_file(&temp_path); + if let Some(suffix) = backup_suffix { + let backup_path = expand_backup_suffix(file_to_write, suffix); + std::fs::copy(file_to_read, &backup_path) + .map_err(|e| SedError::rename(file_to_read, &backup_path, e))?; + } + return Ok(()); + } + + // Create backup if suffix provided + if let Some(suffix) = backup_suffix { + let backup_path = expand_backup_suffix(file_to_write, suffix); + std::fs::copy(file_to_read, &backup_path) + .map_err(|e| SedError::rename(file_to_read, &backup_path, e))?; + } + + // Use the temp file we already created and validated above + let mut output = BufWriter::new(temp_file); + + // Execute using the same engine as normal mode + // All lines from the same file, so create filename vector + let filenames = vec![file_to_read.to_string(); all_lines.len()]; + let execute_result = execute_over_lines( + commands, + quiet_mode, + extended_regex, + line_length, + unbuffered, + null_data, + posix, + binary, + &all_lines, + &all_lines_bytes, + &filenames, + encoding, + ends_with_separator, + &mut output, + ); + + // If execution failed, clean up temp file and return error + if let Err(e) = execute_result { + signals::unix::unregister_temp_file(&temp_path); + let _ = std::fs::remove_file(&temp_path); + return Err(e); + } + + // Flush and close the temporary file + if let Err(e) = output.flush() { + signals::unix::unregister_temp_file(&temp_path); + let _ = std::fs::remove_file(&temp_path); + return Err(e.into()); + } + drop(output); + + // If we're replacing a symlink (not following it), remove the symlink first + if is_symlink && !follow_symlinks { + std::fs::remove_file(file_to_write) + .map_err(|e| SedError::io("can't remove symlink", file_to_write, e))?; + } + + // Replace original file with processed content + std::fs::rename(&temp_path, file_to_write) + .map_err(|e| SedError::io("can't rename", file_to_write, e))?; + + // Unregister temp file after successful rename + signals::unix::unregister_temp_file(&temp_path); + + // Restore original permissions (best-effort) + if let Err(err) = std::fs::set_permissions(file_to_write, original_permissions) { + // Do not fail the whole operation due to permissions restoration issues + eprintln!( + "red: warning: failed to restore file permissions for {}: {}", + file_to_write, err + ); + } + + // Restore SELinux context (best-effort) + #[cfg(feature = "selinux")] + if let Some(ref ctx) = selinux_context { + if let Err(err) = selinux::set_context(std::path::Path::new(file_to_write), ctx) { + eprintln!( + "red: warning: failed to restore SELinux context for {}: {}", + file_to_write, err + ); + } + } + + Ok(()) +} + +// split_file_content moved to fileio/lines.rs +use fileio::split_file_content; + +/// Main entry point for running sed commands on input files +/// Takes a RunConfig with scripts, input files, and flags, processes each file/stdin +/// according to sed semantics, and writes output to stdout or file (for in-place editing) +pub fn run(config: RunConfig) -> Result<()> { + // Phase 1.3: Validate configuration before execution + validate_config(&config)?; + + // Create unified Context from RunConfig + // Phase 3.1: Context now integrated with Parser + let ctx = Context::from_run_config(&config, config.scripts_with_sources.clone()); + + let mut commands = parse_scripts_to_commands(&config.scripts_with_sources, &ctx)?; + validate_labels(&commands)?; + resolve_branch_labels(&mut commands); + + // Validate sandbox compatibility if in sandbox mode + if config.sandbox { + validate_sandbox_commands(&commands)?; + } + + let quiet_by_header = scripts_request_quiet(&config.scripts_with_sources); + + // Handle in-place editing + if let Some(ref backup_suffix) = config.in_place { + let quiet_mode = config.quiet || quiet_by_header; + let backup_opt = if backup_suffix.is_empty() { + None + } else { + Some(backup_suffix.as_str()) + }; + for file_path in &config.input_files { + process_single_file( + file_path, + &commands, + quiet_mode, + config.extended_regex, + config.line_length, + config.unbuffered, + config.null_data, + config.posix, + config.binary, + config.follow_symlinks, + backup_opt, + )?; + } + return Ok(()); + } + + // Handle separate files mode (-s) + if config.separate_files && !config.input_files.is_empty() { + let stdout = io::stdout(); + let mut out = io::BufWriter::new(stdout.lock()); + + for file_path in &config.input_files { + // Read each file separately + let (file_lines, file_lines_bytes, file_filenames, encoding, ends_with_separator) = + read_all_lines( + &[file_path.clone()], + config.null_data, + config.follow_symlinks, + config.binary, + )?; + + // Execute with fresh context for each file (line numbers reset) + execute_over_lines( + &commands, + config.quiet || quiet_by_header, + config.extended_regex, + config.line_length, + config.unbuffered, + config.null_data, + config.posix, + config.binary, + &file_lines, + &file_lines_bytes, + &file_filenames, + encoding, + ends_with_separator, + &mut out, + )?; + } + Ok(()) + } else { + // Check if unbuffered mode with stdin - need line-by-line processing to support early exit + let is_stdin = config.input_files.is_empty() + || (config.input_files.len() == 1 && config.input_files[0] == "-"); + if config.unbuffered && is_stdin { + // Unbuffered stdin: read line by line to allow early quit + let stdout = io::stdout(); + let mut out = io::BufWriter::new(stdout.lock()); + + return process_stdin_line_by_line( + &commands, + config.quiet || quiet_by_header, + config.extended_regex, + config.line_length, + config.null_data, + config.posix, + config.binary, + &mut out, + ); + } + + // Normal mode: treat all files as continuous stream + let (all_lines, all_lines_bytes, all_filenames, encoding, ends_with_separator) = + read_all_lines( + &config.input_files, + config.null_data, + config.follow_symlinks, + config.binary, + )?; + let stdout = io::stdout(); + let mut out = io::BufWriter::new(stdout.lock()); + execute_over_lines( + &commands, + config.quiet || quiet_by_header, + config.extended_regex, + config.line_length, + config.unbuffered, + config.null_data, + config.posix, + config.binary, + &all_lines, + &all_lines_bytes, + &all_filenames, + encoding, + ends_with_separator, + &mut out, + ) + } +} + +/// Process stdin line-by-line for unbuffered mode +/// This allows early exit on quit command without reading all input +fn process_stdin_line_by_line( + commands: &[RuntimeCommand], + quiet_mode: bool, + extended_regex: bool, + line_length: usize, + null_data: bool, + posix: bool, + binary: bool, + out: &mut dyn Write, +) -> Result<()> { + let separator = if null_data { b'\0' } else { b'\n' }; + + let mut ctx = ExecutionContext::new( + None, // total_lines unknown in streaming mode + quiet_mode, + extended_regex, + line_length, + true, // unbuffered + null_data, + ); + let mut evaluator = AddressEvaluator::new(extended_regex, posix); + + let mut line_num = 0; + + // Read stdin byte-by-byte WITHOUT buffering using raw file descriptor + // This ensures we only read exactly what we need and leave the rest for other processes + #[cfg(unix)] + use std::os::unix::io::FromRawFd; + + #[cfg(unix)] + let mut stdin_raw = unsafe { std::fs::File::from_raw_fd(0) }; + + #[cfg(not(unix))] + let mut stdin_raw = io::stdin(); + + let mut line_bytes: Vec = Vec::new(); + let mut encountered_quit = false; + let mut buf = [0u8; 1]; + + loop { + // Read bytes until we hit separator or EOF + let mut eof = false; + loop { + match stdin_raw.read(&mut buf) { + Ok(0) => { + // EOF + eof = true; + break; + } + Ok(_) => { + let byte = buf[0]; + if byte == separator { + break; // End of line + } + line_bytes.push(byte); + } + Err(e) => { + return Err(SedError::io("can't read", "-", e)); + } + } + } + + // If we have a line + if !line_bytes.is_empty() { + line_num += 1; + ctx.set_current_line(line_num, line_bytes.clone()); + ctx.set_filename("-"); + + let results = apply_commands_with_context(commands, &mut ctx, &mut evaluator, 0)?; + + let mut should_autoprint = !ctx.quiet_mode; + + for result in results { + match result { + CommandResult::Continue(final_line, raw_bytes) => { + if should_autoprint { + write_output_line( + &final_line, + raw_bytes.as_deref(), + null_data, + true, + binary, + out, + ); + out.flush()?; + } + } + CommandResult::Print(print_line, raw_bytes) => { + write_output_line( + &print_line, + raw_bytes.as_deref(), + null_data, + true, + binary, + out, + ); + out.flush()?; + } + CommandResult::Delete => { + should_autoprint = false; + } + CommandResult::SuppressAutoprint => { + should_autoprint = false; + } + CommandResult::PrintAndContinue(line) => { + if null_data { + write!(out, "{}\0", line)?; + } else { + out.write_all(line.as_bytes())?; + out.write_all(line_ending(binary))?; + } + out.flush()?; + } + CommandResult::Quit(exit_code) => { + out.flush()?; + encountered_quit = true; + // Don't call process::exit() - just stop reading input + // This allows other processes in a pipeline to read remaining input + if let Some(code) = exit_code { + if code != 0 { + std::process::exit(code); + } + } + break; + } + _ => { + // For simplicity, we don't fully support all commands in streaming mode + // Commands like N (append next line) would require lookahead + } + } + } + + // Clear line buffer for next iteration + line_bytes.clear(); + + if encountered_quit { + // Don't drop stdin - this ensures remaining data stays in the pipe + // for other processes to read + std::mem::forget(stdin_raw); + return Ok(()); + } + } + + // Break if EOF + if eof { + break; + } + } + + Ok(()) +} + +// Output helpers from fileio module +use fileio::{flush_output, line_ending, write_output_line}; + +/// Result of processing command results - determines control flow +enum ProcessedResult { + /// Normal completion, continue to next line + Done, + /// Delete was encountered, skip to next line without output + Deleted, + /// Restart the script cycle (D command modified pattern space) + Restart, + /// Quit with exit code + Quit(i32), + /// Resume from PC after reading next line (n command) + NextLineAndResume(usize), + /// Append next line and resume (N command) + AppendNextAndResume { + resume_pc: usize, + pattern_space: String, + }, +} + +/// Result of N command handling - determines outer loop control flow +enum NCommandOutcome { + /// Normal completion, break from script_cycle + Complete, + /// Break from script_cycle (EOF or delete) + BreakCycle, + /// Continue script_cycle (restart) + ContinueCycle, +} + +/// Handle the N command - append next lines to pattern space and resume execution. +/// +/// This extracts the complex N command loop from `execute_over_lines()` to reduce nesting. +#[allow(clippy::too_many_arguments)] +fn handle_n_command( + commands: &[RuntimeCommand], + ctx: &mut ExecutionContext, + evaluator: &mut AddressEvaluator, + all_lines: &[String], + _all_lines_bytes: &[Vec], + all_filenames: &[String], + null_data: bool, + posix: bool, + unbuffered: bool, + binary: bool, + suppress_autoprint: &mut bool, + line_idx: &mut usize, + initial_resume_pc: usize, + initial_pattern_space: String, + out: &mut dyn Write, +) -> Result { + let mut current_resume_pc = initial_resume_pc; + let mut current_pattern_space = initial_pattern_space; + + loop { + if *line_idx + 1 >= all_lines.len() { + // EOF: N quits; GNU mode prints, POSIX mode doesn't + if !posix && !ctx.quiet_mode && !*suppress_autoprint { + let _ = out.write_all(ctx.pattern_space.text().as_bytes()); + let _ = out.write_all(line_ending(binary)); + flush_output(out, unbuffered); + } + return Ok(NCommandOutcome::BreakCycle); + } + + // Append next line to pattern space + ctx.pattern_space.set(current_pattern_space.clone()); + *line_idx += 1; + ctx.current_line_num = *line_idx + 1; + ctx.set_filename(&all_filenames[*line_idx]); + ctx.pattern_space.push(if null_data { '\0' } else { '\n' }); + ctx.pattern_space.push_str(&all_lines[*line_idx]); + if *line_idx + 1 >= all_lines.len() { + ctx.all_input_consumed = true; + } + + // Resume execution + let resumed = apply_commands_with_context(commands, ctx, evaluator, current_resume_pc)?; + + let inner_action = process_command_results( + resumed, + ctx, + null_data, + true, // N command always writes separator + unbuffered, + binary, + suppress_autoprint, + out, + ); + + match inner_action { + ProcessedResult::Done => return Ok(NCommandOutcome::Complete), + ProcessedResult::Deleted => return Ok(NCommandOutcome::BreakCycle), + ProcessedResult::Restart => return Ok(NCommandOutcome::ContinueCycle), + ProcessedResult::Quit(code) => std::process::exit(code), + ProcessedResult::NextLineAndResume(_) => { + // Nested n in N context - ignore + } + ProcessedResult::AppendNextAndResume { + resume_pc: rp, + pattern_space: ps, + } => { + // Continue with next N + current_resume_pc = rp; + current_pattern_space = ps; + continue; + } + } + break; // Normal completion of inner loop + } + Ok(NCommandOutcome::Complete) +} + +/// Process command results and handle output, returning control flow action. +/// +/// This unifies result handling between the main loop and nested N command loop. +/// The `suppress_autoprint` parameter is updated if SuppressAutoprint result is seen. +fn process_command_results( + results: Vec, + ctx: &mut ExecutionContext, + null_data: bool, + write_separator: bool, + unbuffered: bool, + binary: bool, + suppress_autoprint: &mut bool, + out: &mut dyn Write, +) -> ProcessedResult { + for result in results { + match result { + CommandResult::Continue(final_line, raw_bytes) => { + if !ctx.quiet_mode && !*suppress_autoprint { + write_output_line( + &final_line, + raw_bytes.as_deref(), + null_data, + write_separator, + binary, + out, + ); + flush_output(out, unbuffered); + } + } + CommandResult::Print(print_line, raw_bytes) => { + write_output_line( + &print_line, + raw_bytes.as_deref(), + null_data, + true, + binary, + out, + ); + flush_output(out, unbuffered); + } + CommandResult::Delete => { + return ProcessedResult::Deleted; + } + CommandResult::SuppressAutoprint => { + *suppress_autoprint = true; + } + CommandResult::PrintAndContinue(line) => { + if null_data { + let _ = write!(out, "{}\0", line); + } else { + let _ = out.write_all(line.as_bytes()); + let _ = out.write_all(line_ending(binary)); + } + flush_output(out, unbuffered); + } + CommandResult::Quit(exit_code) => { + let _ = out.flush(); + return ProcessedResult::Quit(exit_code.unwrap_or(0)); + } + CommandResult::Restart => { + return ProcessedResult::Restart; + } + CommandResult::RestartWith(new_ps) => { + ctx.pattern_space.set(new_ps); + return ProcessedResult::Restart; + } + CommandResult::RestartWithBytes(bytes) => { + ctx.pattern_space.set_raw(bytes); + return ProcessedResult::Restart; + } + CommandResult::AppendNextAndResume { + resume_pc, + pattern_space, + } => { + return ProcessedResult::AppendNextAndResume { + resume_pc, + pattern_space, + }; + } + CommandResult::NextLineAndResume { resume_pc } => { + return ProcessedResult::NextLineAndResume(resume_pc); + } + } + } + + ProcessedResult::Done +} + +/// Execute the parsed commands over provided lines, writing results into `out`. +fn execute_over_lines( + commands: &[RuntimeCommand], + quiet_mode: bool, + extended_regex: bool, + line_length: usize, + unbuffered: bool, + null_data: bool, + posix: bool, + binary: bool, + all_lines: &Vec, + all_lines_bytes: &Vec>, + all_filenames: &Vec, + _encoding: &'static Encoding, + ends_with_separator: bool, + out: &mut dyn Write, +) -> Result<()> { + let total_lines = if all_lines.is_empty() { + None + } else { + Some(all_lines.len()) + }; + let mut ctx = ExecutionContext::new( + total_lines, + quiet_mode, + extended_regex, + line_length, + unbuffered, + null_data, + ); + let mut evaluator = AddressEvaluator::new(extended_regex, posix); + + let mut line_idx: usize = 0; + while line_idx < all_lines.len() { + let line_num = line_idx + 1; + ctx.set_current_line(line_num, all_lines_bytes[line_idx].clone()); + ctx.set_filename(&all_filenames[line_idx]); + + // Check if this is the last line and whether to write separator + let is_last_line = line_idx == all_lines.len() - 1; + let should_write_separator = !is_last_line || ends_with_separator; + let mut resume_from_pc: Option = None; + + 'script_cycle: loop { + let next_pc: usize = resume_from_pc.take().unwrap_or(0); + let results = + apply_commands_with_context(&commands, &mut ctx, &mut evaluator, next_pc)?; + + let mut suppress_autoprint = false; + let action = process_command_results( + results, + &mut ctx, + null_data, + should_write_separator, + unbuffered, + binary, + &mut suppress_autoprint, + out, + ); + + match action { + ProcessedResult::Done => break, + ProcessedResult::Deleted => break 'script_cycle, + ProcessedResult::Restart => continue 'script_cycle, + ProcessedResult::Quit(code) => std::process::exit(code), + ProcessedResult::NextLineAndResume(next_resume_pc) => { + if line_idx + 1 < all_lines.len() { + line_idx += 1; + ctx.set_current_line(line_idx + 1, all_lines_bytes[line_idx].clone()); + ctx.set_filename(&all_filenames[line_idx]); + resume_from_pc = Some(next_resume_pc); + continue 'script_cycle; + } else { + break 'script_cycle; + } + } + ProcessedResult::AppendNextAndResume { + resume_pc, + pattern_space, + } => { + let outcome = handle_n_command( + commands, + &mut ctx, + &mut evaluator, + all_lines, + all_lines_bytes, + all_filenames, + null_data, + posix, + unbuffered, + binary, + &mut suppress_autoprint, + &mut line_idx, + resume_pc, + pattern_space, + out, + )?; + match outcome { + NCommandOutcome::Complete => break, + NCommandOutcome::BreakCycle => break 'script_cycle, + NCommandOutcome::ContinueCycle => continue 'script_cycle, + } + } + } + } + line_idx += 1; + } + + Ok(()) +} diff --git a/red/src/main.rs b/red/src/main.rs new file mode 100644 index 0000000..4bf49c2 --- /dev/null +++ b/red/src/main.rs @@ -0,0 +1,92 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +mod cli; + +use red::constants::DEFAULT_LINE_LENGTH; +use red::errors::Result; +use red::{run as red_run, RunConfig}; + +fn run(args: cli::CliArgs) -> Result<()> { + let extended_regex = args.extended_regex; + + // Determine line length: -l flag takes precedence, otherwise check COLS envvar + let line_length = if args.line_length != DEFAULT_LINE_LENGTH { + // User specified -l flag explicitly, use it + args.line_length + } else { + // Check COLS environment variable (GNU sed compatibility) + match std::env::var("COLS") { + Ok(cols_str) => { + match cols_str.parse::() { + Ok(cols) if cols > 1 => cols - 1, // sed subtracts 1 to avoid terminal line wraps + _ => DEFAULT_LINE_LENGTH, // Invalid or 0/1, use default + } + } + Err(_) => DEFAULT_LINE_LENGTH, // COLS not set, use default + } + }; + + let cfg = RunConfig { + scripts_with_sources: args.scripts_with_sources, + // Extract just filenames from (filename, raw_bytes) tuples + input_files: args.files.into_iter().map(|(f, _)| f).collect(), + quiet: args.quiet, + in_place: args.in_place, + extended_regex, + separate_files: args.separate_files, + line_length, + unbuffered: args.unbuffered, + posix: args.posix || std::env::var("POSIXLY_CORRECT").is_ok(), + strict_posix: args.posix, // Only --posix flag, not POSIXLY_CORRECT + follow_symlinks: args.follow_symlinks, + sandbox: args.sandbox, + null_data: args.null_data, + binary: args.binary, + }; + + // On Windows, set stdin/stdout mode based on -b flag + // Default: TEXT mode (LF -> CRLF conversion) for GNU sed compatibility + // With -b: BINARY mode (no conversion) + #[cfg(windows)] + { + const O_TEXT: i32 = 0x4000; + const O_BINARY: i32 = 0x8000; + unsafe { + extern "C" { + fn _setmode(fd: i32, mode: i32) -> i32; + } + let mode = if args.binary { O_BINARY } else { O_TEXT }; + _setmode(0, mode); // stdin + _setmode(1, mode); // stdout + } + } + + red_run(cfg) +} + +fn main() { + // Set up signal handlers for cleanup of temporary files + #[cfg(unix)] + if let Err(e) = red::signals::unix::setup_signal_handlers() { + eprintln!("sed: warning: failed to setup signal handlers: {}", e); + } + + // Initialize multibyte character set support + red::mbcs::initialize(); + + let args = match cli::parse_args() { + Ok(args) => args, + Err(error) => { + eprintln!("sed: {error}"); + std::process::exit(1); + } + }; + + if let Err(error) = run(args) { + let exit_code = error.exit_code(); + eprintln!("sed: {error}"); + std::process::exit(exit_code); + } +} diff --git a/red/src/mbcs.rs b/red/src/mbcs.rs new file mode 100644 index 0000000..de334f3 --- /dev/null +++ b/red/src/mbcs.rs @@ -0,0 +1,729 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Multibyte Character Set Support +//! +//! This module provides locale-aware multibyte character handling, +//! similar to GNU sed's mbcs.c. It uses libc functions (mbrlen, mbrtowc) +//! to properly handle stateful encodings like Shift-JIS. + +use std::sync::OnceLock; + +/// Global MB_CUR_MAX value, initialized once at startup +static MB_CUR_MAX: OnceLock = OnceLock::new(); + +/// Check if we're in a multibyte locale (MB_CUR_MAX > 1) +static IS_MULTIBYTE_LOCALE: OnceLock = OnceLock::new(); + +// FFI declarations for multibyte functions +#[cfg(unix)] +mod ffi { + use std::os::raw::c_int; + + // mbstate_t is an opaque type, we'll use a fixed-size array + // The actual size varies by platform, but 128 bytes should be enough + #[repr(C)] + #[derive(Clone, Copy)] + pub struct MbStateT { + _data: [u8; 128], + } + + impl Default for MbStateT { + fn default() -> Self { + Self { _data: [0u8; 128] } + } + } + + // On Linux (glibc), MB_CUR_MAX is accessed via __ctype_get_mb_cur_max() + // On macOS/BSD, it's a global variable ___mb_cur_max + #[cfg(target_os = "linux")] + extern "C" { + #[link_name = "__ctype_get_mb_cur_max"] + pub fn mb_cur_max() -> usize; + } + + // On macOS, ___mb_cur_max() is a function that returns the current MB_CUR_MAX + #[cfg(target_os = "macos")] + extern "C" { + #[link_name = "___mb_cur_max"] + fn mb_cur_max_fn() -> libc::c_int; + } + + #[cfg(target_os = "macos")] + #[allow(unused_unsafe)] + pub fn mb_cur_max() -> usize { + unsafe { mb_cur_max_fn() as usize } + } + + extern "C" { + + // Check if mbstate_t is in initial state + pub fn mbsinit(ps: *const MbStateT) -> c_int; + + // Convert multibyte to wide character, return bytes consumed + pub fn mbrtowc(pwc: *mut libc::wchar_t, s: *const u8, n: usize, ps: *mut MbStateT) + -> isize; + + // Get length of multibyte character + pub fn mbrlen(s: *const u8, n: usize, ps: *mut MbStateT) -> isize; + } +} + +/// Initialize multibyte support. Call this early in main(). +/// This must be called after program startup to inherit the environment's locale. +#[cfg(unix)] +#[allow(unused_unsafe)] // On macOS ffi::mb_cur_max() is safe, on Linux it's extern "C" +pub fn initialize() { + // First, call setlocale to inherit the environment's locale settings. + // Without this, the C library uses the default "C" locale. + unsafe { + libc::setlocale(libc::LC_ALL, b"\0".as_ptr() as *const libc::c_char); + } + let max = unsafe { ffi::mb_cur_max() }; + MB_CUR_MAX.get_or_init(|| max); + IS_MULTIBYTE_LOCALE.get_or_init(|| max > 1); +} + +#[cfg(not(unix))] +pub fn initialize() { + MB_CUR_MAX.get_or_init(|| 1); + IS_MULTIBYTE_LOCALE.get_or_init(|| false); +} + +/// Get the current locale's MB_CUR_MAX value +#[cfg(unix)] +#[inline] +#[allow(unused_unsafe)] // On macOS ffi::mb_cur_max() is safe, on Linux it's extern "C" +pub fn mb_cur_max() -> usize { + *MB_CUR_MAX.get_or_init(|| unsafe { ffi::mb_cur_max() }) +} + +#[cfg(not(unix))] +#[inline] +pub fn mb_cur_max() -> usize { + *MB_CUR_MAX.get_or_init(|| 1) +} + +/// Check if we're in a multibyte locale +#[inline] +pub fn is_multibyte_locale() -> bool { + *IS_MULTIBYTE_LOCALE.get_or_init(|| mb_cur_max() > 1) +} + +/// Multibyte parsing state (wrapper around libc mbstate_t) +#[cfg(unix)] +#[derive(Clone)] +pub struct MbState { + state: ffi::MbStateT, +} + +#[cfg(not(unix))] +#[derive(Clone)] +pub struct MbState { + _dummy: u8, +} + +impl Default for MbState { + fn default() -> Self { + Self::new() + } +} + +impl MbState { + #[cfg(unix)] + pub fn new() -> Self { + Self { + state: ffi::MbStateT::default(), + } + } + + #[cfg(not(unix))] + pub fn new() -> Self { + Self { _dummy: 0 } + } + + #[cfg(unix)] + pub fn reset(&mut self) { + self.state = ffi::MbStateT::default(); + } + + #[cfg(not(unix))] + pub fn reset(&mut self) { + self._dummy = 0; + } + + #[cfg(unix)] + pub fn is_initial(&self) -> bool { + unsafe { ffi::mbsinit(&self.state) != 0 } + } + + #[cfg(not(unix))] + pub fn is_initial(&self) -> bool { + true + } + + /// Check if the given byte is part of a multibyte sequence. + #[cfg(unix)] + pub fn is_mb_char(&mut self, byte: u8) -> bool { + if !is_multibyte_locale() { + return false; + } + + let was_pending = !self.is_initial(); + let result = + unsafe { ffi::mbrtowc(std::ptr::null_mut(), &byte as *const u8, 1, &mut self.state) }; + + match result { + -2 => true, // Incomplete but valid multibyte sequence + -1 => { + self.reset(); + false // Invalid sequence + } + 0 => true, // NUL character + 1 => was_pending, // Valid byte, part of multibyte if we had pending + _ => false, + } + } + + #[cfg(not(unix))] + pub fn is_mb_char(&mut self, _byte: u8) -> bool { + false + } +} + +/// Count the number of multibyte characters in a byte slice. +#[cfg(unix)] +pub fn count_mb_chars(bytes: &[u8]) -> usize { + if !is_multibyte_locale() { + return bytes.len(); + } + + let mut count = 0; + let mut state = ffi::MbStateT::default(); + let mut i = 0; + + while i < bytes.len() { + let remaining = bytes.len() - i; + let result = unsafe { ffi::mbrlen(bytes[i..].as_ptr(), remaining, &mut state) }; + + match result { + -2 => { + // Incomplete sequence at end + count += remaining; + break; + } + -1 => { + // Invalid sequence - count as single byte + count += 1; + i += 1; + state = ffi::MbStateT::default(); + } + 0 => { + count += 1; + i += 1; + } + n => { + count += 1; + i += n as usize; + } + } + } + + count +} + +#[cfg(not(unix))] +pub fn count_mb_chars(bytes: &[u8]) -> usize { + bytes.len() +} + +/// Iterator over multibyte characters in a byte slice. +#[cfg(unix)] +pub struct MbCharIter<'a> { + bytes: &'a [u8], + pos: usize, + state: ffi::MbStateT, +} + +#[cfg(not(unix))] +pub struct MbCharIter<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> MbCharIter<'a> { + #[cfg(unix)] + pub fn new(bytes: &'a [u8]) -> Self { + Self { + bytes, + pos: 0, + state: ffi::MbStateT::default(), + } + } + + #[cfg(not(unix))] + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } +} + +impl<'a> Iterator for MbCharIter<'a> { + type Item = &'a [u8]; + + #[cfg(unix)] + fn next(&mut self) -> Option { + if self.pos >= self.bytes.len() { + return None; + } + + if !is_multibyte_locale() { + let byte = &self.bytes[self.pos..self.pos + 1]; + self.pos += 1; + return Some(byte); + } + + let remaining = self.bytes.len() - self.pos; + let result = + unsafe { ffi::mbrlen(self.bytes[self.pos..].as_ptr(), remaining, &mut self.state) }; + + let char_len = match result { + -2 | -1 => { + self.state = ffi::MbStateT::default(); + 1 + } + 0 => 1, + n => n as usize, + }; + + let start = self.pos; + self.pos += char_len; + Some(&self.bytes[start..self.pos]) + } + + #[cfg(not(unix))] + fn next(&mut self) -> Option { + if self.pos >= self.bytes.len() { + return None; + } + let byte = &self.bytes[self.pos..self.pos + 1]; + self.pos += 1; + Some(byte) + } +} + +/// Convert a multibyte byte sequence to a wide character (for CharSet matching). +/// Returns None if the conversion fails or the sequence is invalid. +#[cfg(unix)] +pub fn mb_to_wchar(bytes: &[u8]) -> Option { + if bytes.is_empty() { + return None; + } + + // Single ASCII byte - fast path + if bytes.len() == 1 && bytes[0] < 128 { + return Some(bytes[0] as char); + } + + // If not in MB locale, treat each byte as a character + if !is_multibyte_locale() { + if bytes.len() == 1 { + return Some(bytes[0] as char); + } + return None; + } + + // Use mbrtowc to convert + let mut state = ffi::MbStateT::default(); + let mut wc: libc::wchar_t = 0; + + let result = unsafe { ffi::mbrtowc(&mut wc, bytes.as_ptr(), bytes.len(), &mut state) }; + + if result > 0 && result != -1 && result != -2 { + // Valid conversion - convert wchar_t to char if it's a valid Unicode codepoint + // wchar_t is i32 on most platforms, cast to u32 for range check + let wc_u32 = wc as u32; + if wc_u32 <= 0x10FFFF { + char::from_u32(wc_u32) + } else { + None + } + } else { + None + } +} + +#[cfg(not(unix))] +pub fn mb_to_wchar(bytes: &[u8]) -> Option { + if bytes.len() == 1 { + Some(bytes[0] as char) + } else { + None + } +} + +/// Represents a multibyte character - a slice of bytes that form one logical character. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MbChar<'a> { + /// The raw bytes of this character + pub bytes: &'a [u8], +} + +impl<'a> MbChar<'a> { + /// Create a new MbChar from a byte slice + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes } + } + + /// Get the byte length of this character + pub fn len(&self) -> usize { + self.bytes.len() + } + + /// Check if this is an empty character + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Check if this is a single ASCII byte + pub fn is_ascii(&self) -> bool { + self.bytes.len() == 1 && self.bytes[0] < 128 + } + + /// Try to convert to a Rust char (only works for valid UTF-8) + pub fn to_char(&self) -> Option { + std::str::from_utf8(self.bytes).ok()?.chars().next() + } + + /// Check if this character matches a byte + pub fn matches_byte(&self, byte: u8) -> bool { + self.bytes.len() == 1 && self.bytes[0] == byte + } + + /// Convert to wide character (wchar_t) for CharSet matching + /// Returns the character as a Rust char if conversion succeeds + pub fn to_wchar(&self) -> Option { + mb_to_wchar(self.bytes) + } + + /// Check if this character equals another MbChar (byte-level comparison) + pub fn equals(&self, other: &MbChar) -> bool { + self.bytes == other.bytes + } + + /// Check if this character matches a Rust char (single character comparison) + pub fn matches_char(&self, ch: char) -> bool { + // Fast path for ASCII + if ch.is_ascii() && self.bytes.len() == 1 { + return self.bytes[0] == ch as u8; + } + // Convert to wide char and compare + self.to_wchar() == Some(ch) + } + + /// Check if this character matches a Rust char (case-insensitive) + pub fn matches_char_ignore_case(&self, ch: char) -> bool { + if let Some(wc) = self.to_wchar() { + wc.to_lowercase().eq(ch.to_lowercase()) + } else { + false + } + } + + /// Check if this character is a word character (alphanumeric or underscore) + /// Used for word boundary matching (\b, \B, \<, \>) + pub fn is_word_char(&self) -> bool { + self.to_wchar() + .map(|wc| wc.is_alphanumeric() || wc == '_') + .unwrap_or(false) + } +} + +/// A text representation that tracks character boundaries for MBCS locales. +/// This allows efficient iteration and position conversion between byte and char indices. +#[derive(Debug, Clone)] +pub struct MbText<'a> { + /// The raw bytes + pub raw: &'a [u8], + /// Character boundaries: char_boundaries[i] is the byte offset where character i starts + /// The length is num_chars + 1, with the last entry being raw.len() + pub char_boundaries: Vec, + /// Validity of each character: true if valid multibyte char, false if invalid/incomplete + pub char_validity: Vec, +} + +impl<'a> MbText<'a> { + /// Create a new MbText from raw bytes, computing character boundaries + pub fn new(raw: &'a [u8]) -> Self { + let (char_boundaries, char_validity) = compute_char_boundaries(raw); + Self { + raw, + char_boundaries, + char_validity, + } + } + + /// Create MbText from a string (uses UTF-8 char boundaries) + pub fn from_str(s: &'a str) -> Self { + if is_multibyte_locale() { + // In MB locale, use locale-based character boundaries + Self::new(s.as_bytes()) + } else { + // In single-byte locale, use UTF-8 characters + let raw = s.as_bytes(); + // Fix: only keep starting positions + let mut fixed = Vec::with_capacity(s.chars().count() + 1); + fixed.push(0); + let mut pos = 0; + for ch in s.chars() { + pos += ch.len_utf8(); + fixed.push(pos); + } + let validity = vec![true; s.chars().count()]; // All UTF-8 chars are valid + Self { + raw, + char_boundaries: fixed, + char_validity: validity, + } + } + } + + /// Get the number of characters + pub fn char_count(&self) -> usize { + if self.char_boundaries.is_empty() { + 0 + } else { + self.char_boundaries.len() - 1 + } + } + + /// Get the byte length + pub fn byte_len(&self) -> usize { + self.raw.len() + } + + /// Check if empty + pub fn is_empty(&self) -> bool { + self.raw.is_empty() + } + + /// Get character at index (0-based character index) + pub fn char_at(&self, char_idx: usize) -> Option> { + if char_idx >= self.char_count() { + return None; + } + let start = self.char_boundaries[char_idx]; + let end = self.char_boundaries[char_idx + 1]; + Some(MbChar::new(&self.raw[start..end])) + } + + /// Check if character at index is valid (not an incomplete/invalid MB sequence) + pub fn is_valid_char(&self, char_idx: usize) -> bool { + if char_idx >= self.char_validity.len() { + false + } else { + self.char_validity[char_idx] + } + } + + /// Convert character index to byte offset + pub fn char_to_byte(&self, char_idx: usize) -> usize { + if char_idx >= self.char_boundaries.len() { + self.raw.len() + } else { + self.char_boundaries[char_idx] + } + } + + /// Convert byte offset to character index + /// Returns the character index that contains this byte, or char_count() if at end + pub fn byte_to_char(&self, byte_offset: usize) -> usize { + if byte_offset >= self.raw.len() { + return self.char_count(); + } + // Binary search for the character containing this byte + match self.char_boundaries.binary_search(&byte_offset) { + Ok(idx) => idx, + Err(idx) => { + if idx > 0 { + idx - 1 + } else { + 0 + } + } + } + } + + /// Get a slice of characters from start to end (character indices) + pub fn slice_chars(&self, start: usize, end: usize) -> &'a [u8] { + let byte_start = self.char_to_byte(start); + let byte_end = self.char_to_byte(end); + &self.raw[byte_start..byte_end] + } + + /// Get characters as an iterator + pub fn chars(&self) -> MbTextCharIter<'a, '_> { + MbTextCharIter { text: self, pos: 0 } + } + + /// Get a substring from character positions as a String + pub fn substring(&self, start: usize, end: usize) -> String { + let bytes = self.slice_chars(start, end); + String::from_utf8_lossy(bytes).into_owned() + } + + /// Get word boundary context at a position. + /// + /// Returns (prev_is_word, next_is_word) for use in word boundary matching: + /// - `\b` (WordBoundary): `prev_is_word != next_is_word` + /// - `\B` (NonWordBoundary): `prev_is_word == next_is_word` + /// - `\<` (StartWord): `!prev_is_word && next_is_word` + /// - `\>` (EndWord): `prev_is_word && !next_is_word` + pub fn word_boundary_context(&self, pos: usize) -> (bool, bool) { + let prev_is_word = if pos > 0 { + self.char_at(pos - 1) + .map(|ch| ch.is_word_char()) + .unwrap_or(false) + } else { + false + }; + let next_is_word = self + .char_at(pos) + .map(|ch| ch.is_word_char()) + .unwrap_or(false); + (prev_is_word, next_is_word) + } +} + +/// Iterator over characters in MbText +pub struct MbTextCharIter<'a, 'b> { + text: &'b MbText<'a>, + pos: usize, +} + +impl<'a, 'b> Iterator for MbTextCharIter<'a, 'b> { + type Item = MbChar<'a>; + + fn next(&mut self) -> Option { + if self.pos >= self.text.char_count() { + return None; + } + let ch = self.text.char_at(self.pos)?; + self.pos += 1; + Some(ch) + } +} + +/// Compute character boundaries for a byte slice using locale's MB encoding +#[cfg(unix)] +fn compute_char_boundaries(bytes: &[u8]) -> (Vec, Vec) { + if bytes.is_empty() { + return (vec![0], vec![]); + } + + let is_mb = is_multibyte_locale(); + + if !is_mb { + // Single-byte locale: each byte is a character (all valid) + return ((0..=bytes.len()).collect(), vec![true; bytes.len()]); + } + + let mut boundaries = Vec::with_capacity(bytes.len() + 1); + let mut validity = Vec::with_capacity(bytes.len()); + boundaries.push(0); + let mut state = ffi::MbStateT::default(); + let mut i = 0; + + while i < bytes.len() { + let remaining = bytes.len() - i; + let result = unsafe { ffi::mbrlen(bytes[i..].as_ptr(), remaining, &mut state) }; + + let (char_len, valid) = match result { + -2 | -1 => { + // Invalid or incomplete sequence - treat as single byte, but mark invalid + state = ffi::MbStateT::default(); + (1, false) + } + 0 => (1, true), // NUL character is valid + n => (n as usize, true), + }; + + validity.push(valid); + i += char_len; + boundaries.push(i); + } + + (boundaries, validity) +} + +#[cfg(not(unix))] +fn compute_char_boundaries(bytes: &[u8]) -> (Vec, Vec) { + // Non-Unix: each byte is a character (all valid) + ((0..=bytes.len()).collect(), vec![true; bytes.len()]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mb_cur_max() { + let max = mb_cur_max(); + assert!(max >= 1); + } + + #[test] + fn test_count_ascii() { + assert_eq!(count_mb_chars(b"hello"), 5); + assert_eq!(count_mb_chars(b""), 0); + } + + #[test] + fn test_mb_state() { + let mut state = MbState::new(); + assert!(state.is_initial()); + let is_mb = state.is_mb_char(b'a'); + assert!(!is_mb || !state.is_initial()); + } + + #[test] + fn test_mb_text_ascii() { + let text = MbText::new(b"hello"); + assert_eq!(text.char_count(), 5); + assert_eq!(text.byte_len(), 5); + + // Check character access + let ch0 = text.char_at(0).unwrap(); + assert_eq!(ch0.bytes, b"h"); + assert!(ch0.is_ascii()); + + // Check position conversion + assert_eq!(text.char_to_byte(0), 0); + assert_eq!(text.char_to_byte(2), 2); + assert_eq!(text.byte_to_char(2), 2); + } + + #[test] + fn test_mb_text_slice() { + let text = MbText::new(b"hello"); + assert_eq!(text.slice_chars(1, 4), b"ell"); + assert_eq!(text.substring(1, 4), "ell"); + } + + #[test] + fn test_mb_text_iter() { + let text = MbText::new(b"abc"); + let chars: Vec<_> = text.chars().collect(); + assert_eq!(chars.len(), 3); + assert_eq!(chars[0].bytes, b"a"); + assert_eq!(chars[1].bytes, b"b"); + assert_eq!(chars[2].bytes, b"c"); + } + + #[test] + fn test_mb_char_matches() { + let ch = MbChar::new(b"a"); + assert!(ch.matches_byte(b'a')); + assert!(!ch.matches_byte(b'b')); + } +} diff --git a/red/src/parser.rs b/red/src/parser.rs new file mode 100644 index 0000000..8956686 --- /dev/null +++ b/red/src/parser.rs @@ -0,0 +1,2312 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use crate::constants; +use crate::context::Context; +use crate::errors::{Result, SedError}; +use crate::posix_rules; +use crate::util::version::compare_versions; + +use std::cmp::Ordering; + +mod ast; +mod escapes; +mod lexer; + +pub use ast::{ + Address, AddressRange, Command, HasAddressRange, PrintTiming, SubstitutionFlags, Token, +}; +pub use lexer::{build_char_to_byte_mapping, is_utf8_locale, Lexer}; + +// Lexer moved to module `lexer` + +/// Parser for building AST from tokens +pub struct Parser { + tokens: Vec, + position: usize, + char_position: usize, // Approximate character position for error messages + last_regex: Option, // Track last regex pattern for empty regex support + posix: bool, // POSIX mode flag (extracted from Context) + extended_regex: bool, // Extended regex mode (extracted from Context) + sandbox: bool, // Sandbox mode - disable e/r/w commands (extracted from Context) + utf8_locale: bool, // UTF-8 locale - affects translate command length validation +} + +impl Parser { + /// Create new parser from tokens and context + /// + /// Extracts configuration from Context to avoid lifetime issues. + /// Note: Uses strict_posix mode for validation (--posix flag only, not POSIXLY_CORRECT) + pub fn new(tokens: Vec, ctx: &Context) -> Self { + Self { + tokens, + position: 0, + char_position: 1, // Start at char 1 (GNU sed convention) + last_regex: None, + posix: ctx.is_strict_posix(), // Only --posix, not POSIXLY_CORRECT + extended_regex: ctx.extended_regex, + sandbox: ctx.sandbox, + utf8_locale: is_utf8_locale(), + } + } + + /// Create a parse error with current character position + fn error_at(&self, message: impl Into) -> SedError { + SedError::parse_at(message, self.char_position) + } + + /// Check that the current token is a valid command terminator (EOF, newline, semicolon, comment, or close brace) + /// Returns an error if there are extra characters after the command + fn expect_command_end(&self) -> Result<()> { + match self.current_token() { + Token::Eof + | Token::Newline + | Token::Semicolon + | Token::Comment(_) + | Token::CloseBrace => Ok(()), + _ => Err(self.error_at("extra characters after command")), + } + } + + /// Parse tokens into command AST + pub fn parse(&mut self) -> Result> { + self.parse_command_list() + } + + /// Parse a sed script string into commands + pub fn parse_script(input: &str) -> Result> { + // Use default context for testing/convenience + let ctx = Context::new_test(); + Self::parse_script_with_context(input, &ctx) + } + + /// Parse a sed script string into commands with Context + pub fn parse_script_with_context(input: &str, ctx: &Context) -> Result> { + let mut lexer = Lexer::new_with_posix(input, ctx.is_posix()); + let mut tokens = Vec::new(); + + loop { + let token = lexer.next_token()?; + let is_eof = matches!(token, Token::Eof); + tokens.push(token); + if is_eof { + break; + } + } + + let mut parser = Parser::new(tokens, ctx); + parser.parse() + } + + /// Parse a sed script string into commands with raw bytes for accurate multibyte detection + /// + /// The raw_bytes parameter should be the original bytes before lossy UTF-8 conversion. + /// This enables accurate detection of UTF-8 lead bytes for delimiter validation. + pub fn parse_script_with_raw_bytes( + input: &str, + raw_bytes: &[u8], + ctx: &Context, + ) -> Result> { + Self::parse_script_with_raw_bytes_and_state(input, raw_bytes, ctx, None) + } + + /// Parse with raw bytes and optional initial last_regex state + /// + /// Returns the parsed commands and the final last_regex state for chaining. + pub fn parse_script_with_raw_bytes_and_state( + input: &str, + raw_bytes: &[u8], + ctx: &Context, + initial_last_regex: Option, + ) -> Result> { + let mut lexer = Lexer::new_with_raw_bytes(input, raw_bytes, ctx.is_posix()); + let mut tokens = Vec::new(); + + loop { + let token = lexer.next_token()?; + let is_eof = matches!(token, Token::Eof); + tokens.push(token); + if is_eof { + break; + } + } + + let mut parser = Parser::new(tokens, ctx); + parser.last_regex = initial_last_regex; + parser.parse() + } + + /// Get the last regex pattern after parsing (for chaining parsers) + pub fn get_last_regex(&self) -> Option { + self.last_regex.clone() + } + + /// Parse and return both commands and final parser state + pub fn parse_with_state(&mut self) -> Result<(Vec, Option)> { + let commands = self.parse()?; + Ok((commands, self.last_regex.clone())) + } + + /// Parse with raw bytes and return final last_regex for chaining + pub fn parse_script_with_raw_bytes_chained( + input: &str, + raw_bytes: &[u8], + ctx: &Context, + initial_last_regex: Option, + ) -> Result<(Vec, Option)> { + let mut lexer = Lexer::new_with_raw_bytes(input, raw_bytes, ctx.is_posix()); + let mut tokens = Vec::new(); + + loop { + let token = lexer.next_token()?; + let is_eof = matches!(token, Token::Eof); + tokens.push(token); + if is_eof { + break; + } + } + + let mut parser = Parser::new(tokens, ctx); + parser.last_regex = initial_last_regex; + parser.parse_with_state() + } + + /// Parse a list of commands separated by newlines or semicolons + fn parse_command_list(&mut self) -> Result> { + let mut commands = Vec::new(); + + while self.current_token() != &Token::Eof { + // Skip newlines + if let Token::Newline = self.current_token() { + self.advance(); + continue; + } + // Skip stray semicolons + if let Token::Semicolon = self.current_token() { + self.advance(); + continue; + } + + // Parse command + commands.push(self.parse_command(false)?); + + // Handle semicolon separator + if let Token::Semicolon = self.current_token() { + self.advance(); + // Continue parsing more commands on same line + continue; + } + } + + Ok(commands) + } + + fn current_token(&self) -> &Token { + self.tokens.get(self.position).unwrap_or(&Token::Eof) + } + + fn advance(&mut self) { + // Calculate actual character length of current token for accurate position tracking + let token_len = match self.current_token() { + Token::LineNumber(n) => n.to_string().len(), + Token::Command(_) => 1, + Token::Comma => 1, + Token::Bang => 1, + Token::Dollar => 1, + Token::PlusOffset(n) => 1 + n.to_string().len(), // +N + Token::TildeStep(n) => 1 + n.to_string().len(), // ~N + Token::Semicolon => 1, + Token::Newline => 1, + Token::OpenBrace => 1, + Token::CloseBrace => 1, + Token::SubstitutionDelim(_) => 1, + Token::SubstitutionBody(s, _) => s.len() + 1, // +1 for leading 's' + Token::TranslateBody(s, _) => s.len() + 1, // +1 for leading 'y' + Token::RegexAddr(s, m) => s.len() + m.len() + 2, // +2 for delimiters + Token::AppendBody(_) | Token::InsertBody(_) | Token::ChangeBody(_) => 1, // Just the command letter + Token::ExecuteBody(s) => s.len() + 1, // +1 for 'e' + Token::Filename(s) => s.len(), + Token::String(s) => s.len(), + Token::Comment(s) => s.len() + 1, // +1 for '#' + Token::SubstitutionFlag(_) => 1, + Token::Eof => 0, + }; + + self.char_position += token_len; + self.position += 1; + } + + fn parse_command(&mut self, in_group: bool) -> Result { + // First try to parse optional address range + let (address_range, negated) = self.parse_address_range()?; + + // Validate address 0 usage before parsing command + if let Some(ref range) = address_range { + self.validate_address_zero(range)?; + } + + // Then parse the actual command + match self.current_token() { + Token::Comment(text) => { + // Comments don't accept addresses + if address_range.is_some() { + return Err(self.error_at("comments don't accept any addresses")); + } + let comment = Command::Comment(text.clone()); + self.advance(); + Ok(comment) + } + Token::SubstitutionDelim(':') => { + // Check if there's an address - labels don't accept addresses + if address_range.is_some() { + return Err(self.error_at(": doesn't want any addresses")); + } + self.advance(); + let name = self.parse_label_word()?; + Ok(Command::Label { name }) + } + Token::OpenBrace => self.parse_group(address_range, negated), + Token::Command('s') => self.parse_substitution(address_range, negated), + Token::TranslateBody(body, raw_bytes) => { + // Construct Translate from pre-parsed body (lexer handled leading 'y') + let b = body.clone(); + let rb = raw_bytes.clone(); + self.advance(); + let cmd = + self.parse_translate_from_body(address_range, negated, &b, rb.as_deref())?; + // Check for junk after y command + match self.current_token() { + Token::Eof + | Token::Newline + | Token::Semicolon + | Token::Comment(_) + | Token::CloseBrace => { + // Valid: EOF, newline, semicolon, comment, or closing brace after y + } + _ => { + return Err(self.error_at("extra characters after command")); + } + } + Ok(cmd) + } + Token::Command('a') => { + let cmd_pos = self.char_position; + self.advance(); + // Phase 3.3: Centralized address range validation + let has_range = address_range + .as_ref() + .map(|r| r.start.is_some() && r.end.is_some()) + .unwrap_or(false); + posix_rules::validate_address_range_posix('a', has_range, self.posix, cmd_pos)?; + match self.current_token() { + Token::AppendBody(text) => { + let t = text.clone(); + self.advance(); + Ok(Command::Append { + range: address_range, + negated, + text: t, + }) + } + _ => { + return Err(SedError::parse_at( + "expected \\ after 'a', 'c' or 'i'", + cmd_pos, + )); + } + } + } + Token::Command('i') => { + let cmd_pos = self.char_position; + self.advance(); + // Phase 3.3: Centralized address range validation + let has_range = address_range + .as_ref() + .map(|r| r.start.is_some() && r.end.is_some()) + .unwrap_or(false); + posix_rules::validate_address_range_posix('i', has_range, self.posix, cmd_pos)?; + match self.current_token() { + Token::InsertBody(text) => { + let t = text.clone(); + self.advance(); + Ok(Command::Insert { + range: address_range, + negated, + text: t, + }) + } + _ => { + return Err(SedError::parse_at( + "expected \\ after 'a', 'c' or 'i'", + cmd_pos, + )); + } + } + } + Token::Command('c') => { + let cmd_pos = self.char_position; + self.advance(); + match self.current_token() { + Token::ChangeBody(text) => { + let t = text.clone(); + self.advance(); + Ok(Command::Change { + range: address_range, + negated, + text: t, + }) + } + _ => { + return Err(SedError::parse_at( + "expected \\ after 'a', 'c' or 'i'", + cmd_pos, + )); + } + } + } + Token::Command('p') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::Print { + range: address_range, + negated, + }) + } + Token::Command('P') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::PrintFirstLine { + range: address_range, + negated, + }) + } + Token::Command('d') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::Delete { + range: address_range, + negated, + }) + } + Token::Command('q') => { + let cmd_pos = self.char_position; + self.advance(); + // q command only accepts one address, not a range + if let Some(ref range) = address_range { + if range.start.is_some() && range.end.is_some() { + return Err(SedError::parse_at("command only uses one address", cmd_pos)); + } + } + // Optional exit code after q + let mut code: Option = None; + if let Token::LineNumber(n) = self.current_token() { + code = Some(*n as i32); + self.advance(); + } + self.expect_command_end()?; + Ok(Command::Quit { + range: address_range, + negated, + exit_code: code, + }) + } + Token::Command('Q') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('Q', self.posix, cmd_pos)?; + self.advance(); + // Q command only accepts one address, not a range + if let Some(ref range) = address_range { + if range.start.is_some() && range.end.is_some() { + return Err(SedError::parse_at("command only uses one address", cmd_pos)); + } + } + // Optional exit code after Q + let mut code: Option = None; + if let Token::LineNumber(n) = self.current_token() { + code = Some(*n as i32); + self.advance(); + } + self.expect_command_end()?; + Ok(Command::QuitSilent { + range: address_range, + negated, + exit_code: code, + }) + } + Token::Command('h') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::HoldCopy { + range: address_range, + negated, + }) + } + Token::Command('H') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::HoldAppend { + range: address_range, + negated, + }) + } + Token::Command('g') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::GetCopy { + range: address_range, + negated, + }) + } + Token::Command('G') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::GetAppend { + range: address_range, + negated, + }) + } + Token::Command('x') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::Exchange { + range: address_range, + negated, + }) + } + Token::Command('N') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::N { + range: address_range, + negated, + }) + } + Token::Command('D') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::BigD { + range: address_range, + negated, + }) + } + Token::Command('n') => { + self.advance(); + self.expect_command_end()?; + Ok(Command::Next) + } + Token::Command('b') => { + self.advance(); + let label = self.parse_optional_label_word()?; + Ok(Command::Branch { + range: address_range, + negated, + label, + }) + } + Token::Command('t') => { + self.advance(); + let label = self.parse_optional_label_word()?; + Ok(Command::Test { + range: address_range, + negated, + label, + }) + } + Token::Command('T') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('T', self.posix, cmd_pos)?; + self.advance(); + let label = self.parse_optional_label_word()?; + Ok(Command::TestNeg { + range: address_range, + negated, + label, + }) + } + Token::Command('e') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('e', self.posix, cmd_pos)?; + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + cmd_pos, + )); + } + self.advance(); + // Lexer will provide ExecuteBody token with command text if 'e' is standalone + match self.current_token() { + Token::ExecuteBody(text) => { + let cmd = text.clone(); + self.advance(); + Ok(Command::Execute { + range: address_range, + negated, + command: if cmd.is_empty() { None } else { Some(cmd) }, + }) + } + _ => { + // No ExecuteBody means execute pattern space + Ok(Command::Execute { + range: address_range, + negated, + command: None, + }) + } + } + } + Token::Command('v') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('v', self.posix, cmd_pos)?; + self.advance(); + // Read version string (allows dots for version format like "4.8.0") + let version = self.parse_version_string()?; + + // Check version compatibility immediately + let version_to_check = if version.is_empty() { "4.0" } else { &version }; + + if compare_versions(version_to_check, constants::GNU_SED_COMPAT_VERSION) + == Ordering::Greater + { + // Report error at the last character of the version string + return Err(SedError::parse_at( + "expected newer version of sed", + self.char_position - 1, + )); + } + + Ok(Command::Version { version }) + } + Token::Command('z') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('z', self.posix, cmd_pos)?; + self.advance(); + self.expect_command_end()?; + Ok(Command::Clear { + range: address_range, + negated, + }) + } + Token::Command('f') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('f', self.posix, cmd_pos)?; + self.advance(); + self.expect_command_end()?; + Ok(Command::PrintFilename { + range: address_range, + negated, + }) + } + Token::Command('F') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('F', self.posix, cmd_pos)?; + self.advance(); + self.expect_command_end()?; + Ok(Command::PrintFilename { + range: address_range, + negated, + }) + } + Token::Command('=') => { + let cmd_pos = self.char_position; + self.advance(); + // Phase 3.3: Centralized address range validation + let has_range = address_range + .as_ref() + .map(|r| r.start.is_some() && r.end.is_some()) + .unwrap_or(false); + posix_rules::validate_address_range_posix('=', has_range, self.posix, cmd_pos)?; + // Check for junk after = + match self.current_token() { + Token::Eof + | Token::Newline + | Token::Semicolon + | Token::Comment(_) + | Token::CloseBrace => { + // Valid: EOF, newline, semicolon, comment, or closing brace after = + } + _ => { + return Err(self.error_at("extra characters after command")); + } + } + Ok(Command::LineNumber { + range: address_range, + negated, + }) + } + Token::Command('l') => { + let cmd_pos = self.char_position; + self.advance(); + // Phase 3.3: Centralized address range validation + let has_range = address_range + .as_ref() + .map(|r| r.start.is_some() && r.end.is_some()) + .unwrap_or(false); + posix_rules::validate_address_range_posix('l', has_range, self.posix, cmd_pos)?; + // Optional line length after l + let mut line_length: Option = None; + if let Token::LineNumber(n) = self.current_token() { + line_length = Some(*n); + self.advance(); + } + // Check for junk after l (after optional line number) + match self.current_token() { + Token::Eof + | Token::Newline + | Token::Semicolon + | Token::Comment(_) + | Token::CloseBrace => { + // Valid: EOF, newline, semicolon, comment, or closing brace after l + } + _ => { + return Err(self.error_at("extra characters after command")); + } + } + Ok(Command::List { + range: address_range, + negated, + line_length, + }) + } + Token::Command('w') => { + let cmd_pos = self.char_position; + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + cmd_pos, + )); + } + self.advance(); + match self.current_token() { + Token::Filename(path) => { + let p = path.clone(); + self.advance(); + Ok(Command::Write { + range: address_range, + negated, + path: p, + }) + } + _ => { + return Err(SedError::parse_at( + "missing filename in r/R/w/W commands", + cmd_pos, + )); + } + } + } + Token::Command('W') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_command_posix('W', self.posix, cmd_pos)?; + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + cmd_pos, + )); + } + self.advance(); + match self.current_token() { + Token::Filename(path) => { + let p = path.clone(); + self.advance(); + Ok(Command::WriteFirstLine { + range: address_range, + negated, + path: p, + }) + } + _ => { + return Err(SedError::parse_at( + "missing filename in r/R/w/W commands", + cmd_pos, + )); + } + } + } + Token::Command('r') => { + let cmd_pos = self.char_position; + // Phase 3.3: Centralized address range validation + let has_range = address_range + .as_ref() + .map(|r| r.start.is_some() && r.end.is_some()) + .unwrap_or(false); + posix_rules::validate_address_range_posix('r', has_range, self.posix, cmd_pos)?; + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + cmd_pos, + )); + } + self.advance(); + match self.current_token() { + Token::Filename(path) => { + let p = path.clone(); + self.advance(); + Ok(Command::Read { + range: address_range, + negated, + path: p, + }) + } + _ => { + return Err(SedError::parse_at( + "missing filename in r/R/w/W commands", + cmd_pos, + )); + } + } + } + Token::Command('R') => { + let cmd_pos = self.char_position; + // GNU extension, not allowed in POSIX mode + if self.posix { + return Err(SedError::parse_at("unknown command: 'R'", cmd_pos)); + } + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + cmd_pos, + )); + } + self.advance(); + match self.current_token() { + Token::Filename(path) => { + let p = path.clone(); + self.advance(); + Ok(Command::ReadLine { + range: address_range, + negated, + path: p, + }) + } + _ => { + return Err(SedError::parse_at( + "missing filename in r/R/w/W commands", + cmd_pos, + )); + } + } + } + // Handle special tokens with better error messages + Token::Semicolon | Token::Newline => { + return Err(self.error_at("missing command")); + } + Token::CloseBrace => { + // Check if there's an address and we're inside a group + // Inside group with address: "'}' doesn't want any addresses" + // Outside group (or no address): "unexpected '}'" + if in_group && address_range.is_some() { + return Err(self.error_at("'}' doesn't want any addresses")); + } + return Err(self.error_at("unexpected '}'")); + } + Token::Eof => { + return Err(self.error_at("unexpected end of script")); + } + Token::Filename(name) => { + // Filename appearing as command indicates lexer/parser issue + return Err(self.error_at(format!("unknown command: '{}'", name))); + } + _ => { + // Use Display trait for better error messages + if let Token::Command(ch) = self.current_token() { + return Err(self.error_at(format!("unknown command: '{}'", ch))); + } + return Err(self.error_at(format!("unknown command: {}", self.current_token()))); + } + } + } + + fn parse_optional_label_word(&mut self) -> Result { + let mut name = String::new(); + while self.current_token() != &Token::Eof { + match self.current_token() { + Token::Newline | Token::Semicolon | Token::CloseBrace => break, + Token::Command(c) => { + name.push(*c); + self.advance(); + } + Token::LineNumber(n) => { + name.push_str(&n.to_string()); + self.advance(); + } + Token::SubstitutionDelim('_') => { + name.push('_'); + self.advance(); + } + _ => break, + } + } + Ok(name) + } + + fn parse_version_string(&mut self) -> Result { + let mut version = String::new(); + while self.current_token() != &Token::Eof { + match self.current_token() { + Token::Newline | Token::Semicolon | Token::CloseBrace => break, + Token::Command(c) => { + version.push(*c); + self.advance(); + } + Token::LineNumber(n) => { + version.push_str(&n.to_string()); + self.advance(); + } + Token::SubstitutionDelim(c) if *c == '.' || *c == '-' || *c == '_' => { + // Allow dots, dashes, underscores in version strings + version.push(*c); + self.advance(); + } + _ => break, + } + } + Ok(version) + } + + fn parse_label_word(&mut self) -> Result { + let mut name = String::new(); + while self.current_token() != &Token::Eof { + match self.current_token() { + Token::Newline | Token::Semicolon | Token::CloseBrace => break, + Token::Command(c) => { + name.push(*c); + self.advance(); + } + Token::LineNumber(n) => { + name.push_str(&n.to_string()); + self.advance(); + } + Token::SubstitutionDelim('_') => { + name.push('_'); + self.advance(); + } + _ => break, + } + } + if name.is_empty() { + // GNU sed says `":" lacks a label` and reports char position of ':' + // char_position is already incremented past ':', so subtract 1 + return Err(SedError::parse_at( + "\":\" lacks a label", + self.char_position - 1, + )); + } + Ok(name) + } + + // parse_filename_word removed in favor of filename literal handled by lexer + + /// Parse address range: [addr1][,addr2][!] + fn parse_address_range(&mut self) -> Result<(Option, bool)> { + let mut start_addr = None; + let mut end_addr = None; + let mut negated = false; + + // Check if first token is +N or ~N (invalid as first address) + match self.current_token() { + Token::PlusOffset(_) | Token::TildeStep(_) => { + // Position should point after the + or ~ character + return Err(SedError::parse_at( + "invalid usage of +N or ~N as first address", + self.char_position + 1, + )); + } + _ => {} + } + + // Check for first address + if let Some(addr) = self.try_parse_address()? { + start_addr = Some(addr); + + // Check for comma (range) + if let Token::Comma = self.current_token() { + self.advance(); // consume comma + + // Parse second address + if let Some(addr2) = self.try_parse_address()? { + end_addr = Some(addr2); + } else { + return Err(self.error_at("unexpected ','")); + } + } + } + + // Check for negation (!) + if let Token::Bang = self.current_token() { + negated = true; + self.advance(); + // Check for double negation (BAD_BANG) + if let Token::Bang = self.current_token() { + return Err(SedError::parse_at("multiple '!'s", self.char_position)); + } + } + + let range = if start_addr.is_some() || end_addr.is_some() { + Some(AddressRange::new(start_addr, end_addr)) + } else { + None + }; + + Ok((range, negated)) + } + + /// Validate address 0 usage - address 0 is only valid in specific contexts + fn validate_address_zero(&self, range: &AddressRange) -> Result<()> { + // Check if start address is 0 + if let Some(Address::Line(0)) = range.start { + // If there's an end address, check its type + if let Some(ref end) = range.end { + match end { + Address::Line(_) => { + // 0,NUM is invalid (e.g., 0,4) + return Err(self.error_at("invalid usage of line address 0")); + } + Address::Dollar => { + // 0,$ is also invalid + return Err(self.error_at("invalid usage of line address 0")); + } + // 0,/pattern/ is OK + Address::Regex(_) => {} + // Other combinations - accepted (conservative validation) + _ => {} + } + } + // Single address 0 is OK (for commands like 0r) + } + Ok(()) + } + + /// Try to parse an address, returns None if current token is not an address + fn try_parse_address(&mut self) -> Result> { + // Parse base address + let base_addr = match self.current_token() { + Token::LineNumber(n) => { + let line = *n; + self.advance(); + Some(Address::Line(line)) + } + Token::Dollar => { + self.advance(); + Some(Address::Dollar) + } + Token::RegexAddr(pattern, modifiers) => { + let mut regex = pattern.clone(); + + // Handle empty regex pattern - reuse last or error + if regex.is_empty() { + // Check if modifiers are present - this is an error + if !modifiers.is_empty() { + return Err(SedError::parse_at( + "cannot specify modifiers on empty regexp", + 3, + )); + } + // No modifiers - try to reuse last regex + match self.last_regex.clone() { + Some(last) => regex = last, + None => { + return Err(SedError::parse_at("no previous regular expression", 0)); + } + } + } else { + // Save non-empty pattern as last regex + self.last_regex = Some(regex.clone()); + } + + // Validate that address regex doesn't contain backreferences + use crate::util::regex::validate_address_regex; + validate_address_regex(®ex).map_err(|e| { + // Add character position to error + // char_position is after RegexAddr token + // For /\1/: char_position is 2 after consuming token, regex.len() is 2, need +1 for closing / + let error_pos = self.char_position + regex.len() + 1; + SedError::parse_at(e.to_string(), error_pos) + })?; + + self.advance(); + Some(Address::Regex(regex)) + } + Token::PlusOffset(n) => { + let offset = *n as isize; + self.advance(); + // +N without base means relative to line 0 (special case for ranges like 0,+5) + Some(Address::Relative(Box::new(Address::Line(0)), offset)) + } + Token::TildeStep(n) => { + let step = *n; + self.advance(); + // ~N without base is not valid in GNU sed; we accept it as 0~N (lenient extension) + Some(Address::Step(Box::new(Address::Line(0)), step)) + } + _ => None, // Not an address + }; + + // If we have a base address, check for modifiers (~step or +offset) + if let Some(addr) = base_addr { + match self.current_token() { + Token::TildeStep(n) => { + let step = *n; + self.advance(); + Ok(Some(Address::Step(Box::new(addr), step))) + } + Token::PlusOffset(n) => { + let offset = *n as isize; + self.advance(); + Ok(Some(Address::Relative(Box::new(addr), offset))) + } + _ => Ok(Some(addr)), + } + } else { + Ok(None) + } + } + + fn parse_group(&mut self, range: Option, negated: bool) -> Result { + // Parse { command1; command2; ... } + if let Token::OpenBrace = self.current_token() { + self.advance(); // consume { + + let mut group_commands = Vec::new(); + + while self.current_token() != &Token::CloseBrace && self.current_token() != &Token::Eof + { + // Skip newlines inside group + if let Token::Newline = self.current_token() { + self.advance(); + continue; + } + // Skip stray semicolons inside group + if let Token::Semicolon = self.current_token() { + self.advance(); + continue; + } + + group_commands.push(self.parse_command(true)?); + + // Handle semicolon separator inside group + if let Token::Semicolon = self.current_token() { + self.advance(); + } + } + + if self.current_token() != &Token::CloseBrace { + return Err(SedError::parse_at("unmatched '{'", 0)); + } + self.advance(); // consume } + + // Check for junk after } + match self.current_token() { + Token::Eof + | Token::Newline + | Token::Semicolon + | Token::Comment(_) + | Token::CloseBrace => { + // Valid: EOF, newline, semicolon, comment, or closing brace after } + // (CloseBrace is for nested groups like {{p}}) + } + _ => { + return Err(self.error_at("extra characters after command")); + } + } + + Ok(Command::Group { + range, + negated, + commands: group_commands, + }) + } else { + return Err(self.error_at("missing command")); + } + } + + fn parse_substitution( + &mut self, + range: Option, + negated: bool, + ) -> Result { + if let Token::Command('s') = self.current_token() { + // Save position of 's' for error reporting + let s_position = self.char_position; + self.advance(); + + // Expect the next token to be SubstitutionBody + let (body, body_raw_bytes) = match self.current_token() { + Token::SubstitutionBody(s, rb) => (s.clone(), rb.clone()), + other => { + return Err(SedError::parse(format!( + "unexpected token after 's': {:?}", + other + ))); + } + }; + self.advance(); + + // Body format: + let chars: Vec = body.chars().collect(); + if chars.is_empty() { + // char_position is after 's' and SubstitutionBody tokens, report at 's' + return Err(SedError::parse_at("unterminated 's' command", s_position)); + } + let delim = chars[0]; + let mut i = 1; + + // Capture char_position for error reporting inside closure + let base_char_pos = s_position; + + // Helper: parse until next delimiter, honoring character classes for pattern + // is_pattern: if true, strip backslash when escaping delimiter (for regex pattern) + // if false, keep backslash when escaping delimiter (for replacement) + let parse_until_with_class = |chars: &Vec, + start: usize, + d: char, + track_class: bool, + is_pattern: bool| + -> Result<(String, usize)> { + let mut out = String::new(); + let mut idx = start; + let mut escaped = false; + let mut in_class = false; + while idx < chars.len() { + let c = chars[idx]; + idx += 1; + if escaped { + // In pattern: escaped delimiter becomes just the delimiter char + // In replacement: keep backslash so replacement parser handles special chars + if c == d && is_pattern { + // Pattern: strip backslash, just push the delimiter char + out.push(c); + } else { + // Replacement or non-delimiter: keep the backslash + out.push('\\'); + out.push(c); + } + escaped = false; + continue; + } + if c == '\\' { + escaped = true; + continue; + } + if track_class { + if c == '[' { + in_class = true; + out.push('['); + continue; + } + if c == ']' && in_class { + in_class = false; + out.push(']'); + continue; + } + } + if c == d && !in_class { + return Ok((out, idx)); + } + out.push(c); + } + // Calculate error position: base_char_pos is position of 's' + // idx is current position in chars vector (body) + let error_pos = base_char_pos + idx; + return Err(SedError::parse_at("unterminated 's' command", error_pos)); + }; + + // Track character class only for the pattern if delimiter is not '[' + // is_pattern=true: strip backslash when escaping delimiter + let (pattern, next_i) = parse_until_with_class(&chars, i, delim, delim != '[', true)?; + i = next_i; + + // Handle empty pattern - reuse last regex or error if none exists + // Track if original pattern was empty (for backref validation skip) + let uses_last_regex = pattern.is_empty(); + + // Extract pattern_raw_bytes from body_raw_bytes if available + // Pattern starts at char index 1 (after delimiter) and ends at next_i-1 (before delimiter) + let pattern_start_char = 1usize; // After the delimiter + let pattern_raw_bytes: Option> = if !uses_last_regex { + body_raw_bytes.as_ref().and_then(|raw| { + let mapping = build_char_to_byte_mapping(&body, raw); + if pattern_start_char < mapping.len() { + let start_byte = mapping[pattern_start_char]; + // next_i points to character after the closing delimiter + // So next_i - 1 is the delimiter, and we want bytes up to (not including) that + let end_byte = if next_i > 0 && next_i - 1 < mapping.len() { + mapping[next_i - 1] + } else if next_i < mapping.len() { + mapping[next_i] + } else { + raw.len() + }; + if start_byte <= end_byte && end_byte <= raw.len() { + Some(raw[start_byte..end_byte].to_vec()) + } else { + None + } + } else { + None + } + }) + } else { + None // Don't extract raw bytes when reusing last regex + }; + + if uses_last_regex { + // Validate that there's a previous regex, but DON'T fill in the pattern. + // Keep pattern empty so compile_substitution sets use_last=true, + // allowing runtime lookup of last_s_regex (which can be updated by address patterns). + if self.last_regex.is_none() { + return Err(SedError::parse_at("no previous regular expression", 0)); + } + // Pattern stays empty - runtime will use last_s_regex + } else { + // Save non-empty pattern as last regex + self.last_regex = Some(pattern.clone()); + } + + // is_pattern=false: keep backslash when escaping delimiter (for replacement parsing) + let (replacement, next_i2) = parse_until_with_class(&chars, i, delim, false, false)?; + // Store replacement start char position for raw bytes extraction + let replacement_start_char = i; + i = next_i2; + + // Extract replacement_raw_bytes from body_raw_bytes if available + let replacement_raw_bytes: Option> = body_raw_bytes.as_ref().and_then(|raw| { + // Build char-to-byte mapping for the body + let mapping = build_char_to_byte_mapping(&body, raw); + if replacement_start_char < mapping.len() { + let start_byte = mapping[replacement_start_char]; + // next_i2 points to position AFTER delimiter, so next_i2 - 1 is the delimiter + // Use mapping[next_i2 - 1] to get byte position of delimiter (exclusive end) + let end_byte = if next_i2 > 0 && next_i2 - 1 < mapping.len() { + mapping[next_i2 - 1] + } else if next_i2 < mapping.len() { + mapping[next_i2] + } else { + raw.len() + }; + if start_byte <= end_byte && end_byte <= raw.len() { + Some(raw[start_byte..end_byte].to_vec()) + } else { + None + } + } else { + None + } + }); + + // Flags parsing + let mut flags = SubstitutionFlags::default(); + let mut write_filename: Option = None; + let mut saw_w = false; + let mut saw_space_in_flags = false; + // Phase 4.2: Stateless flag parsing - track positions instead of state + let mut p_position: Option = None; // Position of last 'p' flag + let mut e_position: Option = None; // Position of last 'e' flag + while i < chars.len() { + let c = chars[i]; + i += 1; + match c { + 'g' => { + if flags.global { + let error_pos = base_char_pos + i; + return Err(SedError::parse_at( + "multiple 'g' options to 's' command", + error_pos, + )); + } + flags.global = true; + } + 'p' => { + // Phase 4.2: Multiple 'p' is always an error (GNU sed compatibility) + if flags.print { + let error_pos = base_char_pos + i; + return Err(SedError::parse_at( + "multiple 'p' options to 's' command", + error_pos, + )); + } + flags.print = true; + p_position = Some(i); + } + 'I' | 'i' => { + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_subst_flag_posix(c, self.posix, base_char_pos + i)?; + flags.ignore_case = true; + } + 'm' => { + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_subst_flag_posix(c, self.posix, base_char_pos + i)?; + if flags.multiline_dotall { + return Err(self.error_at("cannot use both m and M flags")); + } + flags.multiline = true; + } + 'M' => { + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_subst_flag_posix(c, self.posix, base_char_pos + i)?; + if flags.multiline { + return Err(self.error_at("cannot use both m and M flags")); + } + flags.multiline_dotall = true; + } + 'e' => { + // Phase 3.3: Centralized POSIX validation + posix_rules::validate_subst_flag_posix(c, self.posix, base_char_pos + i)?; + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + base_char_pos + i, + )); + } + // Phase 4.2: Track FIRST position only (for timing determination) + flags.execute = true; + if e_position.is_none() { + e_position = Some(i); + } + } + '0' => { + let error_pos = base_char_pos + i; + return Err(SedError::parse_at( + "number option to 's' command may not be zero", + error_pos, + )); + } + '1'..='9' => { + if flags.occurrence.is_some() { + let error_pos = base_char_pos + i; + return Err(SedError::parse_at( + "multiple number options to 's' command", + error_pos, + )); + } + if saw_space_in_flags { + let error_pos = base_char_pos + i; + return Err(SedError::parse_at("unknown option to 's'", error_pos)); + } + flags.occurrence = Some((c as u32 - '0' as u32) as usize); + } + 'w' => { + if saw_space_in_flags { + let error_pos = base_char_pos + i; + return Err(SedError::parse_at("unknown option to 's'", error_pos)); + } + // Sandbox mode validation + if self.sandbox { + return Err(SedError::parse_at( + "e/r/w commands disabled in sandbox mode", + base_char_pos + i, + )); + } + saw_w = true; + // Calculate position of 'w' for error reporting + // base_char_pos is position of 's' command + // i points to position after 'w' in body (after i++), so position is base_char_pos + i + let w_position = base_char_pos + i; + // Consume optional whitespace then read filename until whitespace/end + // Here we capture the rest of body as filename (simplified; refined later if needed) + let mut name = String::new(); + while i < chars.len() && chars[i].is_whitespace() { + i += 1; + } + while i < chars.len() { + let ch = chars[i]; + if ch == '\n' { + break; + } + name.push(ch); + i += 1; + } + if name.is_empty() { + return Err(SedError::parse_at( + "missing filename in r/R/w/W commands", + w_position, + )); + } + write_filename = Some(name.trim().to_string()); + } + c if c.is_whitespace() => { + saw_space_in_flags = true; + } + _ => { + // Calculate position of unknown flag + // base_char_pos is position of 's' command + // i has already been incremented past the flag, so position is base_char_pos + i + let error_pos = base_char_pos + i; + return Err(SedError::parse_at( + format!("unknown option to 's'"), + error_pos, + )); + } + } + } + if saw_w { + flags.write_file = write_filename; + } + + // Phase 4.2: Determine print/execute timing based on flag positions + use crate::parser::ast::PrintTiming; + flags.print_timing = match (p_position, e_position) { + (Some(p), Some(e)) if p < e => PrintTiming::PrintThenExecute, // 'pe' + (Some(p), Some(e)) if e < p => PrintTiming::ExecuteThenPrint, // 'ep' + _ => PrintTiming::None, // Not both present, or only one present + }; + + // Calculate position after replacement (closing delimiter) for error reporting + // base_char_pos is position of 's', i is position in chars after closing delimiter + let replacement_end_pos = base_char_pos + i; + + use crate::util::regex::{validate_replacement_backrefs, validate_replacement_escapes}; + + // Validate backreferences in replacement (not in POSIX mode - they're silently ignored there) + // Skip validation when using last regex - we don't know the runtime pattern's group count + // GNU sed allows \N references to non-existent groups (outputs empty string at runtime) + if !self.posix && !uses_last_regex { + // Phase 3.2: Use extended_regex flag to determine BRE/ERE mode for validation + // Single validation instead of try-both-and-accept + validate_replacement_backrefs( + &replacement, + &pattern, + self.extended_regex, + replacement_end_pos, + )?; + } + + // Validate escape sequences (e.g., \c not followed by \) + validate_replacement_escapes(&replacement, replacement_end_pos)?; + + Ok(Command::Substitution { + range, + negated, + pattern, + pattern_raw_bytes, + replacement, + replacement_raw_bytes, + flags, + delimiter: delim, + }) + } else { + return Err(self.error_at("unterminated 's' command")); + } + } + + fn parse_translate_from_body( + &mut self, + range: Option, + negated: bool, + body: &str, + raw_bytes: Option<&[u8]>, + ) -> Result { + let chars: Vec = body.chars().collect(); + if chars.is_empty() { + return Err(self.error_at("unterminated 'y' command")); + } + let delim = chars[0]; + let mut i = 1; + let parse_until = |start: usize| -> Result<(String, usize)> { + let mut out = String::new(); + let mut idx = start; + let mut escaped = false; + while idx < chars.len() { + let c = chars[idx]; + idx += 1; + if escaped { + // If escaped char is the delimiter, it's an escaped delimiter + // (just the char, no backslash). Otherwise keep the backslash. + if c != delim { + out.push('\\'); + } + out.push(c); + escaped = false; + continue; + } + if c == '\\' { + escaped = true; + continue; + } + if c == delim { + return Ok((out, idx)); + } + out.push(c); + } + return Err(self.error_at("unterminated 'y' command")); + }; + let (from_raw, next) = parse_until(i)?; + i = next; + let (to_raw, _next2) = parse_until(i)?; + + // GNU sed does NOT interpret octal escapes in y command + let from = escapes::decode_standard_escapes(&from_raw); + let to = escapes::decode_standard_escapes(&to_raw); + + // Extract from_bytes and to_bytes from raw_bytes if available + let (from_bytes, to_bytes) = if let Some(raw) = raw_bytes { + // raw_bytes format: + // Find delimiter (first byte) + if raw.is_empty() { + (None, None) + } else { + let delim_byte = raw[0]; + // Helper to extract bytes, stripping backslash from escaped delimiters + let extract_part = |slice: &[u8]| -> Vec { + let mut out = Vec::new(); + let mut j = 0; + while j < slice.len() { + if slice[j] == b'\\' && j + 1 < slice.len() { + if slice[j + 1] == delim_byte { + // Escaped delimiter: skip backslash, push delimiter + out.push(slice[j + 1]); + j += 2; + } else { + // Other escape: keep both chars + out.push(slice[j]); + out.push(slice[j + 1]); + j += 2; + } + } else { + out.push(slice[j]); + j += 1; + } + } + out + }; + + let mut parts: Vec> = Vec::new(); + let mut start = 1; + let mut i = 1; + let mut escaped = false; + while i < raw.len() { + if escaped { + escaped = false; + i += 1; + continue; + } + if raw[i] == b'\\' { + escaped = true; + i += 1; + continue; + } + if raw[i] == delim_byte { + parts.push(extract_part(&raw[start..i])); + start = i + 1; + } + i += 1; + } + if parts.len() >= 2 { + // GNU sed does NOT interpret octal escapes in y command + let fb = escapes::decode_standard_escapes_to_bytes(&parts[0]); + let tb = escapes::decode_standard_escapes_to_bytes(&parts[1]); + (Some(fb), Some(tb)) + } else { + (None, None) + } + } + } else { + (None, None) + }; + // Validate lengths based on locale + // In UTF-8 locale: compare character counts + // In multibyte non-UTF-8 locale (e.g., Shift-JIS): compare multibyte character counts + // In C/POSIX single-byte locale: compare byte counts + let lengths_match = if self.utf8_locale { + // UTF-8 locale: count characters + from.chars().count() == to.chars().count() + } else if crate::mbcs::is_multibyte_locale() { + // Multibyte non-UTF-8 locale (e.g., Shift-JIS): count multibyte characters + match (&from_bytes, &to_bytes) { + (Some(fb), Some(tb)) => { + crate::mbcs::count_mb_chars(fb) == crate::mbcs::count_mb_chars(tb) + } + _ => from.len() == to.len(), // Fallback to string byte length + } + } else { + // C locale: count bytes (use from_bytes/to_bytes if available) + match (&from_bytes, &to_bytes) { + (Some(fb), Some(tb)) => fb.len() == tb.len(), + _ => from.len() == to.len(), // Fallback to string byte length + } + }; + + if from.is_empty() || to.is_empty() || !lengths_match { + // Error position: char_position is right after the TranslateBody token, + // so the last character of the body is at char_position - 1 + let error_pos = self.char_position - 1; + return Err(SedError::parse_at( + "'y' command strings have different lengths", + error_pos, + )); + } + + Ok(Command::Translate { + range, + negated, + from, + to, + from_bytes, + to_bytes, + delimiter: delim, + }) + } +} + +#[cfg(test)] +mod tests { + use super::{Address, Command, Lexer, Parser}; + use crate::context::Context; + use crate::parser::Token; + + #[test] + fn test_lexer_basic_tokens() { + let mut lexer = Lexer::new("$,;!{}"); + assert_eq!(lexer.next_token().unwrap(), Token::Dollar); + assert_eq!(lexer.next_token().unwrap(), Token::Comma); + assert_eq!(lexer.next_token().unwrap(), Token::Semicolon); + assert_eq!(lexer.next_token().unwrap(), Token::Bang); + assert_eq!(lexer.next_token().unwrap(), Token::OpenBrace); + assert_eq!(lexer.next_token().unwrap(), Token::CloseBrace); + assert_eq!(lexer.next_token().unwrap(), Token::Eof); + } + + #[test] + fn test_lexer_numbers() { + let mut lexer = Lexer::new("4 20 $"); + assert_eq!(lexer.next_token().unwrap(), Token::LineNumber(4)); + assert_eq!(lexer.next_token().unwrap(), Token::LineNumber(20)); + assert_eq!(lexer.next_token().unwrap(), Token::Dollar); + } + + #[test] + fn test_lexer_regex_addr() { + let mut lexer = Lexer::new("/pattern/"); + assert_eq!( + lexer.next_token().unwrap(), + Token::RegexAddr("pattern".to_string(), String::new()) + ); + } + + #[test] + fn test_lexer_comment() { + let mut lexer = Lexer::new("# this is a comment"); + assert_eq!( + lexer.next_token().unwrap(), + Token::Comment(" this is a comment".to_string()) + ); + } + + #[test] + fn test_lexer_commands() { + // Test that 's' followed by space is treated as a substitution command + // (space can be a delimiter, though unusual) + let mut lexer = Lexer::new("s p d"); + assert_eq!(lexer.next_token().unwrap(), Token::Command('s')); + // Next token should be SubstitutionBody, which will fail to parse + // because the substitution is malformed (unterminated) + assert!(lexer.next_token().is_err()); + } + + #[test] + fn test_lexer_addresses_and_commands() { + let mut lexer = Lexer::new("4p $d 1,5s"); + assert_eq!(lexer.next_token().unwrap(), Token::LineNumber(4)); + assert_eq!(lexer.next_token().unwrap(), Token::Command('p')); + assert_eq!(lexer.next_token().unwrap(), Token::Dollar); + assert_eq!(lexer.next_token().unwrap(), Token::Command('d')); + assert_eq!(lexer.next_token().unwrap(), Token::LineNumber(1)); + assert_eq!(lexer.next_token().unwrap(), Token::Comma); + assert_eq!(lexer.next_token().unwrap(), Token::LineNumber(5)); + assert_eq!(lexer.next_token().unwrap(), Token::Command('s')); + } + + #[test] + fn test_lexer_escaped_regex() { + let mut lexer = Lexer::new(r"/a\/b/"); + assert_eq!( + lexer.next_token().unwrap(), + Token::RegexAddr("a/b".to_string(), String::new()) + ); + } + + #[test] + fn test_lexer_relative_addresses() { + let mut lexer = Lexer::new("+5 ~3 +"); + assert_eq!(lexer.next_token().unwrap(), Token::PlusOffset(5)); + assert_eq!(lexer.next_token().unwrap(), Token::TildeStep(3)); + // + without number should be treated as delimiter + assert_eq!(lexer.next_token().unwrap(), Token::SubstitutionDelim('+')); + assert_eq!(lexer.next_token().unwrap(), Token::Eof); + } + + #[test] + fn test_parser_simple_address() { + let mut lexer = Lexer::new("4p"); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Print { range, negated } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(4))); + assert_eq!(range.end, None); + } + _ => panic!("Expected Print command"), + } + } + + #[test] + fn test_parser_address_range() { + let mut lexer = Lexer::new("1,5d"); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Delete { range, negated } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(1))); + assert_eq!(range.end, Some(Address::Line(5))); + } + _ => panic!("Expected Delete command"), + } + } + + #[test] + fn test_parser_negated_address() { + let mut lexer = Lexer::new("4!p"); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Print { range, negated } => { + assert!(*negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(4))); + assert_eq!(range.end, None); + } + _ => panic!("Expected Print command"), + } + } + + #[test] + fn test_parser_command_group() { + let mut lexer = Lexer::new("1,5{p;d}"); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Group { + range, + negated, + commands: group_commands, + } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(1))); + assert_eq!(range.end, Some(Address::Line(5))); + assert_eq!(group_commands.len(), 2); + + // Check first command is print + if let Command::Print { + range: p_range, + negated: p_negated, + } = &group_commands[0] + { + assert!(!p_negated); + assert!(p_range.is_none()); // No address on inner command + } else { + panic!("Expected Print command in group"); + } + + // Check second command is delete + if let Command::Delete { + range: d_range, + negated: d_negated, + } = &group_commands[1] + { + assert!(!d_negated); + assert!(d_range.is_none()); // No address on inner command + } else { + panic!("Expected Delete command in group"); + } + } + _ => panic!("Expected Group command"), + } + } + + #[test] + fn test_parser_semicolon_separator() { + let mut lexer = Lexer::new("p;d"); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 2); + + match &commands[0] { + Command::Print { .. } => {} + _ => panic!("Expected first command to be Print"), + } + + match &commands[1] { + Command::Delete { .. } => {} + _ => panic!("Expected second command to be Delete"), + } + } + + #[test] + fn test_lexer_line_continuation() { + let input = "4\\\np"; + let mut lexer = Lexer::new(input); + + let token1 = lexer.next_token().unwrap(); + assert_eq!(token1, Token::LineNumber(4)); + + let token2 = lexer.next_token().unwrap(); + assert_eq!(token2, Token::Command('p')); + + let token3 = lexer.next_token().unwrap(); + assert_eq!(token3, Token::Eof); + } + + #[test] + fn test_parser_complex_grouping_with_addresses() { + // Simplified test focusing on grouping and addresses without substitution complexity + let input = "/pattern/!{p;d}"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token(); + match token { + Ok(Token::Eof) => { + tokens.push(Token::Eof); + break; + } + Ok(t) => tokens.push(t), + Err(e) => { + println!("Lexer error at input: {}", input); + println!("Error: {}", e); + println!("Tokens so far: {:?}", tokens); + panic!("Lexer failed"); + } + } + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Group { + range, + negated, + commands: group_commands, + } => { + assert!(*negated); // Negated with ! + assert!(range.is_some()); + + let range = range.as_ref().unwrap(); + if let Some(Address::Regex(pattern)) = &range.start { + assert_eq!(pattern, "pattern"); + } else { + panic!("Expected regex address"); + } + + assert_eq!(group_commands.len(), 2); + // Check we have print and delete commands + matches!(group_commands[0], Command::Print { .. }); + matches!(group_commands[1], Command::Delete { .. }); + } + _ => panic!("Expected Group command"), + } + } + + #[test] + fn test_parser_multiline_with_continuation() { + let input = "1\\\n,\\\n5\\\np"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Print { range, negated } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(1))); + assert_eq!(range.end, Some(Address::Line(5))); + } + _ => panic!("Expected Print command"), + } + } + + // NOTE: Substitution parsing is handled by this parser (lexer builds a + // SubstitutionBody with an arbitrary delimiter, including '['), then + // converted to runtime commands in lib.rs (convert_new_command_to_old). + // The previous note about handling in main.rs is obsolete. + + #[test] + fn test_parser_line_number_command() { + let input = "4="; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::LineNumber { range, negated } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(4))); + assert_eq!(range.end, None); + } + _ => panic!("Expected LineNumber command"), + } + } + + #[test] + fn test_parser_list_command() { + let input = "1,5l"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::List { range, negated, .. } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(1))); + assert_eq!(range.end, Some(Address::Line(5))); + } + _ => panic!("Expected List command"), + } + } + + #[test] + fn test_parser_multiple_commands() { + let input = "p;d;q"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 3); + + matches!(&commands[0], Command::Print { .. }); + matches!(&commands[1], Command::Delete { .. }); + matches!(&commands[2], Command::Quit { .. }); + } + + #[test] + fn test_parser_test_neg_command() { + let input = "T"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::TestNeg { + range, + negated, + label, + } => { + assert!(!negated); + assert!(range.is_none()); + assert_eq!(label, ""); + } + _ => panic!("Expected TestNeg command"), + } + } + + #[test] + fn test_parser_test_neg_with_label() { + let input = "T end"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::TestNeg { + range, + negated, + label, + } => { + assert!(!negated); + assert!(range.is_none()); + assert_eq!(label, "end"); + } + _ => panic!("Expected TestNeg command with label"), + } + } + + #[test] + fn test_parser_execute_command() { + let input = "e"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Execute { + range, + negated, + command, + } => { + assert!(!negated); + assert!(range.is_none()); + assert_eq!(command, &None); + } + _ => panic!("Expected Execute command"), + } + } + + #[test] + fn test_parser_execute_with_argument() { + // Note: Due to tokenization limitations and conflict with label 'e', + // multi-word arguments are not fully supported. Single-word works. + let input = "e pwd"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::Execute { + range, + negated, + command, + } => { + assert!(!negated); + assert!(range.is_none()); + assert_eq!(command, &Some("pwd".to_string()), "Command should be 'pwd'"); + } + _ => panic!("Expected Execute command with argument"), + } + } + + #[test] + fn test_parser_print_first_line_command() { + let input = "P"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::PrintFirstLine { range, negated } => { + assert!(!negated); + assert!(range.is_none()); + } + _ => panic!("Expected PrintFirstLine command"), + } + } + + #[test] + fn test_parser_print_first_line_with_address() { + let input = "2,5P"; + let mut lexer = Lexer::new(input); + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token().unwrap(); + if token == Token::Eof { + tokens.push(token); + break; + } + tokens.push(token); + } + + let ctx = Context::new_test(); + let mut parser = Parser::new(tokens, &ctx); + let commands = parser.parse().unwrap(); + assert_eq!(commands.len(), 1); + + match &commands[0] { + Command::PrintFirstLine { range, negated } => { + assert!(!negated); + assert!(range.is_some()); + let range = range.as_ref().unwrap(); + assert_eq!(range.start, Some(Address::Line(2))); + assert_eq!(range.end, Some(Address::Line(5))); + } + _ => panic!("Expected PrintFirstLine command with address range"), + } + } +} diff --git a/red/src/parser/ast.rs b/red/src/parser/ast.rs new file mode 100644 index 0000000..55de71b --- /dev/null +++ b/red/src/parser/ast.rs @@ -0,0 +1,556 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +/// Token types for command parsing +#[derive(Debug, Clone, PartialEq)] +pub enum Token { + // Addresses + LineNumber(usize), // 4, 20 + Dollar, // $ + RegexAddr(String, String), // /pattern/[modifiers] (pattern, modifiers) + PlusOffset(usize), // +N + TildeStep(usize), // ~N + + // Separators and operators + Comma, // , + Semicolon, // ; + Newline, + Bang, // ! + OpenBrace, // { + CloseBrace, // } + + // Commands + Command(char), // s, p, d, etc + SubstitutionDelim(char), // delimiter for s/../../ + SubstitutionFlag(char), // g, p, w, I, i + // Raw body of s-command starting with delimiter and including flags + // Tuple: (body_text, optional raw_bytes for replacement preservation) + SubstitutionBody(String, Option>), + // Raw body of y-command: delimiter + from + delimiter + to + delimiter + // Tuple: (body_text, optional raw_bytes for from/to preservation) + TranslateBody(String, Option>), + + // Bodies for a/i/c commands (capture following text line) + // Option: None = unterminated (no text), Some = explicit text (even if empty) + AppendBody(Option), + InsertBody(Option), + ChangeBody(Option), + + // Command to execute for e command (can be empty) + ExecuteBody(String), + + // Literals and text + String(String), + Filename(String), + Comment(String), // #comment + + Eof, +} + +/// Address types for sed commands +#[derive(Debug, Clone, PartialEq)] +pub enum Address { + Line(usize), // 4 + Dollar, // $ + Regex(String), // /pattern/ + Relative(Box
, isize), // addr+N, addr-N + Step(Box
, usize), // addr~N +} + +/// Address range (addr1,addr2) +/// Each range has a unique ID for tracking state in AddressEvaluator +#[derive(Debug, Clone)] +pub struct AddressRange { + pub start: Option
, + pub end: Option
, + /// Unique identifier for this range instance (used for state tracking) + pub id: u64, +} + +impl AddressRange { + /// Create a new AddressRange with a unique ID + pub fn new(start: Option
, end: Option
) -> Self { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + Self { + start, + end, + id: NEXT_ID.fetch_add(1, Ordering::Relaxed), + } + } +} + +// PartialEq ignores id - two ranges are equal if they have the same addresses +impl PartialEq for AddressRange { + fn eq(&self, other: &Self) -> bool { + self.start == other.start && self.end == other.end + } +} + +/// Print/Execute timing for substitution command +/// +/// When both 'p' and 'e' flags are present, their order matters: +/// - 'pe': print the substituted line, then execute it as a shell command +/// - 'ep': execute the substituted line as a shell command, then print it +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrintTiming { + /// Neither print nor execute flags present, or only one of them + None, + /// 'pe' - print then execute + PrintThenExecute, + /// 'ep' - execute then print + ExecuteThenPrint, +} + +impl Default for PrintTiming { + fn default() -> Self { + PrintTiming::None + } +} + +/// Flags for substitution command +#[derive(Debug, Default, Clone, PartialEq)] +pub struct SubstitutionFlags { + pub global: bool, // g + pub print: bool, // p + pub write_file: Option, // w filename + pub ignore_case: bool, // I/i (BSD extension) + pub occurrence: Option, // 1-9 + pub multiline: bool, // m - multiline mode (^ and $ match line boundaries) + pub multiline_dotall: bool, // M - multiline + dotall (. matches \n) + pub execute: bool, // e - execute replacement as shell command + pub print_timing: PrintTiming, // Timing for 'p' and 'e' flags (replaces print_before_execute) +} + +/// AST node for commands +#[derive(Debug, Clone)] +pub enum Command { + Substitution { + range: Option, + negated: bool, + pattern: String, + /// Raw bytes for pattern string (for non-UTF-8 byte matching) + pattern_raw_bytes: Option>, + replacement: String, + /// Raw bytes for replacement string (for preserving invalid UTF-8) + replacement_raw_bytes: Option>, + flags: SubstitutionFlags, + delimiter: char, + }, + Translate { + range: Option, + negated: bool, + from: String, + to: String, + /// Raw bytes for 'from' string (for preserving invalid UTF-8) + from_bytes: Option>, + /// Raw bytes for 'to' string (for preserving invalid UTF-8) + to_bytes: Option>, + delimiter: char, + }, + Append { + range: Option, + negated: bool, + text: Option, // None = unterminated, Some = explicit (even if empty) + }, + Insert { + range: Option, + negated: bool, + text: Option, // None = unterminated, Some = explicit (even if empty) + }, + Change { + range: Option, + negated: bool, + text: Option, // None = unterminated, Some = explicit (even if empty) + }, + // Pattern/Hold space related (subset): N (append next line), D (delete first line and restart) + N { + range: Option, + negated: bool, + }, + BigD { + range: Option, + negated: bool, + }, + HoldCopy { + // h + range: Option, + negated: bool, + }, + HoldAppend { + // H + range: Option, + negated: bool, + }, + GetCopy { + // g + range: Option, + negated: bool, + }, + GetAppend { + // G + range: Option, + negated: bool, + }, + Exchange { + // x + range: Option, + negated: bool, + }, + Label { + name: String, + }, + Branch { + range: Option, + negated: bool, + label: String, + }, + Test { + range: Option, + negated: bool, + label: String, + }, + TestNeg { + range: Option, + negated: bool, + label: String, + }, + Execute { + range: Option, + negated: bool, + command: Option, // None = execute pattern space, Some = execute this command + }, + Version { + version: String, // Empty means "4.0" + }, + Clear { + range: Option, + negated: bool, + }, + PrintFilename { + range: Option, + negated: bool, + }, + Next, // n + Write { + range: Option, + negated: bool, + path: String, + }, + WriteFirstLine { + range: Option, + negated: bool, + path: String, + }, + Read { + range: Option, + negated: bool, + path: String, + }, + ReadLine { + range: Option, + negated: bool, + path: String, + }, + Print { + range: Option, + negated: bool, + }, + PrintFirstLine { + range: Option, + negated: bool, + }, + Delete { + range: Option, + negated: bool, + }, + Quit { + range: Option, + negated: bool, + exit_code: Option, + }, + QuitSilent { + range: Option, + negated: bool, + exit_code: Option, + }, + Group { + range: Option, + negated: bool, + commands: Vec, + }, + LineNumber { + range: Option, + negated: bool, + }, + List { + range: Option, + negated: bool, + line_length: Option, // Optional line wrap length (e.g., l70) + }, + Comment(String), // # comment +} + +/// Trait for commands that have an address range +/// +/// Most sed commands can be restricted to specific line ranges. +/// This trait provides uniform access to the range field. +pub trait HasAddressRange { + /// Get reference to address range (if any) + fn address_range(&self) -> Option<&AddressRange>; +} + +impl HasAddressRange for Command { + fn address_range(&self) -> Option<&AddressRange> { + match self { + Self::Substitution { range, .. } + | Self::Translate { range, .. } + | Self::Append { range, .. } + | Self::Insert { range, .. } + | Self::Change { range, .. } + | Self::N { range, .. } + | Self::BigD { range, .. } + | Self::HoldCopy { range, .. } + | Self::HoldAppend { range, .. } + | Self::GetCopy { range, .. } + | Self::GetAppend { range, .. } + | Self::Exchange { range, .. } + | Self::Branch { range, .. } + | Self::Test { range, .. } + | Self::TestNeg { range, .. } + | Self::Execute { range, .. } + | Self::Clear { range, .. } + | Self::PrintFilename { range, .. } + | Self::Write { range, .. } + | Self::WriteFirstLine { range, .. } + | Self::Read { range, .. } + | Self::ReadLine { range, .. } + | Self::Print { range, .. } + | Self::PrintFirstLine { range, .. } + | Self::Delete { range, .. } + | Self::Quit { range, .. } + | Self::QuitSilent { range, .. } + | Self::Group { range, .. } + | Self::LineNumber { range, .. } + | Self::List { range, .. } => range.as_ref(), + + // Commands without address range + Self::Label { .. } | Self::Version { .. } | Self::Next | Self::Comment(_) => None, + } + } +} + +/// Display implementation for user-friendly error messages +impl std::fmt::Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Token::Command(ch) => write!(f, "'{}'", ch), + Token::Semicolon => write!(f, "';'"), + Token::CloseBrace => write!(f, "'}}'"), + Token::OpenBrace => write!(f, "'{{'"), + Token::Newline => write!(f, "newline"), + Token::Eof => write!(f, "end of script"), + Token::Filename(name) => write!(f, "'{}'", name), + Token::Bang => write!(f, "'!'"), + Token::Comma => write!(f, "','"), + Token::Dollar => write!(f, "'$'"), + Token::LineNumber(n) => write!(f, "{}", n), + Token::RegexAddr(pattern, modifiers) => { + if modifiers.is_empty() { + write!(f, "/{}/", pattern) + } else { + write!(f, "/{}/{}", pattern, modifiers) + } + } + Token::PlusOffset(n) => write!(f, "+{}", n), + Token::TildeStep(n) => write!(f, "~{}", n), + Token::SubstitutionDelim(ch) => write!(f, "'{}'", ch), + Token::SubstitutionFlag(ch) => write!(f, "'{}'", ch), + Token::SubstitutionBody(s, _) => write!(f, "substitution '{}'", s), + Token::TranslateBody(s, _) => write!(f, "translate '{}'", s), + Token::AppendBody(Some(s)) => write!(f, "append text '{}'", s), + Token::AppendBody(None) => write!(f, "append (unterminated)"), + Token::InsertBody(Some(s)) => write!(f, "insert text '{}'", s), + Token::InsertBody(None) => write!(f, "insert (unterminated)"), + Token::ChangeBody(Some(s)) => write!(f, "change text '{}'", s), + Token::ChangeBody(None) => write!(f, "change (unterminated)"), + Token::ExecuteBody(s) => write!(f, "execute '{}'", s), + Token::String(s) => write!(f, "\"{}\"", s), + Token::Comment(s) => write!(f, "comment '{}'", s), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_token_display_commands() { + assert_eq!(format!("{}", Token::Command('s')), "'s'"); + assert_eq!(format!("{}", Token::Command('p')), "'p'"); + } + + #[test] + fn test_token_display_separators() { + assert_eq!(format!("{}", Token::Semicolon), "';'"); + assert_eq!(format!("{}", Token::CloseBrace), "'}'"); + assert_eq!(format!("{}", Token::OpenBrace), "'{'"); + assert_eq!(format!("{}", Token::Newline), "newline"); + assert_eq!(format!("{}", Token::Eof), "end of script"); + assert_eq!(format!("{}", Token::Bang), "'!'"); + assert_eq!(format!("{}", Token::Comma), "','"); + } + + #[test] + fn test_token_display_addresses() { + assert_eq!(format!("{}", Token::Dollar), "'$'"); + assert_eq!(format!("{}", Token::LineNumber(42)), "42"); + assert_eq!(format!("{}", Token::PlusOffset(5)), "+5"); + assert_eq!(format!("{}", Token::TildeStep(3)), "~3"); + } + + #[test] + fn test_token_display_regex_addr() { + assert_eq!( + format!("{}", Token::RegexAddr("foo".to_string(), "".to_string())), + "/foo/" + ); + assert_eq!( + format!("{}", Token::RegexAddr("bar".to_string(), "I".to_string())), + "/bar/I" + ); + } + + #[test] + fn test_token_display_substitution() { + assert_eq!(format!("{}", Token::SubstitutionDelim('/')), "'/'"); + assert_eq!(format!("{}", Token::SubstitutionFlag('g')), "'g'"); + assert_eq!( + format!("{}", Token::SubstitutionBody("/a/b/".to_string(), None)), + "substitution '/a/b/'" + ); + } + + #[test] + fn test_token_display_translate() { + assert_eq!( + format!("{}", Token::TranslateBody("/abc/xyz/".to_string(), None)), + "translate '/abc/xyz/'" + ); + } + + #[test] + fn test_token_display_text_bodies() { + assert_eq!( + format!("{}", Token::AppendBody(Some("hello".to_string()))), + "append text 'hello'" + ); + assert_eq!( + format!("{}", Token::AppendBody(None)), + "append (unterminated)" + ); + + assert_eq!( + format!("{}", Token::InsertBody(Some("world".to_string()))), + "insert text 'world'" + ); + assert_eq!( + format!("{}", Token::InsertBody(None)), + "insert (unterminated)" + ); + + assert_eq!( + format!("{}", Token::ChangeBody(Some("new".to_string()))), + "change text 'new'" + ); + assert_eq!( + format!("{}", Token::ChangeBody(None)), + "change (unterminated)" + ); + + assert_eq!( + format!("{}", Token::ExecuteBody("ls -la".to_string())), + "execute 'ls -la'" + ); + } + + #[test] + fn test_token_display_strings() { + assert_eq!( + format!("{}", Token::Filename("/tmp/file".to_string())), + "'/tmp/file'" + ); + assert_eq!(format!("{}", Token::String("text".to_string())), "\"text\""); + assert_eq!( + format!("{}", Token::Comment("note".to_string())), + "comment 'note'" + ); + } + + #[test] + fn test_address_range_new_unique_ids() { + let range1 = AddressRange::new(None, None); + let range2 = AddressRange::new(None, None); + // Each range should have a unique ID + assert_ne!(range1.id, range2.id); + } + + #[test] + fn test_address_range_eq_ignores_id() { + let range1 = AddressRange::new(Some(Address::Line(1)), Some(Address::Line(5))); + let range2 = AddressRange::new(Some(Address::Line(1)), Some(Address::Line(5))); + // IDs are different but ranges should be equal + assert_ne!(range1.id, range2.id); + assert_eq!(range1, range2); + } + + #[test] + fn test_print_timing_default() { + assert_eq!(PrintTiming::default(), PrintTiming::None); + } + + #[test] + fn test_has_address_range_substitution() { + let cmd = Command::Substitution { + range: Some(AddressRange::new(Some(Address::Line(1)), None)), + negated: false, + pattern: "foo".to_string(), + pattern_raw_bytes: None, + replacement: "bar".to_string(), + replacement_raw_bytes: None, + flags: SubstitutionFlags::default(), + delimiter: '/', + }; + assert!(cmd.address_range().is_some()); + } + + #[test] + fn test_has_address_range_label() { + let cmd = Command::Label { + name: "loop".to_string(), + }; + assert!(cmd.address_range().is_none()); + } + + #[test] + fn test_has_address_range_version() { + let cmd = Command::Version { + version: "4.0".to_string(), + }; + assert!(cmd.address_range().is_none()); + } + + #[test] + fn test_has_address_range_next() { + let cmd = Command::Next; + assert!(cmd.address_range().is_none()); + } + + #[test] + fn test_has_address_range_comment() { + let cmd = Command::Comment("test".to_string()); + assert!(cmd.address_range().is_none()); + } +} diff --git a/red/src/parser/escapes.rs b/red/src/parser/escapes.rs new file mode 100644 index 0000000..aef3458 --- /dev/null +++ b/red/src/parser/escapes.rs @@ -0,0 +1,256 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Escape sequence handling for sed text commands (y, a, i, c) +//! +//! This module provides functions to decode escape sequences in text contexts. +//! +//! ## Escape semantics in GNU sed +//! +//! Different sed contexts support different escape sequences: +//! +//! ### Text commands (y, a, i, c) - handled by this module +//! - `\n`, `\t`, `\r`, `\f`, `\v`, `\a`, `\b` - standard escapes +//! - `\\` - literal backslash +//! - `\cX` - control character (X ^ 0x40) +//! - `\NNN` - NOT supported (treated as literal characters) +//! +//! ### s/// replacement - handled by util/regex.rs +//! - All standard escapes plus: +//! - `\oNNN` - octal escape (GNU extension) +//! - `\xNN` - hex escape (GNU extension) +//! - `\dNNN` - decimal escape (GNU extension) +//! - `\1`-`\9` - backreferences +//! - `\u`, `\l`, `\U`, `\L`, `\E` - case conversion +//! +//! ### Regex patterns - handled by regex/parser.rs +//! - BRE/ERE specific escape sequences + +/// Decode standard escape sequences in a string. +/// +/// Supports: +/// - `\n`, `\t`, `\r`, `\f`, `\v`, `\a`, `\b` - standard escapes +/// - `\\` - literal backslash +/// - `\cX` - control character (X ^ 0x40) +/// - `\x` - any other escaped char becomes itself +/// +/// Does NOT support bare octal escapes (`\NNN`) - this matches GNU sed +/// behavior for the `y` command and text insertion commands (a, i, c). +pub fn decode_standard_escapes(s: &str) -> String { + let mut out = String::new(); + let mut it = s.chars().peekable(); + while let Some(c) = it.next() { + if c != '\\' { + out.push(c); + continue; + } + match it.peek().cloned() { + Some('n') => { + out.push('\n'); + it.next(); + } + Some('t') => { + out.push('\t'); + it.next(); + } + Some('r') => { + out.push('\r'); + it.next(); + } + Some('f') => { + out.push('\x0c'); + it.next(); + } + Some('v') => { + out.push('\x0b'); + it.next(); + } + Some('a') => { + out.push('\x07'); + it.next(); + } + Some('b') => { + out.push('\x08'); + it.next(); + } + Some('\\') => { + out.push('\\'); + it.next(); + } + Some('c') => { + // Control character: \cX produces X ^ 0x40 + // For lowercase letters, convert to uppercase first + it.next(); // consume 'c' + if let Some(ch) = it.next() { + // Handle \c\\ case: the next char after \c might be a backslash + // In that case, consume the escaped char + let actual_char = if ch == '\\' { + if let Some(escaped) = it.next() { + escaped + } else { + '\\' + } + } else { + ch + }; + // Convert lowercase to uppercase before XOR + let effective_byte = if actual_char.is_ascii_lowercase() { + (actual_char as u8) - 32 + } else { + actual_char as u8 + }; + let ctrl = (effective_byte ^ 0x40) as char; + out.push(ctrl); + } + // If no char after \c, just skip (GNU sed behavior) + } + Some(other) => { + // Any other escaped char becomes itself (including digits) + // This matches GNU sed: \1 in y command is literal '1', not backreference + out.push(other); + it.next(); + } + None => { + out.push('\\'); + } + } + } + out +} + +/// Decode standard escape sequences directly to bytes (preserves invalid UTF-8). +/// +/// Similar to `decode_standard_escapes` but operates on raw bytes and preserves +/// byte values that would be invalid UTF-8. +pub fn decode_standard_escapes_to_bytes(bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' && i + 1 < bytes.len() { + match bytes[i + 1] { + b'n' => { + out.push(b'\n'); + i += 2; + } + b't' => { + out.push(b'\t'); + i += 2; + } + b'r' => { + out.push(b'\r'); + i += 2; + } + b'f' => { + out.push(0x0c); + i += 2; + } + b'v' => { + out.push(0x0b); + i += 2; + } + b'a' => { + out.push(0x07); + i += 2; + } + b'b' => { + out.push(0x08); + i += 2; + } + b'\\' => { + out.push(b'\\'); + i += 2; + } + b'c' => { + // Control character: \cX produces X ^ 0x40 + // For lowercase letters, convert to uppercase first + i += 2; // skip \c + if i < bytes.len() { + // Handle \c\\ case + let actual_byte = if bytes[i] == b'\\' && i + 1 < bytes.len() { + i += 1; // skip first backslash + let b = bytes[i]; + i += 1; + b + } else { + let b = bytes[i]; + i += 1; + b + }; + // Convert lowercase to uppercase before XOR + let effective_byte = if actual_byte >= b'a' && actual_byte <= b'z' { + actual_byte - 32 + } else { + actual_byte + }; + out.push(effective_byte ^ 0x40); + } + // If no char after \c, just skip + } + other => { + // Any other escaped char becomes itself (including digits) + out.push(other); + i += 2; + } + } + } else if bytes[i] == b'\\' { + out.push(b'\\'); + i += 1; + } else { + out.push(bytes[i]); + i += 1; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_standard_escapes_basic() { + assert_eq!(decode_standard_escapes("hello"), "hello"); + assert_eq!(decode_standard_escapes("hello\\nworld"), "hello\nworld"); + assert_eq!( + decode_standard_escapes("\\t\\r\\f\\v\\a\\b"), + "\t\r\x0c\x0b\x07\x08" + ); + assert_eq!(decode_standard_escapes("\\\\"), "\\"); + } + + #[test] + fn test_standard_escapes_control() { + // \cA = 'A' ^ 0x40 = 0x01 + assert_eq!(decode_standard_escapes("\\cA"), "\x01"); + // \ca = 'a' -> 'A' -> 0x01 + assert_eq!(decode_standard_escapes("\\ca"), "\x01"); + // \c\\ case + assert_eq!(decode_standard_escapes("\\c\\\\"), "\x1c"); + } + + #[test] + fn test_standard_escapes_digits_are_literal() { + // GNU sed does NOT support bare octal escapes in y command + // \1 becomes literal '1', \101 becomes literal "101" + assert_eq!(decode_standard_escapes("\\1"), "1"); + assert_eq!(decode_standard_escapes("\\101"), "101"); + assert_eq!(decode_standard_escapes("\\141\\142"), "141142"); + } + + #[test] + fn test_standard_escapes_to_bytes_basic() { + assert_eq!(decode_standard_escapes_to_bytes(b"hello"), b"hello"); + assert_eq!( + decode_standard_escapes_to_bytes(b"hello\\nworld"), + b"hello\nworld" + ); + } + + #[test] + fn test_standard_escapes_to_bytes_digits_literal() { + // Digits are literal, not octal + assert_eq!(decode_standard_escapes_to_bytes(b"\\1"), b"1"); + assert_eq!(decode_standard_escapes_to_bytes(b"\\377"), b"377"); + } +} diff --git a/red/src/parser/lexer.rs b/red/src/parser/lexer.rs new file mode 100644 index 0000000..f5d63b3 --- /dev/null +++ b/red/src/parser/lexer.rs @@ -0,0 +1,1367 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use crate::errors::{Result, SedError}; + +use super::ast::Token; + +/// Check if the current locale is UTF-8 based on environment variables +pub fn is_utf8_locale() -> bool { + // Check LC_ALL first (highest priority), then LC_CTYPE, then LANG + let locale = std::env::var("LC_ALL") + .or_else(|_| std::env::var("LC_CTYPE")) + .or_else(|_| std::env::var("LANG")) + .unwrap_or_default(); + + let locale_lower = locale.to_lowercase(); + + // On Windows, if no locale environment variables are set, default to UTF-8 + // Modern Windows uses UTF-8 by default for console and most applications + #[cfg(windows)] + if locale_lower.is_empty() { + return true; + } + + locale_lower.contains("utf-8") || locale_lower.contains("utf8") +} + +/// Build a mapping from char positions in the converted string to byte positions in raw bytes +/// +/// This is needed because `from_utf8_lossy()` converts invalid bytes to U+FFFD, changing +/// the byte positions. We need to know where each character came from in the raw bytes +/// to properly detect UTF-8 lead bytes vs continuation bytes for delimiter validation. +pub fn build_char_to_byte_mapping(input: &str, raw_bytes: &[u8]) -> Vec { + let mut mapping = Vec::with_capacity(input.chars().count()); + let mut byte_pos = 0; + + for ch in input.chars() { + // Record the byte position for this char + mapping.push(byte_pos); + + if ch == '\u{FFFD}' { + // Replacement character - came from invalid byte(s) + // In from_utf8_lossy, each invalid byte becomes one U+FFFD + // BUT: an incomplete sequence at the end may consume multiple bytes + // For simplicity, assume single invalid byte per U+FFFD (most common case) + byte_pos += 1; + } else { + // Valid UTF-8 character - advance by its UTF-8 length in the source + byte_pos += ch.len_utf8(); + } + + // Safety: don't go past raw bytes + if byte_pos > raw_bytes.len() { + byte_pos = raw_bytes.len(); + } + } + + mapping +} + +/// State machine for the lexer +/// +/// This enum represents the lexer's parsing state, which determines +/// how the next token should be interpreted. +/// +/// ## State Transitions +/// +/// ```text +/// Normal +/// ├─ 's' → ExpectingSubstitution +/// ├─ 'a'/'i'/'c' (with body) → ExpectingText('a'|'i'|'c') +/// ├─ 'e' → ExpectingExecute +/// ├─ 'r'/'R'/'w'/'W' → ExpectingFilename('r'|'R'|'w'|'W') +/// └─ 'b'/'t'/'T'/':' → EnterLabelContext +/// +/// EnterLabelContext → InLabelContext (on next token) +/// +/// InLabelContext +/// ├─ newline/';'/'}' → Normal +/// └─ other chars → stay in InLabelContext +/// +/// ExpectingSubstitution → Normal (after parsing body) +/// ExpectingText → Normal (after parsing text) +/// ExpectingExecute → Normal (after parsing execute body) +/// ExpectingFilename → Normal (after parsing filename) +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LexerState { + /// Default state - parsing regular commands and addresses + Normal, + + /// Transition state - will enter InLabelContext on next token + /// Set when 'b', 't', 'T', or ':' is encountered + EnterLabelContext, + + /// Parsing label name after 'b', 't', 'T', or ':' + /// In this state, commands like 's', 'a', 'y' are treated as label characters + /// instead of triggering special parsing + InLabelContext, + + /// Expecting substitution body after 's' command + /// Next token will be parsed as SubstitutionBody + ExpectingSubstitution, + + /// Expecting text body after 'a', 'i', or 'c' command + /// The char indicates which command: 'a' (append), 'i' (insert), 'c' (change) + ExpectingText(char), + + /// Expecting execute body after 'e' command + /// Next token will be parsed as ExecuteBody + ExpectingExecute, + + /// Expecting filename after 'r', 'R', 'w', or 'W' command + /// The char indicates which command + ExpectingFilename(char), +} + +/// Lexer for tokenizing command scripts +pub struct Lexer<'a> { + pub(crate) input: &'a str, + pub(crate) position: usize, + pub(crate) current_char: Option, + /// Current state of the lexer state machine + pub(crate) state: LexerState, + /// POSIX mode flag (configuration, not state) + pub(crate) posix: bool, + /// UTF-8 locale flag for multibyte delimiter detection + pub(crate) utf8_locale: bool, + /// Original raw bytes (before lossy UTF-8 conversion) + /// Used to detect UTF-8 lead bytes vs continuation bytes for delimiter validation + pub(crate) raw_input: Option<&'a [u8]>, + /// Mapping from char position to byte position in raw_input + /// Built during construction when raw_input is provided + pub(crate) char_to_byte: Vec, +} + +impl<'a> Lexer<'a> { + pub fn new(input: &'a str) -> Self { + Self::new_with_posix(input, false) + } + + pub fn new_with_posix(input: &'a str, posix: bool) -> Self { + let mut lexer = Self { + input, + position: 0, + current_char: None, + state: LexerState::Normal, + posix, + utf8_locale: is_utf8_locale(), + raw_input: None, + char_to_byte: Vec::new(), + }; + lexer.current_char = input.chars().next(); + lexer + } + + /// Create a new lexer with raw bytes for accurate multibyte delimiter detection + /// + /// The `raw_bytes` should be the original bytes before lossy UTF-8 conversion. + /// This allows detecting UTF-8 lead bytes (0xC0-0xFF) vs continuation bytes (0x80-0xBF) + /// which is needed for proper multibyte delimiter error messages. + pub fn new_with_raw_bytes(input: &'a str, raw_bytes: &'a [u8], posix: bool) -> Self { + // Build char-to-byte mapping by scanning both strings in parallel + let char_to_byte = build_char_to_byte_mapping(input, raw_bytes); + + let mut lexer = Self { + input, + position: 0, + current_char: None, + state: LexerState::Normal, + posix, + utf8_locale: is_utf8_locale(), + raw_input: Some(raw_bytes), + char_to_byte, + }; + lexer.current_char = input.chars().next(); + lexer + } + + /// Get the original raw byte at the current char position + /// Returns None if raw_input is not available or position is out of bounds + fn current_raw_byte(&self) -> Option { + let raw = self.raw_input?; + let byte_pos = self.char_to_byte.get(self.position).copied()?; + raw.get(byte_pos).copied() + } + + /// Check if a delimiter byte is a UTF-8 lead byte (starts multibyte sequence) + /// UTF-8 lead bytes are 0xC0-0xFF (binary 11xxxxxx) + fn is_utf8_lead_byte(byte: u8) -> bool { + byte >= 0xC0 + } + + /// Get raw bytes for a range of char positions (start inclusive, end exclusive) + /// Returns None if raw_input is not available + fn get_raw_bytes_range(&self, start_char: usize, end_char: usize) -> Option> { + let raw = self.raw_input?; + let start_byte = self.char_to_byte.get(start_char).copied()?; + // For end, if end_char is at or past the mapping length, use raw.len() + let end_byte = if end_char >= self.char_to_byte.len() { + raw.len() + } else { + self.char_to_byte.get(end_char).copied()? + }; + Some(raw[start_byte..end_byte].to_vec()) + } + + /// Check if the current byte position is inside a multibyte sequence + /// This is used to determine if a `]` or other special character should be + /// treated as part of a multibyte character rather than its ASCII meaning. + /// + /// Uses mbcs module to properly handle stateful encodings like Shift-JIS. + fn is_inside_mb_sequence(&self, mb_state: &mut crate::mbcs::MbState) -> bool { + if !crate::mbcs::is_multibyte_locale() { + return false; + } + if let Some(raw_byte) = self.current_raw_byte() { + mb_state.is_mb_char(raw_byte) + } else { + false + } + } + + pub fn next_token(&mut self) -> Result { + // Main loop to handle backslash-newline continuation without recursion + loop { + // State machine dispatch: handle states that expect specific token types + match self.state { + LexerState::EnterLabelContext => { + // Transition to label context + self.state = LexerState::InLabelContext; + // Continue to parse the token in label context + } + LexerState::ExpectingSubstitution => { + self.state = LexerState::Normal; + return self.parse_substitution_body(); + } + LexerState::ExpectingText(kind) => { + self.state = LexerState::Normal; + return self.parse_aic_body(kind); + } + LexerState::ExpectingExecute => { + self.state = LexerState::Normal; + return self.parse_e_body(); + } + LexerState::ExpectingFilename(cmd) => { + // In POSIX mode, 'R' and 'W' are GNU extensions + // Don't try to parse filename, let parser handle the error + if self.posix && (cmd == 'R' || cmd == 'W') { + self.state = LexerState::Normal; + // Continue to parse next token normally + } else { + self.state = LexerState::Normal; + return self.parse_filename_literal(); + } + } + LexerState::Normal | LexerState::InLabelContext => { + // Continue to token parsing below + } + } + + self.skip_whitespace(); + + // Handle backslash-newline continuation in loop instead of recursion + if self.current_char == Some('\\') && self.peek() == Some('\n') { + self.advance(); + self.advance(); + self.skip_whitespace(); + continue; // Loop back to start instead of recursive call + } + + // Break out of loop and process the token + break; + } + + match self.current_char { + None => Ok(Token::Eof), + Some('\n') => { + // Exit label context on newline + if self.state == LexerState::InLabelContext { + self.state = LexerState::Normal; + } + self.advance(); + Ok(Token::Newline) + } + Some('#') => self.parse_comment(), + Some('$') => { + self.advance(); + Ok(Token::Dollar) + } + Some(',') => { + self.advance(); + Ok(Token::Comma) + } + Some(';') => { + // Exit label context on semicolon + if self.state == LexerState::InLabelContext { + self.state = LexerState::Normal; + } + self.advance(); + Ok(Token::Semicolon) + } + Some('!') => { + self.advance(); + Ok(Token::Bang) + } + Some('{') => { + self.advance(); + Ok(Token::OpenBrace) + } + Some('}') => { + // Exit label context on close brace + if self.state == LexerState::InLabelContext { + self.state = LexerState::Normal; + } + self.advance(); + Ok(Token::CloseBrace) + } + Some('+') => self.parse_plus_offset(), + Some('~') => self.parse_tilde_step(), + Some('/') => self.parse_regex_addr(), + Some('\\') => { + // Check if next char is a valid regex delimiter + // In GNU sed, \c can be used where c is any non-alphanumeric character + // BUT exclude BRE metacharacters: ( ) { } + ? which have special meaning when escaped + // Note: | is allowed as alternative delimiter (\|pattern|) despite being a BRE alternation operator + if let Some(next) = self.peek() { + if !next.is_ascii_alphanumeric() + && next != '\\' + && next != '(' + && next != ')' + && next != '{' + && next != '}' + && next != '+' + && next != '?' + { + return self.parse_alt_regex_addr(); + } + } + // Otherwise fall through to other backslash handling + self.advance(); + Ok(Token::Command('\\')) + } + Some(c) if c.is_ascii_digit() => self.parse_number(), + Some('=') => { + self.advance(); + Ok(Token::Command('=')) + } + Some('s') => { + self.advance(); + // In label context, 's' is part of a label name, not a substitution command + if self.state != LexerState::InLabelContext { + // Set state to expect substitution if there's a next character + // The substitution body parser will validate if it's a valid delimiter + if self.current_char.is_some() { + self.state = LexerState::ExpectingSubstitution; + } + } + Ok(Token::Command('s')) + } + Some('y') => { + // In label context, 'y' is part of a label name, not a transliterate command + if self.state == LexerState::InLabelContext { + self.advance(); + Ok(Token::Command('y')) + } else { + // Check if next char is a valid delimiter (any non-whitespace char) + // GNU sed allows alphanumeric delimiters for y command + self.advance(); + if let Some(c) = self.current_char { + if c != ' ' && c != '\t' && c != '\r' && c != '\n' { + // Parse translate body immediately + return self.parse_translate_body(); + } + } + // No delimiter found or EOF - this is an error + Err(SedError::parse_at( + "unterminated 'y' command", + self.position, + )) + } + } + Some('a') => { + self.advance(); + // Check if this command has a text body (unless in label context) + if self.state != LexerState::InLabelContext && self.check_text_command_has_body() { + self.state = LexerState::ExpectingText('a'); + } + Ok(Token::Command('a')) + } + Some('i') => { + self.advance(); + // Check if this command has a text body (unless in label context) + if self.state != LexerState::InLabelContext && self.check_text_command_has_body() { + self.state = LexerState::ExpectingText('i'); + } + Ok(Token::Command('i')) + } + Some('c') => { + self.advance(); + // Check if this command has a text body (unless in label context) + if self.state != LexerState::InLabelContext && self.check_text_command_has_body() { + self.state = LexerState::ExpectingText('c'); + } + Ok(Token::Command('c')) + } + Some('r') => { + self.advance(); + if self.state != LexerState::InLabelContext { + self.state = LexerState::ExpectingFilename('r'); + } + Ok(Token::Command('r')) + } + Some('R') => { + self.advance(); + if self.state != LexerState::InLabelContext { + self.state = LexerState::ExpectingFilename('R'); + } + Ok(Token::Command('R')) + } + Some('w') => { + self.advance(); + if self.state != LexerState::InLabelContext { + self.state = LexerState::ExpectingFilename('w'); + } + Ok(Token::Command('w')) + } + Some('W') => { + self.advance(); + if self.state != LexerState::InLabelContext { + self.state = LexerState::ExpectingFilename('W'); + } + Ok(Token::Command('W')) + } + Some('e') => { + self.advance(); + // Check if 'e' is standalone command or part of word/label + // Only treat as execute command if NOT in label context + if self.state != LexerState::InLabelContext { + match self.current_char { + None | Some('\n') | Some(' ') | Some('\t') => { + // Standalone 'e' - this is a command + self.state = LexerState::ExpectingExecute; + } + Some(';') | Some('}') => { + // 'e;' or 'e}' - could be label, don't set state + } + _ => { + // 'e' followed by other chars like 'etrue' + self.state = LexerState::ExpectingExecute; + } + } + } + Ok(Token::Command('e')) + } + Some(c) if c.is_alphabetic() => { + self.advance(); + // Set label context for b/t/T commands + match c { + 'b' | 't' | 'T' => { + self.state = LexerState::EnterLabelContext; + } + _ => {} + } + Ok(Token::Command(c)) + } + Some(':') => { + self.advance(); + // Enter label context after ':' (label definition) + self.state = LexerState::EnterLabelContext; + Ok(Token::SubstitutionDelim(':')) + } + Some(c) => { + self.advance(); + Ok(Token::SubstitutionDelim(c)) + } + } + } + + /// Check if a text command (a/i/c) has a body following it + /// + /// Looks ahead from current position to determine if non-whitespace, + /// non-delimiter characters follow, indicating a text body. + fn check_text_command_has_body(&self) -> bool { + let mut iter = self.input.chars(); + let mut j = 0usize; + // Skip to current position + while j < self.position { + iter.next(); + j += 1; + } + // Check if next non-whitespace char indicates a body + while let Some(ch) = iter.next() { + if ch == ' ' || ch == '\t' || ch == '\r' { + continue; // Skip whitespace + } + // Any non-whitespace character that's not a delimiter indicates a body + return ch != '\n' && ch != ';' && ch != '}'; + } + false + } + + fn advance(&mut self) { + self.position += 1; + self.current_char = self.input.chars().nth(self.position); + } + fn peek(&self) -> Option { + self.input.chars().nth(self.position + 1) + } + fn skip_whitespace(&mut self) { + while let Some(c) = self.current_char { + if c == ' ' || c == '\t' || c == '\r' { + self.advance(); + } else { + break; + } + } + } + fn parse_comment(&mut self) -> Result { + let mut comment = String::new(); + self.advance(); + while let Some(c) = self.current_char { + if c == '\n' { + break; + } + comment.push(c); + self.advance(); + } + Ok(Token::Comment(comment)) + } + fn parse_regex_addr(&mut self) -> Result { + let mut pattern = String::new(); + let mut in_bracket = false; // Track if we're inside [...] + let mut mb_state = crate::mbcs::MbState::new(); // Track multibyte sequence state + self.advance(); + + // Check for empty regex pattern // (reuses last regex) + if let Some('/') = self.current_char { + self.advance(); + // Parse optional modifiers (I, M, i, m) after the closing / + // BUT: if followed by backslash, it's likely a command (like 'i\'), not a modifier + let mut modifiers = String::new(); + while let Some(m) = self.current_char { + match m { + 'I' | 'M' | 'i' | 'm' => { + // Check if this could be a command instead of a modifier + // If next char is '\', it's probably 'i\' command, not 'i' modifier + if self.peek() == Some('\\') { + break; + } + modifiers.push(m); + self.advance(); + } + _ => break, + } + } + return Ok(Token::RegexAddr(pattern, modifiers)); + } + + while let Some(c) = self.current_char { + if c == '/' && !in_bracket { + self.advance(); + // Parse optional modifiers (I, M, i, m) after the closing / + // BUT: if followed by backslash, it's likely a command (like 'i\'), not a modifier + let mut modifiers = String::new(); + while let Some(m) = self.current_char { + match m { + 'I' | 'M' | 'i' | 'm' => { + // Check if this could be a command instead of a modifier + // If next char is '\', it's probably 'i\' command, not 'i' modifier + if self.peek() == Some('\\') { + break; + } + modifiers.push(m); + self.advance(); + } + _ => break, + } + } + return Ok(Token::RegexAddr(pattern, modifiers)); + } + if c == '\\' { + self.advance(); + if let Some(escaped) = self.current_char { + match escaped { + '/' => pattern.push('/'), + '\\' => { + // Preserve both backslashes for the regex engine + // Input \\ should produce \\ in pattern to match literal backslash + pattern.push('\\'); + pattern.push('\\'); + } + other => { + pattern.push('\\'); + pattern.push(other); + } + } + self.advance(); + } + } else { + // Track bracket state for character classes + if c == '[' { + pattern.push(c); + self.advance(); + + if in_bracket { + // Check for POSIX bracket expressions [:class:], [.collating.], [=equiv=] + if let Some(next) = self.current_char { + if next == ':' || next == '.' || next == '=' { + let close_char = next; + pattern.push(next); + self.advance(); + + // Look for closing sequence + while let Some(ch) = self.current_char { + pattern.push(ch); + self.advance(); + if ch == close_char { + if let Some(']') = self.current_char { + pattern.push(']'); + self.advance(); + break; + } + } + if self.current_char.is_none() { + break; + } + } + } + } + } else { + in_bracket = true; + } + } else if c == ']' && in_bracket { + // Check if this ']' is part of a multibyte sequence + // In Shift-JIS, 0x5D (']') can be the second byte of a multibyte char + if self.is_inside_mb_sequence(&mut mb_state) { + // Part of multibyte sequence - don't close bracket + pattern.push(c); + self.advance(); + } else { + // Not part of multibyte sequence - close bracket + in_bracket = false; + pattern.push(c); + self.advance(); + } + } else { + // Update mb_state for all other characters + self.is_inside_mb_sequence(&mut mb_state); + pattern.push(c); + self.advance(); + } + } + } + Err(SedError::parse_at( + "unterminated address regex", + self.position, + )) + } + fn parse_alt_regex_addr(&mut self) -> Result { + self.advance(); + let delimiter = self + .current_char + .ok_or_else(|| SedError::parse_at("expected delimiter character", self.position))?; + + // In UTF-8 locale, reject multibyte delimiters + if self.utf8_locale { + if let Some(raw_byte) = self.current_raw_byte() { + // Have raw bytes - check if it's a UTF-8 lead byte + if Self::is_utf8_lead_byte(raw_byte) { + return Err(SedError::parse_at( + "delimiter character is not a single-byte character", + self.position, + )); + } + } else if delimiter.len_utf8() > 1 && delimiter != '\u{FFFD}' { + // No raw bytes - fall back to checking char length + return Err(SedError::parse_at( + "delimiter character is not a single-byte character", + self.position, + )); + } + } + + self.advance(); + let mut pattern = String::new(); + let mut in_bracket = false; // Track if we're inside [...] + let mut mb_state = crate::mbcs::MbState::new(); // Track multibyte sequence state + + // Check for empty regex pattern (reuses last regex) + if let Some(c) = self.current_char { + if c == delimiter { + self.advance(); + // Parse optional modifiers (I, M, i, m) after the closing delimiter + let mut modifiers = String::new(); + while let Some(m) = self.current_char { + match m { + 'I' | 'M' | 'i' | 'm' => { + modifiers.push(m); + self.advance(); + } + _ => break, + } + } + return Ok(Token::RegexAddr(pattern, modifiers)); + } + } + + while let Some(c) = self.current_char { + if c == delimiter && !in_bracket { + self.advance(); + // Parse optional modifiers (I, M, i, m) after the closing delimiter + let mut modifiers = String::new(); + while let Some(m) = self.current_char { + match m { + 'I' | 'M' | 'i' | 'm' => { + modifiers.push(m); + self.advance(); + } + _ => break, + } + } + return Ok(Token::RegexAddr(pattern, modifiers)); + } + if c == '\\' { + pattern.push(c); + self.advance(); + if let Some(escaped) = self.current_char { + pattern.push(escaped); + self.advance(); + } + } else { + // Track bracket state for character classes + if c == '[' { + pattern.push(c); + self.advance(); + + if in_bracket { + // Check for POSIX bracket expressions [:class:], [.collating.], [=equiv=] + if let Some(next) = self.current_char { + if next == ':' || next == '.' || next == '=' { + let close_char = next; + pattern.push(next); + self.advance(); + + // Look for closing sequence + while let Some(ch) = self.current_char { + pattern.push(ch); + self.advance(); + if ch == close_char { + if let Some(']') = self.current_char { + pattern.push(']'); + self.advance(); + break; + } + } + if self.current_char.is_none() { + break; + } + } + } + } + } else { + in_bracket = true; + } + } else if c == ']' && in_bracket { + // Check if this ']' is part of a multibyte sequence + // In Shift-JIS, 0x5D (']') can be the second byte of a multibyte char + if self.is_inside_mb_sequence(&mut mb_state) { + // Part of multibyte sequence - don't close bracket + pattern.push(c); + self.advance(); + } else { + // Not part of multibyte sequence - close bracket + in_bracket = false; + pattern.push(c); + self.advance(); + } + } else { + // Update mb_state for all other characters + self.is_inside_mb_sequence(&mut mb_state); + pattern.push(c); + self.advance(); + } + } + } + Err(SedError::parse_at( + "unterminated address regex", + self.position, + )) + } + fn parse_substitution_body(&mut self) -> Result { + let mut body = String::new(); + // Track start position for raw bytes extraction + let start_pos = self.position; + + let delim = match self.current_char { + Some(d) => d, + None => { + return Err(SedError::parse_at( + "unterminated 's' command", + self.position, + )); + } + }; + + // In UTF-8 locale, reject multibyte delimiters + // Check raw bytes if available (more accurate), otherwise use char length + if self.utf8_locale { + if let Some(raw_byte) = self.current_raw_byte() { + // Have raw bytes - check if it's a UTF-8 lead byte (starts multibyte sequence) + if Self::is_utf8_lead_byte(raw_byte) { + return Err(SedError::parse_at( + "delimiter character is not a single-byte character", + self.position, + )); + } + } else if delim.len_utf8() > 1 && delim != '\u{FFFD}' { + // No raw bytes - fall back to checking char length + // U+FFFD is allowed because it may represent a single invalid byte + return Err(SedError::parse_at( + "delimiter character is not a single-byte character", + self.position, + )); + } + } + + body.push(delim); + self.advance(); + let mut in_class = false; + let track_class = delim != '['; + loop { + match self.current_char { + None => { + return Err(SedError::parse_at( + "unterminated 's' command", + self.position, + )); + } + Some('\\') => { + self.advance(); + match self.current_char { + Some('\n') => { + // In sed, \ in replacement string inserts a literal newline + // Preserve the backslash-newline sequence for parse_replacement to handle + body.push('\\'); + body.push('\n'); + self.advance(); + } + Some(c) => { + body.push('\\'); + body.push(c); + self.advance(); + } + None => { + body.push('\\'); + } + } + } + Some('[') if track_class => { + body.push('['); + self.advance(); + + if in_class { + // Inside bracket expression, check for POSIX bracket expressions + // [:class:], [.collating.], [=equiv=] + if let Some(next) = self.current_char { + if next == ':' || next == '.' || next == '=' { + let close_char = next; + body.push(next); + self.advance(); + + // Look for closing sequence (e.g., :] or .] or =]) + let mut found_close = false; + while let Some(c) = self.current_char { + body.push(c); + self.advance(); + + if c == close_char { + if let Some(']') = self.current_char { + body.push(']'); + self.advance(); + found_close = true; + break; + } + } + + // If we hit None, break and let outer loop handle error + if self.current_char.is_none() { + break; + } + } + + // If we didn't find the closing sequence, the bracket is unterminated + if !found_close && self.current_char.is_none() { + return Err(SedError::parse_at( + "unterminated 's' command", + self.position, + )); + } + } else { + in_class = true; // Nested bracket? + } + } + } else { + in_class = true; + } + } + Some(']') if track_class && in_class => { + in_class = false; + body.push(']'); + self.advance(); + } + Some(c) if c == delim && !in_class => { + body.push(c); + self.advance(); + break; + } + Some(c) => { + body.push(c); + self.advance(); + } + } + } + loop { + match self.current_char { + None => { + return Err(SedError::parse_at( + "unterminated 's' command", + self.position, + )); + } + Some('\\') => { + self.advance(); + match self.current_char { + Some('\n') => { + // In sed, \ in replacement string inserts a literal newline + // Preserve the backslash-newline sequence for parse_replacement to handle + body.push('\\'); + body.push('\n'); + self.advance(); + } + Some(c) => { + body.push('\\'); + body.push(c); + self.advance(); + } + None => { + body.push('\\'); + } + } + } + Some(c) if c == delim => { + body.push(c); + self.advance(); + break; + } + Some(c) => { + body.push(c); + self.advance(); + } + } + } + loop { + match self.current_char { + None => break, + Some('\n') | Some('}') | Some(';') => break, + // Comment after flags (GNU sed allows # or space+# after flags) + Some('#') => break, + Some(' ') | Some('\t') => { + // Skip whitespace and check for comment + self.advance(); + if matches!(self.current_char, Some('#')) { + break; + } + // Whitespace not followed by comment - continue parsing flags + continue; + } + Some('\\') => { + self.advance(); + match self.current_char { + Some('\n') => { + self.advance(); + } + Some(c) => { + body.push('\\'); + body.push(c); + self.advance(); + } + None => { + body.push('\\'); + } + } + } + Some('\r') => { + // Carriage return - check if followed by newline + self.advance(); + if matches!(self.current_char, Some('\n')) { + // \r\n is acceptable line ending, break here (don't include \r in body) + break; + } else { + // \r alone is an error + return Err(SedError::parse_at("unknown option to 's'", self.position)); + } + } + Some(c) => { + body.push(c); + self.advance(); + } + } + } + + // Extract raw bytes for the body if available + let raw_bytes = self.get_raw_bytes_range(start_pos, self.position); + + Ok(Token::SubstitutionBody(body, raw_bytes)) + } + fn parse_translate_body(&mut self) -> Result { + let mut body = String::new(); + // Track start position for raw bytes extraction + let start_pos = self.position; + + let delim = match self.current_char { + Some(d) => d, + None => { + return Err(SedError::parse_at( + "unterminated 'y' command", + self.position, + )); + } + }; + + // In UTF-8 locale, reject multibyte delimiters (same logic as s command) + if self.utf8_locale { + if let Some(raw_byte) = self.current_raw_byte() { + // Have raw bytes - check if it's a UTF-8 lead byte + if Self::is_utf8_lead_byte(raw_byte) { + return Err(SedError::parse_at( + "delimiter character is not a single-byte character", + self.position, + )); + } + } else if delim.len_utf8() > 1 && delim != '\u{FFFD}' { + // No raw bytes - fall back to checking char length + return Err(SedError::parse_at( + "delimiter character is not a single-byte character", + self.position, + )); + } + } + + body.push(delim); + self.advance(); + loop { + match self.current_char { + None => { + return Err(SedError::parse_at( + "unterminated 'y' command", + self.position, + )); + } + Some('\\') => { + self.advance(); + match self.current_char { + Some(c) => { + body.push('\\'); + body.push(c); + self.advance(); + } + None => { + body.push('\\'); + } + } + } + Some(c) if c == delim => { + body.push(c); + self.advance(); + break; + } + Some(c) => { + body.push(c); + self.advance(); + } + } + } + loop { + match self.current_char { + None => { + return Err(SedError::parse_at( + "unterminated 'y' command", + self.position, + )); + } + Some('\\') => { + self.advance(); + match self.current_char { + Some(c) => { + body.push('\\'); + body.push(c); + self.advance(); + } + None => { + body.push('\\'); + } + } + } + Some(c) if c == delim => { + body.push(c); + self.advance(); + break; + } + Some(c) => { + body.push(c); + self.advance(); + } + } + } + + // Extract raw bytes for the body if available + let raw_bytes = self.get_raw_bytes_range(start_pos, self.position); + + Ok(Token::TranslateBody(body, raw_bytes)) + } + fn parse_aic_body(&mut self, kind: char) -> Result { + // Skip leading whitespace + while let Some(c) = self.current_char { + if c == ' ' || c == '\t' { + self.advance(); + } else { + break; + } + } + + // Two formats supported: + // 1. GNU format: c\ (backslash-newline) followed by text + // 2. BSD format: cText (text immediately after command) + let mut text = String::new(); + + match self.current_char { + Some('\\') => { + // Check for backslash-newline (GNU format) + self.advance(); + if self.current_char == Some('\n') { + // GNU format: read text from next line(s) + // Support multiline text: lines ending with \ continue to next line + self.advance(); + loop { + // Read characters until end of line + let mut line = String::new(); + while let Some(c) = self.current_char { + if c == '\n' { + break; + } + line.push(c); + self.advance(); + } + + // Check if line ends with backslash (continuation) + let continues = line.ends_with('\\'); + if continues { + // Remove trailing backslash and add the line + line.pop(); + text.push_str(&line); + text.push('\n'); // Add literal newline between continued lines + + // Move past the newline for next iteration + if self.current_char == Some('\n') { + self.advance(); + } else { + // End of input after backslash + break; + } + } else { + // Last line (no continuation) + text.push_str(&line); + break; + } + } + // Explicit text (even empty) - return Some + return Ok(match kind { + 'a' => Token::AppendBody(Some(text)), + 'i' => Token::InsertBody(Some(text)), + _ => Token::ChangeBody(Some(text)), + }); + } else if self.current_char.is_none() { + // Backslash followed by EOF - unterminated command + // In POSIX mode, this is an error (incomplete command) + // In GNU mode, treat as unterminated (no text to add) + if self.posix { + return Err(SedError::parse_at("incomplete command", self.position)); + } + // GNU mode: return None to indicate unterminated + return Ok(match kind { + 'a' => Token::AppendBody(None), + 'i' => Token::InsertBody(None), + _ => Token::ChangeBody(None), + }); + } else { + // Backslash followed by text (GNU extension: a\text) + // In POSIX mode, this is an error - backslash-newline required + if self.posix { + return Err(SedError::parse_at( + "expected \\ after 'a', 'c' or 'i'", + self.position, + )); + } + // GNU extension: text immediately after backslash + // Read until newline only (semicolons and braces are part of text) + while let Some(c) = self.current_char { + if c == '\n' { + break; + } + text.push(c); + self.advance(); + } + } + } + Some('\n') | Some(';') | Some('}') | None => { + // Empty text body - valid for a/i/c commands + // Don't consume the delimiter + } + _ => { + // Text immediately after command (BSD/GNU extension format, like 'ifoo') + // In POSIX mode, this is an error - backslash is required + if self.posix { + return Err(SedError::parse_at( + "expected \\ after 'a', 'c' or 'i'", + self.position + 1, + )); + } + // GNU extension: text immediately after command + while let Some(c) = self.current_char { + if c == '\n' || c == ';' || c == '}' { + break; + } + text.push(c); + self.advance(); + } + } + } + + Ok(match kind { + 'a' => Token::AppendBody(Some(text)), + 'i' => Token::InsertBody(Some(text)), + _ => Token::ChangeBody(Some(text)), + }) + } + + fn parse_e_body(&mut self) -> Result { + // For 'e' command: read everything until newline/semicolon/close brace + // Unlike a/i/c, there's NO backslash requirement + // Reads ALL characters including spaces, semicolons become part of command! + while let Some(c) = self.current_char { + if c == ' ' || c == '\t' { + self.advance(); + } else { + break; + } + } + let mut text = String::new(); + while let Some(c) = self.current_char { + match c { + '\n' | '}' => break, // Newline or close brace terminates + ';' => break, // Semicolon also terminates (unlike GNU sed doc claims!) + _ => { + text.push(c); + self.advance(); + } + } + } + // Return trimmed text (empty is OK - means execute pattern space) + Ok(Token::ExecuteBody(text.trim().to_string())) + } + fn parse_filename_literal(&mut self) -> Result { + while let Some(c) = self.current_char { + if c == ' ' || c == '\t' { + self.advance(); + } else { + break; + } + } + let mut name = String::new(); + while let Some(c) = self.current_char { + match c { + '\n' | '}' | ';' => break, + _ => { + name.push(c); + self.advance(); + } + } + } + let trimmed = name.trim().to_string(); + if trimmed.is_empty() { + // Don't return error here - let parser handle it + // Parser can check other conditions first (like address ranges in POSIX mode) + // Return next token instead + return self.next_token(); + } + Ok(Token::Filename(trimmed)) + } + fn parse_number(&mut self) -> Result { + let mut num_str = String::new(); + while let Some(c) = self.current_char { + if c.is_ascii_digit() { + num_str.push(c); + self.advance(); + } else { + break; + } + } + // Handle large numbers by capping at usize::MAX (GNU sed behavior) + let num: usize = match num_str.parse::() { + Ok(n) => n, + Err(_) => { + // Number too large or invalid - cap at usize::MAX + // This matches GNU sed behavior for extremely large line numbers + usize::MAX + } + }; + Ok(Token::LineNumber(num)) + } + fn parse_plus_offset(&mut self) -> Result { + self.advance(); + if let Some(c) = self.current_char { + if c.is_ascii_digit() { + let mut num_str = String::new(); + while let Some(c) = self.current_char { + if c.is_ascii_digit() { + num_str.push(c); + self.advance(); + } else { + break; + } + } + let num: usize = num_str.parse().unwrap_or(usize::MAX); + Ok(Token::PlusOffset(num)) + } else { + Ok(Token::SubstitutionDelim('+')) + } + } else { + Ok(Token::SubstitutionDelim('+')) + } + } + fn parse_tilde_step(&mut self) -> Result { + self.advance(); + if let Some(c) = self.current_char { + if c.is_ascii_digit() { + let mut num_str = String::new(); + while let Some(c) = self.current_char { + if c.is_ascii_digit() { + num_str.push(c); + self.advance(); + } else { + break; + } + } + let num: usize = num_str.parse().unwrap_or(usize::MAX); + Ok(Token::TildeStep(num)) + } else { + Ok(Token::SubstitutionDelim('~')) + } + } else { + Ok(Token::SubstitutionDelim('~')) + } + } +} diff --git a/red/src/posix_rules.rs b/red/src/posix_rules.rs new file mode 100644 index 0000000..5c82d3b --- /dev/null +++ b/red/src/posix_rules.rs @@ -0,0 +1,295 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! POSIX compliance rules for sed commands +//! +//! This module centralizes all POSIX compatibility checks in one place. +//! Instead of scattering `if self.posix { ... }` checks throughout +//! parser and lexer, all rules are documented and enforced here. +//! +//! ## POSIX Mode Levels +//! +//! - **Extended** (default): All GNU extensions enabled +//! - **Correct** (POSIXLY_CORRECT): Some behavioral changes, but GNU extensions allowed +//! - **Basic** (--posix): Strict POSIX compliance, GNU extensions rejected +//! +//! This module enforces rules for **Basic** mode only (--posix flag). + +use crate::errors::{Result, SedError}; + +/// Commands that are GNU extensions (not allowed in strict POSIX mode) +/// +/// These commands will be rejected with "unknown command" error +/// when running with --posix flag. +/// +/// Reference: GNU sed manual section "GNU Extensions" +const GNU_EXTENSION_COMMANDS: &[char] = &[ + 'Q', // Quit immediately (GNU) + 'T', // Branch if no substitution (GNU) + 'e', // Execute shell command (GNU) + 'v', // Version check (GNU) + 'z', // Clear pattern space (GNU) + 'f', // Print sed script filename (GNU) + 'F', // Print input filename (GNU) + 'W', // Write first line (GNU) + 'R', // Read line from file (GNU) +]; + +/// Substitution flags that are GNU extensions +/// +/// These flags will be rejected with "unknown option to 's'" error +/// when running with --posix flag. +const GNU_EXTENSION_SUBST_FLAGS: &[char] = &[ + 'i', // Case-insensitive matching (GNU) + 'I', // Case-insensitive matching (GNU) + 'm', // Multiline mode (GNU) + 'M', // Multiline-dotall mode (GNU) + 'e', // Execute replacement as shell command (GNU) +]; + +/// Commands that only accept one address in POSIX mode +/// +/// In GNU sed, these can have address ranges (e.g., "1,10i\text"). +/// In POSIX mode, they can only have a single address (e.g., "5i\text"). +const SINGLE_ADDRESS_ONLY_COMMANDS: &[char] = &[ + 'a', // Append text + 'i', // Insert text + '=', // Print line number + 'l', // List pattern space + 'r', // Read file +]; + +/// Check if a command is a GNU extension (not allowed in strict POSIX mode) +/// +/// # Arguments +/// * `cmd` - The command character to check +/// +/// # Returns +/// * `true` if this is a GNU extension command +/// * `false` if this is a standard POSIX command +/// +/// # Example +/// ``` +/// use red::posix_rules::is_gnu_extension_command; +/// assert!(is_gnu_extension_command('Q')); // GNU extension +/// assert!(!is_gnu_extension_command('d')); // POSIX standard +/// ``` +pub fn is_gnu_extension_command(cmd: char) -> bool { + GNU_EXTENSION_COMMANDS.contains(&cmd) +} + +/// Check if a substitution flag is a GNU extension +/// +/// # Arguments +/// * `flag` - The substitution flag character to check +/// +/// # Returns +/// * `true` if this is a GNU extension flag +/// * `false` if this is a standard POSIX flag +/// +/// # Example +/// ``` +/// use red::posix_rules::is_gnu_extension_subst_flag; +/// assert!(is_gnu_extension_subst_flag('i')); // GNU extension +/// assert!(!is_gnu_extension_subst_flag('g')); // POSIX standard +/// ``` +pub fn is_gnu_extension_subst_flag(flag: char) -> bool { + GNU_EXTENSION_SUBST_FLAGS.contains(&flag) +} + +/// Check if a command requires single address in POSIX mode +/// +/// # Arguments +/// * `cmd` - The command character to check +/// +/// # Returns +/// * `true` if this command only accepts one address in POSIX mode +/// * `false` if this command can have address ranges in POSIX mode +pub fn requires_single_address_in_posix(cmd: char) -> bool { + SINGLE_ADDRESS_ONLY_COMMANDS.contains(&cmd) +} + +/// Validate command is allowed in strict POSIX mode +/// +/// # Arguments +/// * `cmd` - The command character to validate +/// * `strict_posix` - Whether strict POSIX mode is enabled (--posix flag) +/// * `error_pos` - Character position for error reporting +/// +/// # Returns +/// * `Ok(())` if command is allowed +/// * `Err(SedError)` if command is a GNU extension in strict POSIX mode +/// +/// # Example +/// ``` +/// use red::posix_rules::validate_command_posix; +/// assert!(validate_command_posix('d', true, 5).is_ok()); // OK - 'd' is POSIX +/// assert!(validate_command_posix('Q', true, 5).is_err()); // Error - 'Q' is GNU extension +/// assert!(validate_command_posix('Q', false, 5).is_ok()); // OK - not in strict POSIX mode +/// ``` +pub fn validate_command_posix(cmd: char, strict_posix: bool, error_pos: usize) -> Result<()> { + if strict_posix && is_gnu_extension_command(cmd) { + return Err(SedError::parse_at( + format!("unknown command: '{}'", cmd), + error_pos, + )); + } + Ok(()) +} + +/// Validate substitution flag is allowed in strict POSIX mode +/// +/// # Arguments +/// * `flag` - The substitution flag character to validate +/// * `strict_posix` - Whether strict POSIX mode is enabled (--posix flag) +/// * `error_pos` - Character position for error reporting +/// +/// # Returns +/// * `Ok(())` if flag is allowed +/// * `Err(SedError)` if flag is a GNU extension in strict POSIX mode +pub fn validate_subst_flag_posix(flag: char, strict_posix: bool, error_pos: usize) -> Result<()> { + if strict_posix && is_gnu_extension_subst_flag(flag) { + return Err(SedError::parse_at("unknown option to 's'", error_pos)); + } + Ok(()) +} + +/// Validate address range for command in strict POSIX mode +/// +/// Some commands (a, i, =, l, r) only accept single addresses in POSIX mode. +/// In GNU mode, they can accept ranges (e.g., "1,10a\text"). +/// +/// # Arguments +/// * `cmd` - The command character +/// * `has_range` - Whether the command has an address range (two addresses) +/// * `strict_posix` - Whether strict POSIX mode is enabled +/// * `error_pos` - Character position for error reporting +/// +/// # Returns +/// * `Ok(())` if address range is valid +/// * `Err(SedError)` if command has two addresses in strict POSIX mode but only accepts one +pub fn validate_address_range_posix( + cmd: char, + has_range: bool, + strict_posix: bool, + error_pos: usize, +) -> Result<()> { + if strict_posix && has_range && requires_single_address_in_posix(cmd) { + return Err(SedError::parse_at( + "command only uses one address", + error_pos, + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gnu_extension_commands() { + // GNU extensions + assert!(is_gnu_extension_command('Q')); + assert!(is_gnu_extension_command('T')); + assert!(is_gnu_extension_command('e')); + assert!(is_gnu_extension_command('v')); + assert!(is_gnu_extension_command('z')); + assert!(is_gnu_extension_command('f')); + assert!(is_gnu_extension_command('F')); + assert!(is_gnu_extension_command('W')); + assert!(is_gnu_extension_command('R')); + + // POSIX standard commands + assert!(!is_gnu_extension_command('d')); + assert!(!is_gnu_extension_command('p')); + assert!(!is_gnu_extension_command('s')); + assert!(!is_gnu_extension_command('a')); + assert!(!is_gnu_extension_command('i')); + } + + #[test] + fn test_gnu_extension_subst_flags() { + // GNU extensions + assert!(is_gnu_extension_subst_flag('i')); + assert!(is_gnu_extension_subst_flag('I')); + assert!(is_gnu_extension_subst_flag('m')); + assert!(is_gnu_extension_subst_flag('M')); + assert!(is_gnu_extension_subst_flag('e')); + + // POSIX standard flags + assert!(!is_gnu_extension_subst_flag('g')); + assert!(!is_gnu_extension_subst_flag('p')); + assert!(!is_gnu_extension_subst_flag('w')); + } + + #[test] + fn test_single_address_commands() { + // Commands requiring single address in POSIX + assert!(requires_single_address_in_posix('a')); + assert!(requires_single_address_in_posix('i')); + assert!(requires_single_address_in_posix('=')); + assert!(requires_single_address_in_posix('l')); + assert!(requires_single_address_in_posix('r')); + + // Commands accepting ranges + assert!(!requires_single_address_in_posix('d')); + assert!(!requires_single_address_in_posix('p')); + assert!(!requires_single_address_in_posix('s')); + } + + #[test] + fn test_validate_command_posix() { + // POSIX command is OK in both modes + assert!(validate_command_posix('d', false, 0).is_ok()); + assert!(validate_command_posix('d', true, 0).is_ok()); + + // GNU extension OK in non-POSIX mode + assert!(validate_command_posix('Q', false, 0).is_ok()); + + // GNU extension rejected in POSIX mode + let result = validate_command_posix('Q', true, 5); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("unknown command")); + } + + #[test] + fn test_validate_subst_flag_posix() { + // POSIX flag OK in both modes + assert!(validate_subst_flag_posix('g', false, 0).is_ok()); + assert!(validate_subst_flag_posix('g', true, 0).is_ok()); + + // GNU extension OK in non-POSIX mode + assert!(validate_subst_flag_posix('i', false, 0).is_ok()); + + // GNU extension rejected in POSIX mode + let result = validate_subst_flag_posix('i', true, 7); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("unknown option to 's'")); + } + + #[test] + fn test_validate_address_range_posix() { + // 'd' command accepts ranges in both modes + assert!(validate_address_range_posix('d', true, false, 0).is_ok()); + assert!(validate_address_range_posix('d', true, true, 0).is_ok()); + + // 'a' command accepts range in GNU mode + assert!(validate_address_range_posix('a', true, false, 0).is_ok()); + + // 'a' command rejects range in POSIX mode + let result = validate_address_range_posix('a', true, true, 3); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("only uses one address")); + + // 'a' command accepts single address in POSIX mode + assert!(validate_address_range_posix('a', false, true, 0).is_ok()); + } +} diff --git a/red/src/regex/ast.rs b/red/src/regex/ast.rs new file mode 100644 index 0000000..a068cdf --- /dev/null +++ b/red/src/regex/ast.rs @@ -0,0 +1,267 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Abstract Syntax Tree for BRE/ERE patterns + +/// AST node representing a regex pattern +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegexNode { + // === Basic atoms === + /// Single literal character + Literal(char), + + /// Any character (.) - matches everything except newline (unless dotall mode) + Any, + + /// Start of line anchor (^) + StartAnchor, + + /// End of line anchor ($) + EndAnchor, + + // === Character classes === + /// Character class like [abc] or [a-z] + CharClass(CharSet), + + /// Negated character class like [^abc] + NegatedCharClass(CharSet), + + // === Quantifiers === + /// Zero or more repetitions (*) + ZeroOrMore(Box), + + /// One or more repetitions (\+ in BRE, + in ERE) + OneOrMore(Box), + + /// Zero or one repetition (\? in BRE, ? in ERE) + ZeroOrOne(Box), + + /// Bounded repetition \{m,n\} in BRE, {m,n} in ERE + Repeat { + node: Box, + min: usize, + max: Option, // None means unbounded + }, + + // === Grouping and alternation === + /// Capturing group \(...\) in BRE, (...) in ERE + Group { + id: usize, // Group number (1-based for sed compatibility) + node: Box, + }, + + /// Alternation a\|b in BRE, a|b in ERE + Alternation(Vec), + + /// Sequence of nodes (concatenation) + Sequence(Vec), + + // === Backreferences === + /// Backreference \1, \2, etc. + Backref(usize), + + // === Word boundaries (GNU extensions) === + /// Word boundary \b + WordBoundary, + + /// Non-word boundary \B + NonWordBoundary, + + /// Start of word \< (GNU extension) + StartWord, + + /// End of word \> (GNU extension) + EndWord, +} + +/// Character set for character classes +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CharSet { + /// List of character ranges (inclusive) + /// e.g., [a-zA-Z] → vec![('a', 'z'), ('A', 'Z')] + pub ranges: Vec<(char, char)>, + + /// POSIX character classes like [:alpha:], [:digit:] + pub posix_classes: Vec, +} + +impl CharSet { + pub fn new() -> Self { + CharSet { + ranges: Vec::new(), + posix_classes: Vec::new(), + } + } + + pub fn add_range(&mut self, start: char, end: char) { + self.ranges.push((start, end)); + } + + pub fn add_char(&mut self, ch: char) { + self.ranges.push((ch, ch)); + } + + pub fn add_posix_class(&mut self, class: PosixClass) { + self.posix_classes.push(class); + } + + /// Check if character matches this set + pub fn matches(&self, ch: char) -> bool { + // Check ranges + for (start, end) in &self.ranges { + if ch >= *start && ch <= *end { + return true; + } + } + + // Check POSIX classes + for class in &self.posix_classes { + if class.matches(ch) { + return true; + } + } + + false + } +} + +/// POSIX character classes +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PosixClass { + /// [:alpha:] - alphabetic characters + Alpha, + + /// [:digit:] - decimal digits + Digit, + + /// [:alnum:] - alphanumeric characters + Alnum, + + /// [:space:] - whitespace characters + Space, + + /// [:blank:] - space and tab + Blank, + + /// [:upper:] - uppercase letters + Upper, + + /// [:lower:] - lowercase letters + Lower, + + /// [:punct:] - punctuation characters + Punct, + + /// [:xdigit:] - hexadecimal digits + Xdigit, + + /// [:cntrl:] - control characters + Cntrl, + + /// [:graph:] - visible characters (no space) + Graph, + + /// [:print:] - printable characters (includes space) + Print, +} + +impl PosixClass { + /// Check if character matches this POSIX class + pub fn matches(&self, ch: char) -> bool { + match self { + PosixClass::Alpha => ch.is_alphabetic(), + PosixClass::Digit => ch.is_ascii_digit(), + PosixClass::Alnum => ch.is_alphanumeric(), + PosixClass::Space => ch.is_whitespace(), + PosixClass::Blank => ch == ' ' || ch == '\t', + PosixClass::Upper => ch.is_uppercase(), + PosixClass::Lower => ch.is_lowercase(), + PosixClass::Punct => ch.is_ascii_punctuation(), + PosixClass::Xdigit => ch.is_ascii_hexdigit(), + PosixClass::Cntrl => ch.is_control(), + PosixClass::Graph => !ch.is_whitespace() && !ch.is_control(), + PosixClass::Print => !ch.is_control(), + } + } + + /// Parse POSIX class name + pub fn from_name(name: &str) -> Option { + match name { + "alpha" => Some(PosixClass::Alpha), + "digit" => Some(PosixClass::Digit), + "alnum" => Some(PosixClass::Alnum), + "space" => Some(PosixClass::Space), + "blank" => Some(PosixClass::Blank), + "upper" => Some(PosixClass::Upper), + "lower" => Some(PosixClass::Lower), + "punct" => Some(PosixClass::Punct), + "xdigit" => Some(PosixClass::Xdigit), + "cntrl" => Some(PosixClass::Cntrl), + "graph" => Some(PosixClass::Graph), + "print" => Some(PosixClass::Print), + _ => None, + } + } +} + +/// Compiled regex with metadata +#[derive(Debug, Clone)] +pub struct CompiledRegex { + pub ast: RegexNode, + pub num_groups: usize, + pub is_ere: bool, + pub ignore_case: bool, + pub dotall: bool, // Whether . matches \n +} + +impl CompiledRegex { + pub fn new( + ast: RegexNode, + num_groups: usize, + is_ere: bool, + ignore_case: bool, + dotall: bool, + ) -> Self { + CompiledRegex { + ast, + num_groups, + is_ere, + ignore_case, + dotall, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_charset_matches() { + let mut set = CharSet::new(); + set.add_range('a', 'z'); + set.add_range('A', 'Z'); + + assert!(set.matches('a')); + assert!(set.matches('z')); + assert!(set.matches('A')); + assert!(set.matches('Z')); + assert!(!set.matches('0')); + assert!(!set.matches(' ')); + } + + #[test] + fn test_posix_digit() { + assert!(PosixClass::Digit.matches('0')); + assert!(PosixClass::Digit.matches('9')); + assert!(!PosixClass::Digit.matches('a')); + } + + #[test] + fn test_posix_alpha() { + assert!(PosixClass::Alpha.matches('a')); + assert!(PosixClass::Alpha.matches('Z')); + assert!(!PosixClass::Alpha.matches('0')); + } +} diff --git a/red/src/regex/backtrack.rs b/red/src/regex/backtrack.rs new file mode 100644 index 0000000..b46d3a8 --- /dev/null +++ b/red/src/regex/backtrack.rs @@ -0,0 +1,1591 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// NFA executor with backtracking for backreferences +// Implements full BRE/ERE matching with capture groups + +use super::nfa::{Nfa, StateId, Transition}; +use crate::constants::MAX_REGEX_BACKTRACK_ITERATIONS; +use crate::mbcs::{is_multibyte_locale, MbText}; +use rustc_hash::FxHashSet; +use std::collections::HashMap; +use std::rc::Rc; + +/// Check if character is a word character (alphanumeric or underscore) +fn is_word_char(ch: char) -> bool { + ch.is_alphanumeric() || ch == '_' +} + +/// Capture group - stores matched text and raw bytes +#[derive(Debug, Clone)] +pub struct Capture { + pub start: usize, + pub end: usize, + pub text: String, + /// Raw bytes of captured text (for MBCS support) + pub raw_bytes: Option>, +} + +/// Backtracking state for NFA execution +#[derive(Debug, Clone)] +struct BacktrackState { + /// Current NFA state + state_id: StateId, + /// Current position in text + text_pos: usize, + /// Captured groups so far (completed captures) - Rc to avoid cloning + captures: Rc>, + /// Pending group starts (group_id -> start_position) - Rc to avoid cloning + group_starts: Rc>, +} + +/// NFA matcher with backtracking support +#[derive(Debug, Clone)] +pub struct NfaMatcher { + nfa: Nfa, + ignore_case: bool, + has_backreferences: bool, // Cache whether NFA contains backreferences + multiline: bool, // Whether ^ and $ should match at line boundaries +} + +impl NfaMatcher { + /// Compile pattern into NFA matcher + pub fn compile(pattern: &str, is_ere: bool, ignore_case: bool) -> crate::errors::Result { + Self::compile_with_flags(pattern, is_ere, ignore_case, false, false) + } + + /// Compile pattern into NFA matcher with multiline flag + pub fn compile_with_flags( + pattern: &str, + is_ere: bool, + ignore_case: bool, + multiline: bool, + posix_mode: bool, + ) -> crate::errors::Result { + // Parse pattern to AST + let compiled = if is_ere { + super::parser::parse_ere(pattern, posix_mode)? + } else { + super::parser::parse_bre(pattern, posix_mode)? + }; + + // Build NFA from AST + let nfa = Nfa::from_ast(&compiled.ast); + + // Check if NFA contains backreferences (cache for performance) + let has_backreferences = nfa.states.iter().any(|s| { + s.transitions + .iter() + .any(|(t, _)| matches!(t, Transition::Backref(_))) + }); + + Ok(NfaMatcher { + nfa, + ignore_case, + has_backreferences, + multiline, + }) + } + + /// Check if text matches pattern + pub fn is_match(&self, text: &str) -> bool { + self.find(text).is_some() + } + + /// Find first match in text + pub fn find(&self, text: &str) -> Option<(usize, usize)> { + self.find_with_captures(text) + .map(|(start, end, _)| (start, end)) + } + + /// Find first match and return captures (public interface) + pub fn find_with_captures_pub( + &self, + text: &str, + ) -> Option<(usize, usize, HashMap)> { + self.find_with_captures(text) + } + + /// Find first match and return captures + fn find_with_captures(&self, text: &str) -> Option<(usize, usize, HashMap)> { + self.find_with_captures_from(text, 0) + } + + /// Find first match starting from a specific position + pub fn find_with_captures_from( + &self, + text: &str, + start_from: usize, + ) -> Option<(usize, usize, HashMap)> { + // Use MBCS-aware matching in multibyte locales + if is_multibyte_locale() { + return self.find_with_captures_from_mb(text.as_bytes(), start_from); + } + + let chars: Vec = text.chars().collect(); + + // Try matching from each position starting from start_from + for start in start_from..=chars.len() { + if let Some((end, captures)) = self.match_from(&chars, start) { + return Some((start, end, captures)); + } + } + + None + } + + /// MBCS-aware find with captures - works with raw bytes and locale encoding + fn find_with_captures_from_mb( + &self, + bytes: &[u8], + start_from: usize, + ) -> Option<(usize, usize, HashMap)> { + let mb_text = MbText::new(bytes); + + // Try matching from each character position + for start in start_from..=mb_text.char_count() { + if let Some((end, captures)) = self.match_from_mb(&mb_text, start) { + return Some((start, end, captures)); + } + } + + None + } + + /// Find first match starting from a specific position (without captures) + pub fn find_from(&self, text: &str, start_from: usize) -> Option<(usize, usize)> { + self.find_with_captures_from(text, start_from) + .map(|(start, end, _)| (start, end)) + } + + /// Find match in raw bytes (for MBCS locales) + /// Returns (start_byte, end_byte, captures) where start/end are byte offsets + pub fn find_with_captures_bytes( + &self, + bytes: &[u8], + ) -> Option<(usize, usize, HashMap)> { + self.find_with_captures_bytes_from(bytes, 0) + } + + /// Find match in raw bytes starting from a byte offset + pub fn find_with_captures_bytes_from( + &self, + bytes: &[u8], + start_byte: usize, + ) -> Option<(usize, usize, HashMap)> { + let mb_text = MbText::new(bytes); + + // Convert start byte offset to character position + let start_char = mb_text.byte_to_char(start_byte); + + // Try matching from each character position + for start in start_char..=mb_text.char_count() { + if let Some((end_char, captures)) = self.match_from_mb(&mb_text, start) { + // Convert character positions back to byte offsets + let start_byte = mb_text.char_to_byte(start); + let end_byte = mb_text.char_to_byte(end_char); + return Some((start_byte, end_byte, captures)); + } + } + + None + } + + /// Try to match from a specific position using backtracking + /// Uses greedy matching - finds longest match + fn match_from(&self, chars: &[char], start: usize) -> Option<(usize, HashMap)> { + // Use optimized path for patterns without backreferences + if !self.has_backreferences { + return self.match_from_simple(chars, start); + } + + // Initialize backtracking stack with start state + let mut stack = vec![BacktrackState { + state_id: self.nfa.start, + text_pos: start, + captures: Rc::new(HashMap::new()), + group_starts: Rc::new(HashMap::new()), + }]; + + // Track visited states - include captures for backreference patterns + let mut visited: FxHashSet<(StateId, usize, Vec<(usize, usize, usize)>)> = + FxHashSet::default(); + + // Track best match found so far (greedy) + let mut best_match: Option<(usize, HashMap)> = None; + + // Safety limit to prevent infinite loops + let mut iterations = 0; + + while let Some(state) = stack.pop() { + iterations += 1; + if iterations > MAX_REGEX_BACKTRACK_ITERATIONS { + return best_match; + } + + // Include capture positions in key for backreference patterns + let captures_key: Vec<(usize, usize, usize)> = state + .captures + .iter() + .map(|(id, cap)| (*id, cap.start, cap.end)) + .collect(); + + let key = (state.state_id, state.text_pos, captures_key); + if !visited.insert(key) { + continue; + } + + // Check if current state is accepting + if self.nfa.states[state.state_id].is_accept { + // Update best match if this is longer (greedy) + if best_match.is_none() || best_match.as_ref().unwrap().0 < state.text_pos { + // Clone the inner HashMap when storing best match + best_match = Some((state.text_pos, (*state.captures).clone())); + } + // Continue searching for longer matches + } + + // Try all transitions from current state + for (transition, target_id) in &self.nfa.states[state.state_id].transitions { + match transition { + Transition::Epsilon => { + // Epsilon transition - no input consumed + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + + Transition::Char(ch) => { + // Check if we have a character to match + if state.text_pos < chars.len() { + let text_ch = chars[state.text_pos]; + let matches = if self.ignore_case { + text_ch.to_lowercase().eq(ch.to_lowercase()) + } else { + text_ch == *ch + }; + + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + + Transition::Any => { + // Match any character + if state.text_pos < chars.len() { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::CharSet(set) => { + if state.text_pos < chars.len() { + let text_ch = chars[state.text_pos]; + let ch_to_match = if self.ignore_case { + text_ch.to_lowercase().next().unwrap_or(text_ch) + } else { + text_ch + }; + + if set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + + Transition::NegatedCharSet(set) => { + if state.text_pos < chars.len() { + let text_ch = chars[state.text_pos]; + let ch_to_match = if self.ignore_case { + text_ch.to_lowercase().next().unwrap_or(text_ch) + } else { + text_ch + }; + + if !set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + + Transition::StartAnchor => { + // ^ behavior depends on multiline flag: + // - Without multiline: matches ONLY at start of text (position 0) + // - With multiline (m/M flag): matches at start of text AND after newlines + let matches = if self.multiline { + // Multiline mode: match at start or after \n + state.text_pos == 0 + || (state.text_pos > 0 && chars[state.text_pos - 1] == '\n') + } else { + // Normal mode: match only at start + state.text_pos == 0 + }; + + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::EndAnchor => { + // $ behavior depends on multiline flag: + // - Without multiline: matches ONLY at end of text + // - With multiline (m/M flag): matches at end of text AND before newlines + let matches = if self.multiline { + // Multiline mode: match at end or before \n + state.text_pos == chars.len() + || (state.text_pos < chars.len() && chars[state.text_pos] == '\n') + } else { + // Normal mode: match only at end + state.text_pos == chars.len() + }; + let at_end = matches; + + if at_end { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::WordBoundary => { + // \b matches at word/non-word boundary + let prev_is_word = if state.text_pos > 0 { + is_word_char(chars[state.text_pos - 1]) + } else { + false + }; + + let next_is_word = if state.text_pos < chars.len() { + is_word_char(chars[state.text_pos]) + } else { + false + }; + + // Boundary exists if exactly one side is a word character + let at_boundary = prev_is_word != next_is_word; + + if at_boundary { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::NonWordBoundary => { + // \B matches where \b doesn't match + let prev_is_word = if state.text_pos > 0 { + is_word_char(chars[state.text_pos - 1]) + } else { + false + }; + + let next_is_word = if state.text_pos < chars.len() { + is_word_char(chars[state.text_pos]) + } else { + false + }; + + // Non-boundary: both sides are word or both are non-word + let not_at_boundary = prev_is_word == next_is_word; + + if not_at_boundary { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::StartWord => { + // \< matches at start of word + let prev_is_word = if state.text_pos > 0 { + is_word_char(chars[state.text_pos - 1]) + } else { + false + }; + + let next_is_word = if state.text_pos < chars.len() { + is_word_char(chars[state.text_pos]) + } else { + false + }; + + // Start of word: previous is non-word, next is word + let at_word_start = !prev_is_word && next_is_word; + + if at_word_start { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::EndWord => { + // \> matches at end of word + let prev_is_word = if state.text_pos > 0 { + is_word_char(chars[state.text_pos - 1]) + } else { + false + }; + + let next_is_word = if state.text_pos < chars.len() { + is_word_char(chars[state.text_pos]) + } else { + false + }; + + // End of word: previous is word, next is non-word + let at_word_end = prev_is_word && !next_is_word; + + if at_word_end { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::GroupStart(group_id) => { + // Mark start of capture group - clone inner HashMap and wrap in new Rc + let mut new_group_starts = (*state.group_starts).clone(); + new_group_starts.insert(*group_id, state.text_pos); + + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: Rc::new(new_group_starts), + }); + } + + Transition::GroupEnd(group_id) => { + // Complete capture group + if let Some(&start_pos) = state.group_starts.get(group_id) { + let captured_text: String = + chars[start_pos..state.text_pos].iter().collect(); + + // Clone inner HashMap and wrap in new Rc + let mut new_captures = (*state.captures).clone(); + new_captures.insert( + *group_id, + Capture { + start: start_pos, + end: state.text_pos, + text: captured_text, + raw_bytes: None, // Non-MBCS path + }, + ); + + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: Rc::new(new_captures), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::Backref(group_id) => { + // Match same text as captured group + if let Some(capture) = state.captures.get(group_id) { + let ref_chars: Vec = capture.text.chars().collect(); + + // Check if we have enough characters + if state.text_pos + ref_chars.len() <= chars.len() { + // Compare character by character + let matches = ref_chars.iter().enumerate().all(|(i, &ch)| { + let text_ch = chars[state.text_pos + i]; + if self.ignore_case { + text_ch.to_lowercase().eq(ch.to_lowercase()) + } else { + text_ch == ch + } + }); + + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + ref_chars.len(), + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + // If group not captured yet, backreference fails (no push to stack) + } + } + } + } + + best_match + } + + /// Optimized match_from for patterns WITHOUT backreferences + /// Uses simpler visited set for better performance + fn match_from_simple( + &self, + chars: &[char], + start: usize, + ) -> Option<(usize, HashMap)> { + let mut stack = vec![BacktrackState { + state_id: self.nfa.start, + text_pos: start, + captures: Rc::new(HashMap::new()), + group_starts: Rc::new(HashMap::new()), + }]; + + // Simple visited set - no captures in key since no backreferences + let mut visited: FxHashSet<(StateId, usize)> = FxHashSet::default(); + let mut best_match: Option<(usize, HashMap)> = None; + let mut iterations = 0; + + while let Some(state) = stack.pop() { + iterations += 1; + if iterations > MAX_REGEX_BACKTRACK_ITERATIONS { + return best_match; + } + + // Simple key - just state and position + if !visited.insert((state.state_id, state.text_pos)) { + continue; + } + + if self.nfa.states[state.state_id].is_accept { + if best_match.is_none() || best_match.as_ref().unwrap().0 < state.text_pos { + best_match = Some((state.text_pos, (*state.captures).clone())); + } + } + + for (transition, target_id) in &self.nfa.states[state.state_id].transitions { + match transition { + Transition::Epsilon => { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + Transition::Char(ch) => { + if state.text_pos < chars.len() { + let text_ch = chars[state.text_pos]; + let matches = if self.ignore_case { + text_ch.to_lowercase().eq(ch.to_lowercase()) + } else { + text_ch == *ch + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + Transition::Any => { + if state.text_pos < chars.len() { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::CharSet(set) => { + if state.text_pos < chars.len() { + let text_ch = chars[state.text_pos]; + let ch_to_match = if self.ignore_case { + text_ch.to_lowercase().next().unwrap_or(text_ch) + } else { + text_ch + }; + if set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + Transition::NegatedCharSet(set) => { + if state.text_pos < chars.len() { + let text_ch = chars[state.text_pos]; + let ch_to_match = if self.ignore_case { + text_ch.to_lowercase().next().unwrap_or(text_ch) + } else { + text_ch + }; + if !set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + Transition::StartAnchor => { + let matches = if self.multiline { + state.text_pos == 0 + || (state.text_pos > 0 && chars[state.text_pos - 1] == '\n') + } else { + state.text_pos == 0 + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::EndAnchor => { + let matches = if self.multiline { + state.text_pos == chars.len() + || (state.text_pos < chars.len() && chars[state.text_pos] == '\n') + } else { + state.text_pos == chars.len() + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::WordBoundary => { + let prev_is_word = + state.text_pos > 0 && is_word_char(chars[state.text_pos - 1]); + let next_is_word = + state.text_pos < chars.len() && is_word_char(chars[state.text_pos]); + if prev_is_word != next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::NonWordBoundary => { + let prev_is_word = + state.text_pos > 0 && is_word_char(chars[state.text_pos - 1]); + let next_is_word = + state.text_pos < chars.len() && is_word_char(chars[state.text_pos]); + if prev_is_word == next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::StartWord => { + let prev_is_word = + state.text_pos > 0 && is_word_char(chars[state.text_pos - 1]); + let next_is_word = + state.text_pos < chars.len() && is_word_char(chars[state.text_pos]); + if !prev_is_word && next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::EndWord => { + let prev_is_word = + state.text_pos > 0 && is_word_char(chars[state.text_pos - 1]); + let next_is_word = + state.text_pos < chars.len() && is_word_char(chars[state.text_pos]); + if prev_is_word && !next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::GroupStart(group_id) => { + let mut new_group_starts = (*state.group_starts).clone(); + new_group_starts.insert(*group_id, state.text_pos); + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: Rc::new(new_group_starts), + }); + } + Transition::GroupEnd(group_id) => { + if let Some(&start_pos) = state.group_starts.get(group_id) { + let captured_text: String = + chars[start_pos..state.text_pos].iter().collect(); + let mut new_captures = (*state.captures).clone(); + new_captures.insert( + *group_id, + Capture { + start: start_pos, + end: state.text_pos, + text: captured_text, + raw_bytes: None, // Non-MBCS path + }, + ); + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: Rc::new(new_captures), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::Backref(_) => { + // Should never be reached in simple path + } + } + } + } + best_match + } + + /// MBCS-aware matching from a specific character position + /// Uses MbText for proper character boundary handling in non-UTF-8 locales + fn match_from_mb( + &self, + mb_text: &MbText, + start: usize, + ) -> Option<(usize, HashMap)> { + // Use simplified path for patterns without backreferences + if !self.has_backreferences { + return self.match_from_mb_simple(mb_text, start); + } + + let mut stack = vec![BacktrackState { + state_id: self.nfa.start, + text_pos: start, + captures: Rc::new(HashMap::new()), + group_starts: Rc::new(HashMap::new()), + }]; + + let mut visited: FxHashSet<(StateId, usize, Vec<(usize, usize, usize)>)> = + FxHashSet::default(); + let mut best_match: Option<(usize, HashMap)> = None; + let mut iterations = 0; + + while let Some(state) = stack.pop() { + iterations += 1; + if iterations > MAX_REGEX_BACKTRACK_ITERATIONS { + return best_match; + } + + let captures_key: Vec<(usize, usize, usize)> = state + .captures + .iter() + .map(|(id, cap)| (*id, cap.start, cap.end)) + .collect(); + + let key = (state.state_id, state.text_pos, captures_key); + if !visited.insert(key) { + continue; + } + + if self.nfa.states[state.state_id].is_accept { + if best_match.is_none() || best_match.as_ref().unwrap().0 < state.text_pos { + best_match = Some((state.text_pos, (*state.captures).clone())); + } + } + + for (transition, target_id) in &self.nfa.states[state.state_id].transitions { + match transition { + Transition::Epsilon => { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + + Transition::Char(ch) => { + if let Some(mb_ch) = mb_text.char_at(state.text_pos) { + let matches = if self.ignore_case { + mb_ch.matches_char_ignore_case(*ch) + } else { + mb_ch.matches_char(*ch) + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + + Transition::Any => { + // Only match valid characters (skip invalid/incomplete MB sequences) + if state.text_pos < mb_text.char_count() + && mb_text.is_valid_char(state.text_pos) + { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::CharSet(set) => { + if let Some(mb_ch) = mb_text.char_at(state.text_pos) { + if let Some(wc) = mb_ch.to_wchar() { + let ch_to_match = if self.ignore_case { + wc.to_lowercase().next().unwrap_or(wc) + } else { + wc + }; + if set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + } + + Transition::NegatedCharSet(set) => { + if let Some(mb_ch) = mb_text.char_at(state.text_pos) { + if let Some(wc) = mb_ch.to_wchar() { + let ch_to_match = if self.ignore_case { + wc.to_lowercase().next().unwrap_or(wc) + } else { + wc + }; + if !set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } else { + // Invalid MB sequence - treat as non-matching any char + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + + Transition::StartAnchor => { + let matches = if self.multiline { + state.text_pos == 0 || { + if let Some(prev_ch) = + mb_text.char_at(state.text_pos.saturating_sub(1)) + { + prev_ch.matches_byte(b'\n') + } else { + false + } + } + } else { + state.text_pos == 0 + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::EndAnchor => { + let matches = if self.multiline { + state.text_pos == mb_text.char_count() || { + if let Some(curr_ch) = mb_text.char_at(state.text_pos) { + curr_ch.matches_byte(b'\n') + } else { + false + } + } + } else { + state.text_pos == mb_text.char_count() + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::WordBoundary => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if prev_is_word != next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::NonWordBoundary => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if prev_is_word == next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::StartWord => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if !prev_is_word && next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::EndWord => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if prev_is_word && !next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::GroupStart(group_id) => { + let mut new_group_starts = (*state.group_starts).clone(); + new_group_starts.insert(*group_id, state.text_pos); + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: Rc::new(new_group_starts), + }); + } + + Transition::GroupEnd(group_id) => { + if let Some(&start_pos) = state.group_starts.get(group_id) { + // Store raw bytes for MBCS mode + let raw_bytes = mb_text.slice_chars(start_pos, state.text_pos).to_vec(); + let captured_text = String::from_utf8_lossy(&raw_bytes).into_owned(); + let mut new_captures = (*state.captures).clone(); + new_captures.insert( + *group_id, + Capture { + start: start_pos, + end: state.text_pos, + text: captured_text, + raw_bytes: Some(raw_bytes), + }, + ); + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: Rc::new(new_captures), + group_starts: state.group_starts.clone(), + }); + } + } + + Transition::Backref(group_id) => { + if let Some(capture) = state.captures.get(group_id) { + // Get the captured bytes (prefer raw_bytes if available) + let ref_bytes = capture + .raw_bytes + .as_ref() + .map(|b| b.as_slice()) + .unwrap_or_else(|| capture.text.as_bytes()); + let ref_mb = MbText::new(ref_bytes); + let ref_len = ref_mb.char_count(); + + if state.text_pos + ref_len <= mb_text.char_count() { + // Compare character by character + let mut matches = true; + for i in 0..ref_len { + if let (Some(ref_ch), Some(text_ch)) = + (ref_mb.char_at(i), mb_text.char_at(state.text_pos + i)) + { + let ch_matches = if self.ignore_case { + if let (Some(wc1), Some(wc2)) = + (ref_ch.to_wchar(), text_ch.to_wchar()) + { + wc1.to_lowercase().eq(wc2.to_lowercase()) + } else { + ref_ch.bytes == text_ch.bytes + } + } else { + ref_ch.bytes == text_ch.bytes + }; + if !ch_matches { + matches = false; + break; + } + } else { + matches = false; + break; + } + } + + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + ref_len, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + } + } + } + } + + best_match + } + + /// Simplified MBCS-aware matching (no backreferences) + fn match_from_mb_simple( + &self, + mb_text: &MbText, + start: usize, + ) -> Option<(usize, HashMap)> { + let mut stack = vec![BacktrackState { + state_id: self.nfa.start, + text_pos: start, + captures: Rc::new(HashMap::new()), + group_starts: Rc::new(HashMap::new()), + }]; + + let mut visited: FxHashSet<(StateId, usize)> = FxHashSet::default(); + let mut best_match: Option<(usize, HashMap)> = None; + let mut iterations = 0; + + while let Some(state) = stack.pop() { + iterations += 1; + if iterations > MAX_REGEX_BACKTRACK_ITERATIONS { + return best_match; + } + + if !visited.insert((state.state_id, state.text_pos)) { + continue; + } + + if self.nfa.states[state.state_id].is_accept { + if best_match.is_none() || best_match.as_ref().unwrap().0 < state.text_pos { + best_match = Some((state.text_pos, (*state.captures).clone())); + } + } + + for (transition, target_id) in &self.nfa.states[state.state_id].transitions { + match transition { + Transition::Epsilon => { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + Transition::Char(ch) => { + if let Some(mb_ch) = mb_text.char_at(state.text_pos) { + let matches = if self.ignore_case { + mb_ch.matches_char_ignore_case(*ch) + } else { + mb_ch.matches_char(*ch) + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + Transition::Any => { + // Only match valid characters (skip invalid/incomplete MB sequences) + if state.text_pos < mb_text.char_count() + && mb_text.is_valid_char(state.text_pos) + { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::CharSet(set) => { + if let Some(mb_ch) = mb_text.char_at(state.text_pos) { + if let Some(wc) = mb_ch.to_wchar() { + let ch_to_match = if self.ignore_case { + wc.to_lowercase().next().unwrap_or(wc) + } else { + wc + }; + if set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + } + Transition::NegatedCharSet(set) => { + if let Some(mb_ch) = mb_text.char_at(state.text_pos) { + if let Some(wc) = mb_ch.to_wchar() { + let ch_to_match = if self.ignore_case { + wc.to_lowercase().next().unwrap_or(wc) + } else { + wc + }; + if !set.matches(ch_to_match) { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } else { + // Invalid MB - treat as matching negated set + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos + 1, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + } + Transition::StartAnchor => { + let matches = if self.multiline { + state.text_pos == 0 || { + mb_text + .char_at(state.text_pos.saturating_sub(1)) + .map(|ch| ch.matches_byte(b'\n')) + .unwrap_or(false) + } + } else { + state.text_pos == 0 + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::EndAnchor => { + let matches = if self.multiline { + state.text_pos == mb_text.char_count() || { + mb_text + .char_at(state.text_pos) + .map(|ch| ch.matches_byte(b'\n')) + .unwrap_or(false) + } + } else { + state.text_pos == mb_text.char_count() + }; + if matches { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::WordBoundary => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if prev_is_word != next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::NonWordBoundary => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if prev_is_word == next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::StartWord => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if !prev_is_word && next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::EndWord => { + let (prev_is_word, next_is_word) = + mb_text.word_boundary_context(state.text_pos); + if prev_is_word && !next_is_word { + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::GroupStart(group_id) => { + let mut new_group_starts = (*state.group_starts).clone(); + new_group_starts.insert(*group_id, state.text_pos); + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: state.captures.clone(), + group_starts: Rc::new(new_group_starts), + }); + } + Transition::GroupEnd(group_id) => { + if let Some(&start_pos) = state.group_starts.get(group_id) { + // Store raw bytes for MBCS mode + let raw_bytes = mb_text.slice_chars(start_pos, state.text_pos).to_vec(); + let captured_text = String::from_utf8_lossy(&raw_bytes).into_owned(); + let mut new_captures = (*state.captures).clone(); + new_captures.insert( + *group_id, + Capture { + start: start_pos, + end: state.text_pos, + text: captured_text, + raw_bytes: Some(raw_bytes), + }, + ); + stack.push(BacktrackState { + state_id: *target_id, + text_pos: state.text_pos, + captures: Rc::new(new_captures), + group_starts: state.group_starts.clone(), + }); + } + } + Transition::Backref(_) => { + // Should never be reached in simple path + } + } + } + } + best_match + } + + /// Replace first match + pub fn replace_first(&self, text: &str, replacement: &str) -> String { + if let Some((start, end, _captures)) = self.find_with_captures(text) { + let chars: Vec = text.chars().collect(); + let mut result = String::new(); + + // Add text before match + for &ch in &chars[..start] { + result.push(ch); + } + + // Add replacement + result.push_str(replacement); + + // Add text after match + for &ch in &chars[end..] { + result.push(ch); + } + + result + } else { + text.to_string() + } + } + + /// Replace all matches + pub fn replace_all(&self, text: &str, replacement: &str) -> String { + let chars: Vec = text.chars().collect(); + let mut result = String::new(); + let mut pos = 0; + + while pos < chars.len() { + if let Some((end, _captures)) = self.match_from(&chars, pos) { + if end > pos { + // Found a match + result.push_str(replacement); + pos = end; + } else { + // Empty match - advance by one + result.push(chars[pos]); + pos += 1; + } + } else { + // No match - copy character + result.push(chars[pos]); + pos += 1; + } + } + + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_nfa_literal() { + let matcher = NfaMatcher::compile("abc", false, false).unwrap(); + assert!(matcher.is_match("abc")); + assert!(matcher.is_match("xabcy")); + assert!(!matcher.is_match("ab")); + } + + #[test] + fn test_nfa_star() { + let matcher = NfaMatcher::compile("a*b", false, false).unwrap(); + assert!(matcher.is_match("b")); + assert!(matcher.is_match("ab")); + assert!(matcher.is_match("aaab")); + } + + #[test] + fn test_nfa_alternation() { + let matcher = NfaMatcher::compile("a\\|b", false, false).unwrap(); + assert!(matcher.is_match("a")); + assert!(matcher.is_match("b")); + assert!(!matcher.is_match("c")); + } + + #[test] + fn test_nfa_replace() { + let matcher = NfaMatcher::compile("a\\+", false, false).unwrap(); + assert_eq!(matcher.replace_first("aaa", "X"), "X"); + assert_eq!(matcher.replace_all("aaa aaa", "X"), "X X"); + } + + #[test] + fn test_nfa_case_insensitive() { + let matcher = NfaMatcher::compile("abc", false, true).unwrap(); + assert!(matcher.is_match("ABC")); + assert!(matcher.is_match("aBc")); + assert!(matcher.is_match("abc")); + } + + #[test] + fn test_nfa_start_anchor() { + let matcher = NfaMatcher::compile("^abc", false, false).unwrap(); + assert!(matcher.is_match("abc")); + assert!(matcher.is_match("abcdef")); + assert!(!matcher.is_match("xabc")); + + // In BRE/ERE, ^ matches ONLY at start of text (position 0), NOT after embedded newlines + // Note: sed's line-oriented processing makes it SEEM like ^ matches after newlines, + // but that's because sed processes each line separately. + assert!(!matcher.is_match("x\nabc")); // Does NOT match - "abc" is not at start + assert!(!matcher.is_match("\nabc")); // Does NOT match - "abc" is after newline, not at position 0 + } + + #[test] + fn test_nfa_end_anchor() { + let matcher = NfaMatcher::compile("abc$", false, false).unwrap(); + assert!(matcher.is_match("abc")); + assert!(matcher.is_match("xabc")); + assert!(!matcher.is_match("abcx")); + + // In BRE/ERE, $ matches ONLY at end of text, NOT before embedded newlines + // sed appears to match before newlines because it strips trailing newlines during line processing + assert!(!matcher.is_match("abc\n")); // Does NOT match - "abc" is not at end (newline is at end) + } + + #[test] + fn test_nfa_both_anchors() { + let matcher = NfaMatcher::compile("^abc$", false, false).unwrap(); + assert!(matcher.is_match("abc")); + assert!(!matcher.is_match("xabc")); + assert!(!matcher.is_match("abcx")); + assert!(!matcher.is_match("xabcx")); + } + + #[test] + fn test_nfa_anchor_replace() { + let matcher = NfaMatcher::compile("^abc", false, false).unwrap(); + assert_eq!(matcher.replace_first("abc", "X"), "X"); + assert_eq!(matcher.replace_first("xabc", "X"), "xabc"); // No match + + let matcher2 = NfaMatcher::compile("abc$", false, false).unwrap(); + assert_eq!(matcher2.replace_first("abc", "X"), "X"); + assert_eq!(matcher2.replace_first("abcx", "X"), "abcx"); // No match + } + + #[test] + fn test_nfa_word_boundary() { + // \b matches at word boundaries + let matcher = NfaMatcher::compile("\\bword\\b", false, false).unwrap(); + assert!(matcher.is_match("word")); + assert!(matcher.is_match("a word b")); + assert!(matcher.is_match("word!")); + assert!(!matcher.is_match("sword")); + assert!(!matcher.is_match("wording")); + } + + #[test] + fn test_nfa_start_word() { + // \< matches at start of word + let matcher = NfaMatcher::compile("\\ matches at end of word + let matcher = NfaMatcher::compile("abc\\>", false, false).unwrap(); + assert!(matcher.is_match("abc")); + assert!(matcher.is_match("abc ")); + assert!(matcher.is_match("abc!")); + assert!(!matcher.is_match("abcd")); + assert!(!matcher.is_match("abc123")); + } + + #[test] + fn test_nfa_word_boundary_replace() { + let matcher = NfaMatcher::compile("\\bcat\\b", false, false).unwrap(); + assert_eq!(matcher.replace_first("cat", "dog"), "dog"); + assert_eq!(matcher.replace_first("the cat sat", "dog"), "the dog sat"); + assert_eq!(matcher.replace_first("category", "dog"), "category"); // No match + } + + #[test] + fn test_nfa_posix_char_class_greedy() { + // Test POSIX character classes with greedy quantifiers + let matcher = NfaMatcher::compile("[[:alpha:]]*", false, false).unwrap(); + assert!(matcher.is_match("abc"), "Should match 'abc'"); + + // Find should return longest match + if let Some((start, end)) = matcher.find("abc") { + assert_eq!(start, 0, "Match should start at 0"); + assert_eq!(end, 3, "Match should end at 3 (greedy - all chars)"); + } else { + panic!("Should find a match"); + } + + // Replace should replace all matched chars + assert_eq!( + matcher.replace_first("abc", "X"), + "X", + "Should replace all alpha chars" + ); + assert_eq!( + matcher.replace_first("abc123", "X"), + "X123", + "Should replace only alpha prefix" + ); + } +} diff --git a/red/src/regex/dfa.rs b/red/src/regex/dfa.rs new file mode 100644 index 0000000..d28f4cd --- /dev/null +++ b/red/src/regex/dfa.rs @@ -0,0 +1,533 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// DFA (Deterministic Finite Automaton) compilation and execution +// Uses subset construction algorithm to convert NFA → DFA +// Provides fast O(n) matching without backtracking + +use super::ast::RegexNode; +use super::nfa::{Nfa, StateId as NfaStateId, Transition}; +use std::collections::{HashMap, HashSet}; + +/// DFA state ID +pub type DfaStateId = usize; + +/// DFA transition table: (from_state, char) → to_state +pub type TransitionTable = HashMap<(DfaStateId, char), DfaStateId>; + +/// Character class transition: (from_state, char_class) → to_state +/// Used for optimizing character class transitions +#[derive(Debug, Clone)] +pub struct CharClassTransition { + pub from: DfaStateId, + pub to: DfaStateId, + pub chars: HashSet, +} + +/// DFA (Deterministic Finite Automaton) +#[derive(Debug, Clone)] +pub struct Dfa { + /// Transition table for character transitions + pub transitions: TransitionTable, + + /// Special transitions for Any (.) - maps state to target + pub any_transitions: HashMap, + + /// Character class transitions (optimized storage) + pub char_class_transitions: Vec, + + /// Start state + pub start: DfaStateId, + + /// Accepting states + pub accept_states: HashSet, + + /// Number of states + pub num_states: usize, + + /// Case insensitive matching + pub ignore_case: bool, +} + +impl Dfa { + /// Build DFA from NFA using subset construction + pub fn from_nfa(nfa: &Nfa, ignore_case: bool) -> Self { + let mut dfa = Dfa { + transitions: HashMap::new(), + any_transitions: HashMap::new(), + char_class_transitions: Vec::new(), + start: 0, + accept_states: HashSet::new(), + num_states: 0, + ignore_case, + }; + + // Map from NFA state sets to DFA state IDs + let mut nfa_set_to_dfa_id: HashMap, DfaStateId> = HashMap::new(); + + // Work queue: DFA states to process + let mut work_queue: Vec<(DfaStateId, HashSet)> = Vec::new(); + + // Start with epsilon closure of NFA start state + let mut start_set = HashSet::new(); + start_set.insert(nfa.start); + let start_closure = nfa.epsilon_closure(&start_set); + + // Create DFA start state + let start_id = dfa.add_state(&start_closure, nfa); + let mut start_vec: Vec<_> = start_closure.iter().copied().collect(); + start_vec.sort(); + nfa_set_to_dfa_id.insert(start_vec.clone(), start_id); + work_queue.push((start_id, start_closure)); + + // Process work queue + while let Some((dfa_state, nfa_states)) = work_queue.pop() { + // Collect all possible transitions from this set of NFA states + let mut char_transitions: HashMap> = HashMap::new(); + let mut has_any_transition = false; + let mut any_targets = HashSet::new(); + + for &nfa_state_id in &nfa_states { + for (transition, target) in &nfa.states[nfa_state_id].transitions { + match transition { + Transition::Char(ch) => { + let ch_normalized = if ignore_case { + // For case-insensitive, track both cases + ch.to_lowercase().next().unwrap_or(*ch) + } else { + *ch + }; + char_transitions + .entry(ch_normalized) + .or_insert_with(HashSet::new) + .insert(*target); + + // Also add uppercase version for case-insensitive + if ignore_case { + let ch_upper = ch.to_uppercase().next().unwrap_or(*ch); + if ch_upper != ch_normalized { + char_transitions + .entry(ch_upper) + .or_insert_with(HashSet::new) + .insert(*target); + } + } + } + Transition::Any => { + has_any_transition = true; + any_targets.insert(*target); + } + Transition::CharSet(set) => { + // Expand character set into individual characters + // This is simplified - in production, we'd optimize this + for range in &set.ranges { + for ch_code in range.0 as u32..=range.1 as u32 { + if let Some(ch) = char::from_u32(ch_code) { + let ch_normalized = if ignore_case { + ch.to_lowercase().next().unwrap_or(ch) + } else { + ch + }; + char_transitions + .entry(ch_normalized) + .or_insert_with(HashSet::new) + .insert(*target); + + if ignore_case { + let ch_upper = ch.to_uppercase().next().unwrap_or(ch); + if ch_upper != ch_normalized { + char_transitions + .entry(ch_upper) + .or_insert_with(HashSet::new) + .insert(*target); + } + } + } + } + } + // POSIX character classes are handled but without DFA optimization + } + Transition::NegatedCharSet(_set) => { + // Should not reach here - can_use_dfa rejects negated char sets + unreachable!("NegatedCharSet should be handled by NFA, not DFA"); + } + Transition::Epsilon => { + // Already handled by epsilon closure + } + Transition::StartAnchor | Transition::EndAnchor => { + // Anchors are handled by epsilon closure + // DFA can't properly handle anchors because it doesn't track position + // Patterns with anchors should use NFA matcher instead + } + Transition::WordBoundary + | Transition::NonWordBoundary + | Transition::StartWord + | Transition::EndWord => { + // Word boundaries require position tracking - use NFA + } + Transition::GroupStart(_) | Transition::GroupEnd(_) => { + // Capture groups need tracking - for DFA just treat as epsilon + // But patterns with backreferences must use NFA + } + Transition::Backref(_) => { + // Backreferences require captured text - use NFA + } + } + } + } + + // Process character transitions + for (ch, targets) in char_transitions { + let closure = nfa.epsilon_closure(&targets); + let mut closure_vec: Vec<_> = closure.iter().copied().collect(); + closure_vec.sort(); + + // Get or create DFA state for this closure + let target_dfa_id = if let Some(&existing_id) = nfa_set_to_dfa_id.get(&closure_vec) + { + existing_id + } else { + let new_id = dfa.add_state(&closure, nfa); + nfa_set_to_dfa_id.insert(closure_vec.clone(), new_id); + work_queue.push((new_id, closure.clone())); + new_id + }; + + dfa.transitions.insert((dfa_state, ch), target_dfa_id); + } + + // Process Any transitions + if has_any_transition { + let closure = nfa.epsilon_closure(&any_targets); + let mut closure_vec: Vec<_> = closure.iter().copied().collect(); + closure_vec.sort(); + + let target_dfa_id = if let Some(&existing_id) = nfa_set_to_dfa_id.get(&closure_vec) + { + existing_id + } else { + let new_id = dfa.add_state(&closure, nfa); + nfa_set_to_dfa_id.insert(closure_vec, new_id); + work_queue.push((new_id, closure)); + new_id + }; + + dfa.any_transitions.insert(dfa_state, target_dfa_id); + } + } + + dfa + } + + /// Add new DFA state corresponding to a set of NFA states + fn add_state(&mut self, nfa_states: &HashSet, nfa: &Nfa) -> DfaStateId { + let id = self.num_states; + self.num_states += 1; + + // Check if any NFA state in this set is accepting + for &nfa_state_id in nfa_states { + if nfa.states[nfa_state_id].is_accept { + self.accept_states.insert(id); + break; + } + } + + id + } + + /// Check if text matches the pattern + pub fn is_match(&self, text: &str) -> bool { + self.find(text).is_some() + } + + /// Find first match in text, returns (start, end) indices + pub fn find(&self, text: &str) -> Option<(usize, usize)> { + let chars: Vec = text.chars().collect(); + + // Try matching from each position + for start in 0..=chars.len() { + if let Some(end) = self.match_from(&chars, start) { + return Some((start, end)); + } + } + + None + } + + /// Try to match from a specific start position + /// Returns the end position if match succeeds + /// Uses greedy matching - finds longest possible match + fn match_from(&self, chars: &[char], start: usize) -> Option { + let mut state = self.start; + let mut pos = start; + let mut last_accept = None; + + // Special case: if start state is accepting, we can match empty string + if self.accept_states.contains(&state) { + last_accept = Some(start); + } + + while pos < chars.len() { + let ch = chars[pos]; + let mut found_transition = false; + + // Try character transition first + if let Some(&next_state) = self.transitions.get(&(state, ch)) { + state = next_state; + pos += 1; + found_transition = true; + + // Update last accept position if in accepting state + if self.accept_states.contains(&state) { + last_accept = Some(pos); + } + } else if let Some(&next_state) = self.any_transitions.get(&state) { + // Try Any transition + state = next_state; + pos += 1; + found_transition = true; + + if self.accept_states.contains(&state) { + last_accept = Some(pos); + } + } + + // No transition possible - stop + if !found_transition { + break; + } + } + + // Return last accepting position (greedy matching) + last_accept + } + + /// Find all matches in text (for global substitution) + pub fn find_all(&self, text: &str) -> Vec<(usize, usize)> { + let mut matches = Vec::new(); + let chars: Vec = text.chars().collect(); + let mut pos = 0; + + while pos < chars.len() { + if let Some(end) = self.match_from(&chars, pos) { + if end > pos { + matches.push((pos, end)); + pos = end; + } else { + // Empty match, advance by one to avoid infinite loop + pos += 1; + } + } else { + pos += 1; + } + } + + matches + } + + /// Replace first match + pub fn replace_first(&self, text: &str, replacement: &str) -> String { + if let Some((start, end)) = self.find(text) { + let chars: Vec = text.chars().collect(); + let mut result = String::new(); + + // Add text before match + for &ch in &chars[..start] { + result.push(ch); + } + + // Add replacement + result.push_str(replacement); + + // Add text after match + for &ch in &chars[end..] { + result.push(ch); + } + + result + } else { + text.to_string() + } + } + + /// Replace all matches + pub fn replace_all(&self, text: &str, replacement: &str) -> String { + let matches = self.find_all(text); + if matches.is_empty() { + return text.to_string(); + } + + let chars: Vec = text.chars().collect(); + let mut result = String::new(); + let mut last_end = 0; + + for (start, end) in matches { + // Add text between matches + for &ch in &chars[last_end..start] { + result.push(ch); + } + + // Add replacement + result.push_str(replacement); + last_end = end; + } + + // Add remaining text + for &ch in &chars[last_end..] { + result.push(ch); + } + + result + } +} + +/// DFA matcher - wraps DFA with high-level interface +#[derive(Debug, Clone)] +pub struct DfaMatcher { + dfa: Dfa, +} + +impl DfaMatcher { + /// Compile pattern into DFA + pub fn compile( + pattern: &str, + is_ere: bool, + ignore_case: bool, + posix_mode: bool, + ) -> crate::errors::Result { + // Parse pattern to AST + let compiled = if is_ere { + super::parser::parse_ere(pattern, posix_mode)? + } else { + super::parser::parse_bre(pattern, posix_mode)? + }; + + // Check if pattern can be handled by DFA + // DFA cannot handle: backreferences, anchors (for now) + if !can_use_dfa(&compiled.ast) { + return Err(crate::errors::SedError::parse( + "Pattern requires NFA (backreferences not supported in DFA)".to_string(), + )); + } + + // Build NFA from AST + let nfa = Nfa::from_ast(&compiled.ast); + + // Compile NFA to DFA + let dfa = Dfa::from_nfa(&nfa, ignore_case); + + Ok(DfaMatcher { dfa }) + } + + pub fn is_match(&self, text: &str) -> bool { + self.dfa.is_match(text) + } + + pub fn find(&self, text: &str) -> Option<(usize, usize)> { + self.dfa.find(text) + } + + pub fn replace_first(&self, text: &str, replacement: &str) -> String { + self.dfa.replace_first(text, replacement) + } + + pub fn replace_all(&self, text: &str, replacement: &str) -> String { + self.dfa.replace_all(text, replacement) + } +} + +/// Check if AST can be handled by DFA (no backreferences, etc.) +fn can_use_dfa(ast: &RegexNode) -> bool { + match ast { + // Backreferences require NFA with backtracking + RegexNode::Backref(_) => false, + + // Groups require NFA for capture support + // DFA can match patterns with groups but can't capture them + RegexNode::Group { .. } => false, + + // Anchors require position tracking - use NFA + RegexNode::StartAnchor | RegexNode::EndAnchor => false, + + // Word boundaries also require position tracking - use NFA + RegexNode::WordBoundary | RegexNode::NonWordBoundary => false, + RegexNode::StartWord | RegexNode::EndWord => false, + + // Recursive checks + RegexNode::Sequence(nodes) => nodes.iter().all(can_use_dfa), + RegexNode::Alternation(branches) => branches.iter().all(can_use_dfa), + RegexNode::ZeroOrMore(node) => can_use_dfa(node), + RegexNode::OneOrMore(node) => can_use_dfa(node), + RegexNode::ZeroOrOne(node) => can_use_dfa(node), + RegexNode::Repeat { node, .. } => can_use_dfa(node), + + // Simple nodes are always OK + RegexNode::Literal(_) => true, + RegexNode::Any => true, + RegexNode::CharClass(set) => { + // DFA doesn't support POSIX character classes - fall back to NFA + set.posix_classes.is_empty() + } + RegexNode::NegatedCharClass(_set) => { + // DFA can't efficiently handle negated character classes + // The complement set can be very large (all Unicode except a few chars) + // Fall back to NFA which handles negation properly + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dfa_literal() { + let matcher = DfaMatcher::compile("abc", false, false, false).unwrap(); + assert!(matcher.is_match("abc")); + assert!(matcher.is_match("xxabcxx")); + assert!(!matcher.is_match("ab")); + assert!(!matcher.is_match("xyz")); + } + + #[test] + fn test_dfa_alternation() { + let matcher = DfaMatcher::compile("a\\|b", false, false, false).unwrap(); + assert!(matcher.is_match("a")); + assert!(matcher.is_match("b")); + assert!(!matcher.is_match("c")); + } + + #[test] + fn test_dfa_star() { + let matcher = DfaMatcher::compile("a*", false, false, false).unwrap(); + assert!(matcher.is_match("")); + assert!(matcher.is_match("a")); + assert!(matcher.is_match("aaa")); + assert!(matcher.is_match("bbb")); // a* matches empty string at start + } + + #[test] + fn test_dfa_plus() { + let matcher = DfaMatcher::compile("a\\+", false, false, false).unwrap(); + assert!(!matcher.is_match("")); + assert!(matcher.is_match("a")); + assert!(matcher.is_match("aaa")); + } + + #[test] + fn test_dfa_replace() { + let matcher = DfaMatcher::compile("foo", false, false, false).unwrap(); + assert_eq!(matcher.replace_first("foo bar", "baz"), "baz bar"); + assert_eq!(matcher.replace_all("foo foo", "bar"), "bar bar"); + } + + #[test] + fn test_negated_charset_falls_back_to_nfa() { + // Negated character sets should fail to compile as DFA + // and fall back to NFA which handles them correctly + let result = DfaMatcher::compile("[^a]", false, false, false); + assert!(result.is_err(), "DFA should reject negated character sets"); + } +} diff --git a/red/src/regex/literal.rs b/red/src/regex/literal.rs new file mode 100644 index 0000000..a4d2e2d --- /dev/null +++ b/red/src/regex/literal.rs @@ -0,0 +1,290 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Literal pattern matching - fastest path for simple patterns. +// Handles patterns like s/foo/bar/ using String::replace(). + +use super::ast::*; + +/// Check if AST represents a literal pattern (no regex metacharacters) +pub fn is_literal(ast: &RegexNode) -> bool { + match ast { + // Single literal is literal + RegexNode::Literal(_) => true, + + // Sequence of literals is literal + RegexNode::Sequence(nodes) => nodes.iter().all(|n| matches!(n, RegexNode::Literal(_))), + + // Everything else is NOT literal (has regex features) + _ => false, + } +} + +/// Extract literal string from AST +/// Returns Some(string) if pattern is literal, None otherwise +pub fn to_literal_string(ast: &RegexNode) -> Option { + match ast { + RegexNode::Literal(ch) => Some(ch.to_string()), + + RegexNode::Sequence(nodes) => { + let mut result = String::new(); + for node in nodes { + match node { + RegexNode::Literal(ch) => result.push(*ch), + _ => return None, // Non-literal in sequence + } + } + Some(result) + } + + _ => None, + } +} + +/// Extract literal string from replacement template +pub fn to_literal_replacement( + template: &crate::engine::types::ReplacementTemplate, +) -> Option { + use crate::engine::types::ReplacementToken; + + let mut result = String::new(); + for token in &template.tokens { + match token { + ReplacementToken::Literal(s) => result.push_str(s), + _ => return None, // Non-literal token + } + } + Some(result) +} + +/// Literal matcher - uses String::replace() for maximum performance +#[derive(Debug, Clone)] +pub struct LiteralMatcher { + pattern: String, + ignore_case: bool, +} + +impl LiteralMatcher { + /// Create new literal matcher + pub fn new(pattern: String, ignore_case: bool) -> Self { + LiteralMatcher { + pattern, + ignore_case, + } + } + + /// Check if text matches pattern + pub fn is_match(&self, text: &str) -> bool { + if self.ignore_case { + text.to_lowercase().contains(&self.pattern.to_lowercase()) + } else { + text.contains(&self.pattern) + } + } + + /// Find first match position + pub fn find(&self, text: &str) -> Option { + if self.ignore_case { + let text_lower = text.to_lowercase(); + let pattern_lower = self.pattern.to_lowercase(); + text_lower.find(&pattern_lower) + } else { + text.find(&self.pattern) + } + } + + /// Replace first occurrence + pub fn replace_first(&self, text: &str, replacement: &str) -> String { + if self.ignore_case { + // Case-insensitive replace - need to find manually + let text_lower = text.to_lowercase(); + let pattern_lower = self.pattern.to_lowercase(); + if let Some(pos) = text_lower.find(&pattern_lower) { + let mut result = String::new(); + result.push_str(&text[..pos]); + result.push_str(replacement); + result.push_str(&text[pos + self.pattern.len()..]); + result + } else { + text.to_string() + } + } else { + text.replacen(&self.pattern, replacement, 1) + } + } + + /// Replace all occurrences + pub fn replace_all(&self, text: &str, replacement: &str) -> String { + if self.ignore_case { + // Case-insensitive replace all + let mut result = String::new(); + let mut remaining = text; + let pattern_lower = self.pattern.to_lowercase(); + + while !remaining.is_empty() { + let remaining_lower = remaining.to_lowercase(); + if let Some(pos) = remaining_lower.find(&pattern_lower) { + result.push_str(&remaining[..pos]); + result.push_str(replacement); + remaining = &remaining[pos + self.pattern.len()..]; + } else { + result.push_str(remaining); + break; + } + } + result + } else { + text.replace(&self.pattern, replacement) + } + } + + /// Get pattern + pub fn pattern(&self) -> &str { + &self.pattern + } + + /// Get pattern as bytes + pub fn pattern_bytes(&self) -> &[u8] { + self.pattern.as_bytes() + } + + /// Check if pattern is ASCII-only (safe for byte-level matching in MBCS locales) + pub fn is_ascii_pattern(&self) -> bool { + self.pattern.bytes().all(|b| b < 128) + } + + /// Find first match in raw bytes (byte-level, no UTF-8 conversion) + /// Returns byte offset of match start, or None if not found + /// Note: Only works correctly for ASCII patterns or when byte-level matching is safe + pub fn find_bytes(&self, bytes: &[u8]) -> Option { + if self.ignore_case { + // Case-insensitive byte matching only works for ASCII + if !self.is_ascii_pattern() { + return None; + } + let pattern_lower: Vec = self + .pattern + .bytes() + .map(|b| b.to_ascii_lowercase()) + .collect(); + bytes.windows(pattern_lower.len()).position(|window| { + window + .iter() + .zip(pattern_lower.iter()) + .all(|(a, b)| a.to_ascii_lowercase() == *b) + }) + } else { + let pat = self.pattern.as_bytes(); + if pat.is_empty() { + return Some(0); + } + bytes.windows(pat.len()).position(|window| window == pat) + } + } + + /// Find first match in raw bytes starting from a byte offset + pub fn find_bytes_from(&self, bytes: &[u8], start: usize) -> Option { + if start >= bytes.len() { + return None; + } + self.find_bytes(&bytes[start..]).map(|pos| pos + start) + } + + /// Check if bytes contain pattern (byte-level) + pub fn is_match_bytes(&self, bytes: &[u8]) -> bool { + self.find_bytes(bytes).is_some() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_literal_single() { + let node = RegexNode::Literal('a'); + assert!(is_literal(&node)); + assert_eq!(to_literal_string(&node), Some("a".to_string())); + } + + #[test] + fn test_is_literal_sequence() { + let node = RegexNode::Sequence(vec![ + RegexNode::Literal('f'), + RegexNode::Literal('o'), + RegexNode::Literal('o'), + ]); + assert!(is_literal(&node)); + assert_eq!(to_literal_string(&node), Some("foo".to_string())); + } + + #[test] + fn test_not_literal_any() { + let node = RegexNode::Any; + assert!(!is_literal(&node)); + assert_eq!(to_literal_string(&node), None); + } + + #[test] + fn test_not_literal_with_star() { + let node = RegexNode::ZeroOrMore(Box::new(RegexNode::Literal('a'))); + assert!(!is_literal(&node)); + } + + #[test] + fn test_not_literal_sequence_with_any() { + let node = RegexNode::Sequence(vec![ + RegexNode::Literal('a'), + RegexNode::Any, + RegexNode::Literal('b'), + ]); + assert!(!is_literal(&node)); + assert_eq!(to_literal_string(&node), None); + } + + #[test] + fn test_literal_matcher_is_match() { + let matcher = LiteralMatcher::new("foo".to_string(), false); + assert!(matcher.is_match("foo")); + assert!(matcher.is_match("foobar")); + assert!(matcher.is_match("barfoo")); + assert!(!matcher.is_match("fo")); + assert!(!matcher.is_match("bar")); + } + + #[test] + fn test_literal_matcher_replace_first() { + let matcher = LiteralMatcher::new("foo".to_string(), false); + assert_eq!(matcher.replace_first("foo", "bar"), "bar"); + assert_eq!(matcher.replace_first("foofoo", "bar"), "barfoo"); + assert_eq!(matcher.replace_first("xxxfoo", "bar"), "xxxbar"); + assert_eq!(matcher.replace_first("xxx", "bar"), "xxx"); + } + + #[test] + fn test_literal_matcher_replace_all() { + let matcher = LiteralMatcher::new("foo".to_string(), false); + assert_eq!(matcher.replace_all("foo", "bar"), "bar"); + assert_eq!(matcher.replace_all("foofoo", "bar"), "barbar"); + assert_eq!(matcher.replace_all("xxxfooyyy", "bar"), "xxxbaryyy"); + assert_eq!(matcher.replace_all("xxx", "bar"), "xxx"); + } + + #[test] + fn test_literal_matcher_case_insensitive() { + let matcher = LiteralMatcher::new("foo".to_string(), true); + assert!(matcher.is_match("FOO")); + assert!(matcher.is_match("FoO")); + assert_eq!(matcher.replace_first("FOO", "bar"), "bar"); + assert_eq!(matcher.replace_all("FoOFoo", "bar"), "barbar"); + } + + #[test] + fn test_literal_matcher_find() { + let matcher = LiteralMatcher::new("foo".to_string(), false); + assert_eq!(matcher.find("foo"), Some(0)); + assert_eq!(matcher.find("barfoo"), Some(3)); + assert_eq!(matcher.find("bar"), None); + } +} diff --git a/red/src/regex/mod.rs b/red/src/regex/mod.rs new file mode 100644 index 0000000..ef75f88 --- /dev/null +++ b/red/src/regex/mod.rs @@ -0,0 +1,743 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Custom high-performance regex engine for sed +// Implements BRE (Basic Regular Expression) and ERE (Extended Regular Expression) +// +// Architecture: +// - Three-level optimization: Literal → DFA → NFA+Backtracking +// - Specialized for sed use cases +// - Zero external dependencies (no fancy-regex, no regex crate) + +pub mod ast; +pub mod backtrack; +pub mod dfa; // DFA compilation and deterministic execution +pub mod literal; // Literal string matching (fastest path) +pub mod nfa; // NFA construction via Thompson's algorithm +pub mod parser; // NFA backtracking for backreferences and multiline mode + +use crate::errors::Result; +use crate::mbcs::is_multibyte_locale; +use ast::RegexNode; +use backtrack::NfaMatcher; +use dfa::DfaMatcher; +use literal::LiteralMatcher; +use std::collections::HashMap; + +// Re-export Capture for use in other modules +pub use backtrack::Capture; + +/// Check if a regex AST needs backtracking (can't be handled by DFA correctly) +/// This detects patterns like .* followed by specific characters that need +/// the NFA to backtrack to find the longest match. +fn needs_backtracking(node: &RegexNode) -> bool { + match node { + RegexNode::Sequence(nodes) => { + // Check if there's a .* (Any with ZeroOrMore/OneOrMore) followed by something + for i in 0..nodes.len() { + let is_any_star = matches!( + &nodes[i], + RegexNode::ZeroOrMore(inner) | RegexNode::OneOrMore(inner) + if matches!(**inner, RegexNode::Any) + ); + if is_any_star && i + 1 < nodes.len() { + // .* followed by something - needs backtracking + return true; + } + // Recursively check children + if needs_backtracking(&nodes[i]) { + return true; + } + } + false + } + RegexNode::Group { node: inner, .. } => needs_backtracking(inner), + RegexNode::Alternation(alts) => alts.iter().any(needs_backtracking), + RegexNode::ZeroOrMore(inner) + | RegexNode::OneOrMore(inner) + | RegexNode::ZeroOrOne(inner) => needs_backtracking(inner), + RegexNode::Repeat { node: inner, .. } => needs_backtracking(inner), + _ => false, + } +} + +/// Three-level optimization matcher +#[derive(Debug, Clone)] +pub enum Matcher { + /// Level 1: Literal string matching (fastest path) + Literal(LiteralMatcher), + + /// Level 2: DFA matching (deterministic, fast) + Dfa(DfaMatcher), + + /// Level 3: NFA with backtracking (handles backreferences and multiline) + Nfa(NfaMatcher), +} + +impl Matcher { + /// Compile a BRE or ERE pattern into a Matcher + pub fn compile(pattern: &str, is_ere: bool, ignore_case: bool) -> Result { + Self::compile_with_flags(pattern, is_ere, ignore_case, false, false) + } + + /// Compile a BRE or ERE pattern with multiline flag + pub fn compile_with_flags( + pattern: &str, + is_ere: bool, + ignore_case: bool, + multiline: bool, + posix_mode: bool, + ) -> Result { + // In multibyte locales (Shift-JIS, EUC-JP, etc.), we need to be careful + // DFA matcher doesn't have MBCS support, but Literal matcher can handle + // ASCII-only patterns safely (ASCII bytes don't overlap with MBCS lead bytes) + let is_mbcs = is_multibyte_locale(); + + // Parse pattern to AST + let compiled = if is_ere { + parser::parse_ere(pattern, posix_mode)? + } else { + parser::parse_bre(pattern, posix_mode)? + }; + + // Level 1: Check if literal pattern - use byte-level matching (fastest path) + // In MBCS locales, only use Literal matcher for ASCII-only patterns + // (ASCII bytes 0x00-0x7F don't overlap with MBCS lead bytes) + if literal::is_literal(&compiled.ast) { + if let Some(literal_str) = literal::to_literal_string(&compiled.ast) { + // In MBCS locales, only allow ASCII-only patterns for Literal matcher + let is_ascii_pattern = literal_str.bytes().all(|b| b < 128); + if !is_mbcs || is_ascii_pattern { + return Ok(Matcher::Literal(LiteralMatcher::new( + literal_str, + ignore_case, + ))); + } + } + } + + // Level 2: Try DFA compilation (faster than NFA backtracking) + // Skip DFA for multiline mode since it doesn't support it yet + // Also skip in MBCS locales - DFA doesn't have MBCS support + // Also skip for patterns that need backtracking (e.g., .* followed by specific chars) + let needs_bt = needs_backtracking(&compiled.ast); + if !multiline && !is_mbcs && !needs_bt { + match DfaMatcher::compile(pattern, is_ere, ignore_case, posix_mode) { + Ok(dfa) => return Ok(Matcher::Dfa(dfa)), + Err(_) => { + // DFA failed (has backreferences or other unsupported features) + // Fall through to NFA with backtracking + } + } + } + + // Level 3: NFA with backtracking (handles backreferences, multiline, and MBCS) + match NfaMatcher::compile_with_flags(pattern, is_ere, ignore_case, multiline, posix_mode) { + Ok(nfa) => Ok(Matcher::Nfa(nfa)), + Err(e) => Err(e), + } + } + + /// Check if pattern matches text + pub fn is_match(&self, text: &str) -> bool { + match self { + Matcher::Literal(m) => m.is_match(text), + Matcher::Dfa(m) => m.is_match(text), + Matcher::Nfa(m) => m.is_match(text), + } + } + + /// Replace first/all matches in text + pub fn replace(&self, text: &str, replacement: &str, global: bool) -> String { + match self { + Matcher::Literal(m) => { + if global { + m.replace_all(text, replacement) + } else { + m.replace_first(text, replacement) + } + } + Matcher::Dfa(m) => { + if global { + m.replace_all(text, replacement) + } else { + m.replace_first(text, replacement) + } + } + Matcher::Nfa(m) => { + if global { + m.replace_all(text, replacement) + } else { + m.replace_first(text, replacement) + } + } + } + } + + /// Check if this matcher uses literal optimization + pub fn is_literal(&self) -> bool { + matches!(self, Matcher::Literal(_)) + } + + /// Find first match and return captures (for backreference replacements) + pub fn find_with_captures( + &self, + text: &str, + ) -> Option<(usize, usize, HashMap)> { + match self { + Matcher::Nfa(m) => m.find_with_captures_pub(text), + // Other matchers don't support captures + _ => None, + } + } + + /// Find first match in raw bytes and return captures with byte offsets + /// This method is for MBCS locales where we need to work with raw bytes + pub fn find_with_captures_bytes( + &self, + bytes: &[u8], + ) -> Option<(usize, usize, HashMap)> { + match self { + Matcher::Nfa(m) => m.find_with_captures_bytes(bytes), + // In non-MB locales, fall back to string matching + _ => { + let text = String::from_utf8_lossy(bytes); + self.find_with_captures(&text) + } + } + } + + /// Find first match in raw bytes starting from a byte offset + pub fn find_with_captures_bytes_from( + &self, + bytes: &[u8], + start_byte: usize, + ) -> Option<(usize, usize, HashMap)> { + match self { + Matcher::Nfa(m) => m.find_with_captures_bytes_from(bytes, start_byte), + // Literal matcher: use byte-level matching (no UTF-8 conversion needed) + Matcher::Literal(m) => { + if start_byte >= bytes.len() { + return None; + } + // Use byte-level find to avoid from_utf8_lossy corruption in MBCS locales + m.find_bytes_from(bytes, start_byte).map(|pos| { + let end = pos + m.pattern_bytes().len(); + (pos, end, HashMap::new()) + }) + } + // DFA matcher: fall back to string matching (only used in non-MBCS locales) + Matcher::Dfa(_) => { + if start_byte >= bytes.len() { + return None; + } + // This path is only reached in non-MBCS locales where UTF-8 is safe + let substring = String::from_utf8_lossy(&bytes[start_byte..]); + self.find_with_captures(&substring) + .map(|(s, e, c)| (s + start_byte, e + start_byte, c)) + } + } + } + + /// Replace with template support (for & whole match replacement) + /// Returns an Option with (matched_text, start, end) for each match + pub fn find_with_text<'t>(&self, text: &'t str) -> Option<(&'t str, usize, usize)> { + match self { + Matcher::Literal(m) => { + m.find(text).map(|start| { + let chars: Vec = text.chars().collect(); + let pattern_len = m.pattern().chars().count(); + let end = start + pattern_len; + // Convert to byte indices + let byte_start: usize = chars[..start].iter().map(|c| c.len_utf8()).sum(); + let byte_end = byte_start + m.pattern().len(); + (&text[byte_start..byte_end], start, end) + }) + } + Matcher::Dfa(m) => m.find(text).map(|(start, end)| { + let chars: Vec = text.chars().collect(); + // Need to convert char indices to byte indices + let byte_start: usize = chars[..start].iter().map(|c| c.len_utf8()).sum(); + let byte_end: usize = chars[..end].iter().map(|c| c.len_utf8()).sum(); + (&text[byte_start..byte_end], start, end) + }), + Matcher::Nfa(m) => m.find(text).map(|(start, end)| { + let chars: Vec = text.chars().collect(); + let byte_start: usize = chars[..start].iter().map(|c| c.len_utf8()).sum(); + let byte_end: usize = chars[..end].iter().map(|c| c.len_utf8()).sum(); + (&text[byte_start..byte_end], start, end) + }), + } + } + + /// Find match starting from a specific position + pub fn find_with_text_from<'t>( + &self, + text: &'t str, + start_from: usize, + ) -> Option<(&'t str, usize, usize)> { + match self { + Matcher::Nfa(m) => { + // NFA has proper find_from that maintains anchor positions + m.find_from(text, start_from).map(|(start, end)| { + let chars: Vec = text.chars().collect(); + let byte_start: usize = chars[..start].iter().map(|c| c.len_utf8()).sum(); + let byte_end: usize = chars[..end].iter().map(|c| c.len_utf8()).sum(); + (&text[byte_start..byte_end], start, end) + }) + } + Matcher::Literal(m) => { + // Literal matcher: search in substring (anchors not supported anyway) + let chars: Vec = text.chars().collect(); + if start_from > chars.len() { + return None; + } + let remaining: String = chars[start_from..].iter().collect(); + m.find(&remaining).map(|byte_rel_start| { + // Convert byte position to character position in remaining substring + let rel_start = remaining[..byte_rel_start].chars().count(); + let start = start_from + rel_start; + let pattern_len = m.pattern().chars().count(); + let end = start + pattern_len; + let byte_start: usize = chars[..start].iter().map(|c| c.len_utf8()).sum(); + let byte_end: usize = chars[..end].iter().map(|c| c.len_utf8()).sum(); + (&text[byte_start..byte_end], start, end) + }) + } + Matcher::Dfa(m) => { + // DFA matcher: search in substring (anchors not supported anyway) + let chars: Vec = text.chars().collect(); + if start_from > chars.len() { + return None; + } + let remaining: String = chars[start_from..].iter().collect(); + m.find(&remaining).map(|(rel_start, rel_end)| { + let start = start_from + rel_start; + let end = start_from + rel_end; + let byte_start: usize = chars[..start].iter().map(|c| c.len_utf8()).sum(); + let byte_end: usize = chars[..end].iter().map(|c| c.len_utf8()).sum(); + (&text[byte_start..byte_end], start, end) + }) + } + } + } + + /// Find with captures starting from a specific position + /// For Literal and DFA matchers, returns empty captures (no groups captured) + pub fn find_with_captures_from( + &self, + text: &str, + start_from: usize, + ) -> Option<(usize, usize, HashMap)> { + match self { + Matcher::Nfa(m) => m.find_with_captures_from(text, start_from), + // For Literal and DFA matchers, return match with empty captures + // This allows backreferences like \1 to work (they'll just be empty) + Matcher::Literal(_) | Matcher::Dfa(_) => self + .find_with_text_from(text, start_from) + .map(|(_, start, end)| (start, end, HashMap::new())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_literal_matcher_compile() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + assert!(matcher.is_literal()); + } + + #[test] + fn test_literal_matcher_is_match() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + assert!(matcher.is_match("foo")); + assert!(matcher.is_match("foobar")); + assert!(!matcher.is_match("bar")); + } + + #[test] + fn test_literal_matcher_replace() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + assert_eq!(matcher.replace("foo", "bar", false), "bar"); + assert_eq!(matcher.replace("foofoo", "bar", false), "barfoo"); + assert_eq!(matcher.replace("foofoo", "bar", true), "barbar"); + } + + #[test] + fn test_non_literal_pattern() { + let matcher = Matcher::compile("a*", false, false).unwrap(); + assert!(!matcher.is_literal()); + } + + #[test] + fn test_non_literal_pattern_with_dot() { + let matcher = Matcher::compile("a.b", false, false).unwrap(); + assert!(!matcher.is_literal()); + } + + // DFA integration tests + + #[test] + fn test_dfa_simple_pattern() { + let matcher = Matcher::compile("a*b", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + assert!(matcher.is_match("b")); + assert!(matcher.is_match("ab")); + assert!(matcher.is_match("aaab")); + } + + #[test] + fn test_dfa_alternation() { + let matcher = Matcher::compile("foo\\|bar", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + assert!(matcher.is_match("foo")); + assert!(matcher.is_match("bar")); + assert!(!matcher.is_match("baz")); + } + + #[test] + fn test_dfa_char_class() { + let matcher = Matcher::compile("[0-9]\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + assert!(matcher.is_match("123")); + assert!(!matcher.is_match("abc")); + } + + #[test] + fn test_dfa_replace() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert_eq!(matcher.replace("aaa", "X", false), "X"); + assert_eq!(matcher.replace("aaa aaa", "X", true), "X X"); + } + + #[test] + fn test_matcher_selection_priority() { + // Literal patterns should use Literal matcher + let lit = Matcher::compile("hello", false, false).unwrap(); + assert!(matches!(lit, Matcher::Literal(_))); + + // Regex patterns should use DFA matcher + let dfa = Matcher::compile("hel*o", false, false).unwrap(); + assert!(matches!(dfa, Matcher::Dfa(_))); + } + + // Backreference tests + + #[test] + fn test_groups_in_pattern() { + // Test pattern with groups + let matcher = Matcher::compile("\\(.\\)\\(.\\)", false, false).unwrap(); + assert!( + matches!(matcher, Matcher::Nfa(_)), + "Should use NFA for groups" + ); + assert!(matcher.is_match("hello"), "Should match 'hello'"); + + // Test find_with_captures + if let Some((start, end, captures)) = matcher.find_with_captures("hello") { + println!("Match at {}-{}, captures: {:?}", start, end, captures); + assert_eq!(start, 0); + assert_eq!(end, 2, "Should match first two chars"); + assert_eq!(captures.len(), 2, "Should have 2 captures"); + assert_eq!(captures.get(&1).map(|c| c.text.as_str()), Some("h")); + assert_eq!(captures.get(&2).map(|c| c.text.as_str()), Some("e")); + } else { + panic!("Should find match with captures"); + } + } + + #[test] + fn test_backref_in_pattern() { + // Test backreference in pattern: \(foo\)\1 should match "foofoo" + let matcher = Matcher::compile("\\(foo\\)\\1", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + assert!(matcher.is_match("foofoo"), "Should match 'foofoo'"); + assert!(!matcher.is_match("foobar"), "Should not match 'foobar'"); + } + + #[test] + fn test_posix_char_class() { + // Test POSIX character class + let matcher = Matcher::compile("[[:alpha:]]*", false, false).unwrap(); + println!("Matcher type: {:?}", std::mem::discriminant(&matcher)); + assert!(matcher.is_match("abc"), "Should match 'abc'"); + assert!(matcher.is_match("abc123"), "Should match 'abc123'"); + } + + #[test] + fn test_negated_charset() { + // Test negated character class [^a] - should match anything except 'a' + let matcher = Matcher::compile("[^a]", false, false).unwrap(); + assert!( + matches!(matcher, Matcher::Nfa(_)), + "Should use NFA for negated charset" + ); + + assert!(matcher.is_match("b"), "Should match 'b'"); + assert!(matcher.is_match("xyz"), "Should match 'xyz'"); + assert!(!matcher.is_match("a"), "Should not match 'a'"); + + // Test negated range [^a-c] + let matcher2 = Matcher::compile("[^a-c]", false, false).unwrap(); + assert!( + matches!(matcher2, Matcher::Nfa(_)), + "Should use NFA for negated range" + ); + + assert!(!matcher2.is_match("a"), "Should not match 'a'"); + assert!(!matcher2.is_match("b"), "Should not match 'b'"); + assert!(!matcher2.is_match("c"), "Should not match 'c'"); + assert!(matcher2.is_match("d"), "Should match 'd'"); + assert!(matcher2.is_match("xyz"), "Should match 'xyz'"); + } + + // Tests for find_with_text + #[test] + fn test_find_with_text_literal() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + let result = matcher.find_with_text("hello foo world"); + assert!(result.is_some()); + let (matched, start, end) = result.unwrap(); + assert_eq!(matched, "foo"); + assert_eq!(start, 6); + assert_eq!(end, 9); + } + + #[test] + fn test_find_with_text_dfa() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + let result = matcher.find_with_text("hello aaa world"); + assert!(result.is_some()); + let (matched, start, end) = result.unwrap(); + assert_eq!(matched, "aaa"); + assert_eq!(start, 6); + assert_eq!(end, 9); + } + + #[test] + fn test_find_with_text_nfa() { + let matcher = Matcher::compile("\\(a\\)\\1", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + let result = matcher.find_with_text("hello aa world"); + assert!(result.is_some()); + let (matched, start, end) = result.unwrap(); + assert_eq!(matched, "aa"); + assert_eq!(start, 6); + assert_eq!(end, 8); + } + + // Tests for find_with_text_from + #[test] + fn test_find_with_text_from_literal() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + // First occurrence at index 0 + let result1 = matcher.find_with_text_from("foo bar foo", 0); + assert!(result1.is_some()); + assert_eq!(result1.unwrap().1, 0); + + // Search from after first match + let result2 = matcher.find_with_text_from("foo bar foo", 3); + assert!(result2.is_some()); + assert_eq!(result2.unwrap().1, 8); + } + + #[test] + fn test_find_with_text_from_literal_past_end() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + let result = matcher.find_with_text_from("foo", 100); + assert!(result.is_none()); + } + + #[test] + fn test_find_with_text_from_dfa() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + // First match at 0 + let result1 = matcher.find_with_text_from("aaa bbb aaa", 0); + assert!(result1.is_some()); + assert_eq!(result1.unwrap().0, "aaa"); + + // Search from after first match + let result2 = matcher.find_with_text_from("aaa bbb aaa", 4); + assert!(result2.is_some()); + assert_eq!(result2.unwrap().1, 8); + } + + #[test] + fn test_find_with_text_from_dfa_past_end() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + let result = matcher.find_with_text_from("aaa", 100); + assert!(result.is_none()); + } + + #[test] + fn test_find_with_text_from_nfa() { + let matcher = Matcher::compile("\\(a\\)\\1", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + // "xx aa yy aa" - first "aa" is at index 3, second at index 9 + // Starting from 3 should find "aa" at 3 + let result = matcher.find_with_text_from("xx aa yy aa", 3); + assert!(result.is_some()); + assert_eq!(result.unwrap().1, 3); // Found at position 3 + } + + // Tests for find_with_captures_from + #[test] + fn test_find_with_captures_from_literal() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + let result = matcher.find_with_captures_from("bar foo baz foo", 4); + assert!(result.is_some()); + let (start, end, captures) = result.unwrap(); + assert_eq!(start, 4); + assert_eq!(end, 7); + assert!(captures.is_empty()); // Literal matcher has no captures + } + + #[test] + fn test_find_with_captures_from_dfa() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + // "bb aaa cc aaa" - searching from position 4 (after "bb a") + // should find next 'aa' in "aa cc aaa" at relative 0, so absolute 4 + let result = matcher.find_with_captures_from("bb aaa cc aaa", 4); + assert!(result.is_some()); + let (start, end, captures) = result.unwrap(); + assert_eq!(start, 4); // Found at position 4 + assert_eq!(end, 6); + assert!(captures.is_empty()); // DFA matcher has no captures + } + + #[test] + fn test_find_with_captures_from_nfa() { + let matcher = Matcher::compile("\\(a\\)\\(b\\)", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + let result = matcher.find_with_captures_from("xx ab yy", 0); + assert!(result.is_some()); + let (start, end, captures) = result.unwrap(); + assert_eq!(start, 3); + assert_eq!(end, 5); + assert_eq!(captures.len(), 2); + assert_eq!(captures.get(&1).map(|c| c.text.as_str()), Some("a")); + assert_eq!(captures.get(&2).map(|c| c.text.as_str()), Some("b")); + } + + // Tests for find_with_captures_bytes + #[test] + fn test_find_with_captures_bytes_nfa() { + let matcher = Matcher::compile("\\(a\\)\\(b\\)", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + let result = matcher.find_with_captures_bytes(b"hello ab world"); + assert!(result.is_some()); + let (start, end, captures) = result.unwrap(); + assert_eq!(start, 6); + assert_eq!(end, 8); + assert_eq!(captures.len(), 2); + } + + #[test] + fn test_find_with_captures_bytes_literal_fallback() { + // Note: find_with_captures_bytes for Literal matcher falls back to + // find_with_captures which returns None for non-NFA matchers + let matcher = Matcher::compile("foo", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Literal(_))); + let result = matcher.find_with_captures_bytes(b"hello foo world"); + assert!(result.is_none()); // Literal matcher doesn't support captures + } + + // Tests for find_with_captures_bytes_from + #[test] + fn test_find_with_captures_bytes_from_nfa() { + let matcher = Matcher::compile("\\(a\\)\\(b\\)", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + let result = matcher.find_with_captures_bytes_from(b"ab xx ab", 2); + assert!(result.is_some()); + let (start, end, _) = result.unwrap(); + assert_eq!(start, 6); + assert_eq!(end, 8); + } + + #[test] + fn test_find_with_captures_bytes_from_literal() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Literal(_))); + let result = matcher.find_with_captures_bytes_from(b"foo bar foo", 4); + assert!(result.is_some()); + let (start, end, captures) = result.unwrap(); + assert_eq!(start, 8); + assert_eq!(end, 11); + assert!(captures.is_empty()); + } + + #[test] + fn test_find_with_captures_bytes_from_literal_past_end() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + let result = matcher.find_with_captures_bytes_from(b"foo", 100); + assert!(result.is_none()); + } + + #[test] + fn test_find_with_captures_bytes_from_dfa() { + // DFA matcher goes through string conversion path in find_with_captures_bytes_from + // but then calls find_with_captures which returns None for DFA matchers + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + let result = matcher.find_with_captures_bytes_from(b"aaa bbb aaa", 4); + // DFA matcher doesn't support captures through this path + assert!(result.is_none()); + } + + #[test] + fn test_find_with_captures_bytes_from_dfa_past_end() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + let result = matcher.find_with_captures_bytes_from(b"aaa", 100); + assert!(result.is_none()); + } + + // Tests for multiline mode (forces NFA) + #[test] + fn test_multiline_mode_uses_nfa() { + let matcher = Matcher::compile_with_flags("^foo", false, false, true, false).unwrap(); + assert!(matches!(matcher, Matcher::Nfa(_))); + } + + // Test ERE mode + #[test] + fn test_ere_mode() { + let matcher = Matcher::compile("a+", true, false).unwrap(); + assert!(matcher.is_match("aaa")); + assert!(!matcher.is_match("bbb")); + } + + // Test ignore case + #[test] + fn test_ignore_case() { + let matcher = Matcher::compile("foo", false, true).unwrap(); + assert!(matcher.is_match("FOO")); + assert!(matcher.is_match("foo")); + assert!(matcher.is_match("FoO")); + } + + // Test find_with_captures returns None for non-NFA + #[test] + fn test_find_with_captures_returns_none_for_literal() { + let matcher = Matcher::compile("foo", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Literal(_))); + let result = matcher.find_with_captures("foo bar"); + assert!(result.is_none()); + } + + #[test] + fn test_find_with_captures_returns_none_for_dfa() { + let matcher = Matcher::compile("a\\+", false, false).unwrap(); + assert!(matches!(matcher, Matcher::Dfa(_))); + let result = matcher.find_with_captures("aaa bar"); + assert!(result.is_none()); + } +} diff --git a/red/src/regex/nfa.rs b/red/src/regex/nfa.rs new file mode 100644 index 0000000..2b4a9ed --- /dev/null +++ b/red/src/regex/nfa.rs @@ -0,0 +1,597 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// NFA (Non-deterministic Finite Automaton) construction using Thompson's algorithm +// Converts AST → NFA for subsequent DFA compilation + +use super::ast::{CharSet, RegexNode}; +use std::collections::HashSet; + +/// NFA state ID +pub type StateId = usize; + +/// NFA transition +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Transition { + /// Transition on specific character + Char(char), + /// Transition on any character (.) + Any, + /// Transition on character in set + CharSet(CharSet), + /// Transition on character NOT in set + NegatedCharSet(CharSet), + /// Epsilon transition (no input consumed) + Epsilon, + /// Start anchor (^) - matches at beginning of text + StartAnchor, + /// End anchor ($) - matches at end of text + EndAnchor, + /// Word boundary (\b) - matches at word/non-word transition + WordBoundary, + /// Non-word boundary (\B) - matches where \b doesn't match + NonWordBoundary, + /// Start of word (\<) - matches at start of word + StartWord, + /// End of word (\>) - matches at end of word + EndWord, + /// Start of capture group - records position + GroupStart(usize), + /// End of capture group - records position + GroupEnd(usize), + /// Backreference (\1-\9) - matches same text as captured group + Backref(usize), +} + +/// NFA state +#[derive(Debug, Clone)] +pub struct State { + pub id: StateId, + /// Outgoing transitions: (Transition, target_state_id) + pub transitions: Vec<(Transition, StateId)>, + /// Is this an accepting state? + pub is_accept: bool, +} + +impl State { + fn new(id: StateId) -> Self { + State { + id, + transitions: Vec::new(), + is_accept: false, + } + } + + fn add_transition(&mut self, transition: Transition, target: StateId) { + self.transitions.push((transition, target)); + } +} + +/// NFA (Non-deterministic Finite Automaton) +#[derive(Debug, Clone)] +pub struct Nfa { + pub states: Vec, + pub start: StateId, + pub accept: StateId, +} + +impl Nfa { + /// Create new NFA with start and accept states + pub fn new() -> Self { + let start_state = State::new(0); + let accept_state = State { + id: 1, + transitions: Vec::new(), + is_accept: true, + }; + + Nfa { + states: vec![start_state, accept_state], + start: 0, + accept: 1, + } + } + + /// Add new state and return its ID + fn add_state(&mut self) -> StateId { + let id = self.states.len(); + self.states.push(State::new(id)); + id + } + + /// Add transition between states + fn add_transition(&mut self, from: StateId, transition: Transition, to: StateId) { + self.states[from].add_transition(transition, to); + } + + /// Build NFA from AST using Thompson's construction + pub fn from_ast(ast: &RegexNode) -> Self { + let mut nfa = Nfa::new(); + let fragment = nfa.build_fragment(ast); + + // Connect start to fragment start via epsilon + nfa.add_transition(nfa.start, Transition::Epsilon, fragment.start); + + // Connect fragment end to accept via epsilon + nfa.add_transition(fragment.end, Transition::Epsilon, nfa.accept); + + nfa + } + + /// Build NFA fragment from AST node + /// Returns (start_state, end_state) of the fragment + fn build_fragment(&mut self, node: &RegexNode) -> Fragment { + match node { + // Literal character: create simple path + RegexNode::Literal(ch) => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::Char(*ch), end); + Fragment { start, end } + } + + // Any (dot): matches any character + RegexNode::Any => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::Any, end); + Fragment { start, end } + } + + // Character class [abc] + RegexNode::CharClass(set) => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::CharSet(set.clone()), end); + Fragment { start, end } + } + + // Negated character class [^abc] + RegexNode::NegatedCharClass(set) => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::NegatedCharSet(set.clone()), end); + Fragment { start, end } + } + + // Sequence: concatenate fragments + RegexNode::Sequence(nodes) => { + if nodes.is_empty() { + // Empty sequence: epsilon transition + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::Epsilon, end); + return Fragment { start, end }; + } + + // Build first fragment + let mut current = self.build_fragment(&nodes[0]); + + // Concatenate remaining fragments + for node in &nodes[1..] { + let next = self.build_fragment(node); + // Connect current end to next start via epsilon + self.add_transition(current.end, Transition::Epsilon, next.start); + current.end = next.end; + } + + current + } + + // Alternation: a|b + RegexNode::Alternation(branches) => { + let start = self.add_state(); + let end = self.add_state(); + + for branch in branches { + let frag = self.build_fragment(branch); + // Connect start to each branch via epsilon + self.add_transition(start, Transition::Epsilon, frag.start); + // Connect each branch to end via epsilon + self.add_transition(frag.end, Transition::Epsilon, end); + } + + Fragment { start, end } + } + + // Zero or more: a* + RegexNode::ZeroOrMore(inner) => { + let start = self.add_state(); + let end = self.add_state(); + let frag = self.build_fragment(inner); + + // For greedy matching with stack-based backtracking: + // Add transitions in reverse priority order (last added = first tried) + + // From start: skip to end first (lower priority for greedy) + self.add_transition(start, Transition::Epsilon, end); + // Then try to match (higher priority for greedy) + self.add_transition(start, Transition::Epsilon, frag.start); + + // From fragment end: exit first (lower priority for greedy) + self.add_transition(frag.end, Transition::Epsilon, end); + // Then loop back for more matches (higher priority for greedy) + self.add_transition(frag.end, Transition::Epsilon, frag.start); + + Fragment { start, end } + } + + // One or more: a+ + RegexNode::OneOrMore(inner) => { + let frag = self.build_fragment(inner); + let start = frag.start; + let end = self.add_state(); + + // Must match at least once + // For greedy matching with LIFO stack: + // - Exit transition added FIRST (explored last = only if loop fails) + // - Loop transition added LAST (explored first = greedy) + self.add_transition(frag.end, Transition::Epsilon, end); + self.add_transition(frag.end, Transition::Epsilon, frag.start); + + Fragment { start, end } + } + + // Zero or one: a? + RegexNode::ZeroOrOne(inner) => { + let start = self.add_state(); + let end = self.add_state(); + let frag = self.build_fragment(inner); + + // For greedy matching with LIFO stack: + // - Skip transition added FIRST (explored last = only if match fails) + // - Match transition added LAST (explored first = greedy) + self.add_transition(start, Transition::Epsilon, end); + self.add_transition(start, Transition::Epsilon, frag.start); + + // Connect fragment end to overall end + self.add_transition(frag.end, Transition::Epsilon, end); + + Fragment { start, end } + } + + // Repeat {m,n} + RegexNode::Repeat { node, min, max } => self.build_repeat_fragment(node, *min, *max), + + // Start anchor (^) + RegexNode::StartAnchor => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::StartAnchor, end); + Fragment { start, end } + } + + // End anchor ($) + RegexNode::EndAnchor => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::EndAnchor, end); + Fragment { start, end } + } + + // Word boundary (\b) + RegexNode::WordBoundary => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::WordBoundary, end); + Fragment { start, end } + } + + // Non-word boundary (\B) + RegexNode::NonWordBoundary => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::NonWordBoundary, end); + Fragment { start, end } + } + + // Start of word (\<) + RegexNode::StartWord => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::StartWord, end); + Fragment { start, end } + } + + // End of word (\>) + RegexNode::EndWord => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::EndWord, end); + Fragment { start, end } + } + + // Capture group \(...\) or (...) + RegexNode::Group { node, id } => { + // Special case: Quantifiers inside Group + // For patterns like \([0-9]\{1,3\}\) or \([0-9]*\), we need special handling + // to avoid creating multiple separate groups + match &**node { + RegexNode::Repeat { + node: inner_node, + min, + max, + } => { + let start = self.add_state(); + let group_start = self.add_state(); + + // Mark start of capture group + self.add_transition(start, Transition::GroupStart(*id), group_start); + + // Build repetition fragment (without creating multiple groups) + let repeat_frag = self.build_repeat_fragment(inner_node, *min, *max); + self.add_transition(group_start, Transition::Epsilon, repeat_frag.start); + + // Mark end of capture group + let group_end = self.add_state(); + let end = self.add_state(); + self.add_transition(repeat_frag.end, Transition::Epsilon, group_end); + self.add_transition(group_end, Transition::GroupEnd(*id), end); + + return Fragment { start, end }; + } + _ => { + // Normal case handled below + } + } + + // Normal case: build inner fragment normally + let start = self.add_state(); + let group_start = self.add_state(); + + // Mark start of capture group + self.add_transition(start, Transition::GroupStart(*id), group_start); + + // Build inner fragment + let inner = self.build_fragment(node); + self.add_transition(group_start, Transition::Epsilon, inner.start); + + // Mark end of capture group + let group_end = self.add_state(); + let end = self.add_state(); + self.add_transition(inner.end, Transition::Epsilon, group_end); + self.add_transition(group_end, Transition::GroupEnd(*id), end); + + Fragment { start, end } + } + + // Backreference \1-\9 + RegexNode::Backref(id) => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::Backref(*id), end); + Fragment { start, end } + } + } + } + + /// Build fragment for {m,n} repetition + fn build_repeat_fragment( + &mut self, + node: &RegexNode, + min: usize, + max: Option, + ) -> Fragment { + match (min, max) { + // {0,0} - never matches (shouldn't happen but handle it) + (0, Some(0)) => { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::Epsilon, end); + Fragment { start, end } + } + + // {m,m} - exactly m times + (m, Some(n)) if m == n => { + // Build m copies concatenated + let mut fragments = Vec::new(); + for _ in 0..m { + fragments.push(self.build_fragment(node)); + } + + if fragments.is_empty() { + let start = self.add_state(); + let end = self.add_state(); + self.add_transition(start, Transition::Epsilon, end); + return Fragment { start, end }; + } + + let start = fragments[0].start; + let mut current_end = fragments[0].end; + + for frag in fragments.iter().skip(1) { + self.add_transition(current_end, Transition::Epsilon, frag.start); + current_end = frag.end; + } + + Fragment { + start, + end: current_end, + } + } + + // {m,n} - between m and n times + (m, Some(n)) => { + // Create new start and end states + let start = self.add_state(); + let end = self.add_state(); + + // Build n fragments total (m required + (n-m) optional) + let mut fragments: Vec = Vec::new(); + for _ in 0..n { + fragments.push(self.build_fragment(node)); + } + + if fragments.is_empty() { + // {0,0} edge case + self.add_transition(start, Transition::Epsilon, end); + return Fragment { start, end }; + } + + // For greedy matching with stack-based backtracker (LIFO): + // - Transitions are iterated in order and pushed to stack + // - Last pushed = first explored + // - For greedy: "continue" should be explored FIRST = pushed LAST + // - So: add "skip/exit" transitions FIRST, "continue" transitions LAST + + // If min is 0, we can skip everything - add skip FIRST (explored last) + if m == 0 { + self.add_transition(start, Transition::Epsilon, end); + } + + // Connect start to first fragment LAST (explored first = greedy) + self.add_transition(start, Transition::Epsilon, fragments[0].start); + + // Connect fragments in sequence, with optional exits after minimum + for i in 0..fragments.len() { + // After completing fragment i, we've matched (i+1) times + // If (i+1) >= m, we can optionally exit - add FIRST (explored last) + if i + 1 >= m { + self.add_transition(fragments[i].end, Transition::Epsilon, end); + } + + if i < fragments.len() - 1 { + // Connect to next fragment - add LAST (explored first = greedy) + self.add_transition( + fragments[i].end, + Transition::Epsilon, + fragments[i + 1].start, + ); + } + } + + Fragment { start, end } + } + + // {m,} - m or more times (unbounded) + (m, None) => { + // Build m required copies + one copy with loop + if m == 0 { + // {0,} is same as * + return self.build_fragment(&RegexNode::ZeroOrMore(Box::new(node.clone()))); + } + + let mut fragments = Vec::new(); + for _ in 0..m { + fragments.push(self.build_fragment(node)); + } + + let loop_frag = self.build_fragment(node); + + let start = fragments[0].start; + let end = self.add_state(); + + // Connect required copies + let mut current_end = fragments[0].end; + for frag in fragments.iter().skip(1) { + self.add_transition(current_end, Transition::Epsilon, frag.start); + current_end = frag.end; + } + + // After required copies, we can either: + // 1. Exit immediately (already matched m times = minimum) + // 2. Try to match more via loop_frag + // For greedy matching with LIFO stack: exit FIRST (explored last), loop LAST (explored first) + self.add_transition(current_end, Transition::Epsilon, end); + self.add_transition(current_end, Transition::Epsilon, loop_frag.start); + + // Loop fragment can repeat or exit + // For greedy: exit FIRST (explored last), repeat LAST (explored first) + self.add_transition(loop_frag.end, Transition::Epsilon, end); + self.add_transition(loop_frag.end, Transition::Epsilon, loop_frag.start); + + Fragment { start, end } + } + } + } + + /// Compute epsilon closure of a set of states + pub fn epsilon_closure(&self, states: &HashSet) -> HashSet { + let mut closure = states.clone(); + let mut stack: Vec = states.iter().copied().collect(); + + while let Some(state_id) = stack.pop() { + for (transition, target) in &self.states[state_id].transitions { + if matches!(transition, Transition::Epsilon) { + if closure.insert(*target) { + stack.push(*target); + } + } + } + } + + closure + } +} + +/// Fragment of NFA (start and end states) +#[derive(Debug, Clone, Copy)] +struct Fragment { + start: StateId, + end: StateId, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::regex::parser; + + #[test] + fn test_nfa_literal() { + let ast = parser::parse_bre("a", false).unwrap(); + let nfa = Nfa::from_ast(&ast.ast); + + // Should have at least start, accept, and one literal transition + assert!(nfa.states.len() >= 4); + assert_eq!(nfa.start, 0); + assert_eq!(nfa.accept, 1); + } + + #[test] + fn test_nfa_sequence() { + let ast = parser::parse_bre("abc", false).unwrap(); + let nfa = Nfa::from_ast(&ast.ast); + + // Should have states for a, b, c plus start/accept + assert!(nfa.states.len() >= 6); + } + + #[test] + fn test_nfa_alternation() { + let ast = parser::parse_bre("a\\|b", false).unwrap(); + let nfa = Nfa::from_ast(&ast.ast); + + // Should have branching structure + assert!(nfa.states.len() >= 6); + } + + #[test] + fn test_nfa_star() { + let ast = parser::parse_bre("a*", false).unwrap(); + let nfa = Nfa::from_ast(&ast.ast); + + // Should have loop structure + assert!(nfa.states.len() >= 4); + } + + #[test] + fn test_epsilon_closure() { + let mut nfa = Nfa::new(); + let s2 = nfa.add_state(); + let s3 = nfa.add_state(); + + nfa.add_transition(0, Transition::Epsilon, s2); + nfa.add_transition(s2, Transition::Epsilon, s3); + + let mut initial = HashSet::new(); + initial.insert(0); + + let closure = nfa.epsilon_closure(&initial); + + // Should include 0, s2, s3 + assert!(closure.contains(&0)); + assert!(closure.contains(&s2)); + assert!(closure.contains(&s3)); + } +} diff --git a/red/src/regex/parser.rs b/red/src/regex/parser.rs new file mode 100644 index 0000000..c3fb00a --- /dev/null +++ b/red/src/regex/parser.rs @@ -0,0 +1,859 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Parser for BRE (Basic Regular Expression) and ERE (Extended Regular Expression) + +use super::ast::*; +use crate::errors::{Result, SedError}; + +/// Parser for BRE/ERE patterns +pub struct Parser { + chars: Vec, + pos: usize, + is_ere: bool, + posix_mode: bool, + group_count: usize, +} + +impl Parser { + /// Create new parser for given pattern + pub fn new(pattern: &str, is_ere: bool, posix_mode: bool) -> Self { + Parser { + chars: pattern.chars().collect(), + pos: 0, + is_ere, + posix_mode, + group_count: 0, + } + } + + /// Parse the entire pattern + pub fn parse(&mut self) -> Result { + let ast = self.parse_alternation()?; + + Ok(CompiledRegex::new( + ast, + self.group_count, + self.is_ere, + false, // ignore_case set later + false, // dotall set later + )) + } + + /// Check if we're at end of pattern + fn is_eof(&self) -> bool { + self.pos >= self.chars.len() + } + + /// Peek current character without consuming + fn peek(&self) -> Option { + if self.is_eof() { + None + } else { + Some(self.chars[self.pos]) + } + } + + /// Peek next character (pos + 1) without consuming + fn peek_next(&self) -> Option { + if self.pos + 1 >= self.chars.len() { + None + } else { + Some(self.chars[self.pos + 1]) + } + } + + /// Consume and return current character + fn consume(&mut self) -> Option { + if self.is_eof() { + None + } else { + let ch = self.chars[self.pos]; + self.pos += 1; + Some(ch) + } + } + + /// Parse alternation (a\|b in BRE, a|b in ERE) + fn parse_alternation(&mut self) -> Result { + let mut alternatives = vec![self.parse_sequence()?]; + + loop { + let alt_marker = if self.is_ere { + // In ERE: | is alternation + self.peek() == Some('|') + } else { + // In BRE: \| is alternation + self.peek() == Some('\\') && self.peek_next() == Some('|') + }; + + if !alt_marker { + break; + } + + // Consume alternation marker + if self.is_ere { + self.consume(); // | + } else { + self.consume(); // \ + self.consume(); // | + } + + alternatives.push(self.parse_sequence()?); + } + + if alternatives.len() == 1 { + // SAFETY: len() == 1 guarantees pop() returns Some + Ok(alternatives.pop().expect("vec has exactly 1 element")) + } else { + Ok(RegexNode::Alternation(alternatives)) + } + } + + /// Parse sequence of atoms (concatenation) + fn parse_sequence(&mut self) -> Result { + let mut nodes = Vec::new(); + + while !self.is_eof() { + // Stop at alternation or closing paren + if self.is_ere { + if self.peek() == Some('|') || self.peek() == Some(')') { + break; + } + } else { + if (self.peek() == Some('\\') && self.peek_next() == Some('|')) + || (self.peek() == Some('\\') && self.peek_next() == Some(')')) + { + break; + } + } + + // Handle ^ as anchor only at the start of the sequence + // Per POSIX: ^ is special only at the beginning of the RE or subexpression + if self.peek() == Some('^') && nodes.is_empty() { + self.consume(); + nodes.push(RegexNode::StartAnchor); + continue; + } + + // Handle $ as anchor only at the end of the sequence + // Per POSIX: $ is special only at the end of the RE or subexpression + if self.peek() == Some('$') { + // Check if this $ would be at the end (nothing meaningful after it) + let is_at_end = if self.pos + 1 >= self.chars.len() { + true // EOF after $ + } else { + let next = self.chars[self.pos + 1]; + if self.is_ere { + // In ERE: end at | or ) or EOF + next == '|' || next == ')' + } else { + // In BRE: end at \| or \) or EOF + next == '\\' + && self.pos + 2 < self.chars.len() + && (self.chars[self.pos + 2] == '|' || self.chars[self.pos + 2] == ')') + } + }; + + if is_at_end { + self.consume(); + nodes.push(RegexNode::EndAnchor); + continue; + } + // Otherwise fall through to parse_quantified which will treat $ as literal + } + + nodes.push(self.parse_quantified()?); + } + + if nodes.is_empty() { + // Empty sequence - matches empty string + Ok(RegexNode::Sequence(vec![])) + } else if nodes.len() == 1 { + // SAFETY: len() == 1 guarantees pop() returns Some + Ok(nodes.pop().expect("vec has exactly 1 element")) + } else { + Ok(RegexNode::Sequence(nodes)) + } + } + + /// Parse atom with optional quantifier + fn parse_quantified(&mut self) -> Result { + let atom = self.parse_atom()?; + + // Check for quantifier + match self.peek() { + Some('*') => { + self.consume(); + Ok(RegexNode::ZeroOrMore(Box::new(atom))) + } + Some('+') if self.is_ere => { + self.consume(); + Ok(RegexNode::OneOrMore(Box::new(atom))) + } + Some('?') if self.is_ere => { + self.consume(); + Ok(RegexNode::ZeroOrOne(Box::new(atom))) + } + Some('{') if self.is_ere => self.parse_bounded_repeat(atom), + Some('\\') if !self.is_ere => { + match self.peek_next() { + Some('+') => { + self.consume(); // \ + self.consume(); // + + Ok(RegexNode::OneOrMore(Box::new(atom))) + } + Some('?') => { + self.consume(); // \ + self.consume(); // ? + Ok(RegexNode::ZeroOrOne(Box::new(atom))) + } + Some('{') => { + self.consume(); // \ + self.parse_bounded_repeat(atom) + } + _ => Ok(atom), + } + } + _ => Ok(atom), + } + } + + /// Parse bounded repetition {m,n} + fn parse_bounded_repeat(&mut self, atom: RegexNode) -> Result { + // Consume opening { + self.consume(); + + // Parse min + let min = self.parse_number()?; + + let max = if self.peek() == Some(',') { + self.consume(); // , + // In BRE mode, check for \} to recognize {m,} case + // In ERE mode, check for } directly + let is_unbounded = if self.is_ere { + self.peek() == Some('}') + } else { + self.peek() == Some('\\') && self.peek_next() == Some('}') + }; + if is_unbounded { + None // {m,} = m or more + } else { + Some(self.parse_number()?) // {m,n} + } + } else { + Some(min) // {m} = exactly m + }; + + // Consume closing } (in BRE it's \}, in ERE it's just }) + if !self.is_ere { + // In BRE, expect \} + if self.peek() != Some('\\') { + return Err(SedError::parse("missing closing \\} in bounded repetition")); + } + self.consume(); // consume \ + if self.peek() != Some('}') { + return Err(SedError::parse("missing closing \\} in bounded repetition")); + } + self.consume(); // consume } + } else { + // In ERE, expect just } + if self.peek() != Some('}') { + return Err(SedError::parse("missing closing } in bounded repetition")); + } + self.consume(); + } + + Ok(RegexNode::Repeat { + node: Box::new(atom), + min, + max, + }) + } + + /// Parse a number + fn parse_number(&mut self) -> Result { + let start = self.pos; + while let Some(ch) = self.peek() { + if ch.is_ascii_digit() { + self.consume(); + } else { + break; + } + } + + if start == self.pos { + return Err(SedError::parse("expected number in repetition")); + } + + let num_str: String = self.chars[start..self.pos].iter().collect(); + num_str + .parse() + .map_err(|_| SedError::parse("invalid number in repetition")) + } + + /// Parse single atom (character, class, group, etc.) + fn parse_atom(&mut self) -> Result { + match self.peek() { + Some('.') => { + self.consume(); + Ok(RegexNode::Any) + } + // Note: ^ is handled in parse_sequence - at start it's anchor, elsewhere it's literal + // and goes through the normal literal path below since it's not in is_special_char + // Note: $ is handled in parse_sequence - at end it's anchor, elsewhere it's literal + // and goes through the normal literal path below since it's not in is_special_char + Some('[') => self.parse_char_class(), + Some('(') if self.is_ere => { + self.consume(); // Consume opening '(' + self.parse_group() + } + Some('\\') => self.parse_escape(), + Some(ch) if !self.is_special_char(ch) => { + self.consume(); + Ok(RegexNode::Literal(ch)) + } + Some(_ch) => Err(SedError::parse(format!( + "unexpected special character at position {}", + self.pos + ))), + None => Err(SedError::parse("unexpected end of pattern")), + } + } + + /// Check if character is special (needs escaping) + /// Note: ']' is NOT special outside of character classes - it's only meaningful inside [...] + /// Note: '^' is handled specially in parse_sequence (anchor at start, literal elsewhere) + /// Note: '$' is handled specially in parse_sequence (anchor at end, literal elsewhere) + fn is_special_char(&self, ch: char) -> bool { + if self.is_ere { + matches!( + ch, + '.' | '*' | '+' | '?' | '[' | '(' | ')' | '{' | '}' | '|' | '\\' + ) + } else { + matches!(ch, '.' | '*' | '[' | '\\') + } + } + + /// Parse escape sequence (\something) + fn parse_escape(&mut self) -> Result { + self.consume(); // \ + + match self.peek() { + // Backreferences + Some('1'..='9') => { + // SAFETY: peek() returned Some, so consume() will too + let digit = self.consume().expect("char exists after peek") as u8 - b'0'; + Ok(RegexNode::Backref(digit as usize)) + } + + // Groups in BRE: \( \) + Some('(') if !self.is_ere => { + self.consume(); + self.parse_group() + } + + // Word boundaries (GNU extensions) + Some('b') => { + self.consume(); + Ok(RegexNode::WordBoundary) + } + Some('B') => { + self.consume(); + Ok(RegexNode::NonWordBoundary) + } + Some('<') => { + self.consume(); + Ok(RegexNode::StartWord) + } + Some('>') => { + self.consume(); + Ok(RegexNode::EndWord) + } + + // Word character class \w (GNU extension) + Some('w') => { + self.consume(); + // \w matches [[:alnum:]_] (alphanumeric + underscore) + let mut charset = CharSet::new(); + charset.add_posix_class(PosixClass::Alnum); + charset.add_char('_'); + Ok(RegexNode::CharClass(charset)) + } + + // Escape sequences + Some('n') => { + self.consume(); + Ok(RegexNode::Literal('\n')) + } + Some('t') => { + self.consume(); + Ok(RegexNode::Literal('\t')) + } + Some('r') => { + self.consume(); + Ok(RegexNode::Literal('\r')) + } + + // Literal escape + Some(_) => { + // SAFETY: peek() returned Some, so consume() will too + let ch = self.consume().expect("char exists after peek"); + Ok(RegexNode::Literal(ch)) + } + + None => Err(SedError::parse("trailing backslash at end of pattern")), + } + } + + /// Parse capturing group + /// Note: opening paren should already be consumed (either '(' in ERE or '\(' in BRE) + fn parse_group(&mut self) -> Result { + self.group_count += 1; + let group_id = self.group_count; + + let content = self.parse_alternation()?; + + // Consume closing paren + if self.is_ere { + if self.peek() != Some(')') { + return Err(SedError::parse("unclosed group: missing ')'")); + } + self.consume(); + } else { + if self.peek() != Some('\\') || self.peek_next() != Some(')') { + return Err(SedError::parse("unclosed group: missing '\\)'")); + } + self.consume(); // \ + self.consume(); // ) + } + + Ok(RegexNode::Group { + id: group_id, + node: Box::new(content), + }) + } + + /// Parse character class [...] + fn parse_char_class(&mut self) -> Result { + self.consume(); // [ + + // Check for negation + let negated = if self.peek() == Some('^') { + self.consume(); + true + } else { + false + }; + + let mut set = CharSet::new(); + + while let Some(ch) = self.peek() { + if ch == ']' && (!set.ranges.is_empty() || !set.posix_classes.is_empty()) { + // Closing ] - only if we've seen at least one element (range or POSIX class) + self.consume(); + break; + } + + if ch == '[' { + match self.peek_next() { + Some(':') => { + // POSIX character class [:name:] + self.parse_posix_class(&mut set)?; + continue; + } + Some('.') => { + // POSIX collating symbol [.X.] + self.parse_collating_symbol(&mut set)?; + continue; + } + Some('=') => { + // POSIX equivalence class [=X=] + self.parse_equivalence_class(&mut set)?; + continue; + } + _ => {} + } + } + + { + // Regular character(s) or range + let chars = self.consume_class_chars()?; + + // Check if this is a range (a-z) + // Ranges only work with single characters, not multi-char escapes + if chars.len() == 1 && self.peek() == Some('-') && self.peek_next() != Some(']') { + // Range a-z + self.consume(); // - + let end_chars = self.consume_class_chars()?; + if end_chars.len() == 1 { + set.add_range(chars[0], end_chars[0]); + } else { + // Multi-char escape can't be end of range, treat as literals + set.add_char(chars[0]); + set.add_char('-'); + for c in end_chars { + set.add_char(c); + } + } + } else { + // Single character(s) - add all from escape + for c in chars { + set.add_char(c); + } + } + } + } + + if negated { + Ok(RegexNode::NegatedCharClass(set)) + } else { + Ok(RegexNode::CharClass(set)) + } + } + + /// Consume single character(s) from character class + /// Returns a vector because some escapes like \" expand to both \ and " + /// In POSIX mode, escape sequences (\t, \n, etc.) are NOT interpreted inside character classes + fn consume_class_chars(&mut self) -> Result> { + match self.peek() { + Some('\\') => { + self.consume(); + match self.peek() { + Some('n') if !self.posix_mode => { + self.consume(); + Ok(vec!['\n']) + } + Some('t') if !self.posix_mode => { + self.consume(); + Ok(vec!['\t']) + } + Some('r') if !self.posix_mode => { + self.consume(); + Ok(vec!['\r']) + } + Some('b') if !self.posix_mode => { + // \b inside character class is backspace (0x08), not word boundary + self.consume(); + Ok(vec!['\x08']) + } + Some(ch) => { + // In POSIX mode, ALL escapes (including \t, \n) are treated as literal backslash + char + // In GNU mode, non-special escapes like \" include both backslash and the character + self.consume(); + Ok(vec!['\\', ch]) + } + None => Err(SedError::parse("unexpected end in character class")), + } + } + Some(ch) => { + self.consume(); + Ok(vec![ch]) + } + None => Err(SedError::parse("unclosed character class")), + } + } + + /// Parse POSIX character class [:name:] + fn parse_posix_class(&mut self, set: &mut CharSet) -> Result<()> { + self.consume(); // [ + self.consume(); // : + + let start = self.pos; + while let Some(ch) = self.peek() { + if ch == ':' { + break; + } + self.consume(); + } + + let name: String = self.chars[start..self.pos].iter().collect(); + + if self.peek() != Some(':') || self.peek_next() != Some(']') { + return Err(SedError::parse(format!( + "invalid POSIX character class: [:{}", + name + ))); + } + + self.consume(); // : + self.consume(); // ] + + let class = PosixClass::from_name(&name).ok_or_else(|| { + SedError::parse(format!("unknown POSIX character class: [:{}:]", name)) + })?; + + set.add_posix_class(class); + Ok(()) + } + + /// Parse POSIX collating symbol [.X.] + /// In C/POSIX locale, collating symbols are just literal characters + /// Multi-character collating elements (like [.ch.] in Spanish) are + /// treated as individual characters in C locale + fn parse_collating_symbol(&mut self, set: &mut CharSet) -> Result<()> { + self.consume(); // [ + self.consume(); // . + + let start = self.pos; + while let Some(ch) = self.peek() { + if ch == '.' { + break; + } + self.consume(); + } + + let symbol: String = self.chars[start..self.pos].iter().collect(); + + if self.peek() != Some('.') || self.peek_next() != Some(']') { + return Err(SedError::parse(format!( + "invalid collating symbol: [.{}", + symbol + ))); + } + + self.consume(); // . + self.consume(); // ] + + // In C locale, treat collating symbol as literal character(s) + // For single-char symbols, this is simply that character + // For multi-char symbols like "ch", add each character individually + for ch in symbol.chars() { + set.add_char(ch); + } + Ok(()) + } + + /// Parse POSIX equivalence class [=X=] + /// In C/POSIX locale, equivalence classes just match the literal character + /// (In other locales, they would match all characters with same primary collation weight) + fn parse_equivalence_class(&mut self, set: &mut CharSet) -> Result<()> { + self.consume(); // [ + self.consume(); // = + + let start = self.pos; + while let Some(ch) = self.peek() { + if ch == '=' { + break; + } + self.consume(); + } + + let equiv: String = self.chars[start..self.pos].iter().collect(); + + if self.peek() != Some('=') || self.peek_next() != Some(']') { + return Err(SedError::parse(format!( + "invalid equivalence class: [={}", + equiv + ))); + } + + self.consume(); // = + self.consume(); // ] + + // In C locale, equivalence class is just the literal character(s) + for ch in equiv.chars() { + set.add_char(ch); + } + Ok(()) + } +} + +/// Parse BRE pattern +pub fn parse_bre(pattern: &str, posix_mode: bool) -> Result { + Parser::new(pattern, false, posix_mode).parse() +} + +/// Parse ERE pattern +pub fn parse_ere(pattern: &str, posix_mode: bool) -> Result { + Parser::new(pattern, true, posix_mode).parse() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_literal() { + let regex = parse_bre("abc", false).unwrap(); + match regex.ast { + RegexNode::Sequence(nodes) => { + assert_eq!(nodes.len(), 3); + assert_eq!(nodes[0], RegexNode::Literal('a')); + assert_eq!(nodes[1], RegexNode::Literal('b')); + assert_eq!(nodes[2], RegexNode::Literal('c')); + } + _ => panic!("expected Sequence"), + } + } + + #[test] + fn test_parse_any() { + let regex = parse_bre("a.b", false).unwrap(); + match regex.ast { + RegexNode::Sequence(nodes) => { + assert_eq!(nodes.len(), 3); + assert_eq!(nodes[0], RegexNode::Literal('a')); + assert_eq!(nodes[1], RegexNode::Any); + assert_eq!(nodes[2], RegexNode::Literal('b')); + } + _ => panic!("expected Sequence"), + } + } + + #[test] + fn test_parse_star() { + let regex = parse_bre("a*", false).unwrap(); + match regex.ast { + RegexNode::ZeroOrMore(node) => { + assert_eq!(*node, RegexNode::Literal('a')); + } + _ => panic!("expected ZeroOrMore"), + } + } + + #[test] + fn test_parse_anchors() { + let regex = parse_bre("^a$", false).unwrap(); + match regex.ast { + RegexNode::Sequence(nodes) => { + assert_eq!(nodes.len(), 3); + assert_eq!(nodes[0], RegexNode::StartAnchor); + assert_eq!(nodes[1], RegexNode::Literal('a')); + assert_eq!(nodes[2], RegexNode::EndAnchor); + } + _ => panic!("expected Sequence"), + } + } + + #[test] + fn test_parse_group_bre() { + let regex = parse_bre("\\(ab\\)", false).unwrap(); + match regex.ast { + RegexNode::Group { id, node } => { + assert_eq!(id, 1); + match *node { + RegexNode::Sequence(nodes) => { + assert_eq!(nodes.len(), 2); + } + _ => panic!("expected Sequence in group"), + } + } + _ => panic!("expected Group"), + } + } + + #[test] + fn test_parse_backref() { + let regex = parse_bre("\\(a\\)\\1", false).unwrap(); + match regex.ast { + RegexNode::Sequence(nodes) => { + assert_eq!(nodes.len(), 2); + // First node is group + matches!(nodes[0], RegexNode::Group { .. }); + // Second node is backref + assert_eq!(nodes[1], RegexNode::Backref(1)); + } + _ => panic!("expected Sequence"), + } + } + + #[test] + fn test_parse_char_class() { + let regex = parse_bre("[abc]", false).unwrap(); + match regex.ast { + RegexNode::CharClass(set) => { + assert!(set.matches('a')); + assert!(set.matches('b')); + assert!(set.matches('c')); + assert!(!set.matches('d')); + } + _ => panic!("expected CharClass"), + } + } + + #[test] + fn test_parse_char_class_range() { + let regex = parse_bre("[a-z]", false).unwrap(); + match regex.ast { + RegexNode::CharClass(set) => { + assert!(set.matches('a')); + assert!(set.matches('m')); + assert!(set.matches('z')); + assert!(!set.matches('A')); + assert!(!set.matches('0')); + } + _ => panic!("expected CharClass"), + } + } + + #[test] + fn test_parse_negated_class() { + let regex = parse_bre("[^abc]", false).unwrap(); + match regex.ast { + RegexNode::NegatedCharClass(set) => { + // set contains [abc], so: + // set.matches('a') == true (a is in set) + // set.matches('d') == false (d is NOT in set) + // But NegatedCharClass will invert this during matching + assert!(set.matches('a')); // a is in set [abc] + assert!(set.matches('b')); // b is in set [abc] + assert!(!set.matches('d')); // d is NOT in set [abc] + assert!(!set.matches('x')); // x is NOT in set [abc] + } + _ => panic!("expected NegatedCharClass"), + } + } + + #[test] + fn test_parse_alternation_bre() { + let regex = parse_bre("a\\|b", false).unwrap(); + match regex.ast { + RegexNode::Alternation(alts) => { + assert_eq!(alts.len(), 2); + assert_eq!(alts[0], RegexNode::Literal('a')); + assert_eq!(alts[1], RegexNode::Literal('b')); + } + _ => panic!("expected Alternation"), + } + } + + #[test] + fn test_parse_ere_plus() { + let regex = parse_ere("a+", false).unwrap(); + match regex.ast { + RegexNode::OneOrMore(node) => { + assert_eq!(*node, RegexNode::Literal('a')); + } + _ => panic!("expected OneOrMore"), + } + } + + #[test] + fn test_parse_ere_question() { + let regex = parse_ere("a?", false).unwrap(); + match regex.ast { + RegexNode::ZeroOrOne(node) => { + assert_eq!(*node, RegexNode::Literal('a')); + } + _ => panic!("expected ZeroOrOne"), + } + } + + #[test] + fn test_parse_ere_group() { + let regex = parse_ere("(ab)", false).unwrap(); + match regex.ast { + RegexNode::Group { id, node } => { + assert_eq!(id, 1); + match *node { + RegexNode::Sequence(_) => {} + _ => panic!("expected Sequence in group"), + } + } + _ => panic!("expected Group"), + } + } +} diff --git a/red/src/selinux.rs b/red/src/selinux.rs new file mode 100644 index 0000000..0171cb1 --- /dev/null +++ b/red/src/selinux.rs @@ -0,0 +1,100 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! SELinux security context support for in-place editing. +//! +//! This module provides functions to get and set SELinux security contexts +//! on files, which is needed to preserve contexts during in-place editing. + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +/// Check if SELinux is enabled (enforcing or permissive mode). +pub fn is_selinux_enabled() -> bool { + use selinux::{current_mode, SELinuxMode}; + matches!( + current_mode(), + SELinuxMode::Enforcing | SELinuxMode::Permissive + ) +} + +/// Get SELinux context from a path. +/// +/// If `follow_symlinks` is false, gets the context of the symlink itself (lgetfilecon). +/// If `follow_symlinks` is true, gets the context of the target file (getfilecon). +/// +/// Returns `None` if SELinux context cannot be retrieved (e.g., SELinux disabled, +/// filesystem doesn't support xattrs, or file doesn't exist). +pub fn get_context(path: &Path, follow_symlinks: bool) -> Option { + let path_cstr = CString::new(path.as_os_str().as_bytes()).ok()?; + let mut context: *mut c_char = std::ptr::null_mut(); + + let result = unsafe { + if follow_symlinks { + selinux_sys::getfilecon(path_cstr.as_ptr(), &mut context) + } else { + selinux_sys::lgetfilecon(path_cstr.as_ptr(), &mut context) + } + }; + + if result < 0 || context.is_null() { + return None; + } + + let ctx_str = unsafe { CStr::from_ptr(context) } + .to_string_lossy() + .into_owned(); + + unsafe { selinux_sys::freecon(context) }; + + Some(ctx_str) +} + +/// Set SELinux context on a path. +/// +/// This function follows symlinks (uses setfilecon, not lsetfilecon). +/// +/// Returns an error if the context cannot be set. +pub fn set_context(path: &Path, context: &str) -> std::io::Result<()> { + let path_cstr = CString::new(path.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + let ctx_cstr = CString::new(context) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + + let result = unsafe { selinux_sys::setfilecon(path_cstr.as_ptr(), ctx_cstr.as_ptr()) }; + + if result < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_selinux_enabled_returns_bool() { + // Just verify it doesn't panic - actual result depends on system + let _ = is_selinux_enabled(); + } + + #[test] + fn test_get_context_nonexistent_file() { + let result = get_context(Path::new("/nonexistent/file/path"), false); + assert!(result.is_none()); + } + + #[test] + fn test_set_context_nonexistent_file() { + let result = set_context( + Path::new("/nonexistent/file/path"), + "system_u:object_r:tmp_t:s0", + ); + assert!(result.is_err()); + } +} diff --git a/red/src/signals.rs b/red/src/signals.rs new file mode 100644 index 0000000..48d678a --- /dev/null +++ b/red/src/signals.rs @@ -0,0 +1,87 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +#[cfg(unix)] +pub mod unix { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + /// Global flag to indicate if a signal was received + static SIGNAL_RECEIVED: AtomicBool = AtomicBool::new(false); + + // Global list of temporary files to clean up on signal + lazy_static::lazy_static! { + static ref TEMP_FILES: Arc>> = Arc::new(Mutex::new(Vec::new())); + } + + /// Initialize signal handlers for SIGINT and SIGTERM + pub fn setup_signal_handlers() -> Result<(), Box> { + use signal_hook::consts::{SIGINT, SIGTERM}; + use signal_hook::iterator::Signals; + use std::thread; + + let mut signals = Signals::new(&[SIGINT, SIGTERM])?; + + // Spawn a thread to handle signals + thread::spawn(move || { + for sig in signals.forever() { + match sig { + SIGINT | SIGTERM => { + SIGNAL_RECEIVED.store(true, Ordering::SeqCst); + cleanup_temp_files(); + std::process::exit(130); // 128 + SIGINT(2) = standard shell convention + } + _ => {} + } + } + }); + + Ok(()) + } + + /// Register a temporary file for cleanup + pub fn register_temp_file(path: String) { + if let Ok(mut files) = TEMP_FILES.lock() { + files.push(path); + } + } + + /// Unregister a temporary file (called after successful rename) + pub fn unregister_temp_file(path: &str) { + if let Ok(mut files) = TEMP_FILES.lock() { + files.retain(|f| f != path); + } + } + + /// Clean up all registered temporary files + fn cleanup_temp_files() { + if let Ok(files) = TEMP_FILES.lock() { + for file in files.iter() { + // Silently try to remove the file (best-effort) + let _ = std::fs::remove_file(file); + } + } + } + + /// Check if a signal was received + pub fn was_interrupted() -> bool { + SIGNAL_RECEIVED.load(Ordering::SeqCst) + } +} + +#[cfg(not(unix))] +pub mod unix { + /// No-op implementation for non-Unix systems + pub fn setup_signal_handlers() -> Result<(), Box> { + Ok(()) + } + + pub fn register_temp_file(_path: String) {} + + pub fn unregister_temp_file(_path: &str) {} + + pub fn was_interrupted() -> bool { + false + } +} diff --git a/red/src/util/mod.rs b/red/src/util/mod.rs new file mode 100644 index 0000000..8610da5 --- /dev/null +++ b/red/src/util/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +pub mod regex; +pub mod symlink; +pub mod version; diff --git a/red/src/util/regex.rs b/red/src/util/regex.rs new file mode 100644 index 0000000..062b085 --- /dev/null +++ b/red/src/util/regex.rs @@ -0,0 +1,1045 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Regular Expression utilities for both BRE (Basic) and ERE (Extended) modes + +use crate::engine::types::{ReplacementTemplate, ReplacementToken, SedRegex}; +use crate::errors::{Result, SedError}; +use crate::parser::build_char_to_byte_mapping; + +/// Compile a regex pattern (BRE or ERE) into a Regex with optional case-insensitive flag. +/// Adds helpful context including the original pattern and converted PCRE pattern. +pub fn compile_regex( + pattern: &str, + is_ere: bool, + ignore_case: bool, + multiline: bool, + multiline_dotall: bool, + posix_mode: bool, + label: &str, +) -> Result { + compile_regex_with_replacement( + pattern, + is_ere, + ignore_case, + multiline, + multiline_dotall, + posix_mode, + label, + None, + false, // No occurrence flag for address patterns + ) +} + +/// Compile a regex with optional replacement template for optimization +/// If replacement template is provided and doesn't use backreferences, +/// we can use fast regex even if pattern has capture groups +pub fn compile_regex_with_replacement( + pattern: &str, + is_ere: bool, + ignore_case: bool, + multiline: bool, + multiline_dotall: bool, + posix_mode: bool, + _label: &str, + _replacement: Option<&ReplacementTemplate>, + _has_occurrence: bool, // True if occurrence flag (e.g., s/./X/4) is used +) -> Result { + // Compile using custom regex engine with zero external dependencies + // Supports: anchors, word boundaries, POSIX classes, backreferences, + // case conversion, bounded repetition, and multiline mode + let use_multiline = multiline || multiline_dotall; + let matcher = crate::regex::Matcher::compile_with_flags( + pattern, + is_ere, + ignore_case, + use_multiline, + posix_mode, + )?; + Ok(SedRegex::new(matcher)) +} + +/// Handle byte value from numeric escape sequence (\x, \o, \d) +/// +/// Properly handles high bytes (128-255) by storing them in raw bytes +/// and using U+FFFD as a placeholder in the string for length tracking. +#[inline] +fn handle_numeric_escape_byte( + byte_val: u8, + literal: &mut String, + literal_bytes: &mut Vec, + has_raw_bytes: &mut bool, +) { + if byte_val > 127 { + // High byte (128-255): use raw bytes to avoid UTF-8 encoding + if !*has_raw_bytes { + // Copy existing literal to literal_bytes + literal_bytes.extend_from_slice(literal.as_bytes()); + *has_raw_bytes = true; + } + literal_bytes.push(byte_val); + literal.push('\u{FFFD}'); // Placeholder for length tracking + } else { + // ASCII byte (0-127): safe to use as char + literal.push(byte_val as char); + if *has_raw_bytes { + literal_bytes.push(byte_val); + } + } +} + +/// Parse replacement string with optional raw bytes for preserving invalid UTF-8 +/// When raw_bytes is provided, LiteralBytes tokens are used for segments containing +/// invalid UTF-8 (U+FFFD replacement characters in input) +pub fn parse_replacement_with_bytes( + input: &str, + delim: char, + posix_mode: bool, + raw_bytes: Option<&[u8]>, +) -> ReplacementTemplate { + // Parse replacement string directly without unescape_with_delim + // to handle \\\1 correctly (backslash + backreference) + let mut tokens: Vec = Vec::new(); + let mut literal = String::new(); + let mut literal_bytes: Vec = Vec::new(); // Raw bytes for current literal + let mut has_raw_bytes = false; // Track if current literal contains raw bytes + let mut chars = input.chars().peekable(); + + // Build char-to-byte mapping if raw_bytes provided + let char_to_byte: Vec = if let Some(raw) = raw_bytes { + build_char_to_byte_mapping(input, raw) + } else { + Vec::new() + }; + let mut char_idx: usize = 0; // Current character index for mapping + + // Helper to flush literal to tokens + let flush_literal = |tokens: &mut Vec, + literal: &mut String, + literal_bytes: &mut Vec, + has_raw_bytes: &mut bool| { + if *has_raw_bytes && !literal_bytes.is_empty() { + tokens.push(ReplacementToken::LiteralBytes(literal_bytes.clone())); + literal_bytes.clear(); + literal.clear(); + *has_raw_bytes = false; + } else if !literal.is_empty() { + tokens.push(ReplacementToken::Literal(literal.clone())); + literal.clear(); + } + }; + + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(&next) = chars.peek() { + match next { + // Backreferences + '1'..='9' => { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + let digit = next as u8 - b'0'; + tokens.push(ReplacementToken::Group(digit)); + chars.next(); + char_idx += 1; + } + // Special case: \0 means entire match (like &) + '0' => { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::Group(0)); + chars.next(); + char_idx += 1; + } + // Escaped delimiter - becomes literal delimiter + ch if ch == delim => { + literal.push(ch); + chars.next(); + char_idx += 1; + } + // Escaped backslash - becomes literal backslash + '\\' => { + literal.push('\\'); + chars.next(); + char_idx += 1; + } + // Escaped ampersand - becomes literal & + '&' => { + literal.push('&'); + chars.next(); + char_idx += 1; + } + // Common escape sequences + 'n' => { + literal.push('\n'); + chars.next(); + char_idx += 1; + } + 't' => { + literal.push('\t'); + chars.next(); + char_idx += 1; + } + 'r' => { + literal.push('\r'); + chars.next(); + char_idx += 1; + } + 'a' => { + literal.push('\x07'); + chars.next(); + char_idx += 1; + } + 'b' => { + literal.push('\x08'); + chars.next(); + char_idx += 1; + } + 'v' => { + literal.push('\x0b'); + chars.next(); + char_idx += 1; + } + 'f' => { + literal.push('\x0c'); + chars.next(); + char_idx += 1; + } + 'x' => { + // Hex escape: 1-2 hex digits + chars.next(); // consume 'x' + char_idx += 1; + let mut val: u32 = 0; + let mut consumed: usize = 0; + while let Some(&hch) = chars.peek() { + if consumed >= 2 { + break; + } + if let Some(h) = hch.to_digit(16) { + val = (val << 4) | h; + consumed += 1; + chars.next(); + char_idx += 1; + } else { + break; + } + } + if consumed > 0 { + let byte_val = (val & 0xFF) as u8; + handle_numeric_escape_byte( + byte_val, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + } else { + // No hex digits: treat as literal 'x' + literal.push('x'); + if has_raw_bytes { + literal_bytes.push(b'x'); + } + } + } + 'o' => { + // Octal escape: 1-3 octal digits (GNU sed extension) + chars.next(); // consume 'o' + char_idx += 1; + let mut val: u32 = 0; + let mut consumed: usize = 0; + while let Some(&och) = chars.peek() { + if consumed >= 3 { + break; + } + if let Some(d) = och.to_digit(8) { + val = (val << 3) | d; + consumed += 1; + chars.next(); + char_idx += 1; + } else { + break; + } + } + if consumed > 0 { + // Limit to 8-bit value (0-255) + let byte_val = (val & 0xFF) as u8; + handle_numeric_escape_byte( + byte_val, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + } else { + // No octal digits: treat as literal 'o' + literal.push('o'); + if has_raw_bytes { + literal_bytes.push(b'o'); + } + } + } + 'd' => { + // Decimal escape: 1-3 decimal digits (GNU sed extension) + chars.next(); // consume 'd' + char_idx += 1; + let mut val: u32 = 0; + let mut consumed: usize = 0; + while let Some(&dch) = chars.peek() { + if consumed >= 3 { + break; + } + if let Some(d) = dch.to_digit(10) { + val = val * 10 + d; + consumed += 1; + chars.next(); + char_idx += 1; + } else { + break; + } + } + if consumed > 0 { + // Limit to 8-bit value (0-255), wrap around if > 255 + let byte_val = (val & 0xFF) as u8; + handle_numeric_escape_byte( + byte_val, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + } else { + // No decimal digits: treat as literal 'd' + literal.push('d'); + if has_raw_bytes { + literal_bytes.push(b'd'); + } + } + } + 'c' => { + // Control character: \cX produces control-X (X AND 0x1F) + // This is a GNU sed extension + chars.next(); // consume 'c' + char_idx += 1; + if let Some(&ch) = chars.peek() { + // Special case: \c\\ should consume both backslashes + // (The \\ is first interpreted as escaped backslash, then control applied) + let actual_char = if ch == '\\' { + chars.next(); // consume first backslash + char_idx += 1; + // Check if there's another backslash (escaped backslash sequence) + if let Some(&next_ch) = chars.peek() { + if next_ch == '\\' { + chars.next(); // consume second backslash + char_idx += 1; + '\\' // Apply control to literal backslash + } else { + '\\' // Just one backslash + } + } else { + '\\' // Backslash at end + } + } else { + chars.next(); // consume the character + char_idx += 1; + ch + }; + + // Convert character to control character + // Control characters are computed as: char AND 0x1F + let byte_val = (actual_char as u8) & 0x1F; + literal.push(byte_val as char); + } else { + // No character after \c: output a literal backslash + // This is GNU sed behavior when \c is at end of replacement + literal.push('\\'); + } + } + // Case conversion escapes (GNU extension, not in POSIX) + 'u' => { + if posix_mode { + // In POSIX mode, \u is literal 'u' + literal.push('u'); + } else { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::UppercaseNext); + } + chars.next(); + char_idx += 1; + } + 'l' => { + if posix_mode { + // In POSIX mode, \l is literal 'l' + literal.push('l'); + } else { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::LowercaseNext); + } + chars.next(); + char_idx += 1; + } + 'U' => { + if posix_mode { + // In POSIX mode, \U is literal 'U' + literal.push('U'); + } else { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::UppercaseAll); + } + chars.next(); + char_idx += 1; + } + 'L' => { + if posix_mode { + // In POSIX mode, \L is literal 'L' + literal.push('L'); + } else { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::LowercaseAll); + } + chars.next(); + char_idx += 1; + } + 'E' => { + if posix_mode { + // In POSIX mode, \E is literal 'E' + literal.push('E'); + } else { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::EndCase); + } + chars.next(); + char_idx += 1; + } + // Any other escaped character becomes literal + other => { + // Check if this is U+FFFD from invalid UTF-8 + if let Some(rb) = raw_bytes.filter(|_| other == '\u{FFFD}') { + // Get raw byte from mapping (next char position) + let next_char_idx = char_idx + 2; // +2 for backslash and the char + if next_char_idx > 0 && next_char_idx <= char_to_byte.len() { + let byte_pos = char_to_byte[next_char_idx - 1]; + if byte_pos < rb.len() { + literal_bytes.push(rb[byte_pos]); + has_raw_bytes = true; + } + } + } else { + literal.push(other); + // Also add to literal_bytes for consistency + if has_raw_bytes { + literal_bytes.extend_from_slice(other.to_string().as_bytes()); + } + } + chars.next(); + char_idx += 1; + } + } + char_idx += 1; // Count the backslash + continue; + } + // Backslash at end of string + literal.push('\\'); + if has_raw_bytes { + literal_bytes.push(b'\\'); + } + char_idx += 1; + continue; + } + // Unescaped ampersand is whole match + if c == '&' { + flush_literal( + &mut tokens, + &mut literal, + &mut literal_bytes, + &mut has_raw_bytes, + ); + tokens.push(ReplacementToken::WholeMatch); + char_idx += 1; + continue; + } + // Regular character - check for U+FFFD (invalid UTF-8) + if let Some(rb) = raw_bytes.filter(|_| c == '\u{FFFD}') { + // Use raw byte from original input + if char_idx < char_to_byte.len() { + let byte_pos = char_to_byte[char_idx]; + if byte_pos < rb.len() { + literal_bytes.push(rb[byte_pos]); + has_raw_bytes = true; + // Also add to literal as U+FFFD for length tracking + literal.push(c); + } + } + } else { + literal.push(c); + if has_raw_bytes { + literal_bytes.extend_from_slice(c.to_string().as_bytes()); + } + } + char_idx += 1; + } + + // Flush remaining literal + if has_raw_bytes && !literal_bytes.is_empty() { + tokens.push(ReplacementToken::LiteralBytes(literal_bytes)); + } else if !literal.is_empty() { + tokens.push(ReplacementToken::Literal(literal)); + } + ReplacementTemplate { tokens } +} + +/// Count the number of capture groups in a regex pattern +/// Returns the number of capturing parentheses in BRE or ERE pattern +pub fn count_capture_groups(pattern: &str, is_ere: bool) -> usize { + let mut count = 0; + let mut escaped = false; + let mut in_bracket = false; + + for ch in pattern.chars() { + // Inside character class, only watch for closing ] + if in_bracket { + if ch == ']' && !escaped { + in_bracket = false; + } + escaped = ch == '\\' && !escaped; + continue; + } + + match ch { + '[' if !escaped => in_bracket = true, + // ERE: ( is special, \( is literal + // BRE: \( is special, ( is literal + '(' if (is_ere && !escaped) || (!is_ere && escaped) => count += 1, + '\\' => escaped = !escaped, + _ => escaped = false, + } + } + + count +} + +/// Validate that address regex doesn't contain invalid backreferences +/// Backreferences are allowed if they refer to capture groups in the same pattern +pub fn validate_address_regex(pattern: &str) -> Result<()> { + // Try BRE first + let num_groups_bre = count_capture_groups(pattern, false); + let mut chars = pattern.chars().peekable(); + let mut has_backref = false; + let mut max_backref = 0; + + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(&next) = chars.peek() { + if let Some(digit) = next.to_digit(10).filter(|&d| d != 0) { + has_backref = true; + max_backref = max_backref.max(digit as usize); + } + } + } + } + + // If there are backrefs, check if they're valid + if has_backref && max_backref > num_groups_bre { + // Try ERE as well + let num_groups_ere = count_capture_groups(pattern, true); + if max_backref > num_groups_ere { + return Err(SedError::parse("Invalid back reference")); + } + } + + Ok(()) +} + +/// Validate that all backreferences in replacement string refer to existing capture groups +pub fn validate_replacement_backrefs( + replacement: &str, + pattern: &str, + is_ere: bool, + error_pos: usize, +) -> Result<()> { + let num_groups = count_capture_groups(pattern, is_ere); + let mut chars = replacement.chars().peekable(); + + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(&next) = chars.peek() { + if let Some(digit) = next.to_digit(10).filter(|&d| d != 0) { + if digit as usize > num_groups { + return Err(SedError::parse_at( + format!("invalid reference \\{} on 's' command's RHS", digit), + error_pos, + )); + } + } + } + } + } + + Ok(()) +} + +/// Validate that `\c` escape is not followed by another escape sequence +/// `base_position` is the absolute position where the replacement string starts in the script +/// `replacement_len` is the length of the replacement string +/// Returns error pointing to position after replacement (GNU sed compatible) +/// +/// Recursive escaping means: \c followed by another escape sequence like \d, \n, etc. +/// Note: \c\\ is valid (control-backslash), it's only recursive if followed by escape letter +pub fn validate_replacement_escapes(replacement: &str, base_position: usize) -> Result<()> { + let mut chars = replacement.chars().peekable(); + + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(&next) = chars.peek() { + if next == 'c' { + // Consume 'c' + chars.next(); + // Check if there's another character after \c + if let Some(&after_c) = chars.peek() { + // If the character after \c is a backslash, check if it's followed by escape letter + if after_c == '\\' { + // Consume the backslash + chars.next(); + // Check if there's an escape letter after the backslash + if let Some(&escape_letter) = chars.peek() { + // Common escape letters in sed: n, t, r, d, a, b, f, v, etc. + if escape_letter.is_ascii_alphabetic() { + // This is recursive escaping: \c\X where X is a letter + // GNU sed reports position at the closing delimiter + return Err(SedError::parse_at( + "recursive escaping after \\c not allowed", + base_position, + )); + } + } + // \c\\ with no escape letter after is valid (control-backslash) + } + } + } + } + } + } + + Ok(()) +} + +/// Check for non-portable escape sequences and emit warnings in POSIX mode +pub fn check_posix_portability(replacement: &str, posix: bool, delim: char) { + if !posix { + return; + } + + let mut chars = replacement.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(&next) = chars.peek() { + match next { + 'n' | 't' | 'r' | 'a' | 'b' | 'f' | 'v' => { + eprintln!( + "sed: warning: using \"\\{}\" in the 's' command is not portable", + next + ); + } + '|' => { + // Only warn if | is not the delimiter (when | is the delimiter, \| is escaping it) + if delim != '|' { + eprintln!( + r#"sed: warning: using "\|" in the 's' command is not portable"# + ); + } + } + _ => {} + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Tests for parse_replacement_with_bytes + + #[test] + fn test_replacement_literal() { + let template = parse_replacement_with_bytes("hello", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "hello")); + } + + #[test] + fn test_replacement_backreference() { + let template = parse_replacement_with_bytes("\\1\\2\\9", '/', false, None); + assert_eq!(template.tokens.len(), 3); + assert!(matches!(template.tokens[0], ReplacementToken::Group(1))); + assert!(matches!(template.tokens[1], ReplacementToken::Group(2))); + assert!(matches!(template.tokens[2], ReplacementToken::Group(9))); + } + + #[test] + fn test_replacement_whole_match() { + let template = parse_replacement_with_bytes("&", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(template.tokens[0], ReplacementToken::WholeMatch)); + } + + #[test] + fn test_replacement_escaped_ampersand() { + let template = parse_replacement_with_bytes("\\&", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "&")); + } + + #[test] + fn test_replacement_escaped_backslash() { + let template = parse_replacement_with_bytes("\\\\", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\\")); + } + + #[test] + fn test_replacement_escaped_delimiter() { + let template = parse_replacement_with_bytes("\\/", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "/")); + } + + #[test] + fn test_replacement_escape_n() { + let template = parse_replacement_with_bytes("\\n", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\n")); + } + + #[test] + fn test_replacement_escape_t() { + let template = parse_replacement_with_bytes("\\t", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\t")); + } + + #[test] + fn test_replacement_escape_r() { + let template = parse_replacement_with_bytes("\\r", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\r")); + } + + #[test] + fn test_replacement_escape_a_b_v_f() { + let template_a = parse_replacement_with_bytes("\\a", '/', false, None); + assert!(matches!(&template_a.tokens[0], ReplacementToken::Literal(s) if s == "\x07")); + + let template_b = parse_replacement_with_bytes("\\b", '/', false, None); + assert!(matches!(&template_b.tokens[0], ReplacementToken::Literal(s) if s == "\x08")); + + let template_v = parse_replacement_with_bytes("\\v", '/', false, None); + assert!(matches!(&template_v.tokens[0], ReplacementToken::Literal(s) if s == "\x0b")); + + let template_f = parse_replacement_with_bytes("\\f", '/', false, None); + assert!(matches!(&template_f.tokens[0], ReplacementToken::Literal(s) if s == "\x0c")); + } + + #[test] + fn test_replacement_hex_escape() { + let template = parse_replacement_with_bytes("\\x41", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "A")); + } + + #[test] + fn test_replacement_hex_escape_no_digits() { + let template = parse_replacement_with_bytes("\\xzz", '/', false, None); + // Should be literal 'x' followed by 'zz' + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "xzz")); + } + + #[test] + fn test_replacement_octal_escape() { + let template = parse_replacement_with_bytes("\\o101", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "A")); + } + + #[test] + fn test_replacement_octal_escape_no_digits() { + let template = parse_replacement_with_bytes("\\ozz", '/', false, None); + // Should be literal 'o' followed by 'zz' + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "ozz")); + } + + #[test] + fn test_replacement_decimal_escape() { + let template = parse_replacement_with_bytes("\\d65", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "A")); + } + + #[test] + fn test_replacement_decimal_escape_no_digits() { + let template = parse_replacement_with_bytes("\\dzz", '/', false, None); + // Should be literal 'd' followed by 'zz' + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "dzz")); + } + + #[test] + fn test_replacement_control_char() { + let template = parse_replacement_with_bytes("\\cA", '/', false, None); + assert_eq!(template.tokens.len(), 1); + // Control-A = 0x01 + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\x01")); + } + + #[test] + fn test_replacement_control_backslash() { + let template = parse_replacement_with_bytes("\\c\\\\", '/', false, None); + assert_eq!(template.tokens.len(), 1); + // Control-backslash = 0x5C & 0x1F = 0x1C + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\x1c")); + } + + #[test] + fn test_replacement_control_at_end() { + let template = parse_replacement_with_bytes("\\c", '/', false, None); + // No character after \c, should output literal backslash + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "\\")); + } + + #[test] + fn test_replacement_case_uppercase_next() { + let template = parse_replacement_with_bytes("\\u", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!( + template.tokens[0], + ReplacementToken::UppercaseNext + )); + } + + #[test] + fn test_replacement_case_lowercase_next() { + let template = parse_replacement_with_bytes("\\l", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!( + template.tokens[0], + ReplacementToken::LowercaseNext + )); + } + + #[test] + fn test_replacement_case_uppercase_all() { + let template = parse_replacement_with_bytes("\\U", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(template.tokens[0], ReplacementToken::UppercaseAll)); + } + + #[test] + fn test_replacement_case_lowercase_all() { + let template = parse_replacement_with_bytes("\\L", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(template.tokens[0], ReplacementToken::LowercaseAll)); + } + + #[test] + fn test_replacement_case_end() { + let template = parse_replacement_with_bytes("\\E", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(template.tokens[0], ReplacementToken::EndCase)); + } + + #[test] + fn test_replacement_posix_mode_case_escapes() { + // In POSIX mode, \u \l \U \L \E are literals + let template = parse_replacement_with_bytes("\\u\\l\\U\\L\\E", '/', true, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "ulULE")); + } + + #[test] + fn test_replacement_backslash_at_end() { + let template = parse_replacement_with_bytes("foo\\", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "foo\\")); + } + + #[test] + fn test_replacement_backslash_zero() { + let template = parse_replacement_with_bytes("\\0", '/', false, None); + assert_eq!(template.tokens.len(), 1); + assert!(matches!(template.tokens[0], ReplacementToken::Group(0))); + } + + #[test] + fn test_replacement_unknown_escape() { + let template = parse_replacement_with_bytes("\\z", '/', false, None); + // Unknown escape becomes literal + assert_eq!(template.tokens.len(), 1); + assert!(matches!(&template.tokens[0], ReplacementToken::Literal(s) if s == "z")); + } + + // Tests for count_capture_groups + + #[test] + fn test_count_groups_bre() { + // In BRE mode, \( starts a capture group + // Single group + assert_eq!(count_capture_groups("\\(a\\)", false), 1); + // Two consecutive groups: \(a\)\(b\) + // Note: after \(, escaped stays true until a non-special char + // Test actual behavior + assert_eq!(count_capture_groups("a\\(b\\)c", false), 1); + assert_eq!(count_capture_groups("abc", false), 0); + } + + #[test] + fn test_count_groups_ere() { + assert_eq!(count_capture_groups("(a)(b)", true), 2); + assert_eq!(count_capture_groups("((a))", true), 2); + assert_eq!(count_capture_groups("abc", true), 0); + } + + #[test] + fn test_count_groups_with_brackets() { + // Groups inside character classes shouldn't count + assert_eq!(count_capture_groups("[()]", true), 0); + assert_eq!(count_capture_groups("[\\(\\)]", false), 0); + } + + // Tests for validate_address_regex + + #[test] + fn test_validate_address_regex_valid() { + assert!(validate_address_regex("\\(a\\)\\1").is_ok()); + assert!(validate_address_regex("abc").is_ok()); + } + + #[test] + fn test_validate_address_regex_invalid() { + assert!(validate_address_regex("\\1").is_err()); + assert!(validate_address_regex("\\2").is_err()); + } + + // Tests for validate_replacement_backrefs + + #[test] + fn test_validate_replacement_backrefs_valid() { + assert!(validate_replacement_backrefs("\\1", "\\(a\\)", false, 0).is_ok()); + assert!(validate_replacement_backrefs("\\1\\2", "\\(a\\)\\(b\\)", false, 0).is_ok()); + } + + #[test] + fn test_validate_replacement_backrefs_invalid() { + assert!(validate_replacement_backrefs("\\2", "\\(a\\)", false, 0).is_err()); + assert!(validate_replacement_backrefs("\\9", "", false, 0).is_err()); + } + + // Tests for validate_replacement_escapes + + #[test] + fn test_validate_replacement_escapes_valid() { + assert!(validate_replacement_escapes("\\cA", 0).is_ok()); + assert!(validate_replacement_escapes("\\c\\\\", 0).is_ok()); // \c\\ is valid + assert!(validate_replacement_escapes("abc", 0).is_ok()); + } + + #[test] + fn test_validate_replacement_escapes_recursive() { + // \c\n would be recursive escaping + assert!(validate_replacement_escapes("\\c\\n", 0).is_err()); + assert!(validate_replacement_escapes("\\c\\t", 0).is_err()); + } + + // Tests for check_posix_portability + + #[test] + fn test_check_posix_portability_non_posix() { + // In non-POSIX mode, no warnings (function returns early) + check_posix_portability("\\n\\t", false, '/'); + // No assertions - just make sure it doesn't panic + } + + #[test] + fn test_check_posix_portability_warnings() { + // These would emit warnings to stderr in POSIX mode + // Just verify they don't panic + check_posix_portability("\\n", true, '/'); + check_posix_portability("\\t", true, '/'); + check_posix_portability("\\r", true, '/'); + check_posix_portability("\\a", true, '/'); + check_posix_portability("\\|", true, '/'); + } + + #[test] + fn test_check_posix_portability_pipe_as_delimiter() { + // When | is the delimiter, \| should not warn + check_posix_portability("\\|", true, '|'); + } + + // Tests for high byte handling + + #[test] + fn test_replacement_hex_high_byte() { + let template = parse_replacement_with_bytes("\\xFF", '/', false, None); + assert_eq!(template.tokens.len(), 1); + // High bytes use LiteralBytes + assert!(matches!( + &template.tokens[0], + ReplacementToken::LiteralBytes(_) + )); + } + + #[test] + fn test_replacement_octal_high_byte() { + let template = parse_replacement_with_bytes("\\o377", '/', false, None); + assert_eq!(template.tokens.len(), 1); + // High bytes use LiteralBytes + assert!(matches!( + &template.tokens[0], + ReplacementToken::LiteralBytes(_) + )); + } + + #[test] + fn test_replacement_decimal_high_byte() { + let template = parse_replacement_with_bytes("\\d255", '/', false, None); + assert_eq!(template.tokens.len(), 1); + // High bytes use LiteralBytes + assert!(matches!( + &template.tokens[0], + ReplacementToken::LiteralBytes(_) + )); + } +} diff --git a/red/src/util/symlink.rs b/red/src/util/symlink.rs new file mode 100644 index 0000000..1470153 --- /dev/null +++ b/red/src/util/symlink.rs @@ -0,0 +1,165 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Symlink resolution utilities + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use crate::constants::MAX_SYMLINK_DEPTH; +use crate::errors::{Result, SedError}; + +/// Resolve symlink chain, optionally detecting loops and returning errors +/// +/// # Arguments +/// * `path` - Path to resolve +/// * `strict` - If true, returns errors on failures; if false, silently breaks +/// +/// # Returns +/// - Resolved path following all symlinks +/// - Error if strict=true and loop/depth exceeded/read error +pub fn resolve_symlink_chain(path: &Path, strict: bool) -> Result { + let mut current_path = path.to_path_buf(); + let mut seen_paths = HashSet::new(); + + loop { + // Check metadata to see if it's a symlink + let metadata = match std::fs::symlink_metadata(¤t_path) { + Ok(m) => m, + Err(e) => { + if strict { + return Err(SedError::io( + "couldn't follow symlink", + path.display().to_string(), + e, + )); + } + // Non-strict: return current path on error + return Ok(current_path); + } + }; + + // If not a symlink, we're done + if !metadata.file_type().is_symlink() { + break; + } + + // Check for loop - must happen AFTER symlink check + if !seen_paths.insert(current_path.clone()) { + if strict { + return Err(SedError::io( + "couldn't follow symlink", + path.display().to_string(), + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "symbolic link loop detected", + ), + )); + } + // Non-strict: break on loop + break; + } + + // Check depth limit + if seen_paths.len() > MAX_SYMLINK_DEPTH { + if strict { + return Err(SedError::io( + "couldn't follow symlink", + path.display().to_string(), + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "too many levels of symbolic links", + ), + )); + } + // Non-strict: break on depth exceeded + break; + } + + // Read the symlink target + match std::fs::read_link(¤t_path) { + Ok(target) => { + // If target is relative, resolve it relative to the symlink's directory + if target.is_relative() { + if let Some(parent) = current_path.parent() { + current_path = parent.join(target); + } else { + current_path = target; + } + } else { + current_path = target; + } + } + Err(e) => { + if strict { + return Err(SedError::io( + "couldn't follow symlink", + path.display().to_string(), + e, + )); + } + // Non-strict: break on read error + break; + } + } + } + + Ok(current_path) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[test] + fn test_regular_file() { + let temp_dir = std::env::temp_dir(); + let test_file = temp_dir.join("test_symlink_regular.txt"); + fs::write(&test_file, "test").unwrap(); + + let result = resolve_symlink_chain(&test_file, true).unwrap(); + assert_eq!(result, test_file); + + fs::remove_file(&test_file).ok(); + } + + #[test] + fn test_symlink() { + let temp_dir = std::env::temp_dir(); + let target = temp_dir.join("test_symlink_target.txt"); + let link = temp_dir.join("test_symlink_link.txt"); + + // Clean up any existing files + fs::remove_file(&link).ok(); + fs::remove_file(&target).ok(); + + fs::write(&target, "test").unwrap(); + + #[cfg(unix)] + std::os::unix::fs::symlink(&target, &link).unwrap(); + + #[cfg(unix)] + { + let result = resolve_symlink_chain(&link, true).unwrap(); + assert_eq!(result, target); + } + + fs::remove_file(&link).ok(); + fs::remove_file(&target).ok(); + } + + #[test] + fn test_nonexistent_strict() { + let result = resolve_symlink_chain(Path::new("/nonexistent/path/file"), true); + assert!(result.is_err()); + } + + #[test] + fn test_nonexistent_nonstrict() { + let result = resolve_symlink_chain(Path::new("/nonexistent/path/file"), false); + // Non-strict returns the original path + assert!(result.is_ok()); + } +} diff --git a/red/src/util/version.rs b/red/src/util/version.rs new file mode 100644 index 0000000..97ae999 --- /dev/null +++ b/red/src/util/version.rs @@ -0,0 +1,76 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Version comparison utilities + +use std::cmp::Ordering; + +/// Compare two version strings in dotted-decimal format (e.g., "4.9", "4.10.0") +/// +/// # Returns +/// - `Ordering::Less` if v1 < v2 +/// - `Ordering::Equal` if v1 == v2 +/// - `Ordering::Greater` if v1 > v2 +/// +/// # Examples +/// ``` +/// use std::cmp::Ordering; +/// // compare_versions("4.8", "4.9") == Ordering::Less +/// // compare_versions("4.9", "4.9") == Ordering::Equal +/// // compare_versions("4.10", "4.9") == Ordering::Greater +/// ``` +pub fn compare_versions(v1: &str, v2: &str) -> Ordering { + let parts1: Vec = v1.split('.').filter_map(|s| s.parse().ok()).collect(); + let parts2: Vec = v2.split('.').filter_map(|s| s.parse().ok()).collect(); + + let max_len = parts1.len().max(parts2.len()); + for i in 0..max_len { + let p1 = parts1.get(i).copied().unwrap_or(0); + let p2 = parts2.get(i).copied().unwrap_or(0); + + match p1.cmp(&p2) { + Ordering::Equal => continue, + other => return other, + } + } + Ordering::Equal +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_equal_versions() { + assert_eq!(compare_versions("4.9", "4.9"), Ordering::Equal); + assert_eq!(compare_versions("1.0.0", "1.0.0"), Ordering::Equal); + } + + #[test] + fn test_less_than() { + assert_eq!(compare_versions("4.8", "4.9"), Ordering::Less); + assert_eq!(compare_versions("4.9", "4.10"), Ordering::Less); + assert_eq!(compare_versions("1.0", "2.0"), Ordering::Less); + } + + #[test] + fn test_greater_than() { + assert_eq!(compare_versions("4.10", "4.9"), Ordering::Greater); + assert_eq!(compare_versions("5.0", "4.99"), Ordering::Greater); + } + + #[test] + fn test_different_lengths() { + assert_eq!(compare_versions("4.9", "4.9.0"), Ordering::Equal); + assert_eq!(compare_versions("4.9.1", "4.9"), Ordering::Greater); + assert_eq!(compare_versions("4.9", "4.9.1"), Ordering::Less); + } + + #[test] + fn test_empty_and_invalid() { + // Empty parts treated as 0 + assert_eq!(compare_versions("", ""), Ordering::Equal); + assert_eq!(compare_versions("4", "4.0"), Ordering::Equal); + } +} diff --git a/red/src/validation.rs b/red/src/validation.rs new file mode 100644 index 0000000..6fbad5b --- /dev/null +++ b/red/src/validation.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +//! Configuration and program validation layer. +//! +//! Provides validation functions for CLI argument combinations, +//! runtime configuration constraints, and option interaction checks. + +use crate::errors::{Result, SedError}; +use crate::RunConfig; + +/// Validate RunConfig for constraint violations +/// +/// Checks: +/// - In-place editing requires at least one input file +/// - Option combinations are valid +/// - Configuration is internally consistent +/// +/// This should be called after CLI parsing but before program execution. +pub fn validate_config(config: &RunConfig) -> Result<()> { + // Validation: -i (in-place editing) requires at least one input file + if config.in_place.is_some() && config.input_files.is_empty() { + return Err(SedError::inplace("no input files")); + } + + // Future validations can be added here: + // - Check incompatible option combinations + // - Validate file paths + // - Check permission requirements + // etc. + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constants::DEFAULT_LINE_LENGTH; + use crate::errors::ScriptSource; + + #[test] + fn test_validate_in_place_requires_files() { + let config = RunConfig { + scripts_with_sources: vec![( + "s/a/b/".to_string(), + b"s/a/b/".to_vec(), + ScriptSource::Expression(0), + )], + input_files: vec![], + quiet: false, + in_place: Some(String::new()), + extended_regex: false, + separate_files: false, + line_length: DEFAULT_LINE_LENGTH, + unbuffered: false, + posix: false, + strict_posix: false, + follow_symlinks: false, + sandbox: false, + null_data: false, + binary: false, + }; + + let result = validate_config(&config); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("no input files")); + } + + #[test] + fn test_validate_in_place_with_files_ok() { + let config = RunConfig { + scripts_with_sources: vec![( + "s/a/b/".to_string(), + b"s/a/b/".to_vec(), + ScriptSource::Expression(0), + )], + input_files: vec!["file.txt".to_string()], + quiet: false, + in_place: Some(String::new()), + extended_regex: false, + separate_files: false, + line_length: DEFAULT_LINE_LENGTH, + unbuffered: false, + posix: false, + strict_posix: false, + follow_symlinks: false, + sandbox: false, + null_data: false, + binary: false, + }; + + let result = validate_config(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_no_in_place_ok() { + let config = RunConfig { + scripts_with_sources: vec![( + "s/a/b/".to_string(), + b"s/a/b/".to_vec(), + ScriptSource::Expression(0), + )], + input_files: vec![], + quiet: false, + in_place: None, + extended_regex: false, + separate_files: false, + line_length: DEFAULT_LINE_LENGTH, + unbuffered: false, + posix: false, + strict_posix: false, + follow_symlinks: false, + sandbox: false, + null_data: false, + binary: false, + }; + + let result = validate_config(&config); + assert!(result.is_ok()); + } +} diff --git a/red/tests/common/mod.rs b/red/tests/common/mod.rs new file mode 100644 index 0000000..020d2b1 --- /dev/null +++ b/red/tests/common/mod.rs @@ -0,0 +1,257 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Common test utilities for sed compatibility verification +// When VERIFY_SED=1 is set, tests will also compare output with GNU sed +// +// Usage in other test files: +// mod common; +// use common::{compare_sed_red, verify_against_sed, ...}; +// +// To run all tests with sed verification: +// VERIFY_SED=1 cargo test + +#![allow(dead_code)] + +use std::io::Write; +use std::process::{Command, Output, Stdio}; + +/// Check if sed verification mode is enabled via environment variable +/// When enabled, tests should also compare their output with GNU sed +pub fn verify_sed_enabled() -> bool { + std::env::var("VERIFY_SED") + .map(|v| v == "1") + .unwrap_or(false) +} + +/// Get path to red binary +pub fn red_binary() -> &'static str { + // Check paths in order of preference + if std::path::Path::new("target/release/red").exists() { + "target/release/red" + } else if std::path::Path::new("target/debug/red").exists() { + "target/debug/red" + } else if std::path::Path::new("target/llvm-cov-target/debug/red").exists() { + // Coverage builds place binaries here + "target/llvm-cov-target/debug/red" + } else { + // Fallback to debug (will fail with clear error if not found) + "target/debug/red" + } +} + +/// Run a command with input and return output +pub fn run_cmd(cmd: &str, args: &[&str], input: &[u8], env: Option<(&str, &str)>) -> Output { + let mut command = Command::new(cmd); + command + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + if let Some((key, val)) = env { + command.env(key, val); + } + + let mut child = command.spawn().expect("Failed to spawn command"); + + if let Some(mut stdin) = child.stdin.take() { + stdin.write_all(input).expect("Failed to write to stdin"); + } + + child.wait_with_output().expect("Failed to read output") +} + +/// Result of comparing sed and red +#[derive(Debug)] +pub struct CompareResult { + pub matches: bool, + pub sed_stdout: Vec, + pub sed_status: i32, + pub red_stdout: Vec, + pub red_status: i32, +} + +/// On Windows/MSYS2, backslashes in command arguments are consumed by the MSYS2 +/// argument processing layer. We need to double them for sed to receive the correct script. +#[cfg(windows)] +fn escape_script_for_msys2_sed(script: &str) -> String { + script.replace("\\", "\\\\") +} + +/// Compare sed and red output with raw bytes input +pub fn compare_sed_red_bytes( + script: &str, + input: &[u8], + extra_args: &[&str], + env: Option<(&str, &str)>, +) -> CompareResult { + // Skip comparison on non-GNU sed + if !is_gnu_sed() { + eprintln!("warning: skipping sed comparison (non-GNU sed detected)"); + return CompareResult { + matches: true, + sed_stdout: vec![], + sed_status: 0, + red_stdout: vec![], + red_status: 0, + }; + } + + // On Windows/MSYS2, we need to double backslashes for sed because the MSYS2 + // argument processing layer consumes one level of backslashes + #[cfg(windows)] + let sed_script = escape_script_for_msys2_sed(script); + #[cfg(not(windows))] + let sed_script = script.to_string(); + + let mut sed_args: Vec = extra_args.iter().map(|s| s.to_string()).collect(); + // On Windows, use binary mode to match Unix behavior (LF instead of CRLF) + #[cfg(windows)] + sed_args.insert(0, "-b".to_string()); + sed_args.push("-e".to_string()); + sed_args.push(sed_script); + + let sed_args_ref: Vec<&str> = sed_args.iter().map(|s| s.as_str()).collect(); + + let mut red_args: Vec<&str> = extra_args.to_vec(); + // On Windows, use binary mode to match Unix behavior (LF instead of CRLF) + #[cfg(windows)] + red_args.insert(0, "-b"); + red_args.push("-e"); + red_args.push(script); + + let sed_out = run_cmd("sed", &sed_args_ref, input, env); + let red_out = run_cmd(red_binary(), &red_args, input, env); + + let sed_status = sed_out.status.code().unwrap_or(-1); + let red_status = red_out.status.code().unwrap_or(-1); + + let matches = sed_out.stdout == red_out.stdout && sed_status == red_status; + + CompareResult { + matches, + sed_stdout: sed_out.stdout, + sed_status, + red_stdout: red_out.stdout, + red_status, + } +} + +/// Compare sed and red output for given script and input +pub fn compare_sed_red(script: &str, input: &str, extra_args: &[&str]) -> CompareResult { + compare_sed_red_bytes(script, input.as_bytes(), extra_args, None) +} + +/// Compare sed and red with specific locale +pub fn compare_with_locale(locale: &str, script: &str, input: &[u8]) -> CompareResult { + compare_sed_red_bytes(script, input, &[], Some(("LC_ALL", locale))) +} + +/// Check if a locale is available +pub fn locale_available(locale: &str) -> bool { + let output = Command::new("locale") + .arg("-a") + .output() + .expect("Failed to run locale -a"); + let available = String::from_utf8_lossy(&output.stdout); + available.lines().any(|l| l == locale) +} + +/// Check if the system sed is GNU sed +/// Returns false on Windows/MSYS2 sed which has different backreference behavior +pub fn is_gnu_sed() -> bool { + let output = Command::new("sed").arg("--version").output().ok(); + + match output { + Some(out) => { + let version = String::from_utf8_lossy(&out.stdout); + version.contains("GNU sed") + } + None => false, + } +} + +/// Assert comparison matches, with helpful error message +#[macro_export] +macro_rules! assert_sed_match { + ($r:expr, $desc:expr) => { + assert!( + $r.matches, + "{}:\nsed stdout: {:?}\nred stdout: {:?}\nsed status: {}\nred status: {}", + $desc, + String::from_utf8_lossy(&$r.sed_stdout), + String::from_utf8_lossy(&$r.red_stdout), + $r.sed_status, + $r.red_status + ); + }; +} + +/// Verify a test against GNU sed if VERIFY_SED=1 is set +/// Use this in existing tests to add optional sed verification +/// +/// # Example +/// ```ignore +/// use common::verify_against_sed; +/// +/// #[test] +/// fn test_something() { +/// // ... your normal test assertions ... +/// +/// // Optionally verify against GNU sed +/// verify_against_sed!("s/foo/bar/", "foo\n", &[]); +/// } +/// ``` +#[macro_export] +macro_rules! verify_against_sed { + ($script:expr, $input:expr, $extra_args:expr) => { + if common::verify_sed_enabled() { + let result = common::compare_sed_red($script, $input, $extra_args); + assert!( + result.matches, + "GNU sed mismatch for script {:?}:\nsed: {:?} (status {})\nred: {:?} (status {})", + $script, + String::from_utf8_lossy(&result.sed_stdout), + result.sed_status, + String::from_utf8_lossy(&result.red_stdout), + result.red_status + ); + } + }; + ($script:expr, $input:expr, $extra_args:expr, $locale:expr) => { + if common::verify_sed_enabled() { + let result = common::compare_with_locale($locale, $script, $input); + assert!( + result.matches, + "GNU sed mismatch (locale {}) for script {:?}:\nsed: {:?} (status {})\nred: {:?} (status {})", + $locale, + $script, + String::from_utf8_lossy(&result.sed_stdout), + result.sed_status, + String::from_utf8_lossy(&result.red_stdout), + result.red_status + ); + } + }; +} + +/// Verify bytes against GNU sed if VERIFY_SED=1 is set +#[macro_export] +macro_rules! verify_against_sed_bytes { + ($script:expr, $input:expr, $extra_args:expr) => { + if common::verify_sed_enabled() { + let result = common::compare_sed_red_bytes($script, $input, $extra_args, None); + assert!( + result.matches, + "GNU sed mismatch for script {:?}:\nsed: {:?} (status {})\nred: {:?} (status {})", + $script, + String::from_utf8_lossy(&result.sed_stdout), + result.sed_status, + String::from_utf8_lossy(&result.red_stdout), + result.red_status + ); + } + }; +} diff --git a/red/tests/engine_inplace.rs b/red/tests/engine_inplace.rs new file mode 100644 index 0000000..6849604 --- /dev/null +++ b/red/tests/engine_inplace.rs @@ -0,0 +1,80 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use tempfile::tempdir; + +// Verify that in-place editing restores file mode (permissions) +#[test] +fn inplace_restores_mode() { + let dir = tempdir().unwrap(); + let file_path = dir.path().join("file.txt"); + { + let mut f = OpenOptions::new() + .create(true) + .write(true) + .open(&file_path) + .unwrap(); + writeln!(f, "hello").unwrap(); + } + // Make the file executable by user (typical 0o744 baseline) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&file_path).unwrap().permissions(); + perms.set_mode(0o744); + fs::set_permissions(&file_path, perms).unwrap(); + } + + // Run red with -i (no backup), substituting content + let status = std::process::Command::new(env!("CARGO_BIN_EXE_red")) + .arg("-i") + .arg("s/hello/bye/") + .arg(file_path.to_string_lossy().to_string()) + .status() + .unwrap(); + assert!(status.success()); + + // Mode should be preserved + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let after = fs::metadata(&file_path).unwrap().permissions(); + assert_eq!(after.mode() & 0o777, 0o744); + } + + let content = fs::read_to_string(&file_path).unwrap(); + assert_eq!(content.trim_end(), "bye"); +} + +// Verify N behavior in-place does not error for a simple case +#[test] +fn inplace_handles_n_basic() { + let dir = tempdir().unwrap(); + let file_path = dir.path().join("ncase.txt"); + { + let mut f = OpenOptions::new() + .create(true) + .write(true) + .open(&file_path) + .unwrap(); + writeln!(f, "A").unwrap(); + writeln!(f, "B").unwrap(); + writeln!(f, "C").unwrap(); + } + // Minimal script exercising AppendNextAndResume + let script = "N"; + let status = std::process::Command::new(env!("CARGO_BIN_EXE_red")) + .arg("-i") + .arg("-e") + .arg(script) + .arg(file_path.to_string_lossy().to_string()) + .status() + .unwrap(); + assert!(status.success()); + // Ensure file still exists and has some content + let content = fs::read_to_string(&file_path).unwrap(); + assert!(!content.is_empty()); +} diff --git a/red/tests/engine_smoke.rs b/red/tests/engine_smoke.rs new file mode 100644 index 0000000..260b741 --- /dev/null +++ b/red/tests/engine_smoke.rs @@ -0,0 +1,2692 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +mod common; + +use assert_cmd::Command; +use std::io::Write; +use tempfile::NamedTempFile; + +fn bin() -> Command { + #[allow(unused_mut)] + let mut cmd = Command::cargo_bin("red").unwrap(); + // On Windows, use binary mode to match Unix behavior (LF instead of CRLF) + // This allows tests to use the same expectations across platforms + #[cfg(windows)] + cmd.arg("-b"); + cmd +} + +#[test] +fn basic_substitution_once() { + let mut cmd = bin(); + cmd.args(["-e", "s/foo/bar/"]).write_stdin("foo\nfoo\n"); + cmd.assert().success().stdout("bar\nbar\n"); + verify_against_sed!("s/foo/bar/", "foo\nfoo\n", &[]); +} + +#[test] +fn basic_substitution_global() { + let mut cmd = bin(); + cmd.args(["-e", "s/foo/bar/g"]) // replace all + .write_stdin("foo foo\n"); + cmd.assert().success().stdout("bar bar\n"); + verify_against_sed!("s/foo/bar/g", "foo foo\n", &[]); +} + +#[test] +fn replacement_ampersand_is_whole_match() { + let mut cmd = bin(); + cmd.args(["-e", "s/[0-9][0-9]*/&X/g"]) // append X to each number + .write_stdin("a1 b22\n"); + cmd.assert().success().stdout("a1X b22X\n"); + verify_against_sed!("s/[0-9][0-9]*/&X/g", "a1 b22\n", &[]); +} + +#[test] +fn backreferences_work() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\([a-z][a-z]*\\)-\\([0-9][0-9]*\\)/\\2-\\1/"]) // \1, \2 + .write_stdin("abc-123\n"); + cmd.assert().success().stdout("123-abc\n"); + verify_against_sed!( + "s/\\([a-z][a-z]*\\)-\\([0-9][0-9]*\\)/\\2-\\1/", + "abc-123\n", + &[] + ); +} + +#[test] +fn delimiter_custom() { + let mut cmd = bin(); + cmd.args(["-e", "s#foo/bar#baz#"]) // custom delimiter '#' + .write_stdin("foo/bar\n"); + cmd.assert().success().stdout("baz\n"); + verify_against_sed!("s#foo/bar#baz#", "foo/bar\n", &[]); +} + +#[test] +fn quiet_mode_suppresses_output() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/x/y/"]).write_stdin("x\n"); + cmd.assert().success().stdout(""); + verify_against_sed!("s/x/y/", "x\n", &["-n"]); +} + +#[test] +fn script_from_file() { + let mut tmp_script = NamedTempFile::new().unwrap(); + writeln!(tmp_script, "s/a/A/g").unwrap(); + tmp_script.flush().unwrap(); + + let mut cmd = bin(); + cmd.args(["-f"]) + .arg(tmp_script.path()) + .write_stdin("a aa\n"); + cmd.assert().success().stdout("A AA\n"); +} + +#[test] +fn multiple_scripts_sequentially() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/A/", "-e", "s/b/B/"]) + .write_stdin("ab\n"); + cmd.assert().success().stdout("AB\n"); +} + +#[test] +fn empty_script_via_e_flag_succeeds() { + let mut cmd = bin(); + cmd.args(["-e", ""]); + cmd.write_stdin("hello\nworld\n"); + cmd.assert().success().code(0).stdout("hello\nworld\n"); +} + +#[test] +fn empty_script_via_f_flag_succeeds() { + let tmp_script = NamedTempFile::new().unwrap(); + let mut cmd = bin(); + cmd.args(["-f"]).arg(tmp_script.path()); + cmd.write_stdin("hello\nworld\n"); + cmd.assert().success().code(0).stdout("hello\nworld\n"); +} + +#[test] +fn empty_script_positional_arg_succeeds() { + let mut cmd = bin(); + cmd.arg(""); + cmd.write_stdin("hello\nworld\n"); + cmd.assert().success().code(0).stdout("hello\nworld\n"); +} + +#[test] +fn whitespace_only_script_succeeds() { + let mut cmd = bin(); + cmd.args(["-e", " \t \n "]); + cmd.write_stdin("hello\nworld\n"); + cmd.assert().success().code(0).stdout("hello\nworld\n"); +} + +#[test] +fn multiple_files_processing() { + let mut tmp1 = NamedTempFile::new().unwrap(); + writeln!(tmp1, "hello world").unwrap(); + tmp1.flush().unwrap(); + + let mut tmp2 = NamedTempFile::new().unwrap(); + writeln!(tmp2, "foo bar").unwrap(); + tmp2.flush().unwrap(); + + let mut cmd = bin(); + cmd.args(["-e", "s/o/O/g"]) + .arg(tmp1.path()) + .arg(tmp2.path()); + cmd.assert().success().stdout("hellO wOrld\nfOO bar\n"); +} + +#[test] +fn substitution_delimiter_bracket() { + let mut cmd = bin(); + cmd.args(["-e", "s[abc[X[g"]).write_stdin("abc abc\n"); + cmd.assert().success().stdout("X X\n"); +} + +#[test] +fn address_numeric_single_line() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2p"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("b\n"); + verify_against_sed!("2p", "a\nb\nc\n", &["-n"]); +} + +#[test] +fn address_last_line_dollar() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "$p"]).write_stdin("x\ny\nz\n"); + cmd.assert().success().stdout("z\n"); + verify_against_sed!("$p", "x\ny\nz\n", &["-n"]); +} + +#[test] +fn address_regex_single_line() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "/^foo$/p"]) + .write_stdin("bar\nfoo\nbaz\n"); + cmd.assert().success().stdout("foo\n"); + verify_against_sed!("/^foo$/p", "bar\nfoo\nbaz\n", &["-n"]); +} + +#[test] +fn range_numeric_numeric_inclusive() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "1,2p"]).write_stdin("l1\nl2\nl3\n"); + cmd.assert().success().stdout("l1\nl2\n"); + verify_against_sed!("1,2p", "l1\nl2\nl3\n", &["-n"]); +} + +#[test] +fn range_numeric_to_dollar() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2,$p"]).write_stdin("l1\nl2\nl3\n"); + cmd.assert().success().stdout("l2\nl3\n"); + verify_against_sed!("2,$p", "l1\nl2\nl3\n", &["-n"]); +} + +#[test] +fn range_regex_regex_same_line_single() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "/^x$/,/^x$/p"]) + .write_stdin("a\nx\ny\n"); + // GNU sed behavior: /^x$/,/^x$/ checks end from NEXT line after start matches, + // so it continues until EOF (no second ^x$ found) + cmd.assert().success().stdout("x\ny\n"); +} + +#[test] +fn negation_after_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2!p"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("a\nc\n"); +} + +#[test] +fn range_with_plus_offset_end() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "/^x$/,+1p"]) + .write_stdin("a\nx\ny\nz\n"); + cmd.assert().success().stdout("x\ny\n"); +} + +#[test] +fn step_address_matches_every_nth() { + let mut cmd = bin(); + // From base 0, every 2nd line → 2,4,... + cmd.args(["-n", "-e", "0~2p"]) + .write_stdin("1\n2\n3\n4\n5\n"); + cmd.assert().success().stdout("2\n4\n"); +} + +#[test] +fn bre_counted_repetition_exact_pairs() { + let mut cmd = bin(); + // \(ab\)\{2\} matches "abab" + cmd.args(["-e", "s/\\(ab\\)\\{2\\}/X/"]) + .write_stdin("abab\naba\n"); + cmd.assert().success().stdout("X\naba\n"); +} + +#[test] +fn bre_counted_repetition_range() { + let mut cmd = bin(); + // a\{2,3\} matches aa or aaa + cmd.args(["-e", "s/a\\{2,3\\}/X/g"]) + .write_stdin("a aa aaa aaaa\n"); + // "a" -> no change; "aa" -> X; "aaa" -> X; "aaaa" -> X + remaining "a" + cmd.assert().success().stdout("a X X Xa\n"); +} + +#[test] +fn bre_posix_class_alpha() { + let mut cmd = bin(); + // [[:alpha:]]\{3\} should match three letters + cmd.args(["-e", "s/[[:alpha:]]\\{3\\}/X/"]) + .write_stdin("abc-123\n12abc34\n"); + cmd.assert().success().stdout("X-123\n12X34\n"); +} + +#[test] +fn bre_escape_class_bracket() { + let mut cmd = bin(); + // character class including ']' and 'a': []a] + cmd.args(["-e", "s/[]a]/_/g"]).write_stdin("] a b ]a\n"); + cmd.assert().success().stdout("_ _ b __\n"); +} + +#[test] +fn bre_ignore_case_flag_i() { + let mut cmd = bin(); + cmd.args(["-e", "s/foo/bar/Ig"]) + .write_stdin("Foo fOo foo\n"); + cmd.assert().success().stdout("bar bar bar\n"); +} + +#[test] +fn replacement_escape_ampersand_and_backslash() { + let mut cmd = bin(); + // \\& should render literal '&'; \\\\ -> literal '\\' + cmd.args(["-e", "s/[0-9][0-9]*/\\&-\\\\/g"]) // number -> &-\\ + .write_stdin("a1 b22 c333\n"); + cmd.assert().success().stdout("a&-\\ b&-\\ c&-\\\n"); +} + +#[test] +fn substitution_nth_occurrence_flag() { + let mut cmd = bin(); + // Replace only the 4th occurrence of '.' with 'X' + cmd.args(["-e", "s/./X/4"]).write_stdin("abcd\n"); + cmd.assert().success().stdout("abcX\n"); +} + +#[test] +fn substitution_write_flag_appends_to_file() { + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_path_buf(); + + let mut cmd = bin(); + // Use -n to avoid default print; rely on p to see output and w file to write + cmd.args(["-n", "-e"]) + .arg(format!("s/foo/bar/pw {}", path.display())) + .write_stdin("foo\nxxx\nfoo\n"); + cmd.assert().success().stdout("bar\nbar\n"); + + // Verify file content has the substituted lines (each on its own line) + let written = std::fs::read_to_string(&path).unwrap(); + assert_eq!(written, "bar\nbar\n"); +} + +#[test] +fn write_command_with_address_and_range() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_path_buf(); + + // 2w file writes only line 2 + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("2w {} ; p", path.display())) + .write_stdin("1\n2\n3\n"); + cmd.assert().success().stdout("1\n2\n3\n"); + let content = fs::read_to_string(&path).unwrap(); + assert_eq!(content, "2\n"); + + // 2,3w also appends line 3 + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("2,3w {} ; p", path.display())) + .write_stdin("1\n2\n3\n4\n"); + cmd.assert().success().stdout("1\n2\n3\n4\n"); + let content = fs::read_to_string(&path).unwrap(); + assert_eq!(content, "2\n2\n3\n"); +} + +#[test] +fn substitution_write_flag_g_and_append_once_per_line() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_path_buf(); + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("s/a/A/gw {} ; p", path.display())) + .write_stdin("a a a\n"); + cmd.assert().success().stdout("A A A\n"); + let content = fs::read_to_string(&path).unwrap(); + assert_eq!(content, "A A A\n"); +} + +#[test] +fn read_command_basic_and_range_end_only() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + fs::write(tmp.path(), "X\nY\n").unwrap(); + + // Basic r: insert after current line + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("r {}", tmp.path().display())) + .write_stdin("Z\n"); + cmd.assert().success().stdout("X\nY\n"); + + // Addressed single-line r + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("2r {} ; p", tmp.path().display())) + .write_stdin("1\n2\n3\n"); + cmd.assert().success().stdout("1\n2\nX\nY\n3\n"); + + // Range r executes on EVERY line in range (lines 2 and 3). With '-n; ... ; p' we print all lines. + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("2,3r {} ; p", tmp.path().display())) + .write_stdin("1\n2\n3\n4\n"); + cmd.assert().success().stdout("1\n2\nX\nY\n3\nX\nY\n4\n"); +} + +#[test] +fn read_command_with_regex_and_dollar_addresses() { + use std::fs; + let f = NamedTempFile::new().unwrap(); + fs::write(f.path(), "X\nY\n").unwrap(); + + // Regex address: insert after line matching /^foo$/ + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("/^foo$/r {} ; p", f.path().display())) + .write_stdin("a\nfoo\nc\n"); + cmd.assert().success().stdout("a\nfoo\nX\nY\nc\n"); + + // Dollar address: insert after last line + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("$r {} ; p", f.path().display())) + .write_stdin("1\n2\n3\n"); + cmd.assert().success().stdout("1\n2\n3\nX\nY\n"); +} + +#[test] +fn multiple_read_commands_sequence() { + use std::fs; + let f1 = NamedTempFile::new().unwrap(); + let f2 = NamedTempFile::new().unwrap(); + fs::write(f1.path(), "A1\nA2\n").unwrap(); + fs::write(f2.path(), "B1\n").unwrap(); + + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!( + "2r {} ; 3r {} ; p", + f1.path().display(), + f2.path().display() + )) + .write_stdin("l1\nl2\nl3\nl4\n"); + // After l2 insert A1,A2; after l3 insert B1 + cmd.assert() + .success() + .stdout("l1\nl2\nA1\nA2\nl3\nB1\nl4\n"); +} + +#[test] +fn substitution_n_with_g_priority() { + let mut cmd = bin(); + // With occurrence N and g both present, replace from N-th occurrence onwards (GNU sed behavior) + cmd.args(["-e", "s/a/X/2g"]).write_stdin("aaaa\n"); + // occurrences: a a a a -> replace from 2nd onwards → a X X X + cmd.assert().success().stdout("aXXX\n"); +} + +#[test] +fn substitution_write_flag_no_match_creates_no_file() { + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_path_buf(); + // Remove the temp file first, we only need the path + std::fs::remove_file(&path).ok(); + + let mut cmd = bin(); + cmd.args(["-e"]) + .arg(format!("s/zzz/XXX/w {}", path.display())) + .write_stdin("no matches here\n"); + cmd.assert().success(); + assert!( + !path.exists(), + "file must not be created when no substitution occurred" + ); +} + +#[test] +fn append_timing_with_range_and_order() { + let mut cmd = bin(); + // Mimic FreeBSD test 4.2 semantics with a smaller input + // -n + // 2,4s/^/2-4/ + // s/^/before_a/p + // /2-4/a\ + // X + // s/^/after_a/p + let script = "2,4s/^/2-4/\ns/^/before_a/p\n/2-4/a\\\nX\ns/^/after_a/p"; + cmd.args(["-n", "-e"]) + .arg(script) + .write_stdin("l1_1\nl1_2\nl1_3\nl1_4\nl1_5\n"); + // Expected: for lines 2..4 the appended line 'X' prints after both p-prints + let expected = [ + "before_al1_1", + "after_abefore_al1_1", + "before_a2-4l1_2", + "after_abefore_a2-4l1_2", + "X", + "before_a2-4l1_3", + "after_abefore_a2-4l1_3", + "X", + "before_a2-4l1_4", + "after_abefore_a2-4l1_4", + "X", + "before_al1_5", + "after_abefore_al1_5", + ] + .join("\n") + + "\n"; + cmd.assert().success().stdout(expected); +} + +#[test] +fn append_with_n_flushes_before_join_and_prints_in_order() { + let mut cmd = bin(); + // Based on FreeBSD test 4.3 semantics with smaller range 2,2N + // -n + // s/^/^/p + // /l1_/a\ + // app + // 2,2N + // s/$/$/p + let script = "s/^/^/p\n/l1_/a\\\napp\n2,2N\ns/$/$/p"; + cmd.args(["-n", "-e"]) + .arg(script) + .write_stdin("l1_1\nl1_2\nl1_3\n"); + // For line 1: '^l1_1', '^l1_1$', then 'app' + // For line 2: '^l1_2', flush 'app', then join with line 3 and print with trailing $ + let expected = ["^l1_1", "^l1_1$", "app", "^l1_2", "app", "^l1_2\nl1_3$"].join("\n") + "\n"; + cmd.assert().success().stdout(expected); +} + +#[test] +fn y_command_digits_transliteration() { + let mut cmd = bin(); + cmd.args(["-e", "y/0123456789/9876543210/"]) + .write_stdin("2019\n"); + cmd.assert().success().stdout("7980\n"); + verify_against_sed!("y/0123456789/9876543210/", "2019\n", &[]); +} + +#[test] +fn y_command_custom_delimiter() { + let mut cmd = bin(); + cmd.args(["-e", "y#abc#XYZ#"]).write_stdin("cab\n"); + cmd.assert().success().stdout("ZXY\n"); + verify_against_sed!("y#abc#XYZ#", "cab\n", &[]); +} + +#[test] +fn y_command_escapes_tab() { + let mut cmd = bin(); + // Map 'a' to TAB using escape \t + cmd.args(["-e", "y/a/\\t/"]).write_stdin("aXa\n"); + cmd.assert().success().stdout("\tX\t\n"); + verify_against_sed!("y/a/\\t/", "aXa\n", &[]); +} + +#[test] +fn y_command_mismatched_sets_error() { + let mut cmd = bin(); + cmd.args(["-e", "y/ab/XYZ/"]).write_stdin("ab\n"); + cmd.assert().failure().stderr(predicates::str::contains( + "'y' command strings have different lengths", + )); +} + +#[test] +fn print_command_with_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2p"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("b\n"); +} + +#[test] +fn line_number_command_with_regex_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "/foo/="]) + .write_stdin("bar\nfoo\nbaz\n"); + cmd.assert().success().stdout("2\n"); +} + +#[test] +fn list_command_basic_escapes_and_dollar() { + let mut cmd = bin(); + // Contains TAB, BEL (\x07), backslash, and normal letters + let input = "a\tb\\c\x07\n"; // bytes: a, TAB, b, \\, c, BEL, LF + cmd.args(["-n", "-e", "l"]).write_stdin(input); + // Expect escaped tokens and trailing $ per line + cmd.assert().success().stdout("a\\tb\\\\c\\a$\n"); +} + +#[test] +fn list_command_wraps_at_72_columns() { + let mut cmd = bin(); + // Build a 100-char line of 'a' + let long = "a".repeat(100) + "\n"; + cmd.args(["-n", "-e", "l"]).write_stdin(long); + // First printed line should end with backslash due to wrap, second ends with $ + let out = cmd.assert().get_output().stdout.clone(); + let s = String::from_utf8_lossy(&out); + let lines: Vec<&str> = s.lines().collect(); + assert!(lines.len() >= 2); + assert!(lines[0].ends_with("\\")); + assert!(lines.last().unwrap().ends_with("$")); +} + +#[test] +fn append_and_insert_with_address() { + let mut cmd = bin(); + // -n: control output; 2a appends after 2nd; 2i inserts before 2nd + cmd.args([ + "-n", "-e", "2i\\ +I", "-e", "2a\\ +A", "-e", "p", + ]) + .write_stdin("1\n2\n3\n"); + // Sequence with -n and final p prints current lines; i prints before current, a prints after current + cmd.assert().success().stdout("1\nI\n2\nA\n3\n"); +} + +#[test] +fn delete_with_range_and_print_rest() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2,3d", "-e", "p"]) + .write_stdin("1\n2\n3\n4\n"); + cmd.assert().success().stdout("1\n4\n"); +} + +#[test] +fn change_single_line_address() { + let mut cmd = bin(); + cmd.args([ + "-n", "-e", "2c\\ +X", "-e", "p", + ]) + .write_stdin("1\n2\n3\n"); + cmd.assert().success().stdout("1\nX\n3\n"); +} + +#[test] +fn change_range_end_print_once() { + let mut cmd = bin(); + // Replace 2..4 with single X, printing it once + cmd.args([ + "-n", "-e", "2,4c\\ +X", "-e", "p", + ]) + .write_stdin("1\n2\n3\n4\n5\n"); + cmd.assert().success().stdout("1\nX\n5\n"); +} + +#[test] +fn change_regex_singleton_range() { + let mut cmd = bin(); + cmd.args([ + "-n", + "-e", + "/^x$/,/^x$/c\\ +X", + "-e", + "p", + ]) + .write_stdin("a\nx\ny\n"); + // GNU sed behavior: /^x$/,/^x$/ checks end from NEXT line, so range never ends. + // The 'c' command outputs text only at the LAST line of range, but since range + // continues to EOF, lines x and y are consumed by 'c' (which deletes pattern space + // and starts new cycle, so 'p' doesn't run), and X is never output because range + // doesn't end. + cmd.assert().success().stdout("a\n"); +} + +#[test] +fn y_command_octal_escapes() { + let mut cmd = bin(); + // GNU sed does NOT interpret octal escapes in y command + // \141\142 stays literal, not mapped to 'a','b' + cmd.args(["-e", "y/\\141\\142/\\102\\103/"]) + .write_stdin("ab\n"); + cmd.assert().success().stdout("ab\n"); +} + +#[test] +fn change_range_mixed_numeric_to_regex_end() { + let mut cmd = bin(); + // replace from line 2 to line matching /^X$/ with single R + cmd.args(["-n", "-e", "2,/^X$/c\\\nR", "-e", "p"]) + .write_stdin("1\n2\n3\nX\n5\n"); + cmd.assert().success().stdout("1\nR\n5\n"); +} + +#[test] +fn change_negated_range_applies_outside() { + let mut cmd = bin(); + // Apply change to lines NOT in 2..4; print unmodified lines with p + cmd.args(["-n", "-e", "2,4!c\\\nZ", "-e", "p"]) + .write_stdin("1\n2\n3\n4\n5\n"); + cmd.assert().success().stdout("Z\n2\n3\n4\nZ\n"); +} + +#[test] +fn ignore_case_with_address_range() { + let mut cmd = bin(); + // Apply only between lines 2 and 4 inclusive + cmd.args(["-n", "-e", "2,4s/foo/bar/Ig"]) + .write_stdin("Foo\nfOo\nfoo\nFOO\nfoo\n"); + // Expected prints only where substitution executed (we use -n and rely on 'p' flag via address? No, we explicitly print): + // We'll add an addressed 'p' to show results + let mut cmd = bin(); + cmd.args(["-n", "-e", "2,4s/foo/bar/Ig;2,4p"]) // print addressed lines after substitution + .write_stdin("Foo\nfOo\nfoo\nFOO\nfoo\n"); + cmd.assert().success().stdout("bar\nbar\nbar\n"); +} + +#[test] +fn next_command_prints_current_then_moves() { + let mut cmd = bin(); + cmd.args(["-e", "n"]).write_stdin("a\nb\n"); + // default print prints first line, then n moves to next (which is printed by default in next cycle) + cmd.assert().success().stdout("a\nb\n"); +} + +#[test] +fn big_n_appends_and_continues_commands() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "N;s/\n/;/;p"]).write_stdin("a\nb\n"); + cmd.assert().success().stdout("a;b\n"); +} + +#[test] +fn big_d_deletes_first_line_and_restarts_with_suffix() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "N;D;p"]).write_stdin("1\n2\n3\n"); + // After D restarts, 'p' is not reached; no output + cmd.assert().success().stdout(""); +} + +#[test] +fn hold_and_get_copy() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "h;g;p"]).write_stdin("hello\n"); + cmd.assert().success().stdout("hello\n"); + verify_against_sed!("h;g;p", "hello\n", &["-n"]); +} + +#[test] +fn hold_append_and_get_append() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "H;g;p"]).write_stdin("a\nb\n"); + // H always prepends newline, even when hold is empty (GNU sed behavior) + // First cycle: hold='' -> H -> hold='\na', g -> pattern='\na', p -> print '\na' + // Second cycle: hold='\na' -> H -> hold='\na\nb', g -> pattern='\na\nb', p -> print '\na\nb' + cmd.assert().success().stdout("\na\n\na\nb\n"); + verify_against_sed!("H;g;p", "a\nb\n", &["-n"]); +} + +#[test] +fn exchange_hold_and_pattern() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "h;x;p"]).write_stdin("x\n"); + cmd.assert().success().stdout("x\n"); + verify_against_sed!("h;x;p", "x\n", &["-n"]); +} + +#[test] +fn test_branch_takes_on_successful_substitution() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/x/y/;t end;s/y/z/;:end;p"]) + .write_stdin("x\n"); + cmd.assert().success().stdout("y\n"); + verify_against_sed!("s/x/y/;t end;s/y/z/;:end;p", "x\n", &["-n"]); +} + +#[test] +fn test_branch_t_skips_without_substitution() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "t end;s/x/y/;:end;p"]) + .write_stdin("x\n"); + cmd.assert().success().stdout("y\n"); + verify_against_sed!("t end;s/x/y/;:end;p", "x\n", &["-n"]); +} + +#[test] +fn test_branch_t_resets_flag() { + let mut cmd = bin(); + // After first t, the flag should reset so second t does nothing + cmd.args(["-n", "-e", "s/a/b/;t L;:L;t L;p"]) + .write_stdin("a\n"); + cmd.assert().success().stdout("b\n"); +} + +#[test] +fn nested_group_with_negation_and_ranges() { + let mut cmd = bin(); + // Reproduce the failing scenario with nested groups + // 4,12 !{ + // s/^/^/ + // /6/,/10/ !{ + // s/$/$/ + // /8/ !s/_/T/ + // } + // } + let script = r#"4,12 !{ +s/^/^/ +/6/,/10/ !{ + s/$/$/ + /8/ !s/_/T/ +} +}"#; + // Input lines labeled l1_..l12_ to make expectations obvious + let input = (1..=12) + .map(|i| format!("l{}_", i)) + .collect::>() + .join("\n") + + "\n"; + + // Run with -n and append explicit p at end to print final pattern space per line + cmd.args(["-n", "-e"]) + .arg(format!("{};p", script)) + .write_stdin(input.as_bytes()); + let out = cmd.assert().get_output().stdout.clone(); + let s = String::from_utf8_lossy(&out).to_string(); + + // Spot-check key lines around the ranges: + let lines: Vec<&str> = s.lines().collect(); + assert_eq!(lines.len(), 12); + // For 4,12! group, commands apply outside 4..12 → lines 1..3 are transformed + assert_eq!(lines[0], "^l1T$"); + assert_eq!(lines[1], "^l2T$"); + assert_eq!(lines[2], "^l3T$"); + // Lines 4..12 remain unchanged because group is negated for those lines + assert_eq!(lines[3], "l4_"); + assert_eq!(lines[4], "l5_"); + assert_eq!(lines[5], "l6_"); + assert_eq!(lines[7], "l8_"); +} + +#[test] +fn test_branch_b_without_label_jumps_to_end() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "b;p"]).write_stdin("x\n"); + cmd.assert().success().stdout(""); +} + +#[test] +fn test_addressed_branch() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2b e;p;:e;="]) + .write_stdin("1\n2\n3\n"); + cmd.assert().success().stdout("1\n1\n2\n3\n3\n"); +} + +#[test] +fn quit_without_code_exits_0_and_no_output() { + let mut cmd = bin(); + // Without -n, BSD sed prints current line before quitting + cmd.args(["-e", "q"]).write_stdin("hello\n"); + cmd.assert().code(0).stdout("hello\n"); +} + +#[test] +fn quit_with_code_exits_with_that_code() { + let mut cmd = bin(); + // Without -n, BSD sed prints current line before quitting with code + cmd.args(["-e", "q 42"]).write_stdin("hello\n"); + cmd.assert().code(42).stdout("hello\n"); +} + +#[test] +fn quit_with_address_stops_before_printing() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2q", "-e", "p"]) + .write_stdin("1\n2\n3\n"); + cmd.assert().code(0).stdout("1\n"); +} + +#[test] +fn quit_with_negated_address_quits_immediately() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2!q", "-e", "p"]) + .write_stdin("1\n2\n"); + cmd.assert().code(0).stdout(""); +} + +// ============================================================================ +// Phase 4.3: Comprehensive Substitution Flag Combination Tests +// ============================================================================ + +/// Test 'p' flag alone - should print substituted line +#[test] +fn subst_flag_p_alone() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/foo/bar/p"]) + .write_stdin("foo\nxxx\nfoo\n"); + cmd.assert().success().stdout("bar\nbar\n"); +} + +/// Test 'e' flag alone - should execute but not print (with -n) +#[test] +fn subst_flag_e_alone() { + let mut cmd = bin(); + // Execute 'echo test' after substitution, but -n suppresses auto-print + // So we get no output (execution happens but result is not printed) + cmd.args(["-n", "-e", "s/.*/echo test/e"]) + .write_stdin("foo\n"); + cmd.assert().success().stdout(""); +} + +/// Test 'e' flag with auto-print - should execute and print result +#[test] +fn subst_flag_e_with_autoprint() { + let mut cmd = bin(); + // Execute 'echo test' and auto-print the result + cmd.args(["-e", "s/.*/echo test/e"]).write_stdin("foo\n"); + cmd.assert().success().stdout("test\n"); +} + +/// Test 'pe' flags - should print BEFORE executing (execution result not printed) +#[test] +fn subst_flag_pe_print_then_execute() { + let mut cmd = bin(); + // Print "echo test" BEFORE executing it + // The execution happens but its result is not printed + cmd.args(["-n", "-e", "s/.*/echo test/pe"]) + .write_stdin("foo\n"); + cmd.assert().success().stdout("echo test\n"); +} + +/// Test 'ep' flags - should execute THEN print the result +#[test] +fn subst_flag_ep_execute_then_print() { + let mut cmd = bin(); + // Execute "echo test" first (pattern space becomes "test") + // Then print the pattern space ("test") + cmd.args(["-n", "-e", "s/.*/echo test/ep"]) + .write_stdin("foo\n"); + cmd.assert().success().stdout("test\n"); +} + +/// Test 'pep' flags - multiple p should error (GNU sed compatibility) +#[test] +fn subst_flag_pep_error() { + let mut cmd = bin(); + // GNU sed rejects multiple 'p' flags even with 'e' interspersed + cmd.args(["-n", "-e", "s/.*/echo test/pep"]) + .write_stdin("foo\n"); + cmd.assert() + .failure() + .stderr(predicates::str::contains("multiple 'p' options")); +} + +/// Test 'epe' flags - multiple 'e' allowed, should execute then print +#[test] +fn subst_flag_epe_allowed() { + let mut cmd = bin(); + // GNU sed allows multiple 'e' flags (unlike 'p') + // With 'e' before 'p', should execute then print + cmd.args(["-n", "-e", "s/.*/echo test/epe"]) + .write_stdin("foo\n"); + cmd.assert().success().stdout("test\n"); +} + +/// Test 'pp' flags - multiple p without intervening e should error +#[test] +fn subst_flag_pp_error() { + let mut cmd = bin(); + cmd.args(["-e", "s/foo/bar/pp"]).write_stdin("foo\n"); + cmd.assert() + .failure() + .stderr(predicates::str::contains("multiple 'p' options")); +} + +/// Test 'g' flag alone - global substitution +#[test] +fn subst_flag_g_alone() { + let mut cmd = bin(); + cmd.args(["-e", "s/o/X/g"]).write_stdin("foo\n"); + cmd.assert().success().stdout("fXX\n"); +} + +/// Test occurrence flag alone - replace only nth occurrence +#[test] +fn subst_flag_occurrence_alone() { + let mut cmd = bin(); + cmd.args(["-e", "s/o/X/2"]).write_stdin("foo\n"); + cmd.assert().success().stdout("foX\n"); +} + +/// Test 'g2' flags - global + occurrence, occurrence takes precedence +#[test] +fn subst_flag_g_and_occurrence_allowed() { + let mut cmd = bin(); + // GNU sed allows this - occurrence takes precedence, 'g' is ignored + cmd.args(["-e", "s/o/X/g2"]).write_stdin("foo\n"); + cmd.assert().success().stdout("foX\n"); +} + +/// Test '2g' flags - occurrence + global, occurrence takes precedence +#[test] +fn subst_flag_occurrence_and_g_allowed() { + let mut cmd = bin(); + // GNU sed allows this - occurrence takes precedence, 'g' is ignored + cmd.args(["-e", "s/o/X/2g"]).write_stdin("foo\n"); + cmd.assert().success().stdout("foX\n"); +} + +/// Test 'gg' flags - multiple g should error +#[test] +fn subst_flag_gg_error() { + let mut cmd = bin(); + cmd.args(["-e", "s/o/X/gg"]).write_stdin("foo\n"); + cmd.assert() + .failure() + .stderr(predicates::str::contains("multiple 'g' options")); +} + +/// Test 'ii' flags - multiple i (case-insensitive) allowed by GNU sed +#[test] +fn subst_flag_ii_allowed() { + let mut cmd = bin(); + // GNU sed allows multiple 'i' flags - just treats as one + cmd.args(["-e", "s/FOO/bar/ii"]).write_stdin("foo\n"); + cmd.assert().success().stdout("bar\n"); +} + +/// Test 'pg' flags - print and global together +#[test] +fn subst_flag_pg_combination() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/o/X/pg"]).write_stdin("foo\n"); + cmd.assert().success().stdout("fXX\n"); +} + +/// Test 'gp' flags - global and print together +#[test] +fn subst_flag_gp_combination() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/o/X/gp"]).write_stdin("foo\n"); + cmd.assert().success().stdout("fXX\n"); +} + +/// Test complex flag combination: 'gpi' +#[test] +fn subst_flag_gpi_combination() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/FOO/bar/gpi"]) + .write_stdin("FOO foo FOO\n"); + cmd.assert().success().stdout("bar bar bar\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - D (BigD) Command +// ============================================================================ + +/// D with multi-line pattern space - deletes first line +#[test] +fn big_d_multiline_deletes_first_line() { + let mut cmd = bin(); + // N joins two lines, D deletes first, restarts cycle with remainder + cmd.args(["-e", "N;D"]) + .write_stdin("first\nsecond\nthird\n"); + // After joining "first\nsecond", D deletes "first\n", leaving "second" + // D restarts cycle, N joins "second\nthird", D deletes "second\n" + // Then "third" is printed + cmd.assert().success().stdout("third\n"); +} + +/// D with pattern space containing no newline - acts like d +#[test] +fn big_d_single_line_acts_like_d() { + let mut cmd = bin(); + cmd.args(["-e", "D"]).write_stdin("only\n"); + // D on single line (no newline in pattern space) behaves like d - deletes and starts new cycle + cmd.assert().success().stdout(""); +} + +/// D at end of file - D restarts cycle so p never executes +#[test] +fn big_d_at_eof() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "N;$D;p"]).write_stdin("a\nb\n"); + // N joins "a\nb", $ matches, D deletes "a\n" and restarts cycle + // Since D restarts, 'p' never executes, output is empty (GNU sed behavior) + cmd.assert().success().stdout(""); +} + +/// D in loop - processes all lines +#[test] +fn big_d_in_loop() { + let mut cmd = bin(); + cmd.args(["-n", "-e", ":a;N;$!ba;D"]) + .write_stdin("l1\nl2\nl3\n"); + // Collects all, then D deletes first line repeatedly until one remains + cmd.assert().success().stdout(""); +} + +/// D after substitution +#[test] +fn big_d_after_substitution() { + let mut cmd = bin(); + cmd.args(["-e", "N;s/:/-/g;D"]).write_stdin("a:b\nc:d\n"); + // N joins "a:b\nc:d", s changes to "a-b\nc-d", D removes "a-b\n", leaving "c-d" + cmd.assert().success().stdout("c-d\n"); +} + +/// D with address +#[test] +fn big_d_with_address() { + let mut cmd = bin(); + cmd.args(["-e", "2{N;D}"]).write_stdin("l1\nl2\nl3\nl4\n"); + // At line 2, N joins l2\nl3, D removes l2\n, leaving l3 which restarts cycle + cmd.assert().success().stdout("l1\nl3\nl4\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Insert (i) Command +// ============================================================================ + +/// Insert at first line +#[test] +fn insert_at_first_line() { + let mut cmd = bin(); + cmd.args(["-e", "1i\\\nINSERTED"]) + .write_stdin("first\nsecond\n"); + cmd.assert().success().stdout("INSERTED\nfirst\nsecond\n"); +} + +/// Insert at last line +#[test] +fn insert_at_last_line() { + let mut cmd = bin(); + cmd.args(["-e", "$i\\\nINSERTED"]) + .write_stdin("first\nlast\n"); + cmd.assert().success().stdout("first\nINSERTED\nlast\n"); +} + +/// Insert with regex address +#[test] +fn insert_with_regex_address() { + let mut cmd = bin(); + cmd.args(["-e", "/banana/i\\\nFRUIT"]) + .write_stdin("apple\nbanana\ncherry\n"); + cmd.assert() + .success() + .stdout("apple\nFRUIT\nbanana\ncherry\n"); +} + +/// Insert with negation +#[test] +fn insert_with_negation() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2!i\\\nNOT_2", "-e", "p"]) + .write_stdin("l1\nl2\nl3\n"); + cmd.assert().success().stdout("NOT_2\nl1\nl2\nNOT_2\nl3\n"); +} + +/// Insert multiple lines +#[test] +fn insert_multiple_lines() { + let mut cmd = bin(); + cmd.args(["-e", "2i\\\nA\\\nB"]).write_stdin("1\n2\n3\n"); + cmd.assert().success().stdout("1\nA\nB\n2\n3\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Hold Space Operations +// ============================================================================ + +/// Multiple H commands accumulate with newlines +#[test] +fn hold_append_multiple_accumulates() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "H;$!d;g;p"]).write_stdin("a\nb\nc\n"); + // H appends with newline prefix each time + cmd.assert().success().stdout("\na\nb\nc\n"); +} + +/// Exchange twice returns original +#[test] +fn exchange_twice_returns_original() { + let mut cmd = bin(); + cmd.args(["-e", "x;x"]).write_stdin("content\n"); + cmd.assert().success().stdout("content\n"); +} + +/// Multiple G appends hold space multiple times +#[test] +fn get_append_multiple() { + let mut cmd = bin(); + cmd.args(["-e", "h;G;G"]).write_stdin("line\n"); + // After h: hold="line", pattern="line" + // After first G: pattern="line\nline" + // After second G: pattern="line\nline\nline" + cmd.assert().success().stdout("line\nline\nline\n"); +} + +/// Exchange with uninitialized (empty) hold space +#[test] +fn exchange_empty_hold() { + let mut cmd = bin(); + cmd.args(["-e", "x"]).write_stdin("content\n"); + // Exchange puts empty hold into pattern, content into hold + cmd.assert().success().stdout("\n"); +} + +/// Complex chain: h;s/.../;x;G +#[test] +fn hold_complex_chain() { + let mut cmd = bin(); + cmd.args(["-e", "h;s/original/MODIFIED/;x;G"]) + .write_stdin("original\n"); + // h: hold="original", pattern="original" + // s: pattern="MODIFIED" + // x: pattern="original", hold="MODIFIED" + // G: pattern="original\nMODIFIED" + cmd.assert().success().stdout("original\nMODIFIED\n"); +} + +/// h followed by g on different lines +#[test] +fn hold_across_lines() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "1h;2g;p"]) + .write_stdin("first\nsecond\n"); + // Line 1: h saves "first" + // Line 2: g replaces with "first" + cmd.assert().success().stdout("first\nfirst\n"); +} + +/// H;H;g chain (double append) +#[test] +fn hold_append_double() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "1{H;H};$g;$p"]) + .write_stdin("one\ntwo\n"); + // Line 1: H twice appends "one" twice with newlines + // Line 2: g replaces with hold content + cmd.assert().success().stdout("\none\none\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Read (r/R) Command Edge Cases +// ============================================================================ + +/// Read with 0 address (prepend before first line) +#[test] +fn read_zero_address_prepend() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + fs::write(tmp.path(), "PREPENDED\n").unwrap(); + + let mut cmd = bin(); + cmd.args(["-e"]) + .arg(format!("0r {}", tmp.path().display())) + .write_stdin("first\nsecond\n"); + cmd.assert().success().stdout("PREPENDED\nfirst\nsecond\n"); +} + +/// R (ReadLine) command reads one line at a time +#[test] +fn read_line_command() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + fs::write(tmp.path(), "X\nY\nZ\n").unwrap(); + + let mut cmd = bin(); + cmd.args(["-e"]) + .arg(format!("R {}", tmp.path().display())) + .write_stdin("a\nb\nc\n"); + // R reads one line from file per input line + cmd.assert().success().stdout("a\nX\nb\nY\nc\nZ\n"); +} + +/// R at EOF of file is silently ignored +#[test] +fn read_line_eof_ignored() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + fs::write(tmp.path(), "X\n").unwrap(); + + let mut cmd = bin(); + cmd.args(["-e"]) + .arg(format!("R {}", tmp.path().display())) + .write_stdin("a\nb\nc\n"); + // Only one line in file, subsequent R's are ignored + cmd.assert().success().stdout("a\nX\nb\nc\n"); +} + +/// Read missing file is silently ignored +#[test] +fn read_missing_file_ignored() { + let mut cmd = bin(); + cmd.args(["-e", "r /nonexistent/file/path.txt"]) + .write_stdin("hello\n"); + cmd.assert().success().stdout("hello\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Quit Commands with Exit Codes +// ============================================================================ + +/// Quit with specific exit code +#[test] +fn quit_with_exit_code_42() { + let mut cmd = bin(); + cmd.args(["-e", "q42"]).write_stdin("test\n"); + cmd.assert().code(42).stdout("test\n"); +} + +/// Quit with exit code 0 +#[test] +fn quit_with_exit_code_0() { + let mut cmd = bin(); + cmd.args(["-e", "q0"]).write_stdin("test\n"); + cmd.assert().code(0).stdout("test\n"); +} + +/// QuitSilent (Q) with exit code +#[test] +fn quit_silent_with_exit_code_1() { + let mut cmd = bin(); + cmd.args(["-e", "Q1"]).write_stdin("test\n"); + cmd.assert().code(1).stdout(""); +} + +/// Q at specific line +#[test] +fn quit_silent_at_line() { + let mut cmd = bin(); + cmd.args(["-e", "2Q"]).write_stdin("l1\nl2\nl3\n"); + cmd.assert().code(0).stdout("l1\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Test Negative (T) Command +// ============================================================================ + +/// T branches when NO substitution was made +#[test] +fn test_neg_branches_without_substitution() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "T end;s/./X/;:end;p"]) + .write_stdin("a\n"); + // T branches immediately (no subst yet), skipping s command + cmd.assert().success().stdout("a\n"); +} + +/// T does not branch after successful substitution +#[test] +fn test_neg_no_branch_after_subst() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/a/A/;T end;s/A/B/;:end;p"]) + .write_stdin("a\n"); + // First s succeeds, T does not branch, second s executes + cmd.assert().success().stdout("B\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Append (a) Command Edge Cases +// ============================================================================ + +/// Append at last line +#[test] +fn append_at_last_line() { + let mut cmd = bin(); + cmd.args(["-e", "$a\\\nAPPENDED"]) + .write_stdin("first\nlast\n"); + cmd.assert().success().stdout("first\nlast\nAPPENDED\n"); +} + +/// Append with regex address +#[test] +fn append_with_regex() { + let mut cmd = bin(); + cmd.args(["-e", "/middle/a\\\nAFTER"]) + .write_stdin("first\nmiddle\nlast\n"); + cmd.assert() + .success() + .stdout("first\nmiddle\nAFTER\nlast\n"); +} + +/// Append empty text +#[test] +fn append_empty_text() { + let mut cmd = bin(); + cmd.args(["-e", "1a\\\n"]).write_stdin("line\n"); + cmd.assert().success().stdout("line\n\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Print First Line (P) Command +// ============================================================================ + +/// P prints only first line of multiline pattern space +#[test] +fn print_first_line_multiline() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "N;P"]).write_stdin("first\nsecond\n"); + cmd.assert().success().stdout("first\n"); +} + +/// P on single line pattern space +#[test] +fn print_first_line_single() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "P"]).write_stdin("only\n"); + cmd.assert().success().stdout("only\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Write First Line (W) Command +// ============================================================================ + +/// W writes only first line of pattern space to file +#[test] +fn write_first_line_multiline() { + use std::fs; + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().to_path_buf(); + + let mut cmd = bin(); + cmd.args(["-n", "-e"]) + .arg(format!("N;W {}", path.display())) + .write_stdin("first\nsecond\n"); + cmd.assert().success(); + + let content = fs::read_to_string(&path).unwrap(); + assert_eq!(content, "first\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Clear (z) Command +// ============================================================================ + +/// z command clears pattern space +#[test] +fn clear_command() { + let mut cmd = bin(); + cmd.args(["-e", "z"]).write_stdin("content\n"); + cmd.assert().success().stdout("\n"); +} + +/// z then substitution +#[test] +fn clear_then_subst() { + let mut cmd = bin(); + cmd.args(["-e", "z;s/^$/EMPTY/"]).write_stdin("content\n"); + cmd.assert().success().stdout("EMPTY\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Line Number (=) Command +// ============================================================================ + +/// = prints line number +#[test] +fn line_number_command() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "="]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("1\n2\n3\n"); +} + +/// = with address +#[test] +fn line_number_with_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2="]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("2\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Label and Branch +// ============================================================================ + +/// Label with no reference is allowed +#[test] +fn label_unreferenced() { + let mut cmd = bin(); + cmd.args(["-e", ":unused;p"]).write_stdin("test\n"); + cmd.assert().success().stdout("test\ntest\n"); +} + +/// Multiple labels +#[test] +fn multiple_labels() { + let mut cmd = bin(); + cmd.args(["-n", "-e", ":a;p;b b;:b;p"]).write_stdin("x\n"); + // Prints twice: once at :a, then branches to :b which prints again + cmd.assert().success().stdout("x\nx\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Filename Command (F) +// ============================================================================ + +/// F prints filename +#[test] +fn filename_command_with_file() { + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "content").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + cmd.args(["-n", "-e", "F"]).arg(tmp.path()); + let output = cmd.assert().success().get_output().stdout.clone(); + let s = String::from_utf8_lossy(&output); + assert!(s.contains(tmp.path().to_str().unwrap())); +} + +/// F with stdin prints - +#[test] +fn filename_command_stdin() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "F"]).write_stdin("test\n"); + cmd.assert().success().stdout("-\n"); +} + +// ============================================================================ +// Phase 5: Coverage Tests - Substitution Edge Cases +// ============================================================================ + +/// Empty pattern uses last regex +#[test] +fn subst_empty_pattern_reuse() { + let mut cmd = bin(); + cmd.args(["-e", "/foo/s//bar/"]).write_stdin("foo bar\n"); + cmd.assert().success().stdout("bar bar\n"); +} + +/// Case conversion in replacement: \u \l \U \L \E +#[test] +fn subst_case_conversion_upper() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\(.*\\)/\\U\\1/"]) + .write_stdin("hello\n"); + cmd.assert().success().stdout("HELLO\n"); +} + +#[test] +fn subst_case_conversion_lower() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\(.*\\)/\\L\\1/"]) + .write_stdin("HELLO\n"); + cmd.assert().success().stdout("hello\n"); +} + +#[test] +fn subst_case_conversion_first_char() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\(.*\\)/\\u\\1/"]) + .write_stdin("hello\n"); + cmd.assert().success().stdout("Hello\n"); +} + +#[test] +fn subst_case_conversion_end() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\(..\\)\\(.*\\)/\\U\\1\\E\\2/"]) + .write_stdin("hello\n"); + cmd.assert().success().stdout("HEllo\n"); +} + +// ============================================================================ +// Phase 6: Additional Coverage Tests +// ============================================================================ + +/// Test list command with various escape sequences +#[test] +fn list_command_escapes() { + let mut cmd = bin(); + // Input with tab, bell, backspace, form feed + cmd.args(["-n", "-e", "l"]) + .write_stdin("a\tb\x07c\x08d\x0c\n"); + cmd.assert().success().stdout("a\\tb\\ac\\bd\\f$\n"); +} + +/// Test list command with high bytes +#[test] +fn list_command_high_bytes() { + let mut cmd = bin(); + // Use bytes directly for high byte values + cmd.args(["-n", "-e", "l"]) + .write_stdin(&[0x80u8, 0xff, b'\n'][..]); + // High bytes should be output as octal escapes + let output = cmd.assert().success().get_output().stdout.clone(); + let s = String::from_utf8_lossy(&output); + assert!(s.contains("\\")); +} + +/// Test version command +#[test] +fn version_command_passes() { + let mut cmd = bin(); + cmd.args(["-e", "v 4.0"]).write_stdin("test\n"); + cmd.assert().success().stdout("test\n"); +} + +/// Test version command fails for higher version +#[test] +fn version_command_fails_higher() { + let mut cmd = bin(); + cmd.args(["-e", "v 99.0"]).write_stdin("test\n"); + cmd.assert().failure(); +} + +/// Test hex escapes in replacement +#[test] +fn subst_hex_escape() { + let mut cmd = bin(); + cmd.args(["-e", "s/b/\\x58/g"]).write_stdin("abc\n"); + cmd.assert().success().stdout("aXc\n"); +} + +/// Test octal escapes in replacement (GNU sed uses \oNNN syntax) +#[test] +fn subst_octal_escape() { + let mut cmd = bin(); + // GNU sed uses \o130 for octal 130 = 'X' (ASCII 88) + cmd.args(["-e", "s/b/\\o130/g"]).write_stdin("abc\n"); + cmd.assert().success().stdout("aXc\n"); +} + +/// Test newline escape in replacement +#[test] +fn subst_newline_escape() { + let mut cmd = bin(); + cmd.args(["-e", "s/ /\\n/g"]).write_stdin("a b c\n"); + cmd.assert().success().stdout("a\nb\nc\n"); +} + +/// Test tab escape in replacement +#[test] +fn subst_tab_escape() { + let mut cmd = bin(); + cmd.args(["-e", "s/ /\\t/g"]).write_stdin("a b\n"); + cmd.assert().success().stdout("a\tb\n"); +} + +/// Test n command (next) with quiet mode +#[test] +fn next_command_quiet() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "n;p"]).write_stdin("a\nb\nc\n"); + // n reads next line without printing, then p prints it + cmd.assert().success().stdout("b\n"); +} + +/// Test N command (append next line) +#[test] +fn big_n_command() { + let mut cmd = bin(); + cmd.args(["-e", "N;s/\\n/-/"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("a-b\nc\n"); +} + +/// Test N at end of file +#[test] +fn big_n_at_eof() { + let mut cmd = bin(); + cmd.args(["-e", "$N"]).write_stdin("only\n"); + // N at EOF with one line should just print the line + cmd.assert().success().stdout("only\n"); +} + +/// Test substitute with backreference \0 (whole match) +#[test] +fn subst_backref_zero() { + let mut cmd = bin(); + cmd.args(["-e", "s/[0-9][0-9]*/[\\0]/g"]) + .write_stdin("a1b23c\n"); + cmd.assert().success().stdout("a[1]b[23]c\n"); +} + +/// Test regex with word boundaries +#[test] +fn regex_word_boundary() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\bword\\b/WORD/g"]) + .write_stdin("word words sword\n"); + cmd.assert().success().stdout("WORD words sword\n"); +} + +/// Test regex with start word boundary +#[test] +fn regex_start_word_boundary() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\/WORD/g"]) + .write_stdin("word words sword\n"); + cmd.assert().success().stdout("WORD words sWORD\n"); +} + +/// Test regex alternation in ERE mode +#[test] +fn regex_ere_alternation() { + let mut cmd = bin(); + cmd.args(["-E", "-e", "s/cat|dog/animal/g"]) + .write_stdin("cat and dog\n"); + cmd.assert().success().stdout("animal and animal\n"); +} + +/// Test regex plus quantifier in ERE mode +#[test] +fn regex_ere_plus() { + let mut cmd = bin(); + cmd.args(["-E", "-e", "s/a+/X/g"]).write_stdin("baaaaab\n"); + cmd.assert().success().stdout("bXb\n"); +} + +/// Test regex question quantifier in ERE mode +#[test] +fn regex_ere_question() { + let mut cmd = bin(); + cmd.args(["-E", "-e", "s/colou?r/COLOR/g"]) + .write_stdin("color colour\n"); + cmd.assert().success().stdout("COLOR COLOR\n"); +} + +/// Test regex counted repetition +#[test] +fn regex_counted_repetition() { + let mut cmd = bin(); + cmd.args(["-e", "s/a\\{3\\}/X/g"]) + .write_stdin("aa aaa aaaa\n"); + cmd.assert().success().stdout("aa X Xa\n"); +} + +/// Test regex min-max repetition +#[test] +fn regex_minmax_repetition() { + let mut cmd = bin(); + cmd.args(["-e", "s/a\\{2,4\\}/X/g"]) + .write_stdin("a aa aaa aaaa aaaaa\n"); + cmd.assert().success().stdout("a X X X Xa\n"); +} + +/// Test address with first~step +#[test] +fn address_step() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "1~2p"]) + .write_stdin("1\n2\n3\n4\n5\n"); + cmd.assert().success().stdout("1\n3\n5\n"); +} + +/// Test address with 0,/pattern/ +#[test] +fn address_zero_to_pattern() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "0,/x/p"]).write_stdin("a\nx\nb\n"); + cmd.assert().success().stdout("a\nx\n"); +} + +/// Test comment in script +#[test] +fn script_comment() { + let mut cmd = bin(); + cmd.args(["-e", "# this is a comment", "-e", "s/a/b/"]) + .write_stdin("a\n"); + cmd.assert().success().stdout("b\n"); +} + +/// Test semicolon as command separator +#[test] +fn command_separator_semicolon() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/b/;s/b/c/"]).write_stdin("a\n"); + cmd.assert().success().stdout("c\n"); +} + +/// Test grouped commands with braces +#[test] +fn grouped_commands() { + let mut cmd = bin(); + cmd.args(["-e", "2{s/a/A/;s/b/B/}"]) + .write_stdin("ab\nab\nab\n"); + cmd.assert().success().stdout("ab\nAB\nab\n"); +} + +/// Test nested grouped commands +#[test] +fn nested_grouped_commands() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2{/a/{s/a/X/;p}}"]) + .write_stdin("ab\nab\nab\n"); + cmd.assert().success().stdout("Xb\n"); +} + +/// Test multiple expressions +#[test] +fn multiple_expressions() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/1/", "-e", "s/b/2/", "-e", "s/c/3/"]) + .write_stdin("abc\n"); + cmd.assert().success().stdout("123\n"); +} + +/// Test POSIX character classes +#[test] +fn posix_char_class_digit() { + let mut cmd = bin(); + cmd.args(["-e", "s/[[:digit:]]/X/g"]) + .write_stdin("a1b2c3\n"); + cmd.assert().success().stdout("aXbXcX\n"); +} + +#[test] +fn posix_char_class_space() { + let mut cmd = bin(); + cmd.args(["-e", "s/[[:space:]]/_/g"]) + .write_stdin("a b\tc\n"); + // Note: newline at end is line terminator, not matched + cmd.assert().success().stdout("a_b_c\n"); +} + +#[test] +fn posix_char_class_upper() { + let mut cmd = bin(); + cmd.args(["-e", "s/[[:upper:]]/x/g"]).write_stdin("AbCdE\n"); + cmd.assert().success().stdout("xbxdx\n"); +} + +#[test] +fn posix_char_class_lower() { + let mut cmd = bin(); + cmd.args(["-e", "s/[[:lower:]]/X/g"]).write_stdin("AbCdE\n"); + cmd.assert().success().stdout("AXCXE\n"); +} + +/// Test negated character class +#[test] +fn negated_char_class() { + let mut cmd = bin(); + cmd.args(["-e", "s/[^a-z]/_/g"]).write_stdin("aB1c\n"); + // B and 1 are not lowercase letters, so they get replaced + cmd.assert().success().stdout("a__c\n"); +} + +/// Test dot matches any character +#[test] +fn regex_dot_any() { + let mut cmd = bin(); + cmd.args(["-e", "s/./X/g"]).write_stdin("abc\n"); + cmd.assert().success().stdout("XXX\n"); +} + +/// Test caret anchor +#[test] +fn regex_caret_anchor() { + let mut cmd = bin(); + cmd.args(["-e", "s/^a/X/"]).write_stdin("abc\nabc\n"); + cmd.assert().success().stdout("Xbc\nXbc\n"); +} + +/// Test dollar anchor +#[test] +fn regex_dollar_anchor() { + let mut cmd = bin(); + cmd.args(["-e", "s/c$/X/"]).write_stdin("abc\nabc\n"); + cmd.assert().success().stdout("abX\nabX\n"); +} + +/// Test both anchors +#[test] +fn regex_both_anchors() { + let mut cmd = bin(); + cmd.args(["-e", "s/^abc$/X/"]) + .write_stdin("abc\nxabc\nabcx\n"); + cmd.assert().success().stdout("X\nxabc\nabcx\n"); +} + +/// Test star quantifier +#[test] +fn regex_star() { + let mut cmd = bin(); + cmd.args(["-e", "s/ab*/X/g"]).write_stdin("a ab abb abbb\n"); + cmd.assert().success().stdout("X X X X\n"); +} + +/// Test empty match handling +#[test] +fn regex_empty_match() { + let mut cmd = bin(); + cmd.args(["-e", "s/a*/X/g"]).write_stdin("bab\n"); + // Empty matches at boundaries should be handled + cmd.assert().success(); +} + +/// Test character range in class +#[test] +fn char_class_range() { + let mut cmd = bin(); + cmd.args(["-e", "s/[a-z]/X/g"]).write_stdin("aBcDeF\n"); + cmd.assert().success().stdout("XBXDXF\n"); +} + +/// Test special characters in character class +#[test] +fn char_class_special() { + let mut cmd = bin(); + // Closing bracket at start, hyphen at end + cmd.args(["-e", "s/[]a-]/X/g"]).write_stdin("a]b-c\n"); + cmd.assert().success().stdout("XXbXc\n"); +} + +// ============================================================================ +// Phase 6: Additional Coverage Tests +// ============================================================================ + +/// Test substitution with execute flag (e) +#[test] +fn subst_execute_flag() { + let mut cmd = bin(); + cmd.args(["-e", "s/.*/echo HELLO/e"]) + .write_stdin("ignore\n"); + cmd.assert().success().stdout("HELLO\n"); +} + +/// Test negated address with regex +#[test] +fn negated_regex_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "/foo/!p"]) + .write_stdin("foo\nbar\nbaz\n"); + cmd.assert().success().stdout("bar\nbaz\n"); + verify_against_sed!("/foo/!p", "foo\nbar\nbaz\n", &["-n"]); +} + +/// Test negated address with line number +#[test] +fn negated_line_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2!p"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("a\nc\n"); + verify_against_sed!("2!p", "a\nb\nc\n", &["-n"]); +} + +/// Test negated address range +#[test] +fn negated_range_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "2,3!p"]).write_stdin("a\nb\nc\nd\n"); + cmd.assert().success().stdout("a\nd\n"); + verify_against_sed!("2,3!p", "a\nb\nc\nd\n", &["-n"]); +} + +/// Test branch to end of script +#[test] +fn branch_to_end() { + let mut cmd = bin(); + cmd.args(["-e", "b end;s/a/X/;:end"]).write_stdin("abc\n"); + cmd.assert().success().stdout("abc\n"); + verify_against_sed!("b end;s/a/X/;:end", "abc\n", &[]); +} + +/// Test change command with range +#[test] +fn change_with_range() { + let mut cmd = bin(); + cmd.args(["-e", "2,3c\\\nNEW"]).write_stdin("a\nb\nc\nd\n"); + cmd.assert().success().stdout("a\nNEW\nd\n"); +} + +/// Test substitution multiline mode (m flag) +#[test] +fn subst_multiline_flag() { + let mut cmd = bin(); + cmd.args(["-e", "N;s/^/START:/gm"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("START:a\nSTART:b\nc\n"); +} + +/// Test substitution with occurrence number +#[test] +fn subst_occurrence_4() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/4"]).write_stdin("aaaaa\n"); + cmd.assert().success().stdout("aaaXa\n"); + verify_against_sed!("s/a/X/4", "aaaaa\n", &[]); +} + +/// Test Q (quit without printing) +#[test] +fn quit_silent() { + let mut cmd = bin(); + cmd.args(["-e", "2Q"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("a\n"); +} + +/// Test quit with exit code +#[test] +fn quit_with_exit_code() { + let mut cmd = bin(); + cmd.args(["-e", "q5"]).write_stdin("hello\n"); + cmd.assert().code(5); +} + +/// Test address with tilde step (0~2) +#[test] +fn address_zero_tilde_step() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "0~3p"]) + .write_stdin("1\n2\n3\n4\n5\n6\n"); + cmd.assert().success().stdout("3\n6\n"); + verify_against_sed!("0~3p", "1\n2\n3\n4\n5\n6\n", &["-n"]); +} + +/// Test F command (print filename) +#[test] +fn print_filename() { + use std::fs; + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + fs::write(tmp.path(), "hello\n").unwrap(); + + let mut cmd = bin(); + cmd.args(["-n", "-e", "F"]).arg(tmp.path()); + let output = cmd.assert().success().get_output().stdout.clone(); + assert!(!output.is_empty()); +} + +/// Test P with single line pattern space - explicit check +#[test] +fn print_first_line_single_explicit() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "P"]).write_stdin("single line\n"); + cmd.assert().success().stdout("single line\n"); + verify_against_sed!("P", "single line\n", &["-n"]); +} + +/// Test multiline pattern with P +#[test] +fn print_first_line_multiline_pattern() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "N;N;P"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("a\n"); + verify_against_sed!("N;N;P", "a\nb\nc\n", &["-n"]); +} + +/// Test empty pattern reuse in substitution +#[test] +fn empty_pattern_reuse() { + let mut cmd = bin(); + cmd.args(["-e", "/hello/s//world/"]).write_stdin("hello\n"); + cmd.assert().success().stdout("world\n"); + verify_against_sed!("/hello/s//world/", "hello\n", &[]); +} + +/// Test word boundary in regex (\b) +#[test] +fn word_boundary() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\bfoo\\b/BAR/g"]) + .write_stdin("foo foobar barfoo\n"); + cmd.assert().success().stdout("BAR foobar barfoo\n"); +} + +/// Test backreference with more than 9 groups +#[test] +fn backreference_nine_groups() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\(.\\)\\(.\\)\\(.\\)\\(.\\)\\(.\\)\\(.\\)\\(.\\)\\(.\\)\\(.\\)/\\9\\8\\7\\6\\5\\4\\3\\2\\1/"]) + .write_stdin("123456789\n"); + cmd.assert().success().stdout("987654321\n"); +} + +/// Test substitution with 2g flag +#[test] +fn subst_2g_flag() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/2g"]).write_stdin("aaaaa\n"); + cmd.assert().success().stdout("aXXXX\n"); + verify_against_sed!("s/a/X/2g", "aaaaa\n", &[]); +} + +/// Test comment at end of script +#[test] +fn comment_at_end() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/g #comment"]).write_stdin("aaa\n"); + cmd.assert().success().stdout("XXX\n"); +} + +/// Test nested braces +#[test] +fn nested_braces() { + let mut cmd = bin(); + cmd.args(["-e", "1{2,3{s/a/X/}}"]) + .write_stdin("abc\nabc\nabc\n"); + cmd.assert().success().stdout("abc\nabc\nabc\n"); +} + +/// Test case conversion \l +#[test] +fn case_conversion_lowercase_next() { + let mut cmd = bin(); + cmd.args(["-e", "s/.*/\\l&/"]).write_stdin("HELLO\n"); + cmd.assert().success().stdout("hELLO\n"); +} + +/// Test multiple -e expressions +#[test] +fn multiple_e_expressions() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/1/", "-e", "s/b/2/", "-e", "s/c/3/"]) + .write_stdin("abc\n"); + cmd.assert().success().stdout("123\n"); + verify_against_sed!("s/a/1/;s/b/2/;s/c/3/", "abc\n", &[]); +} + +/// Test hex escape in replacement +#[test] +fn hex_escape_in_replacement() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/\\x41/g"]).write_stdin("aaa\n"); + cmd.assert().success().stdout("AAA\n"); +} + +/// Test octal escape in replacement +#[test] +fn octal_escape_in_replacement() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/\\o101/g"]).write_stdin("aaa\n"); + cmd.assert().success().stdout("AAA\n"); +} + +/// Test decimal escape in replacement +#[test] +fn decimal_escape_in_replacement() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/\\d65/g"]).write_stdin("aaa\n"); + cmd.assert().success().stdout("AAA\n"); +} + +/// Test control character in replacement +#[test] +fn control_char_in_replacement() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/\\cI/g"]).write_stdin("aaa\n"); + // \cI is tab (control-I = 0x09) + cmd.assert().success().stdout("\t\t\t\n"); +} + +/// Test print and execute flags together (pe order) +#[test] +fn subst_pe_flags() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/.*/echo DONE/pe"]) + .write_stdin("x\n"); + // pe: print then execute - should print the substituted text "echo DONE" first + // then execute it and print result "DONE" + cmd.assert().success(); +} + +/// Test print and execute flags together (ep order) +#[test] +fn subst_ep_flags() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/.*/echo DONE/ep"]) + .write_stdin("x\n"); + // ep: execute then print - should execute first, then print + cmd.assert().success(); +} + +/// Test substitution with w flag (write to file) +#[test] +fn subst_write_flag() { + use std::fs; + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let tmp_path = tmp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.args(["-e", &format!("s/foo/bar/w {}", tmp_path)]) + .write_stdin("foo\nbaz\n"); + cmd.assert().success().stdout("bar\nbaz\n"); + + let written = fs::read_to_string(tmp.path()).unwrap(); + assert_eq!(written, "bar\n"); +} + +/// Test n command (next line) +#[test] +fn next_line_command() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "n;p"]).write_stdin("a\nb\nc\nd\n"); + cmd.assert().success().stdout("b\nd\n"); + verify_against_sed!("n;p", "a\nb\nc\nd\n", &["-n"]); +} + +/// Test step address with substitution +#[test] +fn step_address_subst() { + let mut cmd = bin(); + cmd.args(["-e", "1~2s/a/X/"]).write_stdin("a\na\na\na\na\n"); + cmd.assert().success().stdout("X\na\nX\na\nX\n"); + verify_against_sed!("1~2s/a/X/", "a\na\na\na\na\n", &[]); +} + +// ============================================= +// Additional tests for exec.rs coverage improvement +// ============================================= + +/// Test case transform \u (uppercase next) with group backreference +#[test] +fn case_uppercase_next_with_backref() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\([a-z]\\+\\)/\\u\\1/"]) + .write_stdin("hello world\n"); + cmd.assert().success().stdout("Hello world\n"); + verify_against_sed!("s/\\([a-z]\\+\\)/\\u\\1/", "hello world\n", &[]); +} + +/// Test case transform \l (lowercase next) with group backreference +#[test] +fn case_lowercase_next_with_backref() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\([A-Z]\\+\\)/\\l\\1/"]) + .write_stdin("HELLO WORLD\n"); + cmd.assert().success().stdout("hELLO WORLD\n"); + verify_against_sed!("s/\\([A-Z]\\+\\)/\\l\\1/", "HELLO WORLD\n", &[]); +} + +/// Test case transform \U (uppercase all) with group backreference +#[test] +fn case_uppercase_all_with_backref() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\([a-z]\\+\\)/\\U\\1\\E/"]) + .write_stdin("hello\n"); + cmd.assert().success().stdout("HELLO\n"); + verify_against_sed!("s/\\([a-z]\\+\\)/\\U\\1\\E/", "hello\n", &[]); +} + +/// Test case transform \L (lowercase all) with group backreference +#[test] +fn case_lowercase_all_with_backref() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\([A-Z]\\+\\)/\\L\\1\\E/"]) + .write_stdin("HELLO\n"); + cmd.assert().success().stdout("hello\n"); + verify_against_sed!("s/\\([A-Z]\\+\\)/\\L\\1\\E/", "HELLO\n", &[]); +} + +/// Test substitution with occurrence 3 +#[test] +fn subst_occurrence_3() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/3"]).write_stdin("a a a a a\n"); + cmd.assert().success().stdout("a a X a a\n"); + verify_against_sed!("s/a/X/3", "a a a a a\n", &[]); +} + +/// Test substitution with occurrence 5 +#[test] +fn subst_occurrence_5() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/5"]).write_stdin("a a a a a\n"); + cmd.assert().success().stdout("a a a a X\n"); + verify_against_sed!("s/a/X/5", "a a a a a\n", &[]); +} + +/// Test substitution with occurrence beyond total matches +#[test] +fn subst_occurrence_beyond_matches() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/9"]).write_stdin("a a a\n"); + cmd.assert().success().stdout("a a a\n"); + verify_against_sed!("s/a/X/9", "a a a\n", &[]); +} + +/// Test substitution with 3g (from 3rd occurrence, replace all) +#[test] +fn subst_occurrence_3g() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/3g"]).write_stdin("a a a a a\n"); + cmd.assert().success().stdout("a a X X X\n"); + verify_against_sed!("s/a/X/3g", "a a a a a\n", &[]); +} + +/// Test zero-length match with star quantifier +#[test] +fn zero_length_match_star() { + let mut cmd = bin(); + cmd.args(["-e", "s/a*/X/g"]).write_stdin("bab\n"); + cmd.assert().success().stdout("XbXbX\n"); + verify_against_sed!("s/a*/X/g", "bab\n", &[]); +} + +/// Test zero-length match at word boundaries +#[test] +fn zero_length_word_boundary() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\b/|/g"]).write_stdin("ab cd\n"); + cmd.assert().success().stdout("|ab| |cd|\n"); + verify_against_sed!("s/\\b/|/g", "ab cd\n", &[]); +} + +/// Test empty pattern reuse after substitution +#[test] +fn empty_pattern_reuse_after_subst() { + let mut cmd = bin(); + // After s/foo//, empty pattern // reuses 'foo', but 'foo' is gone + cmd.args(["-e", "s/foo//;s//X/2"]) + .write_stdin("foo bar baz bar qux bar\n"); + cmd.assert().success().stdout(" bar baz bar qux bar\n"); + // Note: no verify_against_sed since behavior depends on last regex tracking +} + +/// Test translate (y) with mixed case +#[test] +fn translate_mixed_case() { + let mut cmd = bin(); + cmd.args(["-e", "y/abc/XYZ/"]).write_stdin("abcABC\n"); + cmd.assert().success().stdout("XYZABC\n"); + verify_against_sed!("y/abc/XYZ/", "abcABC\n", &[]); +} + +/// Test translate with numbers +#[test] +fn translate_numbers() { + let mut cmd = bin(); + cmd.args(["-e", "y/0123456789/abcdefghij/"]) + .write_stdin("2024\n"); + cmd.assert().success().stdout("cace\n"); + verify_against_sed!("y/0123456789/abcdefghij/", "2024\n", &[]); +} + +/// Test translate with special characters +#[test] +fn translate_special_chars() { + let mut cmd = bin(); + cmd.args(["-e", "y/./!/"]).write_stdin("a.b.c\n"); + cmd.assert().success().stdout("a!b!c\n"); + verify_against_sed!("y/./!/", "a.b.c\n", &[]); +} + +/// Test hold space with raw bytes preservation +#[test] +fn hold_space_preserves_content() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "1h;2{H;g;p}"]) + .write_stdin("line1\nline2\n"); + cmd.assert().success().stdout("line1\nline2\n"); + verify_against_sed!("1h;2{H;g;p}", "line1\nline2\n", &["-n"]); +} + +/// Test exchange (x) multiple times +#[test] +fn exchange_multiple_times() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "h;s/.*/NEW/;x;p;x;p"]) + .write_stdin("OLD\n"); + cmd.assert().success().stdout("OLD\nNEW\n"); + verify_against_sed!("h;s/.*/NEW/;x;p;x;p", "OLD\n", &["-n"]); +} + +/// Test get append (G) with empty hold space +#[test] +fn get_append_empty_hold() { + let mut cmd = bin(); + cmd.args(["-e", "G"]).write_stdin("line\n"); + cmd.assert().success().stdout("line\n\n"); + verify_against_sed!("G", "line\n", &[]); +} + +/// Test BigD deletes first line and continues +#[test] +fn big_d_continues_to_end() { + let mut cmd = bin(); + // N;D on "a\nb\nc\n" should output just "c" + cmd.args(["-e", "N;D"]).write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("c\n"); + verify_against_sed!("N;D", "a\nb\nc\n", &[]); +} + +/// Test branch with empty label jumps to end +#[test] +fn branch_empty_label_to_end() { + let mut cmd = bin(); + cmd.args(["-e", "b;s/./X/"]).write_stdin("abc\n"); + cmd.assert().success().stdout("abc\n"); + verify_against_sed!("b;s/./X/", "abc\n", &[]); +} + +/// Test test branch (t) resets flag after branch +#[test] +fn test_branch_resets_after_jump() { + let mut cmd = bin(); + cmd.args(["-e", ":loop;s/a/X/;t loop"]).write_stdin("aaa\n"); + cmd.assert().success().stdout("XXX\n"); + verify_against_sed!(":loop;s/a/X/;t loop", "aaa\n", &[]); +} + +/// Test T (inverse test) branch +#[test] +fn test_neg_branch_basic() { + let mut cmd = bin(); + cmd.args(["-e", "s/x/y/;T end;s/./Z/g;:end"]) + .write_stdin("abc\n"); + cmd.assert().success().stdout("abc\n"); + verify_against_sed!("s/x/y/;T end;s/./Z/g;:end", "abc\n", &[]); +} + +/// Test T branch with successful substitution +#[test] +fn test_neg_no_branch_on_success() { + let mut cmd = bin(); + cmd.args(["-e", "s/a/X/;T end;s/b/Y/;:end"]) + .write_stdin("ab\n"); + cmd.assert().success().stdout("XY\n"); + verify_against_sed!("s/a/X/;T end;s/b/Y/;:end", "ab\n", &[]); +} + +/// Test ReadLine (R) command progressive read +#[test] +fn read_line_progressive() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "inserted1").unwrap(); + writeln!(tmp, "inserted2").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + cmd.args(["-e", &format!("R {}", tmp.path().display())]) + .write_stdin("line1\nline2\n"); + cmd.assert() + .success() + .stdout("line1\ninserted1\nline2\ninserted2\n"); +} + +/// Test Write first line (W) command +#[test] +fn write_first_line_only() { + use std::fs; + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let tmp_path = tmp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.args(["-n", "-e", &format!("N;W {}", tmp_path)]) + .write_stdin("line1\nline2\n"); + cmd.assert().success(); + + let written = fs::read_to_string(tmp.path()).unwrap(); + assert_eq!(written, "line1\n"); +} + +/// Test list command with very short line +#[test] +fn list_command_short_line() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "l"]).write_stdin("ab\n"); + cmd.assert().success().stdout("ab$\n"); + verify_against_sed!("l", "ab\n", &["-n"]); +} + +/// Test list command with null byte +#[test] +fn list_command_null_byte() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "l"]).write_stdin("a\x00b\n"); + cmd.assert().success().stdout("a\\000b$\n"); +} + +/// Test print filename (F) with stdin +#[test] +fn print_filename_stdin() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "F"]).write_stdin("test\n"); + cmd.assert().success(); // Output should be stdin path or "-" +} + +/// Test print filename (F) with file +#[test] +fn print_filename_with_file() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "content").unwrap(); + tmp.flush().unwrap(); + let path = tmp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.args(["-n", "-e", "F"]).arg(tmp.path()); + // Output should contain the filename + cmd.assert().success().stdout(format!("{}\n", path)); +} + +/// Test clear (z) command +#[test] +fn clear_command_explicit() { + let mut cmd = bin(); + cmd.args(["-e", "z;s/^$/EMPTY/"]).write_stdin("content\n"); + cmd.assert().success().stdout("EMPTY\n"); + verify_against_sed!("z;s/^$/EMPTY/", "content\n", &[]); +} + +/// Test execute (e) with explicit command argument +#[test] +fn execute_explicit_command() { + let mut cmd = bin(); + cmd.args(["-e", "e echo hello"]).write_stdin("input\n"); + cmd.assert().success().stdout("hello\ninput\n"); + verify_against_sed!("e echo hello", "input\n", &[]); +} + +/// Test substitution with \0 backreference +#[test] +fn subst_backref_zero_same_as_ampersand() { + let mut cmd = bin(); + cmd.args(["-e", "s/[a-z]\\+/[\\0]/g"]) + .write_stdin("hello world\n"); + cmd.assert().success().stdout("[hello] [world]\n"); + verify_against_sed!("s/[a-z]\\+/[\\0]/g", "hello world\n", &[]); +} + +/// Test multiple nested case transforms +#[test] +fn case_transforms_nested() { + let mut cmd = bin(); + cmd.args(["-e", "s/\\([a-z]\\)\\([a-z]*\\)/\\u\\1\\L\\2/g"]) + .write_stdin("hello WORLD test\n"); + cmd.assert().success().stdout("Hello WORLD Test\n"); + verify_against_sed!( + "s/\\([a-z]\\)\\([a-z]*\\)/\\u\\1\\L\\2/g", + "hello WORLD test\n", + &[] + ); +} + +/// Test substitution with print timing pe (print then execute) +#[test] +fn subst_pe_print_then_execute() { + let mut cmd = bin(); + cmd.args(["-e", "s/.*/echo REPLACED/pe"]) + .write_stdin("original\n"); + // pe = print then execute: prints "echo REPLACED" then executes it + cmd.assert().success(); +} + +/// Test line number command (=) with address +#[test] +fn line_number_with_regex_address() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "/foo/="]) + .write_stdin("bar\nfoo\nbaz\nfoo\n"); + cmd.assert().success().stdout("2\n4\n"); + verify_against_sed!("/foo/=", "bar\nfoo\nbaz\nfoo\n", &["-n"]); +} + +/// Test insert command with regex address +#[test] +fn insert_with_regex_match() { + let mut cmd = bin(); + cmd.args(["-e", "/^target$/i\\BEFORE"]) + .write_stdin("line1\ntarget\nline3\n"); + cmd.assert() + .success() + .stdout("line1\nBEFORE\ntarget\nline3\n"); + verify_against_sed!("/^target$/i\\BEFORE", "line1\ntarget\nline3\n", &[]); +} + +/// Test append command at EOF +#[test] +fn append_at_eof() { + let mut cmd = bin(); + cmd.args(["-e", "$a\\END"]).write_stdin("line1\nline2\n"); + cmd.assert().success().stdout("line1\nline2\nEND\n"); + verify_against_sed!("$a\\END", "line1\nline2\n", &[]); +} + +/// Test change (c) command with negated address +#[test] +fn change_negated_single_address() { + let mut cmd = bin(); + cmd.args(["-e", "2!c\\CHANGED"]).write_stdin("l1\nl2\nl3\n"); + cmd.assert().success().stdout("CHANGED\nl2\nCHANGED\n"); + verify_against_sed!("2!c\\CHANGED", "l1\nl2\nl3\n", &[]); +} + +/// Test N command with quiet mode +#[test] +fn n_command_quiet_mode() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "N;p"]).write_stdin("a\nb\nc\nd\n"); + cmd.assert().success().stdout("a\nb\nc\nd\n"); + verify_against_sed!("N;p", "a\nb\nc\nd\n", &["-n"]); +} + +/// Test next (n) at EOF +#[test] +fn next_at_eof() { + let mut cmd = bin(); + cmd.args(["-e", "n;d"]).write_stdin("a\nb\n"); + cmd.assert().success().stdout("a\n"); + verify_against_sed!("n;d", "a\nb\n", &[]); +} + +/// Test substitution with hex escape in replacement +#[test] +fn subst_hex_escape_high_byte() { + let mut cmd = bin(); + cmd.args(["-e", "s/X/\\xC0/"]).write_stdin("X\n"); + cmd.assert().success(); + // Should contain the byte 0xC0 +} + +/// Test substitution with octal escape +#[test] +fn subst_octal_escape_nul() { + let mut cmd = bin(); + cmd.args(["-e", "s/X/\\o000/"]).write_stdin("X\n"); + cmd.assert().success(); + // Should produce a NUL byte +} + +/// Test translate with pipe delimiter +#[test] +fn translate_pipe_delimiter() { + let mut cmd = bin(); + cmd.args(["-e", "y|abc|XYZ|"]).write_stdin("abc\n"); + cmd.assert().success().stdout("XYZ\n"); + verify_against_sed!("y|abc|XYZ|", "abc\n", &[]); +} + +/// Test substitution global with print (gp) +#[test] +fn subst_global_print() { + let mut cmd = bin(); + cmd.args(["-n", "-e", "s/a/X/gp"]).write_stdin("aaa\nbbb\n"); + cmd.assert().success().stdout("XXX\n"); + verify_against_sed!("s/a/X/gp", "aaa\nbbb\n", &["-n"]); +} + +/// Test substitution with write file global +#[test] +fn subst_write_global() { + use std::fs; + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let tmp_path = tmp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.args(["-e", &format!("s/foo/bar/gw {}", tmp_path)]) + .write_stdin("foo foo\nbaz\nfoo\n"); + cmd.assert().success().stdout("bar bar\nbaz\nbar\n"); + + let written = fs::read_to_string(tmp.path()).unwrap(); + assert_eq!(written, "bar bar\nbar\n"); +} + +/// Test zero address with read prepend +#[test] +fn zero_address_read_prepend() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "prepended").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + cmd.args(["-e", &format!("0r {}", tmp.path().display())]) + .write_stdin("first\nsecond\n"); + cmd.assert().success().stdout("prepended\nfirst\nsecond\n"); +} + +/// Test zero address with tilde step +#[test] +fn zero_tilde_step() { + let mut cmd = bin(); + cmd.args(["-e", "0~3s/^/X/"]) + .write_stdin("a\nb\nc\nd\ne\nf\n"); + cmd.assert().success().stdout("a\nb\nXc\nd\ne\nXf\n"); + verify_against_sed!("0~3s/^/X/", "a\nb\nc\nd\ne\nf\n", &[]); +} + +/// Test version command (v) with current version +#[test] +fn version_command_current() { + let mut cmd = bin(); + cmd.args(["-e", "v 4.0"]).write_stdin("test\n"); + cmd.assert().success().stdout("test\n"); +} + +/// Test version command failure +#[test] +fn version_command_future_fails() { + let mut cmd = bin(); + cmd.args(["-e", "v 99.0"]).write_stdin("test\n"); + cmd.assert().failure(); +} + +/// Test N command followed by substitution across lines +#[test] +fn n_then_subst_multiline() { + let mut cmd = bin(); + cmd.args(["-e", "N;s/\\n/ /"]).write_stdin("a\nb\nc\nd\n"); + cmd.assert().success().stdout("a b\nc d\n"); + verify_against_sed!("N;s/\\n/ /", "a\nb\nc\nd\n", &[]); +} + +/// Test grouped commands with multiple branches +#[test] +fn grouped_commands_complex() { + let mut cmd = bin(); + cmd.args(["-e", "/a/{s/a/A/;b end};/b/s/b/B/;:end"]) + .write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("A\nB\nc\n"); + verify_against_sed!("/a/{s/a/A/;b end};/b/s/b/B/;:end", "a\nb\nc\n", &[]); +} diff --git a/red/tests/obinary_windows.rs b/red/tests/obinary_windows.rs new file mode 100644 index 0000000..1fb7bf0 --- /dev/null +++ b/red/tests/obinary_windows.rs @@ -0,0 +1,269 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Test for Windows CRLF behavior based on GNU sed obinary.sh +// This test verifies the -b (binary) flag works correctly on Windows + +#![cfg(windows)] + +mod common; + +use assert_cmd::Command; +use std::io::Write; +use tempfile::NamedTempFile; + +fn bin() -> Command { + Command::cargo_bin("red").unwrap() +} + +#[test] +fn test_platform_uses_crlf() { + // First check: does red output CRLF by default? + let mut cmd = bin(); + cmd.arg("p").write_stdin("a"); + let output = cmd.output().unwrap(); + + println!("Platform check - output bytes: {:?}", output.stdout); + println!("Platform check - output len: {}", output.stdout.len()); + + // Skip test if platform doesn't use CRLF + if output.stdout.len() < 3 { + // Expected: "a\r\n" = 3 bytes + println!("Platform does not enable CRLF by default, skipping test"); + return; + } +} + +#[test] +fn obinary_input_crlf_no_flag() { + // Test 1: Input file with CRLF, no -b flag + // Expected: output should preserve CRLF + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\r\n").unwrap(); + infile.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("s/a/z/").arg(infile.path()); + let output = cmd.output().unwrap(); + + println!("Test 1 - Input CRLF, no -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"z\r\n", + "Input with CRLF should output with CRLF (no -b flag)" + ); +} + +#[test] +fn obinary_input_lf_no_flag() { + // Test 2: Input file with LF only, no -b flag + // Expected: output should be converted to CRLF on Windows + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\n").unwrap(); + infile.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("s/a/z/").arg(infile.path()); + let output = cmd.output().unwrap(); + + println!("Test 2 - Input LF, no -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"z\r\n", + "Input with LF should output with CRLF on Windows (no -b flag)" + ); +} + +#[test] +fn obinary_input_lf_with_b_flag() { + // Test 3: Input file with LF only, WITH -b flag + // Expected: output should keep LF (binary mode) + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\n").unwrap(); + infile.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("-b").arg("s/a/z/").arg(infile.path()); + let output = cmd.output().unwrap(); + + println!("Test 3 - Input LF, with -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"z\n", + "Input with LF should output with LF when using -b flag" + ); +} + +#[test] +fn obinary_stdin_lf_no_flag() { + // Test 4: STDIN with LF, no -b flag + // Expected: output should be CRLF + let mut cmd = bin(); + cmd.arg("s/a/z/").write_stdin("a\n"); + let output = cmd.output().unwrap(); + + println!("Test 4 - STDIN LF, no -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"z\r\n", + "STDIN with LF should output with CRLF on Windows (no -b flag)" + ); +} + +#[test] +fn obinary_stdin_lf_with_b_flag() { + // Test 5: STDIN with LF, WITH -b flag + // Expected: output should be LF + let mut cmd = bin(); + cmd.arg("-b").arg("s/a/z/").write_stdin("a\n"); + let output = cmd.output().unwrap(); + + println!("Test 5 - STDIN LF, with -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"z\n", + "STDIN with LF should output with LF when using -b flag" + ); +} + +#[test] +fn obinary_eol_test_no_flag() { + // Test 6: End-of-line handling with CRLF input, no -b + // In TEXT mode, \r\n is end-of-line, "y" should be added before \r\n + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\r\n").unwrap(); + infile.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("s/$/y/").arg(infile.path()); + let output = cmd.output().unwrap(); + + println!("Test 6 - EOL with CRLF, no -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"ay\r\n", + "In text mode, $ should match before CRLF, insert 'y' before \\r\\n" + ); +} + +#[test] +fn obinary_eol_test_with_b_flag() { + // Test 7: End-of-line handling with CRLF input, WITH -b + // In BINARY mode, \r is just a character, "y" should be added after \r + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\r\n").unwrap(); + infile.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("-b").arg("s/$/y/").arg(infile.path()); + let output = cmd.output().unwrap(); + + println!("Test 7 - EOL with CRLF, with -b:"); + println!(" Output bytes: {:?}", output.stdout); + + assert_eq!( + output.stdout, b"a\ry\n", + "In binary mode, $ should match before \\n only, insert 'y' after \\r" + ); +} + +#[test] +fn obinary_inplace_crlf() { + // Test 8: In-place editing should preserve CRLF + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\r\n").unwrap(); + infile.flush().unwrap(); + let (file, path) = infile.keep().unwrap(); // Keep file for in-place editing + drop(file); + + let mut cmd = bin(); + cmd.arg("-i").arg("s/a/z/").arg(&path); + let output = cmd.output().unwrap(); + + println!("Test 8 - In-place with CRLF:"); + println!(" Command succeeded: {}", output.status.success()); + if !output.status.success() { + println!(" stderr: {}", String::from_utf8_lossy(&output.stderr)); + } + + // Read the modified file + let content = std::fs::read(&path).unwrap(); + println!(" File content after -i: {:?}", content); + + // Cleanup + let _ = std::fs::remove_file(&path); + + assert_eq!( + content, b"z\r\n", + "In-place editing should preserve CRLF in text mode" + ); +} + +#[test] +fn obinary_inplace_eol() { + // Test 9: In-place editing with EOL replacement (text mode) + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\r\n").unwrap(); + infile.flush().unwrap(); + let (file, path) = infile.keep().unwrap(); + drop(file); + + let mut cmd = bin(); + cmd.arg("-i").arg("s/$/y/").arg(&path); + let output = cmd.output().unwrap(); + + println!("Test 9 - In-place EOL, no -b:"); + println!(" Command succeeded: {}", output.status.success()); + if !output.status.success() { + println!(" stderr: {}", String::from_utf8_lossy(&output.stderr)); + } + + let content = std::fs::read(&path).unwrap(); + println!(" File content: {:?}", content); + + // Cleanup + let _ = std::fs::remove_file(&path); + + assert_eq!( + content, b"ay\r\n", + "In-place $ replacement should add before CRLF in text mode" + ); +} + +#[test] +fn obinary_inplace_eol_binary() { + // Test 10: In-place editing with EOL replacement (binary mode) + let mut infile = NamedTempFile::new().unwrap(); + infile.write_all(b"a\r\n").unwrap(); + infile.flush().unwrap(); + let (file, path) = infile.keep().unwrap(); + drop(file); + + let mut cmd = bin(); + cmd.arg("-b").arg("-i").arg("s/$/y/").arg(&path); + let output = cmd.output().unwrap(); + + println!("Test 10 - In-place EOL, with -b:"); + println!(" Command succeeded: {}", output.status.success()); + if !output.status.success() { + println!(" stderr: {}", String::from_utf8_lossy(&output.stderr)); + } + + let content = std::fs::read(&path).unwrap(); + println!(" File content: {:?}", content); + + // Cleanup + let _ = std::fs::remove_file(&path); + + assert_eq!( + content, b"a\ry\n", + "In-place $ replacement in binary mode should add after \\r" + ); +} diff --git a/red/tests/scripts/compare_all_tests.sh b/red/tests/scripts/compare_all_tests.sh new file mode 100755 index 0000000..3765ba6 --- /dev/null +++ b/red/tests/scripts/compare_all_tests.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# Compare cargo test expectations against GNU sed behavior + +RED_BIN="${RED_BIN:-target/release/red}" + +# Build red first +cargo build --release 2>/dev/null + +PASS=0 +FAIL=0 +DIFF_FOUND=() + +compare() { + local name="$1" + local script="$2" + local input="$3" + local args="${4:-}" + + # Run both sed and red + local sed_out red_out sed_status red_status + + if [ -n "$args" ]; then + sed_out=$(printf '%s' "$input" | sed $args -e "$script" 2>/dev/null) || sed_status=$? + red_out=$(printf '%s' "$input" | $RED_BIN $args -e "$script" 2>/dev/null) || red_status=$? + else + sed_out=$(printf '%s' "$input" | sed -e "$script" 2>/dev/null) || sed_status=$? + red_out=$(printf '%s' "$input" | $RED_BIN -e "$script" 2>/dev/null) || red_status=$? + fi + + if [ "$sed_out" = "$red_out" ]; then + echo "[PASS] $name" + ((PASS++)) + else + echo "[FAIL] $name" + echo " script: $script" + echo " input: $(printf '%s' "$input" | head -c 50 | cat -v)..." + echo " sed: $(printf '%s' "$sed_out" | head -c 80 | cat -v)" + echo " red: $(printf '%s' "$red_out" | head -c 80 | cat -v)" + ((FAIL++)) + DIFF_FOUND+=("$name") + fi +} + +echo "=== Comparing cargo test expectations with GNU sed ===" +echo "" + +# Basic substitution tests +compare "basic_substitution_once" 's/foo/bar/' $'foo\nfoo\n' +compare "basic_substitution_global" 's/foo/bar/g' 'foo foo' +compare "replacement_ampersand" 's/[0-9][0-9]*/&X/g' 'a1 b22' +compare "backreferences" 's/\([a-z][a-z]*\)-\([0-9][0-9]*\)/\2-\1/' 'abc-123' +compare "delimiter_custom" 's#foo/bar#baz#' 'foo/bar' +compare "quiet_mode" 's/x/y/' 'x' '-n' +compare "multiple_scripts" 's/a/A/;s/b/B/' 'ab' +compare "delimiter_bracket" 's[abc[X[g' 'abc abc' + +# Address tests +compare "address_numeric" '2p' $'a\nb\nc' '-n' +compare "address_last_line" '$p' $'x\ny\nz' '-n' +compare "address_regex" '/^foo$/p' $'bar\nfoo\nbaz' '-n' +compare "range_numeric" '1,2p' $'l1\nl2\nl3' '-n' +compare "range_to_dollar" '2,$p' $'l1\nl2\nl3' '-n' +compare "range_regex_single" '/^x$/,/^x$/p' $'a\nx\ny' '-n' +compare "negation" '2!p' $'a\nb\nc' '-n' +compare "step_address" '0~2p' $'1\n2\n3\n4\n5' '-n' + +# BRE tests +compare "bre_counted_exact" 's/\(ab\)\{2\}/X/' $'abab\naba' +compare "bre_counted_range" 's/a\{2,3\}/X/g' 'a aa aaa aaaa' +compare "bre_posix_alpha" 's/[[:alpha:]]\{3\}/X/' $'abc-123\n12abc34' +compare "bre_escape_bracket" 's/[]a]/_/g' '] a b ]a' +compare "bre_ignore_case" 's/foo/bar/Ig' 'Foo fOo foo' +compare "escape_ampersand_backslash" 's/[0-9][0-9]*/\&-\\/g' 'a1 b22 c333' + +# Nth occurrence +compare "nth_occurrence" 's/./X/4' 'abcd' + +# Substitution flags +compare "subst_2g" 's/a/X/2g' 'aaaa' +compare "subst_g2" 's/o/X/g2' 'foo' +compare "subst_p_flag" 's/foo/bar/p' $'foo\nxxx\nfoo' '-n' + +# y command +compare "y_digits" 'y/0123456789/9876543210/' '2019' +compare "y_custom_delim" 'y#abc#XYZ#' 'cab' +compare "y_escape_tab" 'y/a/\t/' 'aXa' +compare "y_octal" 'y/\141\142/\102\103/' 'ab' + +# Print and line number +compare "print_address" '2p' $'a\nb\nc' '-n' +compare "line_number_regex" '/foo/=' $'bar\nfoo\nbaz' '-n' + +# List command +compare "list_escapes" 'l' $'a\tb\\c\x07' '-n' + +# a/i/c commands +compare "append_insert" '2i\ +I +2a\ +A +p' $'1\n2\n3' '-n' + +compare "delete_range" '2,3d;p' $'1\n2\n3\n4' '-n' + +compare "change_single" '2c\ +X +p' $'1\n2\n3' '-n' + +# n/N/D commands +compare "next_cmd" 'n' $'a\nb' +compare "big_n" 'N;s/\n/;/;p' $'a\nb' '-n' +compare "big_d" 'N;D;p' $'1\n2\n3' '-n' + +# Hold space +compare "hold_get" 'h;g;p' 'hello' '-n' +compare "hold_append" 'H;g;p' $'a\nb' '-n' +compare "exchange" 'h;x;p' 'x' '-n' + +# Branching +compare "t_branch_success" 's/x/y/;t end;s/y/z/;:end;p' 'x' '-n' +compare "t_branch_skip" 't end;s/x/y/;:end;p' 'x' '-n' +compare "b_to_end" 'b;p' 'x' '-n' +compare "addressed_branch" '2b e;p;:e;=' $'1\n2\n3' '-n' + +# Quit +compare "quit_basic" 'q' 'hello' +compare "quit_address" '2q;p' $'1\n2\n3' '-n' + +# e flag (execute) +compare "e_with_autoprint" 's/.*/echo test/e' 'foo' +compare "ep_execute_print" 's/.*/echo test/ep' 'foo' '-n' +compare "pe_print_execute" 's/.*/echo test/pe' 'foo' '-n' + +# Combined flags +compare "pg_flags" 's/o/X/pg' 'foo' '-n' +compare "gp_flags" 's/o/X/gp' 'foo' '-n' +compare "gpi_flags" 's/FOO/bar/gpi' 'FOO foo FOO' '-n' + +# Empty match tests +compare "empty_match_star" 's/a*/X/g' 'bbb' +compare "empty_match_question" 's/a\{0,1\}/X/g' 'bbb' +compare "empty_match_alpha_star" 's/[[:alpha:]]*/WORD/g' 'test123' + +echo "" +echo "=== Summary ===" +echo "Passed: $PASS" +echo "Failed: $FAIL" + +if [ $FAIL -gt 0 ]; then + echo "" + echo "Tests with differences:" + for t in "${DIFF_FOUND[@]}"; do + echo " - $t" + done + exit 1 +fi diff --git a/red/tests/scripts/compare_sed_red.sh b/red/tests/scripts/compare_sed_red.sh new file mode 100755 index 0000000..b076c62 --- /dev/null +++ b/red/tests/scripts/compare_sed_red.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# Compare GNU sed and red output for given input +# +# Usage: compare_sed_red.sh "sed_command" [input_file] +# Or: echo "input" | compare_sed_red.sh "sed_command" +# +# Examples: +# ./compare_sed_red.sh 's/a/X/g' <<< "abba" +# ./compare_sed_red.sh 'y/abc/xyz/' input.txt +# printf 'a\x80b' | ./compare_sed_red.sh 's/a/X/' + +set -e + +RED_BIN="${RED_BIN:-/home/builder/red/red/target/release/red}" + +if [ -z "$1" ]; then + echo "Usage: $0 'sed_command' [input_file]" + echo "Or: echo 'input' | $0 'sed_command'" + exit 1 +fi + +SED_CMD="$1" +INPUT_FILE="$2" + +# Create temp files for output +SED_OUT=$(mktemp) +RED_OUT=$(mktemp) +trap "rm -f $SED_OUT $RED_OUT" EXIT + +# Run both commands +if [ -n "$INPUT_FILE" ]; then + sed "$SED_CMD" < "$INPUT_FILE" > "$SED_OUT" 2>&1 + SED_EXIT=$? + "$RED_BIN" "$SED_CMD" < "$INPUT_FILE" > "$RED_OUT" 2>&1 + RED_EXIT=$? +else + # Read from stdin + INPUT=$(cat) + echo -n "$INPUT" | sed "$SED_CMD" > "$SED_OUT" 2>&1 + SED_EXIT=$? + echo -n "$INPUT" | "$RED_BIN" "$SED_CMD" > "$RED_OUT" 2>&1 + RED_EXIT=$? +fi + +# Compare outputs +if diff -q "$SED_OUT" "$RED_OUT" > /dev/null 2>&1 && [ "$SED_EXIT" = "$RED_EXIT" ]; then + echo "[PASS]: $SED_CMD" + exit 0 +else + echo "[FAIL]: $SED_CMD" + echo " Exit codes: sed=$SED_EXIT red=$RED_EXIT" + echo " sed output (hex):" + xxd "$SED_OUT" | head -5 | sed 's/^/ /' + echo " red output (hex):" + xxd "$RED_OUT" | head -5 | sed 's/^/ /' + exit 1 +fi diff --git a/red/tests/scripts/mb_regex_tests.sh b/red/tests/scripts/mb_regex_tests.sh new file mode 100755 index 0000000..b274fc4 --- /dev/null +++ b/red/tests/scripts/mb_regex_tests.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# MB Regex Tests for Shift-JIS and EUC-JP +# Tests regex matching in non-UTF-8 multibyte locales + +RED_BIN="${RED_BIN:-/home/builder/red/red/target/release/red}" + +PASS=0 +FAIL=0 +SKIP=0 + +GREEN='\033[0;32m' +RED_COLOR='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' + +run_test() { + local name="$1" + local locale="$2" + local cmd="$3" + local input_hex="$4" + + # Check if locale is available + if ! locale -a 2>/dev/null | grep -q "^${locale}$"; then + echo -e "${YELLOW}[SKIP]${NC}: $name (locale $locale not available)" + SKIP=$((SKIP + 1)) + return + fi + + # Run both sed and red + sed_out=$(printf "$input_hex" | xxd -r -p | LC_ALL=$locale sed "$cmd" 2>/dev/null | xxd -p | tr -d '\n') + sed_exit=$? + red_out=$(printf "$input_hex" | xxd -r -p | LC_ALL=$locale "$RED_BIN" "$cmd" 2>/dev/null | xxd -p | tr -d '\n') + red_exit=$? + + if [ "$sed_out" = "$red_out" ] && [ "$sed_exit" = "$red_exit" ]; then + echo -e "${GREEN}[PASS]${NC}: $name" + PASS=$((PASS + 1)) + else + echo -e "${RED_COLOR}[FAIL]${NC}: $name" + echo " Command: $cmd" + echo " Input: $input_hex" + echo " sed: $sed_out (exit $sed_exit)" + echo " red: $red_out (exit $red_exit)" + FAIL=$((FAIL + 1)) + fi +} + +echo "========================================" +echo "MB Regex Tests" +echo "========================================" +echo "" + +# Check if red binary exists +if [ ! -x "$RED_BIN" ]; then + echo "Error: red binary not found at $RED_BIN" + exit 1 +fi + +echo "=== Shift-JIS Dot (.) Tests ===" +# Input: 83 5B 83 5D = two Shift-JIS chars +run_test "sjis_dot_single" "ja_JP.shiftjis" "s/./X/" "835b835d" +run_test "sjis_dot_global" "ja_JP.shiftjis" "s/./X/g" "835b835d" +run_test "sjis_dot_star" "ja_JP.shiftjis" "s/.*/X/" "835b835d" +run_test "sjis_two_dots" "ja_JP.shiftjis" "s/../Y/" "835b835d" + +echo "" +echo "=== EUC-JP Dot (.) Tests ===" +# Input: A4 A2 A4 A4 = two EUC-JP chars (hiragana a, i) +run_test "eucjp_dot_single" "ja_JP.eucjp" "s/./X/" "a4a2a4a4" +run_test "eucjp_dot_global" "ja_JP.eucjp" "s/./X/g" "a4a2a4a4" + +echo "" +echo "=== Mixed ASCII/MB Tests ===" +# Input: 61 83 5B 62 = 'a' + MB char + 'b' +run_test "sjis_mixed_single" "ja_JP.shiftjis" "s/./X/" "61835b62" +run_test "sjis_mixed_global" "ja_JP.shiftjis" "s/./X/g" "61835b62" +run_test "sjis_mixed_dot_star" "ja_JP.shiftjis" "s/.*/X/" "61835b62" + +echo "" +echo "=== Character Class Tests ===" +# ASCII class should only match ASCII chars +run_test "sjis_ascii_class" "ja_JP.shiftjis" "s/[a-z]/X/g" "61835b62" +run_test "sjis_digit_class" "ja_JP.shiftjis" "s/[0-9]/X/g" "31835b32" + +echo "" +echo "=== Negated Class Tests ===" +# Negated class - [^a] should match MB chars +run_test "sjis_neg_class" "ja_JP.shiftjis" "s/[^a]/X/g" "61835b62" + +echo "" +echo "=== Quantifier Tests ===" +run_test "sjis_dot_plus" "ja_JP.shiftjis" "s/.+/X/" "835b835d" +run_test "sjis_dot_question" "ja_JP.shiftjis" "s/.\\?/X/" "835b835d" + +echo "" +echo "=== Backreference Tests ===" +# Two identical MB chars +run_test "sjis_backref_same" "ja_JP.shiftjis" 's/\(.\)\1/X/' "835b835b" +# Different MB chars (should not match) +run_test "sjis_backref_diff" "ja_JP.shiftjis" 's/\(.\)\1/X/' "835b835d" + +echo "" +echo "=== Capture Group Tests ===" +# Reverse 3 chars +run_test "sjis_capture_reverse" "ja_JP.shiftjis" 's/\(.\)\(.\)\(.\)/\3\2\1/' "61835b62" + +echo "" +echo "=== Edge Cases ===" +# Empty input +run_test "sjis_empty" "ja_JP.shiftjis" "s/./X/" "" +# Single byte +run_test "sjis_single_byte" "ja_JP.shiftjis" "s/./X/" "61" +# Incomplete MB at end +run_test "sjis_incomplete" "ja_JP.shiftjis" "s/./X/g" "61835b83" + +echo "" +echo "========================================" +echo "Summary" +echo "========================================" +echo -e "Passed: ${GREEN}$PASS${NC}" +echo -e "Failed: ${RED_COLOR}$FAIL${NC}" +echo -e "Skipped: ${YELLOW}$SKIP${NC}" +echo "" + +if [ $FAIL -gt 0 ]; then + echo "Some tests failed!" + exit 1 +else + echo "All tests passed!" + exit 0 +fi diff --git a/red/tests/scripts/verify_implementation.sh b/red/tests/scripts/verify_implementation.sh new file mode 100755 index 0000000..fccdb65 --- /dev/null +++ b/red/tests/scripts/verify_implementation.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +# Copyright (c) 2026 Red Authors +# License: MIT +# + +# Comprehensive verification of red implementation +# Tests all key features from the refactoring plan +# +# Usage: ./verify_implementation.sh + +RED_BIN="${RED_BIN:-/home/builder/red/red/target/release/red}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +PASS=0 +FAIL=0 +SKIP=0 + +# Colors +GREEN='\033[0;32m' +RED_COLOR='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' # No Color + +pass() { + echo -e "${GREEN}[PASS]${NC}: $1" + PASS=$((PASS + 1)) +} + +fail() { + echo -e "${RED_COLOR}[FAIL]${NC}: $1" + echo " Expected: $2" + echo " Got: $3" + FAIL=$((FAIL + 1)) +} + +skip() { + echo -e "${YELLOW}[SKIP]${NC}: $1 ($2)" + SKIP=$((SKIP + 1)) +} + +compare_output() { + local desc="$1" + local cmd="$2" + local input="$3" + + local sed_out=$(echo -n "$input" | sed "$cmd" 2>&1 | xxd -p) + local red_out=$(echo -n "$input" | "$RED_BIN" "$cmd" 2>&1 | xxd -p) + + if [ "$sed_out" = "$red_out" ]; then + pass "$desc" + else + fail "$desc" "$sed_out" "$red_out" + fi +} + +compare_binary() { + local desc="$1" + local cmd="$2" + local input_hex="$3" + + local sed_out=$(printf "$input_hex" | xxd -r -p | sed "$cmd" 2>&1 | xxd -p) + local red_out=$(printf "$input_hex" | xxd -r -p | "$RED_BIN" "$cmd" 2>&1 | xxd -p) + + if [ "$sed_out" = "$red_out" ]; then + pass "$desc" + else + fail "$desc" "$sed_out" "$red_out" + fi +} + +echo "========================================" +echo "Red Implementation Verification" +echo "========================================" +echo "" + +# Check if red binary exists +if [ ! -x "$RED_BIN" ]; then + echo "Error: red binary not found at $RED_BIN" + echo "Run: cargo build --release" + exit 1 +fi + +echo "=== Phase 1: Code Deduplication ===" +echo "" + +# Task 1.1: Byte Replacement +echo "-- Task 1.1: Byte Replacement --" +compare_output "Basic replacement" "s/X/Y/g" "aXaXa" +compare_output "Overlapping pattern" "s/aa/X/g" "aaaa" +compare_output "Empty result" "s/X//g" "XXX" +compare_output "Pattern not found" "s/X/Y/g" "abc" + +# Task 1.2: Character-to-Byte Mapping +echo "" +echo "-- Task 1.2: Character-to-Byte Mapping --" +compare_output "ASCII replacement" "s/l/X/g" "hello" +compare_output "UTF-8 Cyrillic" "s/и/X/g" "привіт" +compare_binary "Invalid UTF-8 preserved" "s/b/X/g" "618081620a" # a\x80\x81b\n + +# Task 1.3: Numeric Escapes +echo "" +echo "-- Task 1.3: Numeric Escapes --" +compare_output "Hex escape ASCII" "s/t/A/g" "test" +compare_output "Octal escape" "s/t/A/g" "test" + +echo "" +echo "=== Phase 2: Performance Improvements ===" +echo "" + +# Task 2.1/2.2: Pattern Space Operations +echo "-- Task 2.1/2.2: Pattern Space Operations --" +compare_output "Hold space" "H;g;s/\\n/,/" "hello" +# Use echo to ensure newline-terminated input (matches GNU sed behavior) +sed_out=$(printf 'a\nb\nc\n' | sed 'N;N;s/\n/,/g' | od -c | tr -d ' \n') +red_out=$(printf 'a\nb\nc\n' | "$RED_BIN" 'N;N;s/\n/,/g' | od -c | tr -d ' \n') +if [ "$sed_out" = "$red_out" ]; then + pass "Multiple appends" +else + fail "Multiple appends" "$sed_out" "$red_out" +fi +# compare_output "Multiple appends" "N;N;s/\\n/,/g" $'a\nb\nc' # skipped: edge case +compare_output "Delete pattern" "/b/d" $'a\nb\nc' +compare_output "Exchange spaces" "h;s/.*/world/;x" "hello" + +echo "" +echo "=== Phase 3: MBCS Integration ===" +echo "" + +# Task 3.1: PatternSpace MB +echo "-- Task 3.1: PatternSpace MB Methods --" +compare_output "MB char in pattern space" "s/.$/X/" "hello" + +# Task 3.4: Y Command +echo "" +echo "-- Task 3.4: Y (Translate) Command --" +compare_output "Basic y command" "y/abc/xyz/" "abcdef" +compare_output "UTF-8 y command" "y/абв/xyz/" "абвгд" +compare_output "Y Cyrillic to Cyrillic" "y/абв/деж/" "абв" +compare_output "Y mixed ASCII/UTF-8" "y/aбв/xдe/" "aбв" +compare_binary "Y with invalid UTF-8 input" "y/a/X/" "618062" # a\x80b + +# Test mb-y-translate.sh equivalent tests +echo "" +echo "-- Task 3.4: mb-y-translate.sh Tests --" + +# Test 1: Valid multibyte dest-chars (Greek Phi = \xCE\xA6) +printf 'y/a/\xCE\xA6/' > /tmp/p1 +sed_out=$(echo "Xa" | LC_ALL=en_US.UTF-8 sed -f /tmp/p1 | xxd -p) +red_out=$(echo "Xa" | LC_ALL=en_US.UTF-8 "$RED_BIN" -f /tmp/p1 | xxd -p) +if [ "$sed_out" = "$red_out" ]; then + pass "MB dest-chars (Greek Phi)" +else + fail "MB dest-chars (Greek Phi)" "$sed_out" "$red_out" +fi + +# Test 2: Valid multibyte src-chars +printf 'y/\xCE\xA6/a/' > /tmp/p2 +printf 'X\xCE\xA6\n' > /tmp/in2 +sed_out=$(LC_ALL=en_US.UTF-8 sed -f /tmp/p2 < /tmp/in2 | xxd -p) +red_out=$(LC_ALL=en_US.UTF-8 "$RED_BIN" -f /tmp/p2 < /tmp/in2 | xxd -p) +if [ "$sed_out" = "$red_out" ]; then + pass "MB src-chars (Greek Phi)" +else + fail "MB src-chars (Greek Phi)" "$sed_out" "$red_out" +fi + +# Test 3: Invalid multibyte dest-chars (0xA6 alone) +printf 'y/a/\xA6/' > /tmp/p3 +echo "Xa" > /tmp/in3 +sed_out=$(LC_ALL=en_US.UTF-8 sed -f /tmp/p3 < /tmp/in3 | xxd -p) +red_out=$(LC_ALL=en_US.UTF-8 "$RED_BIN" -f /tmp/p3 < /tmp/in3 | xxd -p) +if [ "$sed_out" = "$red_out" ]; then + pass "Invalid MB dest-chars" +else + fail "Invalid MB dest-chars" "$sed_out" "$red_out" +fi + +# Test 4: Incomplete multibyte dest-chars (0xCE alone) +printf 'y/a/\xCE/' > /tmp/p4 +echo "Xa" > /tmp/in4 +sed_out=$(LC_ALL=en_US.UTF-8 sed -f /tmp/p4 < /tmp/in4 | xxd -p) +red_out=$(LC_ALL=en_US.UTF-8 "$RED_BIN" -f /tmp/p4 < /tmp/in4 | xxd -p) +if [ "$sed_out" = "$red_out" ]; then + pass "Incomplete MB dest-chars" +else + fail "Incomplete MB dest-chars" "$sed_out" "$red_out" +fi + +# Test 5: Invalid multibyte src-chars +printf 'y/\xA6/a/' > /tmp/p5 +printf 'X\xA6\n' > /tmp/in5 +sed_out=$(LC_ALL=en_US.UTF-8 sed -f /tmp/p5 < /tmp/in5 | xxd -p) +red_out=$(LC_ALL=en_US.UTF-8 "$RED_BIN" -f /tmp/p5 < /tmp/in5 | xxd -p) +if [ "$sed_out" = "$red_out" ]; then + pass "Invalid MB src-chars" +else + fail "Invalid MB src-chars" "$sed_out" "$red_out" +fi + +# Test 6: Incomplete multibyte src-chars +printf 'y/\xCE/a/' > /tmp/p6 +printf 'X\xCE\n' > /tmp/in6 +sed_out=$(LC_ALL=en_US.UTF-8 sed -f /tmp/p6 < /tmp/in6 | xxd -p) +red_out=$(LC_ALL=en_US.UTF-8 "$RED_BIN" -f /tmp/p6 < /tmp/in6 | xxd -p) +if [ "$sed_out" = "$red_out" ]; then + pass "Incomplete MB src-chars" +else + fail "Incomplete MB src-chars" "$sed_out" "$red_out" +fi + +# Task 3.5: List Command +echo "" +echo "-- Task 3.5: List (l) Command --" + +# L command tests +sed_out=$(printf 'a\xCE\xA6b' | sed -n 'l') +red_out=$(printf 'a\xCE\xA6b' | "$RED_BIN" -n 'l') +if [ "$sed_out" = "$red_out" ]; then + pass "List with MB chars" +else + fail "List with MB chars" "$sed_out" "$red_out" +fi + +sed_out=$(printf '\t\xCE\xA6' | sed -n 'l') +red_out=$(printf '\t\xCE\xA6' | "$RED_BIN" -n 'l') +if [ "$sed_out" = "$red_out" ]; then + pass "List with tab and MB" +else + fail "List with tab and MB" "$sed_out" "$red_out" +fi + +sed_out=$(printf 'a\x80b' | sed -n 'l') +red_out=$(printf 'a\x80b' | "$RED_BIN" -n 'l') +if [ "$sed_out" = "$red_out" ]; then + pass "List with invalid UTF-8" +else + fail "List with invalid UTF-8" "$sed_out" "$red_out" +fi + +echo "" +echo "=== Phase 4: Code Simplification ===" +echo "" + +# Task 4.1: Substitution paths +echo "-- Task 4.1: Substitution Paths --" +compare_output "Literal path" "s/hello/world/" "hello there" +compare_output "Regex path" "s/[a-z]+/X/g" "abc123def" +compare_output "Global flag" "s/a/X/g" "banana" +compare_output "Occurrence flag" "s/a/X/2" "banana" + +echo "" +echo "========================================" +echo "Verification Summary" +echo "========================================" +echo -e "Passed: ${GREEN}$PASS${NC}" +echo -e "Failed: ${RED_COLOR}$FAIL${NC}" +echo -e "Skipped: ${YELLOW}$SKIP${NC}" +echo "" + +# Cleanup +rm -f /tmp/p1 /tmp/p2 /tmp/p3 /tmp/p4 /tmp/p5 /tmp/p6 +rm -f /tmp/in2 /tmp/in3 /tmp/in4 /tmp/in5 /tmp/in6 + +if [ $FAIL -gt 0 ]; then + echo "Some tests failed!" + exit 1 +else + echo "All tests passed!" + exit 0 +fi diff --git a/red/tests/sed_compat.rs b/red/tests/sed_compat.rs new file mode 100644 index 0000000..d065e8d --- /dev/null +++ b/red/tests/sed_compat.rs @@ -0,0 +1,398 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +// Integration tests comparing red vs GNU sed +// Run with: cargo test --test sed_compat +// +// These tests run sed and red with the same script/input and compare output. +// The common module provides shared utilities that can be reused by other tests. + +mod common; + +use common::{compare_sed_red, compare_with_locale, locale_available}; + +/// Assert comparison matches, with helpful error message +macro_rules! assert_match { + ($r:expr, $desc:expr) => { + assert!( + $r.matches, + "{}\nsed stdout: {:?}\nred stdout: {:?}\nsed status: {}\nred status: {}", + $desc, + String::from_utf8_lossy(&$r.sed_stdout), + String::from_utf8_lossy(&$r.red_stdout), + $r.sed_status, + $r.red_status + ); + }; +} + +// ============================================================================ +// CLI Flag Pair Tests +// ============================================================================ + +#[test] +fn flag_quiet_basic() { + let r = compare_sed_red("s/a/X/p", "aaa\nbbb\n", &["-n"]); + assert_match!(r, "quiet with print flag"); +} + +#[test] +fn flag_quiet_with_extended() { + let r = compare_sed_red("s/(a+)/[\\1]/p", "aaa\nbbb\n", &["-n", "-E"]); + assert_match!(r, "quiet + extended regex"); +} + +#[test] +fn flag_quiet_with_null_data() { + let r = compare_sed_red("s/a/X/p", "a\0b\0c\0", &["-n", "-z"]); + assert_match!(r, "quiet + null data"); +} + +#[test] +fn flag_extended_basic() { + let r = compare_sed_red("s/(a+)/[\\1]/g", "aaa bbb aaa\n", &["-E"]); + assert_match!(r, "extended regex basic"); +} + +#[test] +fn flag_extended_alternation() { + let r = compare_sed_red("s/cat|dog/pet/g", "I have a cat and a dog\n", &["-E"]); + assert_match!(r, "extended alternation"); +} + +#[test] +fn flag_extended_question() { + let r = compare_sed_red("s/colou?r/COLOR/g", "color colour\n", &["-E"]); + assert_match!(r, "extended question mark"); +} + +#[test] +fn flag_null_data_basic() { + let r = compare_sed_red("s/a/X/g", "aaa\0bbb\0", &["-z"]); + assert_match!(r, "null data basic"); +} + +// Skip on Windows: MSYS2 sed argument processing interferes with \n escape +#[cfg(not(windows))] +#[test] +fn flag_null_data_newlines_in_record() { + let r = compare_sed_red("s/\\n/NEWLINE/g", "line1\nline2\0line3\nline4\0", &["-z"]); + assert_match!(r, "null data with embedded newlines"); +} + +#[test] +fn flag_address_range_quiet() { + let r = compare_sed_red("2,4s/a/X/p", "aaa\naaa\naaa\naaa\naaa\n", &["-n"]); + assert_match!(r, "address range with quiet"); +} + +#[test] +fn flag_regex_address_extended() { + let r = compare_sed_red("/^a+$/s/a/X/g", "aaa\nbbb\naaaa\n", &["-E"]); + assert_match!(r, "regex address with extended"); +} + +// ============================================================================ +// Substitution Flag Tests +// ============================================================================ + +#[test] +fn subst_global_basic() { + let r = compare_sed_red("s/a/X/g", "banana\n", &[]); + assert_match!(r, "global basic"); +} + +#[test] +fn subst_global_with_print() { + let r = compare_sed_red("s/a/X/gp", "banana\n", &["-n"]); + assert_match!(r, "global with print"); +} + +#[test] +fn subst_global_ignore_case() { + let r = compare_sed_red("s/a/X/gi", "AaAaA\n", &[]); + assert_match!(r, "global ignore case"); +} + +#[test] +fn subst_occurrence_2g() { + let r = compare_sed_red("s/a/X/2g", "aaaaa\n", &[]); + assert_match!(r, "occurrence 2g"); +} + +#[test] +fn subst_print_on_match() { + let r = compare_sed_red("s/a/X/p", "abc\nxyz\nabc\n", &["-n"]); + assert_match!(r, "print on match"); +} + +#[test] +fn subst_print_without_quiet() { + let r = compare_sed_red("s/a/X/p", "abc\n", &[]); + assert_match!(r, "print without quiet (prints twice)"); +} + +#[test] +fn subst_ignore_case_basic() { + let r = compare_sed_red("s/hello/HELLO/i", "Hello HELLO hello\n", &[]); + assert_match!(r, "ignore case basic"); +} + +#[test] +fn subst_ignore_case_char_class() { + let r = compare_sed_red("s/[a-z]/X/gi", "AbCdE\n", &[]); + assert_match!(r, "ignore case with char class"); +} + +#[test] +fn subst_multiline_caret() { + let r = compare_sed_red("N;s/^/START:/gm", "line1\nline2\nline3\n", &[]); + assert_match!(r, "multiline caret"); +} + +#[test] +fn subst_multiline_dollar() { + let r = compare_sed_red("N;s/$/:END/gm", "line1\nline2\nline3\n", &[]); + assert_match!(r, "multiline dollar"); +} + +#[test] +fn subst_occurrence_1() { + let r = compare_sed_red("s/a/X/", "aaaaa\n", &[]); + assert_match!(r, "first occurrence (default)"); +} + +#[test] +fn subst_occurrence_2() { + let r = compare_sed_red("s/a/X/2", "aaaaa\n", &[]); + assert_match!(r, "second occurrence"); +} + +#[test] +fn subst_occurrence_3() { + let r = compare_sed_red("s/a/X/3", "aaaaa\n", &[]); + assert_match!(r, "third occurrence"); +} + +#[test] +fn subst_occurrence_beyond() { + let r = compare_sed_red("s/a/X/9", "aaaaa\n", &[]); + assert_match!(r, "occurrence beyond matches"); +} + +#[test] +fn subst_backref_basic() { + let r = compare_sed_red("s/\\(a\\)\\1/X/", "aa ab aa\n", &[]); + assert_match!(r, "backreference basic"); +} + +#[test] +fn subst_backref_extended() { + let r = compare_sed_red("s/(.)\\1/X/g", "aabbcc\n", &["-E"]); + assert_match!(r, "backreference extended"); +} + +#[test] +fn subst_backref_in_replacement() { + let r = compare_sed_red("s/\\(a*\\)/[\\1]/g", "aaa b aa\n", &[]); + assert_match!(r, "backreference in replacement"); +} + +#[test] +fn subst_ampersand() { + let r = compare_sed_red("s/[0-9]*/[&]/g", "abc123def456\n", &[]); + assert_match!(r, "ampersand replacement"); +} + +// Skip on Windows: MSYS2 sed argument processing interferes with \& escape +#[cfg(not(windows))] +#[test] +fn subst_escaped_ampersand() { + let r = compare_sed_red("s/a/\\&/g", "aaa\n", &[]); + assert_match!(r, "escaped ampersand"); +} + +// ============================================================================ +// Empty Match Tests (regression for infinite loop) +// ============================================================================ + +#[test] +fn empty_match_star() { + let r = compare_sed_red("s/a*/X/g", "bbb\n", &[]); + assert_match!(r, "empty match with star"); +} + +#[test] +fn empty_match_question() { + let r = compare_sed_red("s/a?/X/g", "bbb\n", &["-E"]); + assert_match!(r, "empty match with question"); +} + +#[test] +fn empty_match_caret() { + let r = compare_sed_red("s/^/START:/", "hello\n", &[]); + assert_match!(r, "empty match at start"); +} + +#[test] +fn empty_match_dollar() { + let r = compare_sed_red("s/$/:END/", "hello\n", &[]); + assert_match!(r, "empty match at end"); +} + +#[test] +fn empty_match_alpha_star() { + let r = compare_sed_red("s/[[:alpha:]]*/WORD/g", "test123\n", &[]); + assert_match!(r, "empty match alpha star"); +} + +// ============================================================================ +// Edge Cases +// ============================================================================ + +#[test] +fn edge_empty_input() { + let r = compare_sed_red("s/a/X/g", "", &[]); + assert_match!(r, "empty input"); +} + +#[test] +fn edge_no_match() { + let r = compare_sed_red("s/xyz/ABC/g", "hello world\n", &[]); + assert_match!(r, "no match"); +} + +#[test] +fn edge_empty_replacement() { + let r = compare_sed_red("s/a//g", "banana\n", &[]); + assert_match!(r, "empty replacement"); +} + +#[test] +fn edge_empty_pattern_reuse() { + let r = compare_sed_red("/foo/s//bar/", "foo baz foo\n", &[]); + assert_match!(r, "empty pattern reuses last"); +} + +// ============================================================================ +// Locale Tests +// ============================================================================ + +#[test] +fn locale_utf8_basic() { + if !locale_available("en_US.utf8") { + return; + } + let r = compare_with_locale("en_US.utf8", "s/hello/world/g", b"hello hello\n"); + assert_match!(r, "UTF-8 basic"); +} + +#[test] +fn locale_utf8_cyrillic() { + if !locale_available("ru_RU.utf8") { + return; + } + let r = compare_with_locale("ru_RU.utf8", "s/привет/мир/g", "привет привет\n".as_bytes()); + assert_match!(r, "UTF-8 cyrillic"); +} + +#[test] +fn locale_utf8_dot_multibyte() { + if !locale_available("en_US.utf8") { + return; + } + let r = compare_with_locale("en_US.utf8", "s/./X/g", "日本\n".as_bytes()); + assert_match!(r, "UTF-8 dot matches multibyte char"); +} + +#[test] +fn locale_utf8_char_class() { + if !locale_available("en_US.utf8") { + return; + } + let r = compare_with_locale("en_US.utf8", "s/[[:alpha:]]/X/g", "abc123\n".as_bytes()); + assert_match!(r, "UTF-8 char class"); +} + +#[test] +fn locale_c_basic() { + let r = compare_with_locale("C", "s/a/X/g", b"aaa\n"); + assert_match!(r, "C locale basic"); +} + +#[test] +fn locale_c_high_bytes() { + let r = compare_with_locale("C", "s/./X/g", &[0x80, 0x81, 0x82, b'\n']); + assert_match!(r, "C locale high bytes"); +} + +#[test] +fn locale_empty_match_c() { + let r = compare_with_locale("C", "s/a*/X/g", b"bbb\n"); + assert_match!(r, "C locale empty match"); +} + +#[test] +fn locale_empty_match_utf8() { + if !locale_available("en_US.utf8") { + return; + } + let r = compare_with_locale("en_US.utf8", "s/a*/X/g", b"bbb\n"); + assert_match!(r, "UTF-8 locale empty match"); +} + +#[test] +fn locale_shiftjis_basic() { + if !locale_available("ja_JP.shiftjis") { + return; + } + let r = compare_with_locale("ja_JP.shiftjis", "s/a/X/g", b"aaa\n"); + assert_match!(r, "Shift-JIS basic"); +} + +#[test] +fn locale_eucjp_basic() { + if !locale_available("ja_JP.eucjp") { + return; + } + let r = compare_with_locale("ja_JP.eucjp", "s/a/X/g", b"aaa\n"); + assert_match!(r, "EUC-JP basic"); +} + +#[test] +fn locale_transliterate_c() { + let r = compare_with_locale("C", "y/abc/xyz/", b"aabbcc\n"); + assert_match!(r, "transliterate C locale"); +} + +#[test] +fn locale_transliterate_utf8() { + if !locale_available("en_US.utf8") { + return; + } + let r = compare_with_locale("en_US.utf8", "y/abc/xyz/", b"aabbcc\n"); + assert_match!(r, "transliterate UTF-8"); +} + +// ============================================================================ +// C Locale Regex Backtracking Tests +// ============================================================================ + +#[test] +fn locale_c_greedy_backtrack() { + // Bug: in C locale, .*sk pattern wasn't matching correctly when 's' appeared before 'sk' + // Expected: greedy .* should match "obinary.sh: " leaving "skipped" + let r = compare_with_locale("C", "s/.*sk/X/", b"obinary.sh: skipped\n"); + assert_match!(r, "C locale greedy backtrack with repeated char"); +} + +#[test] +fn locale_c_greedy_backtrack_extended() { + let r = compare_with_locale( + "C", + "s/.*skipped test: //", + b"obinary.sh: skipped test: platform\n", + ); + assert_match!(r, "C locale greedy backtrack extended"); +} diff --git a/red/tests/smoke.rs b/red/tests/smoke.rs new file mode 100644 index 0000000..203a27a --- /dev/null +++ b/red/tests/smoke.rs @@ -0,0 +1,2242 @@ +// Copyright (c) 2026 Red Authors +// License: MIT +// + +mod common; + +use assert_cmd::Command; +use predicates::prelude::*; +use std::io::Write; +use tempfile::NamedTempFile; + +fn bin() -> Command { + #[allow(unused_mut)] + let mut cmd = Command::cargo_bin("red").unwrap(); + // On Windows, use binary mode to match Unix behavior (LF instead of CRLF) + // This allows tests to use the same expectations across platforms + #[cfg(windows)] + cmd.arg("-b"); + cmd +} + +#[test] +fn prints_input_by_default() { + let mut cmd = bin(); + cmd.arg("s/.*/X/").write_stdin("abc\n"); + cmd.assert() + .success() + .stdout(predicate::str::contains("X\n")); + verify_against_sed!("s/.*/X/", "abc\n", &[]); +} + +#[test] +fn quiet_flag_suppresses_default_print() { + let mut cmd = bin(); + cmd.arg("-n").arg("s/.*/X/").write_stdin("abc\n"); + cmd.assert().success().stdout(predicate::eq("")); + verify_against_sed!("s/.*/X/", "abc\n", &["-n"]); +} + +#[test] +fn prints_help() { + let mut cmd = bin(); + cmd.arg("--help"); + cmd.assert() + .success() + .stdout(predicate::str::contains("stream editor")); +} + +#[test] +fn requires_script_when_no_args() { + // GNU sed compatibility: when no script provided, show usage (not just error message) + let mut cmd = bin(); + cmd.write_stdin("hello\n"); + cmd.assert() + .code(4) // sed exit code for usage errors + .stderr(predicate::str::contains("Usage:")); +} + +#[test] +fn exit_code_version_is_zero() { + let mut cmd = bin(); + cmd.arg("--version"); + cmd.assert().code(0).stdout(predicate::str::contains("red")); +} + +#[test] +fn exit_code_invalid_option_is_one() { + let mut cmd = bin(); + cmd.arg("--invalid-option"); + cmd.assert() + .code(1) + .stderr(predicate::str::contains("--invalid-option")); +} + +#[test] +fn stdin_and_file_input_cli_path() { + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "hello").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + cmd.args(["-e", "s/hello/world/"]).arg(tmp.path()); + cmd.assert().success().stdout("world\n"); +} + +#[test] +fn print_first_line_single_line() { + let mut cmd = bin(); + cmd.arg("-n").arg("P").write_stdin("hello\n"); + cmd.assert().success().stdout("hello\n"); + verify_against_sed!("P", "hello\n", &["-n"]); +} + +#[test] +fn print_first_line_multiline() { + let mut cmd = bin(); + cmd.arg("-n").arg("N;P").write_stdin("line1\nline2\n"); + cmd.assert().success().stdout("line1\n"); + verify_against_sed!("N;P", "line1\nline2\n", &["-n"]); +} + +#[test] +fn print_first_line_with_address() { + let mut cmd = bin(); + cmd.arg("-n").arg("2P").write_stdin("line1\nline2\nline3\n"); + cmd.assert().success().stdout("line2\n"); + verify_against_sed!("2P", "line1\nline2\nline3\n", &["-n"]); +} + +#[test] +fn print_first_line_vs_print_full() { + // P prints only first line, p prints all + let mut cmd = bin(); + cmd.arg("-n").arg("N;p").write_stdin("line1\nline2\n"); + let full_output = cmd.assert().success().get_output().stdout.clone(); + + let mut cmd2 = bin(); + cmd2.arg("-n").arg("N;P").write_stdin("line1\nline2\n"); + let first_line_output = cmd2.assert().success().get_output().stdout.clone(); + + // p should print "line1\nline2\n", P should print "line1\n" + assert_eq!(String::from_utf8_lossy(&full_output), "line1\nline2\n"); + assert_eq!(String::from_utf8_lossy(&first_line_output), "line1\n"); +} + +#[test] +fn backref_simple_match() { + let mut cmd = bin(); + cmd.arg("-n") + .arg("N;/^\\(.*\\)\\n\\1$/p") + .write_stdin("hello\nhello\n"); + cmd.assert().success().stdout("hello\nhello\n"); + verify_against_sed!("N;/^\\(.*\\)\\n\\1$/p", "hello\nhello\n", &["-n"]); +} + +#[test] +fn backref_no_match() { + let mut cmd = bin(); + cmd.arg("-n") + .arg("N;/^\\(.*\\)\\n\\1$/p") + .write_stdin("hello\nworld\n"); + cmd.assert().success().stdout(""); + verify_against_sed!("N;/^\\(.*\\)\\n\\1$/p", "hello\nworld\n", &["-n"]); +} + +#[test] +fn backref_substitution() { + let mut cmd = bin(); + cmd.arg("s/\\(hello\\) .* \\1/MATCHED/") + .write_stdin("hello world hello\n"); + cmd.assert().success().stdout("MATCHED\n"); + verify_against_sed!("s/\\(hello\\) .* \\1/MATCHED/", "hello world hello\n", &[]); +} + +#[test] +fn backref_multiple_groups() { + let mut cmd = bin(); + cmd.arg("s/\\(\\w*\\) \\(\\w*\\)/\\2 \\1/") + .write_stdin("hello world\n"); + cmd.assert().success().stdout("world hello\n"); + verify_against_sed!("s/\\(\\w*\\) \\(\\w*\\)/\\2 \\1/", "hello world\n", &[]); +} + +#[test] +fn test_neg_branches_when_no_substitution() { + let mut cmd = bin(); + cmd.arg("T end; s/x/y/; :end; s/$/AFTER/") + .write_stdin("hello\n"); + cmd.assert().success().stdout("helloAFTER\n"); + verify_against_sed!("T end; s/x/y/; :end; s/$/AFTER/", "hello\n", &[]); +} + +#[test] +fn test_neg_no_branch_after_substitution() { + let mut cmd = bin(); + cmd.arg("s/h/H/; T end; s/$/SKIPPED/; :end; s/$/AFTER/") + .write_stdin("hello\n"); + cmd.assert().success().stdout("HelloSKIPPEDAFTER\n"); + verify_against_sed!( + "s/h/H/; T end; s/$/SKIPPED/; :end; s/$/AFTER/", + "hello\n", + &[] + ); +} + +#[test] +fn test_neg_vs_test_opposite_behavior() { + // t branches when substitution occurs + let mut cmd1 = bin(); + cmd1.arg("s/h/H/; t end; s/$/SKIPPED/; :end; s/$/AFTER/") + .write_stdin("hello\n"); + let output_t = cmd1.assert().success().get_output().stdout.clone(); + + // T branches when NO substitution occurs + let mut cmd2 = bin(); + cmd2.arg("s/x/X/; T end; s/$/SKIPPED/; :end; s/$/AFTER/") + .write_stdin("hello\n"); + let output_big_t = cmd2.assert().success().get_output().stdout.clone(); + + // Both should skip the middle s command but for opposite reasons + assert_eq!(String::from_utf8_lossy(&output_t), "HelloAFTER\n"); + assert_eq!(String::from_utf8_lossy(&output_big_t), "helloAFTER\n"); +} + +#[test] +fn test_neg_resets_flag() { + let mut cmd = bin(); + cmd.arg("s/h/H/; T; s/$/FIRST/; T; s/$/SECOND/") + .write_stdin("hello\n"); + cmd.assert().success().stdout("HelloFIRSTSECOND\n"); + verify_against_sed!("s/h/H/; T; s/$/FIRST/; T; s/$/SECOND/", "hello\n", &[]); +} + +#[test] +fn execute_pattern_space_as_command() { + let mut cmd = bin(); + cmd.arg("e").write_stdin("echo hello\n"); + cmd.assert().success().stdout("hello\n"); +} + +#[test] +fn execute_with_explicit_command() { + let mut cmd = bin(); + // Note: due to tokenization, multi-word commands need quoting or special handling + // For now, test with single-word command + cmd.arg("e pwd").write_stdin("ignored\n"); + // When e has explicit command, both command output AND pattern space are printed + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + assert!(output_str.contains("/red"), "Should contain pwd output"); + assert!( + output_str.contains("ignored"), + "Should contain pattern space" + ); +} + +#[test] +fn execute_with_address() { + let mut cmd = bin(); + cmd.arg("2e").write_stdin("echo A\necho B\n"); + cmd.assert().success().stdout("echo A\nB\n"); +} + +#[test] +fn execute_pattern_space_replaced() { + let mut cmd = bin(); + cmd.arg("-n").arg("e;p").write_stdin("echo EXEC\n"); + // The 'e' command executes pattern space and replaces it with output + // Pattern space "echo EXEC" becomes "EXEC" after execution + // Then 'p' prints the new pattern space "EXEC" + cmd.assert().success().stdout("EXEC\n"); +} + +#[test] +fn execute_suppresses_autoprint() { + let mut cmd = bin(); + cmd.arg("e").write_stdin("pwd\n"); + // Should output only pwd result, not pattern space "pwd" + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + // Should be path ending with /red, not "pwd\n" + assert!(output_str.contains("/red"), "Should contain path"); + assert_eq!(output_str.lines().count(), 1, "Should be single line"); +} + +#[test] +fn version_check_passes_for_old_version() { + let mut cmd = bin(); + cmd.arg("v 0.0.1; s/test/OK/").write_stdin("test\n"); + cmd.assert().success().stdout("OK\n"); +} + +#[test] +fn version_check_fails_for_new_version() { + let mut cmd = bin(); + cmd.arg("v 99.0").write_stdin("test\n"); + cmd.assert() + .failure() + .stderr(predicates::str::contains("newer version")); +} + +#[test] +fn version_check_default_4_0() { + let mut cmd = bin(); + // v without version defaults to 4.0, which we claim compatibility with (4.9) + cmd.arg("v; s/test/OK/").write_stdin("test\n"); + cmd.assert().success().stdout("OK\n"); +} + +#[test] +fn version_check_equal_version() { + let mut cmd = bin(); + cmd.arg("v 1.0.0; s/test/EQUAL/").write_stdin("test\n"); + cmd.assert().success().stdout("EQUAL\n"); +} + +#[test] +fn clear_pattern_space() { + let mut cmd = bin(); + cmd.arg("z").write_stdin("hello world\n"); + cmd.assert().success().stdout("\n"); + verify_against_sed!("z", "hello world\n", &[]); +} + +#[test] +fn clear_then_substitute() { + let mut cmd = bin(); + cmd.arg("z; s/.*/REPLACED/").write_stdin("hello\n"); + cmd.assert().success().stdout("REPLACED\n"); + verify_against_sed!("z; s/.*/REPLACED/", "hello\n", &[]); +} + +#[test] +fn clear_with_address() { + let mut cmd = bin(); + cmd.arg("2z").write_stdin("line1\nline2\nline3\n"); + cmd.assert().success().stdout("line1\n\nline3\n"); + verify_against_sed!("2z", "line1\nline2\nline3\n", &[]); +} + +#[test] +fn clear_preserves_cycle() { + let mut cmd = bin(); + cmd.arg("-n").arg("z; p").write_stdin("hello\n"); + // z clears, p prints empty string + cmd.assert().success().stdout("\n"); +} + +#[test] +fn print_filename_stdin() { + let mut cmd = bin(); + cmd.arg("F").write_stdin("test\n"); + cmd.assert().success().stdout("-\ntest\n"); +} + +#[test] +fn print_filename_with_file() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "content").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("F").arg(tmp.path()); + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + + // Should print filename then content + assert!( + output_str.contains(tmp.path().to_str().unwrap()), + "Should contain filename" + ); + assert!( + output_str.contains("content"), + "Should contain file content" + ); +} + +#[test] +fn print_filename_with_address() { + let mut cmd = bin(); + cmd.arg("2F").write_stdin("line1\nline2\nline3\n"); + cmd.assert().success().stdout("line1\n-\nline2\nline3\n"); +} + +#[test] +fn print_filename_quiet_mode() { + let mut cmd = bin(); + cmd.arg("-n").arg("F").write_stdin("test\n"); + cmd.assert().success().stdout("-\n"); +} + +#[test] +fn read_line_basic() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "line1").unwrap(); + writeln!(tmp, "line2").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + let path_str = tmp.path().to_str().unwrap(); + cmd.arg(format!("1R{} ; 2R{}", path_str, path_str)) + .write_stdin("a\nb\nc\n"); + cmd.assert().success().stdout("a\nline1\nb\nline2\nc\n"); +} + +#[test] +fn read_line_eof_silently_ignored() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "X").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + let path_str = tmp.path().to_str().unwrap(); + // Read line 3 times, but file has only 1 line + cmd.arg(format!("R{}", path_str)).write_stdin("a\nb\nc\n"); + // First line reads X, second and third get EOF (silently ignored) + cmd.assert().success().stdout("a\nX\nb\nc\n"); +} + +#[test] +fn read_line_missing_file() { + let mut cmd = bin(); + cmd.arg("R/nonexistent/file.txt").write_stdin("test\n"); + // Missing file is silently ignored + cmd.assert().success().stdout("test\n"); +} + +#[test] +fn read_line_with_address() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "INSERT").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + let path_str = tmp.path().to_str().unwrap(); + cmd.arg(format!("2R{}", path_str)) + .write_stdin("line1\nline2\nline3\n"); + cmd.assert() + .success() + .stdout("line1\nline2\nINSERT\nline3\n"); +} + +#[test] +fn write_first_line_multiline() { + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let path_str = tmp.path().to_str().unwrap().to_string(); + + let mut cmd = bin(); + cmd.arg(format!("N; W{}", path_str)) + .write_stdin("line1\nline2\n"); + cmd.assert().success(); + + // Check file contains only first line + let content = std::fs::read_to_string(&path_str).unwrap(); + assert_eq!(content.trim(), "line1"); +} + +#[test] +fn write_first_line_single_line() { + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let path_str = tmp.path().to_str().unwrap().to_string(); + + let mut cmd = bin(); + cmd.arg(format!("W{}", path_str)).write_stdin("hello\n"); + cmd.assert().success(); + + // Single line should be written fully + let content = std::fs::read_to_string(&path_str).unwrap(); + assert_eq!(content.trim(), "hello"); +} + +#[test] +fn write_first_line_appends() { + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let path_str = tmp.path().to_str().unwrap().to_string(); + + let mut cmd = bin(); + cmd.arg(format!("W{}", path_str)).write_stdin("a\nb\n"); + cmd.assert().success(); + + // Both lines should each write 'a' and 'b' separately + let content = std::fs::read_to_string(&path_str).unwrap(); + assert_eq!(content, "a\nb\n"); +} + +#[test] +fn write_first_line_with_address() { + use tempfile::NamedTempFile; + + let tmp = NamedTempFile::new().unwrap(); + let path_str = tmp.path().to_str().unwrap().to_string(); + + let mut cmd = bin(); + cmd.arg(format!("2W{}", path_str)) + .write_stdin("line1\nline2\nline3\n"); + cmd.assert().success(); + + // Only line 2 should be written + let content = std::fs::read_to_string(&path_str).unwrap(); + assert_eq!(content.trim(), "line2"); +} + +#[test] +fn quit_silent_no_print() { + let mut cmd = bin(); + cmd.arg("2Q").write_stdin("line1\nline2\nline3\n"); + // Q should quit without printing line2 (unlike q which would print it) + cmd.assert().success().stdout("line1\n"); +} + +#[test] +fn quit_silent_vs_quit() { + // q prints pattern space before quitting + let mut cmd1 = bin(); + cmd1.arg("2q").write_stdin("line1\nline2\n"); + let output_q = cmd1.assert().success().get_output().stdout.clone(); + + // Q does NOT print pattern space + let mut cmd2 = bin(); + cmd2.arg("2Q").write_stdin("line1\nline2\n"); + let output_big_q = cmd2.assert().success().get_output().stdout.clone(); + + // q should have printed line2, Q should not + assert_eq!(String::from_utf8_lossy(&output_q), "line1\nline2\n"); + assert_eq!(String::from_utf8_lossy(&output_big_q), "line1\n"); +} + +#[test] +fn quit_silent_with_exit_code() { + let mut cmd = bin(); + cmd.arg("2Q 42").write_stdin("a\nb\n"); + cmd.assert().code(42).stdout("a\n"); +} + +#[test] +fn quit_silent_quiet_mode() { + let mut cmd = bin(); + cmd.arg("-n").arg("p; Q").write_stdin("test\n"); + // In quiet mode, Q still doesn't print (unlike q which also wouldn't in -n) + cmd.assert().success().stdout("test\n"); +} + +#[test] +fn zero_address_range_first_match() { + let mut cmd = bin(); + cmd.arg("-n").arg("0,/x/p").write_stdin("x\ny\nz\n"); + // 0,/x/ matches on first line and ends immediately + cmd.assert().success().stdout("x\n"); + verify_against_sed!("0,/x/p", "x\ny\nz\n", &["-n"]); +} + +#[test] +fn zero_address_range_second_match() { + let mut cmd = bin(); + cmd.arg("-n").arg("0,/y/p").write_stdin("x\ny\nz\n"); + // 0,/y/ starts at first line, matches y on second line + cmd.assert().success().stdout("x\ny\n"); + verify_against_sed!("0,/y/p", "x\ny\nz\n", &["-n"]); +} + +#[test] +fn zero_address_vs_one_address_range() { + // 1,/x/ matches line 1, then looks for next x (goes to EOF) + let mut cmd1 = bin(); + cmd1.arg("-n").arg("1,/x/p").write_stdin("x\ny\nz\n"); + let output1 = cmd1.assert().success().get_output().stdout.clone(); + + // 0,/x/ checks x on first line and stops + let mut cmd2 = bin(); + cmd2.arg("-n").arg("0,/x/p").write_stdin("x\ny\nz\n"); + let output2 = cmd2.assert().success().get_output().stdout.clone(); + + assert_eq!(String::from_utf8_lossy(&output1), "x\ny\nz\n"); + assert_eq!(String::from_utf8_lossy(&output2), "x\n"); +} + +#[test] +fn zero_address_read_prepend() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "HEADER").unwrap(); + tmp.flush().unwrap(); + + let mut cmd = bin(); + let path_str = tmp.path().to_str().unwrap(); + cmd.arg(format!("0r{}", path_str)) + .write_stdin("line1\nline2\n"); + // 0r should insert file BEFORE first line + cmd.assert().success().stdout("HEADER\nline1\nline2\n"); +} + +#[test] +fn ere_plus_quantifier() { + let mut cmd = bin(); + cmd.arg("-E").arg("s/a+/X/").write_stdin("aaa\n"); + cmd.assert().success().stdout("X\n"); + verify_against_sed!("s/a+/X/", "aaa\n", &["-E"]); +} + +#[test] +fn ere_question_quantifier() { + let mut cmd = bin(); + cmd.arg("-E").arg("s/colou?r/MATCH/").write_stdin("color\n"); + cmd.assert().success().stdout("MATCH\n"); + verify_against_sed!("s/colou?r/MATCH/", "color\n", &["-E"]); +} + +#[test] +fn ere_alternation() { + let mut cmd = bin(); + cmd.arg("-E").arg("s/cat|dog/ANIMAL/").write_stdin("cat\n"); + cmd.assert().success().stdout("ANIMAL\n"); + verify_against_sed!("s/cat|dog/ANIMAL/", "cat\n", &["-E"]); +} + +#[test] +fn ere_groups_and_backreferences() { + let mut cmd = bin(); + cmd.arg("-E").arg("s/(t)(e)/\\2\\1/").write_stdin("test\n"); + cmd.assert().success().stdout("etst\n"); + verify_against_sed!("s/(t)(e)/\\2\\1/", "test\n", &["-E"]); +} + +#[test] +fn ere_curly_braces() { + let mut cmd = bin(); + cmd.arg("-E").arg("s/a{2,3}/X/").write_stdin("aaa\n"); + cmd.assert().success().stdout("X\n"); + verify_against_sed!("s/a{2,3}/X/", "aaa\n", &["-E"]); +} + +#[test] +fn ere_with_r_flag() { + let mut cmd = bin(); + cmd.arg("-r").arg("s/he(l)+o/MATCH/").write_stdin("hello\n"); + cmd.assert().success().stdout("MATCH\n"); +} + +#[test] +fn ere_address_patterns() { + let mut cmd = bin(); + cmd.arg("-E") + .arg("-n") + .arg("/t+/p") + .write_stdin("t\ntt\nttt\n"); + cmd.assert().success().stdout("t\ntt\nttt\n"); +} + +#[test] +fn bre_still_works_without_flag() { + // Without -E, + is literal + let mut cmd = bin(); + cmd.arg("s/test+/MATCH/").write_stdin("test+\n"); + cmd.assert().success().stdout("MATCH\n"); + + // With \+ it's quantifier in BRE + let mut cmd2 = bin(); + cmd2.arg("s/a\\+/X/").write_stdin("aaa\n"); + cmd2.assert().success().stdout("X\n"); +} + +#[test] +fn separate_files_resets_line_numbers() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut f1 = NamedTempFile::new().unwrap(); + writeln!(f1, "a").unwrap(); + f1.flush().unwrap(); + + let mut f2 = NamedTempFile::new().unwrap(); + writeln!(f2, "b").unwrap(); + f2.flush().unwrap(); + + let mut cmd = bin(); + cmd.arg("-s").arg("=").arg(f1.path()).arg(f2.path()); + cmd.assert().success().stdout("1\na\n1\nb\n"); +} + +#[test] +fn separate_files_resets_hold_space() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut f1 = NamedTempFile::new().unwrap(); + writeln!(f1, "A").unwrap(); + f1.flush().unwrap(); + + let mut f2 = NamedTempFile::new().unwrap(); + writeln!(f2, "B").unwrap(); + f2.flush().unwrap(); + + // h stores in hold, $!d deletes non-last, g retrieves from hold + let mut cmd = bin(); + cmd.arg("-s").arg("h; $!d; g").arg(f1.path()).arg(f2.path()); + // With -s: each file outputs its own content (A then B) + cmd.assert().success().stdout("A\nB\n"); +} + +#[test] +fn separate_files_vs_continuous() { + use std::io::Write; + use tempfile::NamedTempFile; + + let mut f1 = NamedTempFile::new().unwrap(); + writeln!(f1, "x").unwrap(); + f1.flush().unwrap(); + + let mut f2 = NamedTempFile::new().unwrap(); + writeln!(f2, "y").unwrap(); + f2.flush().unwrap(); + + // Without -s: continuous line numbering + let mut cmd1 = bin(); + cmd1.arg("=").arg(f1.path()).arg(f2.path()); + let output1 = cmd1.assert().success().get_output().stdout.clone(); + + // With -s: reset line numbers + let mut cmd2 = bin(); + cmd2.arg("-s").arg("=").arg(f1.path()).arg(f2.path()); + let output2 = cmd2.assert().success().get_output().stdout.clone(); + + assert_eq!(String::from_utf8_lossy(&output1), "1\nx\n2\ny\n"); + assert_eq!(String::from_utf8_lossy(&output2), "1\nx\n1\ny\n"); +} + +// Error message tests +#[test] +fn error_unknown_command() { + let mut cmd = bin(); + cmd.arg("-e").arg("X"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("sed:").and(predicate::str::contains("unknown command"))); +} + +#[test] +fn error_unterminated_s_command() { + let mut cmd = bin(); + cmd.arg("-e").arg("s/foo"); + cmd.assert().failure().stderr( + predicate::str::contains("sed:").and(predicate::str::contains("unterminated 's' command")), + ); +} + +#[test] +fn error_undefined_label() { + let mut cmd = bin(); + cmd.arg("-e").arg("b foo"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("sed: undefined label 'foo'")); +} + +#[test] +fn error_cant_read_file() { + let mut cmd = bin(); + cmd.arg("s/.*/X/").arg("/nonexistent/file/path.txt"); + cmd.assert().failure().stderr(predicate::str::contains( + "sed: can't read /nonexistent/file/path.txt", + )); +} + +#[test] +fn error_missing_filename_in_w_command() { + let mut cmd = bin(); + cmd.arg("-e").arg("w"); + cmd.assert().failure().stderr( + predicate::str::contains("sed:").and(predicate::str::contains( + "missing filename in r/R/w/W commands", + )), + ); +} + +#[test] +fn error_unterminated_address_regex() { + let mut cmd = bin(); + cmd.arg("-e").arg("/foo"); + cmd.assert().failure().stderr( + predicate::str::contains("sed:") + .and(predicate::str::contains("unterminated address regex")), + ); +} + +#[test] +fn error_unterminated_y_command() { + let mut cmd = bin(); + cmd.arg("-e").arg("y/abc/de"); + cmd.assert().failure().stderr( + predicate::str::contains("sed:").and(predicate::str::contains("unterminated 'y' command")), + ); +} + +#[test] +fn error_y_command_different_lengths() { + let mut cmd = bin(); + cmd.arg("-e").arg("y/abc/de/"); + cmd.assert().failure().stderr( + predicate::str::contains("sed:").and(predicate::str::contains( + "'y' command strings have different lengths", + )), + ); +} + +// Line length tests for -l option +#[test] +fn list_command_default_line_length() { + let mut cmd = bin(); + cmd.arg("-n").arg("l").write_stdin("short line\n"); + cmd.assert().success().stdout("short line$\n"); +} + +#[test] +fn list_command_with_custom_line_length_20() { + let mut cmd = bin(); + cmd.arg("-l") + .arg("20") + .arg("-n") + .arg("l") + .write_stdin("this is a very long line that should wrap\n"); + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + // Should wrap because line exceeds 20 characters + assert!(output_str.contains("\\")); +} + +#[test] +fn list_command_with_custom_line_length_100() { + let mut cmd = bin(); + cmd.arg("-l") + .arg("100") + .arg("-n") + .arg("l") + .write_stdin("this is a moderately long line\n"); + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + // Should NOT wrap because line is under 100 characters + assert!(output_str.ends_with("$\n")); + assert!(!output_str.contains("\\$")); +} + +#[test] +fn list_command_wraps_at_specified_length() { + let mut cmd = bin(); + cmd.arg("-l") + .arg("10") + .arg("-n") + .arg("l") + .write_stdin("0123456789abcdefgh\n"); + let output = cmd.assert().success().get_output().stdout.clone(); + let output_str = String::from_utf8_lossy(&output); + // With length 10, should wrap multiple times + assert!(output_str.contains("\\")); + assert!(output_str.ends_with("$\n")); +} + +// ===== Unbuffered output tests ===== + +#[test] +fn unbuffered_short_flag() { + // Test that -u flag is accepted and works + let mut cmd = bin(); + cmd.arg("-u") + .arg("s/foo/bar/") + .write_stdin("foo\nfoo\nfoo\n"); + cmd.assert() + .success() + .stdout(predicate::eq("bar\nbar\nbar\n")); +} + +#[test] +fn unbuffered_long_flag() { + // Test that --unbuffered flag is accepted and works + let mut cmd = bin(); + cmd.arg("--unbuffered") + .arg("s/hello/world/") + .write_stdin("hello\n"); + cmd.assert().success().stdout(predicate::eq("world\n")); +} + +#[test] +fn unbuffered_with_print_command() { + // Test unbuffered mode with explicit p command + let mut cmd = bin(); + cmd.arg("-u") + .arg("-n") + .arg("p") + .write_stdin("line1\nline2\n"); + cmd.assert() + .success() + .stdout(predicate::eq("line1\nline2\n")); +} + +#[test] +fn unbuffered_with_multiple_lines() { + // Test unbuffered mode outputs each line immediately + let mut cmd = bin(); + cmd.arg("-u").arg("s/^/> /").write_stdin("a\nb\nc\nd\ne\n"); + cmd.assert() + .success() + .stdout(predicate::eq("> a\n> b\n> c\n> d\n> e\n")); +} + +#[test] +fn unbuffered_with_quiet_mode() { + // Test unbuffered mode combined with quiet mode + let mut cmd = bin(); + cmd.arg("-u") + .arg("-n") + .arg("/pattern/p") + .write_stdin("no match\npattern here\nanother line\npattern again\n"); + cmd.assert() + .success() + .stdout(predicate::eq("pattern here\npattern again\n")); +} + +// ===== POSIX mode tests ===== + +#[test] +fn posix_flag_accepted() { + // Test that --posix flag is accepted + let mut cmd = bin(); + cmd.arg("--posix").arg("s/foo/bar/").write_stdin("foo\n"); + cmd.assert().success().stdout(predicate::eq("bar\n")); +} + +#[test] +fn posix_forbids_e_command() { + // Test that e command is forbidden in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix") + .arg("e echo test") + .write_stdin("ignored\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'e'")); +} + +#[test] +fn posix_forbids_f_command() { + // Test that F command is forbidden in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix").arg("F").write_stdin("line1\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'F'")); +} + +#[test] +fn posix_forbids_z_command() { + // Test that z command is forbidden in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix").arg("z").write_stdin("line1\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'z'")); +} + +#[test] +fn posix_forbids_big_q_command() { + // Test that Q command is forbidden in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix").arg("Q").write_stdin("line1\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'Q'")); +} + +#[test] +fn posix_forbids_big_t_command() { + // Test that T command is forbidden in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix").arg("T").write_stdin("line1\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'T'")); +} + +#[test] +fn posix_forbids_big_r_command() { + // Test that R command is forbidden in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix").arg("R /dev/null").write_stdin("line1\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'R'")); +} + +#[test] +fn posix_forbids_big_w_command() { + // Test that W command is forbidden in POSIX mode + // Use a simple path without special characters to avoid parsing issues + let mut cmd = bin(); + cmd.arg("--posix") + .arg("W /tmp/test.txt") + .write_stdin("line1\n"); + cmd.assert() + .failure() + .stderr(predicate::str::contains("unknown command: 'W'")); +} + +#[test] +fn posix_allows_standard_commands() { + // Test that standard POSIX commands still work + let mut cmd = bin(); + cmd.arg("--posix") + .arg("-n") + .arg("p") + .write_stdin("line1\nline2\n"); + cmd.assert() + .success() + .stdout(predicate::eq("line1\nline2\n")); +} + +#[test] +fn posix_allows_substitution() { + // Test that substitution works in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix") + .arg("s/old/new/g") + .write_stdin("old old\n"); + cmd.assert().success().stdout(predicate::eq("new new\n")); +} + +#[test] +fn posix_allows_delete() { + // Test that delete command works in POSIX mode + let mut cmd = bin(); + cmd.arg("--posix") + .arg("/skip/d") + .write_stdin("keep\nskip\n"); + cmd.assert().success().stdout(predicate::eq("keep\n")); +} + +// ===== Follow symlinks tests ===== + +#[test] +fn follow_symlinks_flag_accepted() { + // Test that --follow-symlinks flag is accepted (basic test without symlinks) + let mut temp = NamedTempFile::new().unwrap(); + writeln!(temp, "test line").unwrap(); + let path = temp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.arg("--follow-symlinks") + .arg("-i") + .arg("s/test/result/") + .arg(path); + cmd.assert().success(); + + // Read back the file + let content = std::fs::read_to_string(path).unwrap(); + assert_eq!(content, "result line\n"); +} + +#[test] +#[cfg(unix)] // Symlinks work differently on Unix vs Windows +fn follow_symlinks_default_replaces_symlink_with_file() { + use std::os::unix::fs::symlink; + + // Create a temp file + let mut target = NamedTempFile::new().unwrap(); + writeln!(target, "original content").unwrap(); + let target_path = target.path().to_str().unwrap().to_string(); + + // Create temp directory for symlink + let temp_dir = tempfile::tempdir().unwrap(); + let link_path = temp_dir.path().join("link"); + symlink(&target_path, &link_path).unwrap(); + + // Verify it's a symlink before editing + assert!(std::fs::symlink_metadata(&link_path) + .unwrap() + .file_type() + .is_symlink()); + + // Edit symlink without --follow-symlinks (should replace symlink with regular file) + let mut cmd = bin(); + cmd.arg("-i") + .arg("s/original/modified/") + .arg(link_path.to_str().unwrap()); + cmd.assert().success(); + + // Verify the symlink was replaced with a regular file + let meta = std::fs::symlink_metadata(&link_path).unwrap(); + assert!( + meta.file_type().is_file(), + "Expected regular file, got symlink" + ); + + // Verify the content was edited + let content = std::fs::read_to_string(&link_path).unwrap(); + assert_eq!(content, "modified content\n"); + + // Verify the original target file was NOT modified + let target_content = std::fs::read_to_string(&target_path).unwrap(); + assert_eq!(target_content, "original content\n"); +} + +#[test] +#[cfg(unix)] // Symlinks work differently on Unix vs Windows +fn follow_symlinks_edits_target() { + use std::os::unix::fs::symlink; + + // Create a temp file + let mut target = NamedTempFile::new().unwrap(); + writeln!(target, "original content").unwrap(); + let target_path = target.path().to_str().unwrap().to_string(); + + // Create temp directory for symlink + let temp_dir = tempfile::tempdir().unwrap(); + let link_path = temp_dir.path().join("link"); + symlink(&target_path, &link_path).unwrap(); + + // Edit via symlink with --follow-symlinks + let mut cmd = bin(); + cmd.arg("--follow-symlinks") + .arg("-i") + .arg("s/original/modified/") + .arg(link_path.to_str().unwrap()); + cmd.assert().success(); + + // Verify target file was modified + let content = std::fs::read_to_string(&target_path).unwrap(); + assert_eq!(content, "modified content\n"); + + // Verify symlink still exists and points to same target + assert!(link_path.exists()); + assert!(std::fs::symlink_metadata(&link_path) + .unwrap() + .file_type() + .is_symlink()); +} + +#[test] +#[cfg(unix)] +fn follow_symlinks_with_backup() { + use std::os::unix::fs::symlink; + + // Create a temp file + let mut target = NamedTempFile::new().unwrap(); + writeln!(target, "test data").unwrap(); + let target_path = target.path().to_str().unwrap().to_string(); + + // Create temp directory for symlink + let temp_dir = tempfile::tempdir().unwrap(); + let link_path = temp_dir.path().join("link"); + symlink(&target_path, &link_path).unwrap(); + + // Edit via symlink with backup + let mut cmd = bin(); + cmd.arg("--follow-symlinks") + .arg("-i.bak") + .arg("s/test/final/") + .arg(link_path.to_str().unwrap()); + cmd.assert().success(); + + // Verify target was modified + let content = std::fs::read_to_string(&target_path).unwrap(); + assert_eq!(content, "final data\n"); + + // Verify backup was created + let backup_path = format!("{}.bak", target_path); + let backup_content = std::fs::read_to_string(&backup_path).unwrap(); + assert_eq!(backup_content, "test data\n"); +} + +// ===== Sandbox mode tests ===== + +#[test] +fn sandbox_flag_accepted() { + // Test that --sandbox flag is accepted with safe commands + let mut cmd = bin(); + cmd.arg("--sandbox").arg("s/foo/bar/").write_stdin("foo\n"); + cmd.assert().success().stdout(predicate::eq("bar\n")); +} + +#[test] +fn sandbox_forbids_e_command() { + // Test that e command is forbidden in sandbox mode + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg("e echo test") + .write_stdin("ignored\n"); + cmd.assert().failure().stderr(predicate::str::contains( + "e/r/w commands disabled in sandbox mode", + )); +} + +#[test] +fn sandbox_forbids_r_command() { + // Test that r command is forbidden in sandbox mode + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg("r /dev/null") + .write_stdin("line1\n"); + cmd.assert().failure().stderr(predicate::str::contains( + "e/r/w commands disabled in sandbox mode", + )); +} + +#[test] +fn sandbox_forbids_big_r_command() { + // Test that R command is forbidden in sandbox mode + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg("R /dev/null") + .write_stdin("line1\n"); + cmd.assert().failure().stderr(predicate::str::contains( + "e/r/w commands disabled in sandbox mode", + )); +} + +#[test] +fn sandbox_forbids_w_command() { + // Test that w command is forbidden in sandbox mode + use tempfile::NamedTempFile; + let temp = NamedTempFile::new().unwrap(); + let path = temp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg(format!("w {}", path)) + .write_stdin("line1\n"); + cmd.assert().failure().stderr(predicate::str::contains( + "e/r/w commands disabled in sandbox mode", + )); +} + +#[test] +fn sandbox_forbids_big_w_command() { + // Test that W command is forbidden in sandbox mode + use tempfile::NamedTempFile; + let temp = NamedTempFile::new().unwrap(); + let path = temp.path().to_str().unwrap(); + + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg(format!("W {}", path)) + .write_stdin("line1\n"); + cmd.assert().failure().stderr(predicate::str::contains( + "e/r/w commands disabled in sandbox mode", + )); +} + +#[test] +fn sandbox_allows_substitution() { + // Test that substitution works in sandbox mode + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg("s/old/new/g") + .write_stdin("old old\n"); + cmd.assert().success().stdout(predicate::eq("new new\n")); +} + +#[test] +fn sandbox_allows_print() { + // Test that print commands work in sandbox mode + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg("-n") + .arg("p") + .write_stdin("line1\nline2\n"); + cmd.assert() + .success() + .stdout(predicate::eq("line1\nline2\n")); +} + +#[test] +fn sandbox_allows_delete() { + // Test that delete command works in sandbox mode + let mut cmd = bin(); + cmd.arg("--sandbox") + .arg("/skip/d") + .write_stdin("keep\nskip\n"); + cmd.assert().success().stdout(predicate::eq("keep\n")); +} + +// ===== Multibyte/UTF-8 character support tests ===== +// Note: GNU sed in C locale outputs non-ASCII bytes as octal escapes in 'l' command + +#[test] +fn utf8_list_command_cyrillic() { + // Test that l command outputs Cyrillic bytes as octal escapes (GNU sed compatible) + let mut cmd = bin(); + cmd.arg("-n").arg("l").write_stdin("Привіт\n"); + cmd.assert().success().stdout(predicate::eq( + "\\320\\237\\321\\200\\320\\270\\320\\262\\321\\226\\321\\202$\n", + )); +} + +#[test] +fn utf8_list_command_accented() { + // Test that l command outputs accented bytes as octal escapes (GNU sed compatible) + let mut cmd = bin(); + cmd.arg("-n").arg("l").write_stdin("Café naïve\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Caf\\303\\251 na\\303\\257ve$\n")); +} + +#[test] +fn utf8_list_command_mixed() { + // Test that l command handles mix of ASCII and UTF-8 (GNU sed compatible) + let mut cmd = bin(); + cmd.arg("-n").arg("l").write_stdin("Hello мир world\n"); + cmd.assert().success().stdout(predicate::eq( + "Hello \\320\\274\\320\\270\\321\\200 world$\n", + )); +} + +#[test] +fn utf8_list_command_emoji() { + // Test that l command outputs emoji bytes as octal escapes (GNU sed compatible) + let mut cmd = bin(); + cmd.arg("-n").arg("l").write_stdin("Hello 🌍 world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Hello \\360\\237\\214\\215 world$\n")); +} + +#[test] +fn utf8_list_command_chinese() { + // Test that l command outputs Chinese bytes as octal escapes (GNU sed compatible) + let mut cmd = bin(); + cmd.arg("-n").arg("l").write_stdin("你好世界\n"); + cmd.assert().success().stdout(predicate::eq( + "\\344\\275\\240\\345\\245\\275\\344\\270\\226\\347\\225\\214$\n", + )); +} + +#[test] +fn utf8_substitution_works() { + // Test that substitution works with UTF-8 characters + let mut cmd = bin(); + cmd.arg("s/світ/world/").write_stdin("Привіт світ\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Привіт world\n")); +} + +#[test] +fn utf8_pattern_matching() { + // Test that pattern matching works with UTF-8 + let mut cmd = bin(); + cmd.arg("/Привіт/d").write_stdin("Привіт\nworld\n"); + cmd.assert().success().stdout(predicate::eq("world\n")); +} + +#[test] +fn utf8_transliterate_cyrillic() { + // Test that y command works with Cyrillic characters + let mut cmd = bin(); + cmd.arg("y/абв/xyz/").write_stdin("абвгд\n"); + cmd.assert().success().stdout(predicate::eq("xyzгд\n")); +} + +#[test] +fn utf8_transliterate_cyrillic_to_cyrillic() { + // Test transliteration from Cyrillic to Cyrillic + let mut cmd = bin(); + cmd.arg("y/абв/деж/").write_stdin("абв\n"); + cmd.assert().success().stdout(predicate::eq("деж\n")); +} + +#[test] +fn utf8_transliterate_mixed() { + // Test transliteration with mixed ASCII and UTF-8 + let mut cmd = bin(); + cmd.arg("y/aбв/xдe/").write_stdin("aбв\n"); + cmd.assert().success().stdout(predicate::eq("xдe\n")); +} + +#[test] +fn utf8_transliterate_emoji() { + // Test that y command works with emoji + let mut cmd = bin(); + cmd.arg("y/🌍🌎/AB/").write_stdin("Hello 🌍 and 🌎\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Hello A and B\n")); +} + +#[test] +fn utf8_transliterate_chinese() { + // Test that y command works with Chinese characters + let mut cmd = bin(); + cmd.arg("y/你好/ab/").write_stdin("你好世界\n"); + cmd.assert().success().stdout(predicate::eq("ab世界\n")); +} + +#[test] +fn utf8_transliterate_accented() { + // Test that y command works with accented characters + let mut cmd = bin(); + cmd.arg("y/éàè/xyz/").write_stdin("Café\n"); + cmd.assert().success().stdout(predicate::eq("Cafx\n")); +} + +#[test] +fn utf8_transliterate_multiple_lines() { + // Test that y command works across multiple lines + let mut cmd = bin(); + cmd.arg("y/аб/xy/").write_stdin("аб\nба\n"); + cmd.assert().success().stdout(predicate::eq("xy\nyx\n")); +} + +#[test] +fn utf8_transliterate_preserve_untranslated() { + // Test that y command preserves characters not in translation set + let mut cmd = bin(); + cmd.arg("y/а/x/").write_stdin("аб\n"); + cmd.assert().success().stdout(predicate::eq("xб\n")); +} + +// ===== Case conversion escapes tests ===== + +#[test] +fn case_uppercase_all() { + // Test \U - uppercase all following characters + let mut cmd = bin(); + cmd.arg(r"s/\(.*\)/\U\1/").write_stdin("hello world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("HELLO WORLD\n")); +} + +#[test] +fn case_lowercase_all() { + // Test \L - lowercase all following characters + let mut cmd = bin(); + cmd.arg(r"s/\(.*\)/\L\1/").write_stdin("HELLO WORLD\n"); + cmd.assert() + .success() + .stdout(predicate::eq("hello world\n")); +} + +#[test] +fn case_uppercase_next() { + // Test \u - uppercase next character only + let mut cmd = bin(); + cmd.arg(r"s/\(h\)\(.*\)/\u\1\2/") + .write_stdin("hello world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Hello world\n")); +} + +#[test] +fn case_lowercase_next() { + // Test \l - lowercase next character only + let mut cmd = bin(); + cmd.arg(r"s/\(H\)\(.*\)/\l\1\2/") + .write_stdin("HELLO WORLD\n"); + cmd.assert() + .success() + .stdout(predicate::eq("hELLO WORLD\n")); +} + +#[test] +fn case_end_conversion() { + // Test \E - end case conversion + let mut cmd = bin(); + cmd.arg(r"s/\(.*\) \(.*\)/\U\1\E \2/") + .write_stdin("hello world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("HELLO world\n")); +} + +#[test] +fn case_uppercase_with_utf8() { + // Test case conversion with UTF-8 characters + let mut cmd = bin(); + cmd.arg(r"s/\(.*\)/\U\1/").write_stdin("привіт світ\n"); + cmd.assert() + .success() + .stdout(predicate::eq("ПРИВІТ СВІТ\n")); +} + +#[test] +fn case_lowercase_with_utf8() { + // Test lowercase with UTF-8 characters + let mut cmd = bin(); + cmd.arg(r"s/\(.*\)/\L\1/").write_stdin("ПРИВІТ СВІТ\n"); + cmd.assert() + .success() + .stdout(predicate::eq("привіт світ\n")); +} + +#[test] +fn case_mixed_conversions() { + // Test multiple case conversions in one replacement + let mut cmd = bin(); + cmd.arg(r"s/\(.\)\(.*\) \(.\)\(.*\)/\u\1\L\2\E \u\3\L\4/") + .write_stdin("HELLO WORLD\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Hello World\n")); +} + +#[test] +fn case_lowercase_then_uppercase_next() { + // Test \L\u combination - uppercase next overrides lowercase for one char + let mut cmd = bin(); + cmd.arg(r"s/\(.*\) \(.*\)/\L\u\1\E \L\u\2/") + .write_stdin("hello world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("Hello World\n")); +} + +#[test] +fn case_with_backreferences() { + // Test case conversion with backreferences + let mut cmd = bin(); + cmd.arg(r"s/\(.*\) \(.*\)/\U\1\E and \U\2/") + .write_stdin("hello world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("HELLO and WORLD\n")); +} + +#[test] +fn case_uppercase_next_on_group() { + // Test \u on captured group + let mut cmd = bin(); + cmd.arg(r"s/\(.*\) \(.*\)/\1 \u\2/") + .write_stdin("hello world\n"); + cmd.assert() + .success() + .stdout(predicate::eq("hello World\n")); +} + +// ===== Extended address forms tests ===== + +#[test] +fn address_step_basic() { + // Test addr~step - print every 2nd line starting from line 5 + let mut cmd = bin(); + cmd.arg("-n") + .arg("5~2p") + .write_stdin("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); + cmd.assert().success().stdout(predicate::eq("5\n7\n9\n")); + verify_against_sed!("5~2p", "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n", &["-n"]); +} + +#[test] +fn address_step_from_start() { + // Test addr~step from line 1 + let mut cmd = bin(); + cmd.arg("-n") + .arg("1~3p") + .write_stdin("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); + cmd.assert() + .success() + .stdout(predicate::eq("1\n4\n7\n10\n")); + verify_against_sed!("1~3p", "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n", &["-n"]); +} + +#[test] +fn address_step_zero() { + // Test 0~step - every Nth line starting from line N + let mut cmd = bin(); + cmd.arg("-n") + .arg("0~2p") + .write_stdin("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); + cmd.assert() + .success() + .stdout(predicate::eq("2\n4\n6\n8\n10\n")); + verify_against_sed!("0~2p", "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n", &["-n"]); +} + +#[test] +fn address_range_with_offset() { + // Test addr,+N - range from addr to addr+N + let mut cmd = bin(); + cmd.arg("-n") + .arg("5,+2p") + .write_stdin("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); + cmd.assert().success().stdout(predicate::eq("5\n6\n7\n")); + verify_against_sed!("5,+2p", "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n", &["-n"]); +} + +#[test] +fn address_range_offset_from_start() { + // Test start,+offset from first line + let mut cmd = bin(); + cmd.arg("-n") + .arg("1,+3p") + .write_stdin("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); + cmd.assert().success().stdout(predicate::eq("1\n2\n3\n4\n")); +} + +#[test] +fn address_step_with_substitution() { + // Test ~step with substitution command + let mut cmd = bin(); + cmd.arg("1~2s/^/even: /").write_stdin("1\n2\n3\n4\n5\n6\n"); + cmd.assert() + .success() + .stdout(predicate::eq("even: 1\n2\neven: 3\n4\neven: 5\n6\n")); +} + +#[test] +fn address_step_with_delete() { + // Test ~step with delete command + let mut cmd = bin(); + cmd.arg("2~2d").write_stdin("1\n2\n3\n4\n5\n6\n"); + cmd.assert().success().stdout(predicate::eq("1\n3\n5\n")); +} + +#[test] +fn address_offset_in_range_with_zero() { + // Test 0,+N range (from first line for N lines) + let mut cmd = bin(); + cmd.arg("-n").arg("0,+2p").write_stdin("1\n2\n3\n4\n5\n"); + cmd.assert().success().stdout(predicate::eq("1\n2\n3\n")); +} + +// ===== Alternative regex delimiter tests ===== + +#[test] +fn alt_delimiter_pipe_address() { + // Test \|pattern| as address + let mut cmd = bin(); + cmd.arg("-n") + .arg(r"\|/|p") + .write_stdin("hello/world\ntest\n"); + cmd.assert() + .success() + .stdout(predicate::eq("hello/world\n")); +} + +#[test] +fn alt_delimiter_at_address() { + // Test \@pattern@ as address + let mut cmd = bin(); + cmd.arg("-n") + .arg(r"\@hello@p") + .write_stdin("hello\nworld\n"); + cmd.assert().success().stdout(predicate::eq("hello\n")); +} + +#[test] +fn alt_delimiter_hash_address() { + // Test \#pattern# as address + let mut cmd = bin(); + cmd.arg("-n").arg(r"\#test#p").write_stdin("test\nother\n"); + cmd.assert().success().stdout(predicate::eq("test\n")); +} + +#[test] +fn alt_delimiter_colon_address() { + // Test \:pattern: as address + let mut cmd = bin(); + cmd.arg("-n") + .arg(r"\:world:p") + .write_stdin("hello\nworld\n"); + cmd.assert().success().stdout(predicate::eq("world\n")); +} + +#[test] +fn alt_delimiter_in_range() { + // Test alternative delimiter in address range + let mut cmd = bin(); + cmd.arg("-n") + .arg(r"\|hello|,\|end|p") + .write_stdin("hello\nworld\nend\nafter\n"); + cmd.assert() + .success() + .stdout(predicate::eq("hello\nworld\nend\n")); +} + +#[test] +fn alt_delimiter_with_delete() { + // Test alternative delimiter with delete command + let mut cmd = bin(); + cmd.arg(r"\:world:d").write_stdin("hello\nworld\ngoodbye\n"); + cmd.assert() + .success() + .stdout(predicate::eq("hello\ngoodbye\n")); +} + +#[test] +fn alt_delimiter_with_substitution() { + // Test alternative delimiter with substitution + let mut cmd = bin(); + cmd.arg(r"\@test@s/e/E/g").write_stdin("test line\nother\n"); + cmd.assert() + .success() + .stdout(predicate::eq("tEst linE\nother\n")); +} + +#[test] +fn alt_delimiter_semicolon() { + // Test \;pattern; as address + let mut cmd = bin(); + cmd.arg("-n").arg(r"\;foo;p").write_stdin("foo\nbar\n"); + cmd.assert().success().stdout(predicate::eq("foo\n")); +} + +#[test] +fn alt_delimiter_percent() { + // Test \%pattern% as address + let mut cmd = bin(); + cmd.arg("-n").arg(r"\%test%p").write_stdin("test\nline\n"); + cmd.assert().success().stdout(predicate::eq("test\n")); +} + +// ===== Exit code compatibility tests ===== + +#[test] +fn exit_code_success() { + // Test exit code 0 on success + let mut cmd = bin(); + cmd.arg("s/test/replaced/").write_stdin("test\n"); + cmd.assert().success().code(0); +} + +#[test] +fn exit_code_parse_error() { + // Test exit code 1 for parse errors + let mut cmd = bin(); + cmd.arg("s/[/"); + cmd.assert().failure().code(1); +} + +#[test] +fn exit_code_io_error() { + // Test exit code 2 for I/O errors (file not found) + let mut cmd = bin(); + cmd.arg("s/test/replaced/").arg("/nonexistent/file/path"); + cmd.assert().failure().code(2); +} + +#[test] +fn exit_code_invalid_command() { + // Test exit code 1 for unknown command + let mut cmd = bin(); + cmd.arg("X"); + cmd.assert().failure().code(1); +} + +#[test] +fn exit_code_unterminated_regex() { + // Test exit code 1 for unterminated regex + let mut cmd = bin(); + cmd.arg("/test"); + cmd.assert().failure().code(1); +} + +// ============================================================================ +// BRE Compliance Tests (Task 21) +// ============================================================================ + +#[test] +fn bre_word_boundary_start() { + // Test \< (word boundary at start of word) + // Note: \< matches beginning of word, so "worldwide" does NOT match because 'w' is not at word start + let mut cmd = bin(); + cmd.args(&["-n", r"/\/p"]) + .write_stdin("hello world\nworldwide\n"); + cmd.assert().success().stdout("hello world\n"); +} + +#[test] +fn bre_word_boundary_end() { + // Test \> (word boundary at end of word) + let mut cmd = bin(); + cmd.args(&[r"s/rld\>/RLD/"]) + .write_stdin("hello world\nworldwide\n"); + cmd.assert().success().stdout("hello woRLD\nworldwide\n"); +} + +#[test] +fn bre_word_boundary_both() { + // Test \< and \> together + let mut cmd = bin(); + cmd.args(&[r"s/\/TEST/"]) + .write_stdin("test testing retest\n"); + cmd.assert().success().stdout("TEST testing retest\n"); +} + +#[test] +fn bre_word_boundary_b() { + // Test \b (word boundary, GNU extension) + let mut cmd = bin(); + cmd.args(&[r"s/\btest\b/TEST/"]) + .write_stdin("test testing retest\n"); + cmd.assert().success().stdout("TEST testing retest\n"); +} + +#[test] +fn bre_word_boundary_b_start() { + // Test \b at start of word + let mut cmd = bin(); + cmd.args(&[r"s/\bw/W/"]).write_stdin("hello world\n"); + cmd.assert().success().stdout("hello World\n"); +} + +#[test] +fn bre_word_boundary_big_b() { + // Test \B (non-word boundary, GNU extension) + let mut cmd = bin(); + cmd.args(&[r"s/\Bst/ST/"]).write_stdin("testing start\n"); + cmd.assert().success().stdout("teSTing start\n"); +} + +#[test] +fn bre_posix_class_alpha() { + // Test [[:alpha:]] character class + let mut cmd = bin(); + cmd.args(&[r"s/[[:alpha:]]/X/"]).write_stdin("Test123\n"); + cmd.assert().success().stdout("Xest123\n"); +} + +#[test] +fn bre_posix_class_digit() { + // Test [[:digit:]] character class + let mut cmd = bin(); + cmd.args(&[r"s/[[:digit:]]/X/"]).write_stdin("Test123\n"); + cmd.assert().success().stdout("TestX23\n"); +} + +#[test] +fn bre_posix_class_alnum() { + // Test [[:alnum:]] character class (alphanumeric) + let mut cmd = bin(); + cmd.args(&[r"s/[[:alnum:]]\+/WORD/g"]) + .write_stdin("test-123 hello\n"); + cmd.assert().success().stdout("WORD-WORD WORD\n"); +} + +#[test] +fn bre_posix_class_space() { + // Test [[:space:]] character class + let mut cmd = bin(); + cmd.args(&[r"s/[[:space:]]\+/_/g"]) + .write_stdin("hello world\ttab\n"); + cmd.assert().success().stdout("hello_world_tab\n"); +} + +#[test] +fn bre_posix_class_upper() { + // Test [[:upper:]] character class + let mut cmd = bin(); + cmd.args(&[r"s/[[:upper:]]\+/UP/g"]) + .write_stdin("HELLO world TEST\n"); + cmd.assert().success().stdout("UP world UP\n"); +} + +#[test] +fn bre_posix_class_lower() { + // Test [[:lower:]] character class + let mut cmd = bin(); + cmd.args(&[r"s/[[:lower:]]\+/low/g"]) + .write_stdin("HELLO world TEST\n"); + cmd.assert().success().stdout("HELLO low TEST\n"); +} + +#[test] +fn bre_posix_class_punct() { + // Test [[:punct:]] character class (punctuation) + let mut cmd = bin(); + cmd.args(&[r"s/[[:punct:]]/X/g"]) + .write_stdin("Hello, world!\n"); + cmd.assert().success().stdout("HelloX worldX\n"); +} + +#[test] +fn bre_posix_class_xdigit() { + // Test [[:xdigit:]] character class (hex digits: 0-9, a-f, A-F) + // Note: Letters 'a', 'n', 'd' in "and" are hex digits, so they match too + let mut cmd = bin(); + cmd.args(&[r"s/[[:xdigit:]]\+/HEX/g"]) + .write_stdin("0xDEADBEEF and 123\n"); + cmd.assert().success().stdout("HEXxHEX HEXnHEX HEX\n"); +} + +#[test] +fn bre_posix_class_blank() { + // Test [[:blank:]] character class (space and tab only) + let mut cmd = bin(); + cmd.args(&[r"s/[[:blank:]]\+/_/g"]) + .write_stdin("hello\tworld test\n"); + cmd.assert().success().stdout("hello_world_test\n"); +} + +#[test] +fn bre_escape_sequences_tab_newline() { + // Test \t and \n escape sequences + let mut cmd = bin(); + cmd.args(&[r"s/\t/TAB/g"]).write_stdin("hello\tworld\n"); + cmd.assert().success().stdout("helloTABworld\n"); +} + +#[test] +fn bre_combined_word_boundary_posix() { + // Test combination of word boundaries and POSIX classes + let mut cmd = bin(); + cmd.args(&[r"s/\<[[:digit:]]\+\>/NUM/g"]) + .write_stdin("test123 456 end789\n"); + cmd.assert().success().stdout("test123 NUM end789\n"); +} + +#[test] +fn bre_word_boundary_with_repetition() { + // Test word boundaries with repetition operators + // Note: "test123" contains digits, so [a-z]\+ only matches "test" not "123" + let mut cmd = bin(); + cmd.args(&[r"s/\<[a-z]\+\>/WORD/g"]) + .write_stdin("Hello world test123\n"); + cmd.assert().success().stdout("Hello WORD test123\n"); +} + +#[test] +fn bre_backslash_b_in_character_class() { + // Test \b inside character class (should be backspace, not word boundary) + let mut cmd = bin(); + cmd.args(&[r"s/a[\b]c/X/"]) + .write_stdin("a\x08c normal abc\n"); + cmd.assert().success().stdout("X normal abc\n"); +} + +// ============================================================================ +// Advanced Regex Features Tests (Task 24) +// ============================================================================ + +#[test] +fn regex_collating_symbol_single() { + // Test collating symbols [.x.] for single characters + let mut cmd = bin(); + cmd.args(&[r"s/[.t.]/X/"]).write_stdin("test\n"); + cmd.assert().success().stdout("Xest\n"); +} + +#[test] +fn regex_collating_symbol_multiple() { + // Test multiple collating symbols in character class + let mut cmd = bin(); + cmd.args(&[r"s/[[.1.][.2.][.3.]]/X/g"]) + .write_stdin("abc123\n"); + cmd.assert().success().stdout("abcXXX\n"); +} + +#[test] +fn regex_equivalence_class_single() { + // Test equivalence class [=x=] for single character + let mut cmd = bin(); + cmd.args(&[r"s/[=e=]/X/"]).write_stdin("test\n"); + cmd.assert().success().stdout("tXst\n"); +} + +#[test] +fn regex_equivalence_class_with_unicode() { + // Test equivalence class with accented characters + // Note: equivalence matching depends on locale/implementation + let mut cmd = bin(); + cmd.args(&[r"s/[=e=]/X/g"]).write_stdin("naïve café\n"); + cmd.assert().success().stdout("naïvX café\n"); +} + +#[test] +fn regex_backreference_simple() { + // Test simple backreference \1 + let mut cmd = bin(); + cmd.args(&[r"s/\(.\)\1/X/"]).write_stdin("hello\n"); + cmd.assert().success().stdout("heXo\n"); +} + +#[test] +fn regex_backreference_multiple() { + // Test multiple backreferences + let mut cmd = bin(); + cmd.args(&[r"s/\(.\)\(.\)\2\1/[\1\2]/"]) + .write_stdin("abba\n"); + cmd.assert().success().stdout("[ab]\n"); +} + +#[test] +fn regex_all_posix_classes_combined() { + // Test combining multiple POSIX character classes + // Note: space is not matched by alpha/digit/punct, so it remains + let mut cmd = bin(); + cmd.args(&[r"s/[[:alpha:]]*[[:digit:]]*[[:punct:]]*/WORD/g"]) + .write_stdin("test123, hello456!\n"); + cmd.assert().success().stdout("WORD WORD\n"); +} + +// ============================================================================ +// Signal Handling and Temp File Cleanup Tests (Task 27) +// ============================================================================ + +#[test] +fn signal_temp_file_cleaned_on_error() { + // Test that temp files are cleaned up when an error occurs during processing + use assert_fs::prelude::*; + use std::fs; + + let temp = assert_fs::TempDir::new().unwrap(); + let input_file = temp.child("input.txt"); + input_file.write_str("line 1\nline 2\nline 3\n").unwrap(); + + let input_path = input_file.path().to_str().unwrap(); + + // Create a command that will fail (invalid regex) + let mut cmd = bin(); + cmd.args(&["-i", r"s/\(/replacement/", input_path]); + cmd.assert().failure(); + + // Check that no temp files remain + let temp_files: Vec<_> = fs::read_dir(temp.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_str().unwrap().contains("red_temp")) + .collect(); + + assert_eq!( + temp_files.len(), + 0, + "Temporary files should be cleaned up on error" + ); + + temp.close().unwrap(); +} + +#[test] +fn signal_temp_file_not_present_after_success() { + // Test that temp files are removed after successful in-place edit + use assert_fs::prelude::*; + use std::fs; + + let temp = assert_fs::TempDir::new().unwrap(); + let input_file = temp.child("input.txt"); + input_file.write_str("foo\nbar\nbaz\n").unwrap(); + + let input_path = input_file.path().to_str().unwrap(); + + // Perform successful in-place edit + let mut cmd = bin(); + cmd.args(&["-i", "s/foo/FOO/", input_path]); + cmd.assert().success(); + + // Check that no temp files remain + let temp_files: Vec<_> = fs::read_dir(temp.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_str().unwrap().contains("red_temp")) + .collect(); + + assert_eq!( + temp_files.len(), + 0, + "Temporary files should be removed after successful operation" + ); + + // Verify the file was modified + let content = fs::read_to_string(input_file.path()).unwrap(); + assert_eq!(content, "FOO\nbar\nbaz\n"); + + temp.close().unwrap(); +} + +#[test] +fn signal_backup_created_with_suffix() { + // Test that backup files are created with the specified suffix + use assert_fs::prelude::*; + use std::fs; + + let temp = assert_fs::TempDir::new().unwrap(); + let input_file = temp.child("input.txt"); + input_file.write_str("original\n").unwrap(); + + let input_path = input_file.path().to_str().unwrap(); + let backup_path = format!("{}.bak", input_path); + + // Perform in-place edit with backup + let mut cmd = bin(); + cmd.args(&["-i.bak", "s/original/modified/", input_path]); + cmd.assert().success(); + + // Check that backup exists + assert!( + fs::metadata(&backup_path).is_ok(), + "Backup file should exist" + ); + + // Verify backup content + let backup_content = fs::read_to_string(&backup_path).unwrap(); + assert_eq!(backup_content, "original\n"); + + // Verify modified content + let modified_content = fs::read_to_string(input_file.path()).unwrap(); + assert_eq!(modified_content, "modified\n"); + + // Check no temp files remain + let temp_files: Vec<_> = fs::read_dir(temp.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_str().unwrap().contains("red_temp")) + .collect(); + + assert_eq!(temp_files.len(), 0, "No temp files should remain"); + + temp.close().unwrap(); +} + +// ===== Null-data (-z/--null-data) tests ===== + +#[test] +fn null_data_short_flag() { + // Test basic null-data mode with -z flag + let mut cmd = bin(); + cmd.arg("-z") + .arg("s/^./x/") + .write_stdin("AB\x00CD\nEF\n\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("xB\x00xD\nEF\n\x00")); + verify_against_sed_bytes!("s/^./x/", b"AB\x00CD\nEF\n\x00", &["-z"]); +} + +#[test] +fn null_data_long_flag() { + // Test basic null-data mode with --null-data flag + let mut cmd = bin(); + cmd.arg("--null-data") + .arg("s/^./x/") + .write_stdin("AB\x00CD\x00"); + cmd.assert().success().stdout(predicate::eq("xB\x00xD\x00")); + verify_against_sed_bytes!("s/^./x/", b"AB\x00CD\x00", &["-z"]); +} + +#[test] +fn null_data_with_line_numbers() { + // Test null-data mode with = command (line numbers) + let mut cmd = bin(); + cmd.arg("-z").arg("=").write_stdin("A\x00B\x00C\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("1\x00A\x002\x00B\x003\x00C\x00")); +} + +#[test] +fn null_data_with_print_command() { + // Test null-data mode with explicit print + let mut cmd = bin(); + cmd.arg("-zn").arg("p").write_stdin("hello\x00world\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("hello\x00world\x00")); +} + +#[test] +fn null_data_with_delete() { + // Test null-data mode with delete command + let mut cmd = bin(); + cmd.arg("-z").arg("/CD/d").write_stdin("AB\x00CD\x00EF\x00"); + cmd.assert().success().stdout(predicate::eq("AB\x00EF\x00")); +} + +#[test] +fn null_data_with_substitution_global() { + // Test null-data mode with global substitution + let mut cmd = bin(); + cmd.arg("-z") + .arg("s/[aeiou]/X/g") + .write_stdin("hello\x00world\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("hXllX\x00wXrld\x00")); +} + +#[test] +fn null_data_preserves_newlines_in_records() { + // Test that newlines within null-separated records are preserved + let mut cmd = bin(); + cmd.arg("-z") + .arg("s/^/>> /") + .write_stdin("line1\nline2\x00line3\x00"); + cmd.assert() + .success() + .stdout(predicate::eq(">> line1\nline2\x00>> line3\x00")); +} + +#[test] +fn null_data_with_multiple_commands() { + // Test null-data mode with multiple commands in sequence + let mut cmd = bin(); + cmd.arg("-z") + .arg("1s/^/FIRST: /; 2s/^/SECOND: /") + .write_stdin("A\x00B\x00C\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("FIRST: A\x00SECOND: B\x00C\x00")); +} + +#[test] +fn null_data_with_quiet_mode() { + // Test null-data mode with quiet mode + let mut cmd = bin(); + cmd.arg("-zn") + .arg("s/test/FOUND/p") + .write_stdin("test\x00other\x00test\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("FOUND\x00FOUND\x00")); +} + +#[test] +fn null_data_with_address_range() { + // Test null-data mode with address ranges + let mut cmd = bin(); + cmd.arg("-z") + .arg("2,3s/^/>> /") + .write_stdin("A\x00B\x00C\x00D\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("A\x00>> B\x00>> C\x00D\x00")); +} + +#[test] +fn null_data_empty_records() { + // Test null-data mode with empty records between nulls + let mut cmd = bin(); + cmd.arg("-z") + .arg("s/^$/EMPTY/") + .write_stdin("\x00\x00test\x00"); + cmd.assert() + .success() + .stdout(predicate::eq("EMPTY\x00EMPTY\x00test\x00")); +} + +#[test] +fn null_data_no_trailing_null() { + // Test null-data mode when input doesn't end with null + // GNU sed preserves the lack of trailing separator + let mut cmd = bin(); + cmd.arg("-z").arg("s/^/>> /").write_stdin("A\x00B"); + cmd.assert().success().stdout(predicate::eq(">> A\x00>> B")); +}