diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index b47540ed9..dd83c66df 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -5,7 +5,7 @@ name: CICD # spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain # spell-checker:ignore (names) CodeCOV MacOS MinGW Peltoche rivy # spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot gmake grcov halium lcov libssl mkdir popd printf pushd rsync rustc rustfmt rustup shopt xargs -# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils gnueabihf issuecomment maint nullglob onexitbegin onexitend pell runtest tempfile testsuite uutils DESTDIR sizemulti +# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils gnueabihf issuecomment maint nullglob onexitbegin onexitend pell runtest tempfile testsuite uutils DESTDIR sizemulti Swatinem # ToDO: [2021-06; rivy] change from `cargo-tree` to `cargo tree` once MSRV is >= 1.45 @@ -37,6 +37,7 @@ jobs: - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -94,6 +95,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -157,6 +159,7 @@ jobs: - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -214,6 +217,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -265,6 +269,7 @@ jobs: # - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -309,6 +314,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -385,6 +391,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Install `rust` toolchain uses: actions-rs/toolchain@v1 with: @@ -409,6 +416,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Install `rust` toolchain uses: actions-rs/toolchain@v1 with: @@ -436,6 +444,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Install dependencies shell: bash run: | @@ -486,10 +495,15 @@ jobs: # - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_selinux , use-cross: use-cross } # - { os: ubuntu-18.04 , target: i586-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } ## note: older windows platform; not required, dev-FYI only # - { os: ubuntu-18.04 , target: i586-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } ## note: older windows platform; not required, dev-FYI only - - { os: ubuntu-18.04 , target: i686-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } - - { os: ubuntu-18.04 , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: ubuntu-18.04 , target: x86_64-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } - - { os: ubuntu-18.04 , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } + - { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } + - { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } + # Commented until https://github.com/uutils/coreutils/issues/3210 is fixed + #- { os: ubuntu-18.04 , target: i686-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } + #- { os: ubuntu-18.04 , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } + #- { os: ubuntu-18.04 , target: x86_64-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } + #- { os: ubuntu-18.04 , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - { os: macos-latest , target: x86_64-apple-darwin , features: feat_os_macos } - { os: windows-latest , target: i686-pc-windows-gnu , features: feat_os_windows } - { os: windows-latest , target: i686-pc-windows-msvc , features: feat_os_windows } @@ -497,6 +511,7 @@ jobs: - { os: windows-latest , target: x86_64-pc-windows-msvc , features: feat_os_windows } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize workflow variables id: vars shell: bash @@ -745,6 +760,7 @@ jobs: - { os: ubuntu-latest } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Install/setup prerequisites shell: bash run: | @@ -780,6 +796,7 @@ jobs: mem: 2048 steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Prepare, build and test ## spell-checker:ignore (ToDO) sshfs usesh vmactions uses: vmactions/freebsd-vm@v0.1.5 @@ -848,6 +865,7 @@ jobs: - { os: windows-latest , features: windows } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 # - name: Reattach HEAD ## may be needed for accurate code coverage info # run: git checkout ${{ github.head_ref }} - name: Initialize workflow variables diff --git a/.github/workflows/FixPR.yml b/.github/workflows/FixPR.yml index ead0b5e81..2a5382e27 100644 --- a/.github/workflows/FixPR.yml +++ b/.github/workflows/FixPR.yml @@ -1,5 +1,7 @@ name: FixPR +# spell-checker:ignore Swatinem + # Trigger automated fixes for PRs being merged (with associated commits) # ToDO: [2021-06; rivy] change from `cargo-tree` to `cargo tree` once MSRV is >= 1.45 @@ -27,6 +29,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize job variables id: vars shell: bash @@ -98,6 +101,7 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v2 + - uses: Swatinem/rust-cache@v1 - name: Initialize job variables id: vars shell: bash diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index eef8567c7..9582b2af7 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -67,7 +67,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq + sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq valgrind libexpect-perl - name: Add various locales shell: bash run: | @@ -160,36 +160,117 @@ jobs: - name: Compare test failures VS reference shell: bash run: | + have_new_failures="" REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_LOG_FILE}"; then - echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - for LINE in $REF_FAILING + for LINE in ${REF_FAILING} do - if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then - echo "::warning ::Congrats! The gnu test $LINE is now passing!" + if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then + echo "::warning ::Congrats! The gnu test ${LINE} is now passing!" fi done - for LINE in $NEW_FAILING + for LINE in ${NEW_FAILING} do - if ! grep -Fxq $LINE<<<"$REF_FAILING" + if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then - echo "::error ::GNU test failed: $LINE. $LINE is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + echo "::error ::GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + have_new_failures="true" fi done else echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." fi + if test -n "${have_new_failures}" ; then exit -1 ; fi - name: Compare test summary VS reference + if: success() || failure() # run regardless of prior step success/failure shell: bash run: | REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_SUMMARY_FILE}"; then - echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" mv "${REF_SUMMARY_FILE}" main-gnu-result.json python uutils/util/compare_gnu_result.py else echo "::warning ::Skipping test summary comparison; no prior reference summary is available." fi + + gnu_coverage: + name: Run GNU tests with coverage + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + steps: + - name: Checkout code uutil + uses: actions/checkout@v2 + with: + path: 'uutils' + - name: Checkout GNU coreutils + uses: actions/checkout@v2 + with: + repository: 'coreutils/coreutils' + path: 'gnu' + ref: 'v9.0' + submodules: recursive + - name: Install `rust` toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + default: true + profile: minimal # minimal component installation (ie, no documentation) + components: rustfmt + - name: Install dependencies + run: | + sudo apt update + sudo apt install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq valgrind libexpect-perl -y + - name: Add various locales + run: | + echo "Before:" + locale -a + ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' + ## Some others need a French locale + sudo locale-gen + sudo locale-gen fr_FR + sudo locale-gen fr_FR.UTF-8 + sudo update-locale + echo "After:" + locale -a + - name: Build binaries + env: + CARGO_INCREMENTAL: "0" + RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" + RUSTDOCFLAGS: "-Cpanic=abort" + run: | + cd uutils + UU_MAKE_PROFILE=debug bash util/build-gnu.sh + - name: Run GNU tests + run: bash uutils/util/run-gnu-test.sh + - name: "`grcov` ~ install" + uses: actions-rs/install@v0.1 + with: + crate: grcov + version: latest + use-tool-cache: false + - name: Generate coverage data (via `grcov`) + id: coverage + run: | + ## Generate coverage data + cd uutils + COVERAGE_REPORT_DIR="target/debug" + COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" + mkdir -p "${COVERAGE_REPORT_DIR}" + sudo chown -R "$(whoami)" "${COVERAGE_REPORT_DIR}" + # display coverage files + grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique + # generate coverage report + grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" + echo ::set-output name=report::${COVERAGE_REPORT_FILE} + - name: Upload coverage results (to Codecov.io) + uses: codecov/codecov-action@v2 + with: + file: ${{ steps.coverage.outputs.report }} + flags: gnutests + name: gnutests + working-directory: uutils diff --git a/.vscode/cspell.dictionaries/workspace.wordlist.txt b/.vscode/cspell.dictionaries/workspace.wordlist.txt index e41aba979..99ac20ea2 100644 --- a/.vscode/cspell.dictionaries/workspace.wordlist.txt +++ b/.vscode/cspell.dictionaries/workspace.wordlist.txt @@ -44,6 +44,7 @@ termsize termwidth textwrap thiserror +ureq walkdir winapi xattr diff --git a/Cargo.lock b/Cargo.lock index 90b71d2a7..fd064dc76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.4.7" @@ -73,6 +79,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bigdecimal" version = "0.3.0" @@ -167,6 +179,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byte-unit" version = "4.0.13" @@ -228,6 +246,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "clang-sys" version = "1.3.0" @@ -329,6 +353,7 @@ dependencies = [ "time", "unindent", "unix_socket", + "ureq", "users", "uu_arch", "uu_base32", @@ -432,6 +457,7 @@ dependencies = [ "uu_yes", "uucore", "walkdir", + "zip", ] [[package]] @@ -546,6 +572,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.2" @@ -617,11 +652,12 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" dependencies = [ "generic-array", + "typenum", ] [[package]] @@ -684,13 +720,12 @@ checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" [[package]] name = "digest" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ "block-buffer", "crypto-common", - "generic-array", "subtle", ] @@ -792,12 +827,34 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.2.0" @@ -900,9 +957,9 @@ dependencies = [ [[package]] name = "hex" -version = "0.2.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-literal" @@ -927,6 +984,17 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "if_rust_version" version = "1.0.0" @@ -967,6 +1035,15 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.0" @@ -1045,10 +1122,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] -name = "md5" -version = "0.3.8" +name = "matches" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "md-5" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" +dependencies = [ + "digest", +] [[package]] name = "memchr" @@ -1089,6 +1175,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.14" @@ -1238,9 +1334,9 @@ checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" [[package]] name = "onig" -version = "4.3.3" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8518fcb2b1b8c2f45f0ad499df4fda6087fc3475ca69a185c173b8315d2fb383" +checksum = "67ddfe2c93bb389eea6e6d713306880c7f6dcc99a75b659ce145d962c861b225" dependencies = [ "bitflags", "lazy_static", @@ -1250,9 +1346,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.1.0" +version = "69.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388410bf5fa341f10e58e6db3975f4bea1ac30247dd79d37a9e5ced3cb4cc3b0" +checksum = "5dd3eee045c84695b53b20255bb7317063df090b68e18bfac0abb6c39cf7f33e" dependencies = [ "cc", "pkg-config", @@ -1375,6 +1471,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "phf" version = "0.10.1" @@ -1655,6 +1757,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rlimit" version = "0.4.0" @@ -1681,6 +1798,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustls" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b323592e3164322f5b193dc4302e4e36cd8d37158a712d664efae1a5c2791700" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "same-file" version = "1.0.6" @@ -1696,6 +1831,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "selinux" version = "0.2.5" @@ -1744,9 +1889,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cc229fb94bcb689ffc39bd4ded842f6ff76885efede7c6d1ffb62582878bea" +checksum = "c77f4e7f65455545c2153c1253d25056825e77ee2533f0e41deb65a93a34852f" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -1755,9 +1900,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -1766,9 +1911,9 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f935e31cf406e8c0e96c2815a5516181b7004ae8c5f296293221e9b1e356bd" +checksum = "881bf8156c87b6301fc5ca6b27f11eeb2761224c7081e69b409d5a1951a70c86" dependencies = [ "digest", "keccak", @@ -1838,6 +1983,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -1858,19 +2009,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strum" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" [[package]] name = "strum_macros" -version = "0.21.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" dependencies = [ "heck", "proc-macro2", "quote 1.0.14", + "rustversion", "syn", ] @@ -2009,6 +2161,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "toml" version = "0.5.8" @@ -2024,6 +2191,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + [[package]] name = "unicode-linebreak" version = "0.1.2" @@ -2033,6 +2206,15 @@ dependencies = [ "regex", ] +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -2073,6 +2255,41 @@ dependencies = [ "libc", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "users" version = "0.10.0" @@ -2410,7 +2627,7 @@ dependencies = [ "digest", "hex", "libc", - "md5", + "md-5", "memchr 2.4.1", "regex", "regex-syntax", @@ -3171,6 +3388,89 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote 1.0.14", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +dependencies = [ + "webpki", +] + [[package]] name = "which" version = "4.2.2" @@ -3248,3 +3548,15 @@ name = "z85" version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af896e93db81340b74b65f74276a99b210c086f3d34ed0abf433182a462af856" + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "crc32fast", + "flate2", + "thiserror", +] diff --git a/Cargo.toml b/Cargo.toml index 336729813..4222f1749 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,8 @@ lazy_static = { version="1.3" } textwrap = { version="0.14", features=["terminal_size"] } uucore = { version=">=0.0.11", package="uucore", path="src/uucore" } selinux = { version="0.2", optional = true } +ureq = "2.4.0" +zip = { version = "0.5.13", default_features=false, features=["deflate"] } # * uutils uu_test = { optional=true, version="0.0.12", package="uu_test", path="src/uu/test" } # diff --git a/DEVELOPER_INSTRUCTIONS.md b/DEVELOPER_INSTRUCTIONS.md index 027d4dca1..c007fba7e 100644 --- a/DEVELOPER_INSTRUCTIONS.md +++ b/DEVELOPER_INSTRUCTIONS.md @@ -21,7 +21,7 @@ Running GNU tests At the end you should have uutils, gnu and gnulib checked out next to each other. - Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while) -- Finally, you can run `tests with bash uutils/util/run-gnu-test.sh `. Instead of `` insert the test you want to run, e.g. `tests/misc/wc-proc`. +- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the test you want to run, e.g. `tests/misc/wc-proc.sh`. Code Coverage Report Generation @@ -33,7 +33,7 @@ Code coverage report can be generated using [grcov](https://github.com/mozilla/g ### Using Nightly Rust -To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-cc) coverage report +To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report ```bash $ export CARGO_INCREMENTAL=0 diff --git a/build.rs b/build.rs index f4d4acbc1..c4f135e64 100644 --- a/build.rs +++ b/build.rs @@ -116,6 +116,8 @@ pub fn main() { phf_map.entry("sha3-512sum", &map_value); phf_map.entry("shake128sum", &map_value); phf_map.entry("shake256sum", &map_value); + phf_map.entry("b2sum", &map_value); + phf_map.entry("b3sum", &map_value); tf.write_all( format!( "#[path=\"{dir}/test_{krate}.rs\"]\nmod test_{krate};\n", diff --git a/docs/Makefile b/docs/Makefile index dd700bcb0..7372b3868 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ clean: - rm -rf _build + rm -rf book rm -f src/SUMMARY.md rm -f src/utils/* diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 7afee2db7..a6bc16cc2 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -2,15 +2,27 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +// spell-checker:ignore tldr use clap::App; use std::ffi::OsString; use std::fs::File; -use std::io::{self, Write}; +use std::io::Cursor; +use std::io::{self, Read, Seek, Write}; +use zip::ZipArchive; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn main() -> io::Result<()> { + println!("Downloading tldr archive"); + let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip") + .call() + .unwrap() + .into_reader(); + let mut buffer = Vec::new(); + zip_reader.read_to_end(&mut buffer).unwrap(); + let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap(); + let utils = util_map::>>(); match std::fs::create_dir("docs/src/utils/") { Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), @@ -40,7 +52,7 @@ fn main() -> io::Result<()> { } let p = format!("docs/src/utils/{}.md", name); if let Ok(f) = File::create(&p) { - write_markdown(f, &mut app(), name)?; + write_markdown(f, &mut app(), name, &mut tldr_zip)?; println!("Wrote to '{}'", p); } else { println!("Error writing to {}", p); @@ -50,12 +62,18 @@ fn main() -> io::Result<()> { Ok(()) } -fn write_markdown(mut w: impl Write, app: &mut App, name: &str) -> io::Result<()> { +fn write_markdown( + mut w: impl Write, + app: &mut App, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { write!(w, "# {}\n\n", name)?; write_version(&mut w, app)?; write_usage(&mut w, app, name)?; write_description(&mut w, app)?; - write_options(&mut w, app) + write_options(&mut w, app)?; + write_examples(&mut w, name, tldr_zip) } fn write_version(w: &mut impl Write, app: &App) -> io::Result<()> { @@ -76,7 +94,7 @@ fn write_usage(w: &mut impl Write, app: &mut App, name: &str) -> io::Result<()> .filter(|l| !l.is_empty()) .collect::>() .join("\n"); - usage = usage.replace(app.get_name(), name); + usage = usage.replace(uucore::execution_phrase(), name); writeln!(w, "{}", usage)?; writeln!(w, "```") } @@ -89,6 +107,51 @@ fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> { } } +fn write_examples( + w: &mut impl Write, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { + let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) { + f + } else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) { + f + } else { + return Ok(()); + }; + + writeln!(w, "## Examples")?; + writeln!(w)?; + for line in content.lines().skip_while(|l| !l.starts_with('-')) { + if let Some(l) = line.strip_prefix("- ") { + writeln!(w, "{}", l)?; + } else if line.starts_with('`') { + writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; + } else if line.is_empty() { + writeln!(w)?; + } else { + println!("Not sure what to do with this line:"); + println!("{}", line); + } + } + writeln!(w)?; + writeln!( + w, + "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)." + )?; + writeln!(w, ">")?; + writeln!( + w, + "> Please note that, as uutils is a work in progress, some examples might fail." + ) +} + +fn get_zip_content(archive: &mut ZipArchive, name: &str) -> Option { + let mut s = String::new(); + archive.by_name(name).ok()?.read_to_string(&mut s).unwrap(); + Some(s) +} + fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { writeln!(w, "

Options

")?; write!(w, "
")?; @@ -138,7 +201,11 @@ fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { write!(w, "")?; } writeln!(w, "")?; - writeln!(w, "
\n\n{}\n\n
", arg.get_help().unwrap_or_default())?; + writeln!( + w, + "
\n\n{}\n\n
", + arg.get_help().unwrap_or_default().replace('\n', "
") + )?; } - writeln!(w, "
") + writeln!(w, "\n") } diff --git a/src/uu/base32/src/base32.rs b/src/uu/base32/src/base32.rs index 006a796f0..ff1ba502a 100644 --- a/src/uu/base32/src/base32.rs +++ b/src/uu/base32/src/base32.rs @@ -22,16 +22,13 @@ to attempt to recover from any other non-alphabet bytes in the encoded stream. "; -fn usage() -> String { - format!("{0} [OPTION]... [FILE]", uucore::execution_phrase()) -} +const USAGE: &str = "{} [OPTION]... [FILE]"; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let format = Format::Base32; - let usage = usage(); - let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, &usage)?; + let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; // Create a reference to stdin so we can return a locked stdin from // parse_base_cmd_args @@ -48,5 +45,5 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } pub fn uu_app<'a>() -> App<'a> { - base_common::base_app(ABOUT) + base_common::base_app(ABOUT, USAGE) } diff --git a/src/uu/base32/src/base_common.rs b/src/uu/base32/src/base_common.rs index 39a46e315..1d4d8a19d 100644 --- a/src/uu/base32/src/base_common.rs +++ b/src/uu/base32/src/base_common.rs @@ -12,7 +12,7 @@ use std::io::{stdout, Read, Write}; use uucore::display::Quotable; use uucore::encoding::{wrap_print, Data, Format}; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; -use uucore::InvalidEncodingHandling; +use uucore::{format_usage, InvalidEncodingHandling}; use std::fs::File; use std::io::{BufReader, Stdin}; @@ -86,17 +86,18 @@ impl Config { } pub fn parse_base_cmd_args(args: impl uucore::Args, about: &str, usage: &str) -> UResult { - let app = base_app(about).override_usage(usage); + let app = base_app(about, usage); let arg_list = args .collect_str(InvalidEncodingHandling::ConvertLossy) .accept_any(); Config::from(&app.get_matches_from(arg_list)) } -pub fn base_app(about: &str) -> App { +pub fn base_app<'a>(about: &'a str, usage: &'a str) -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(about) + .override_usage(format_usage(usage)) .setting(AppSettings::InferLongArgs) // Format arguments. .arg( diff --git a/src/uu/base64/src/base64.rs b/src/uu/base64/src/base64.rs index 20a9f55a5..1bfe29312 100644 --- a/src/uu/base64/src/base64.rs +++ b/src/uu/base64/src/base64.rs @@ -23,16 +23,13 @@ to attempt to recover from any other non-alphabet bytes in the encoded stream. "; -fn usage() -> String { - format!("{0} [OPTION]... [FILE]", uucore::execution_phrase()) -} +const USAGE: &str = "{0} [OPTION]... [FILE]"; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let format = Format::Base64; - let usage = usage(); - let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, &usage)?; + let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; // Create a reference to stdin so we can return a locked stdin from // parse_base_cmd_args diff --git a/src/uu/basename/src/basename.rs b/src/uu/basename/src/basename.rs index b3afe7d7d..bf13fde7e 100644 --- a/src/uu/basename/src/basename.rs +++ b/src/uu/basename/src/basename.rs @@ -11,18 +11,13 @@ use clap::{crate_version, App, AppSettings, Arg}; use std::path::{is_separator, PathBuf}; use uucore::display::Quotable; use uucore::error::{UResult, UUsageError}; -use uucore::InvalidEncodingHandling; +use uucore::{format_usage, InvalidEncodingHandling}; static SUMMARY: &str = "Print NAME with any leading directory components removed If specified, also remove a trailing SUFFIX"; -fn usage() -> String { - format!( - "{0} NAME [SUFFIX] - {0} OPTION... NAME...", - uucore::execution_phrase() - ) -} +const USAGE: &str = "{} NAME [SUFFIX] + {} OPTION... NAME..."; pub mod options { pub static MULTIPLE: &str = "multiple"; @@ -36,11 +31,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args .collect_str(InvalidEncodingHandling::ConvertLossy) .accept_any(); - let usage = usage(); // // Argument parsing // - let matches = uu_app().override_usage(&usage[..]).get_matches_from(args); + let matches = uu_app().get_matches_from(args); // too few arguments if !matches.is_present(options::NAME) { @@ -97,6 +91,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(SUMMARY) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::MULTIPLE) diff --git a/src/uu/basenc/src/basenc.rs b/src/uu/basenc/src/basenc.rs index ef133b151..9e48cf80c 100644 --- a/src/uu/basenc/src/basenc.rs +++ b/src/uu/basenc/src/basenc.rs @@ -38,12 +38,10 @@ const ENCODINGS: &[(&str, Format)] = &[ ("z85", Format::Z85), ]; -fn usage() -> String { - format!("{0} [OPTION]... [FILE]", uucore::execution_phrase()) -} +const USAGE: &str = "{} [OPTION]... [FILE]"; pub fn uu_app<'a>() -> App<'a> { - let mut app = base_common::base_app(ABOUT); + let mut app = base_common::base_app(ABOUT, USAGE); for encoding in ENCODINGS { app = app.arg(Arg::new(encoding.0).long(encoding.0)); } @@ -51,8 +49,7 @@ pub fn uu_app<'a>() -> App<'a> { } fn parse_cmd_args(args: impl uucore::Args) -> UResult<(Config, Format)> { - let usage = usage(); - let matches = uu_app().override_usage(&usage[..]).get_matches_from( + let matches = uu_app().get_matches_from( args.collect_str(InvalidEncodingHandling::ConvertLossy) .accept_any(), ); diff --git a/src/uu/cat/src/cat.rs b/src/uu/cat/src/cat.rs index 498e5e8ad..0a1ba2559 100644 --- a/src/uu/cat/src/cat.rs +++ b/src/uu/cat/src/cat.rs @@ -36,10 +36,10 @@ use std::net::Shutdown; use std::os::unix::fs::FileTypeExt; #[cfg(unix)] use unix_socket::UnixStream; -use uucore::InvalidEncodingHandling; +use uucore::{format_usage, InvalidEncodingHandling}; static NAME: &str = "cat"; -static SYNTAX: &str = "[OPTION]... [FILE]..."; +static USAGE: &str = "{} [OPTION]... [FILE]..."; static SUMMARY: &str = "Concatenate FILE(s), or standard input, to standard output With no FILE, or when FILE is -, read standard input."; @@ -243,7 +243,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .name(NAME) .version(crate_version!()) - .override_usage(SYNTAX) + .override_usage(format_usage(USAGE)) .about(SUMMARY) .setting(AppSettings::InferLongArgs) .arg( diff --git a/src/uu/chcon/src/chcon.rs b/src/uu/chcon/src/chcon.rs index 84de681cd..d033b62c1 100644 --- a/src/uu/chcon/src/chcon.rs +++ b/src/uu/chcon/src/chcon.rs @@ -3,6 +3,7 @@ #![allow(clippy::upper_case_acronyms)] use uucore::error::{UResult, USimpleError, UUsageError}; +use uucore::format_usage; use uucore::{display::Quotable, show_error, show_warning}; use clap::{App, AppSettings, Arg}; @@ -22,6 +23,10 @@ use errors::*; static VERSION: &str = env!("CARGO_PKG_VERSION"); static ABOUT: &str = "Change the SELinux security context of each FILE to CONTEXT. \n\ With --reference, change the security context of each FILE to that of RFILE."; +const USAGE: &str = "\ + {} [OPTION]... CONTEXT FILE... \n \ + {} [OPTION]... [-u USER] [-r ROLE] [-l RANGE] [-t TYPE] FILE... \n \ + {} [OPTION]... --reference=RFILE FILE..."; pub mod options { pub static VERBOSE: &str = "verbose"; @@ -52,20 +57,9 @@ pub mod options { } } -fn get_usage() -> String { - format!( - "{0} [OPTION]... CONTEXT FILE... \n \ - {0} [OPTION]... [-u USER] [-r ROLE] [-l RANGE] [-t TYPE] FILE... \n \ - {0} [OPTION]... --reference=RFILE FILE...", - uucore::execution_phrase() - ) -} - #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let usage = get_usage(); - - let config = uu_app().override_usage(usage.as_ref()); + let config = uu_app(); let options = match parse_command_line(config, args) { Ok(r) => r, @@ -164,6 +158,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(VERSION) .about(ABOUT) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::dereference::DEREFERENCE) diff --git a/src/uu/chgrp/src/chgrp.rs b/src/uu/chgrp/src/chgrp.rs index d228155b6..affaf86a3 100644 --- a/src/uu/chgrp/src/chgrp.rs +++ b/src/uu/chgrp/src/chgrp.rs @@ -10,6 +10,7 @@ use uucore::display::Quotable; pub use uucore::entries; use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::format_usage; use uucore::perms::{chown_base, options, IfFrom}; use clap::{App, AppSettings, Arg, ArgMatches}; @@ -20,12 +21,9 @@ use std::os::unix::fs::MetadataExt; static ABOUT: &str = "Change the group of each FILE to GROUP."; static VERSION: &str = env!("CARGO_PKG_VERSION"); -fn get_usage() -> String { - format!( - "{0} [OPTION]... GROUP FILE...\n {0} [OPTION]... --reference=RFILE FILE...", - uucore::execution_phrase() - ) -} +const USAGE: &str = "\ + {} [OPTION]... GROUP FILE...\n \ + {} [OPTION]... --reference=RFILE FILE..."; fn parse_gid_and_uid(matches: &ArgMatches) -> UResult<(Option, Option, IfFrom)> { let dest_gid = if let Some(file) = matches.value_of(options::REFERENCE) { @@ -53,21 +51,14 @@ fn parse_gid_and_uid(matches: &ArgMatches) -> UResult<(Option, Option, #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let usage = get_usage(); - - chown_base( - uu_app().override_usage(&usage[..]), - args, - options::ARG_GROUP, - parse_gid_and_uid, - true, - ) + chown_base(uu_app(), args, options::ARG_GROUP, parse_gid_and_uid, true) } pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(VERSION) .about(ABOUT) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::verbosity::CHANGES) diff --git a/src/uu/chmod/src/chmod.rs b/src/uu/chmod/src/chmod.rs index 9b14c867f..84c5850d6 100644 --- a/src/uu/chmod/src/chmod.rs +++ b/src/uu/chmod/src/chmod.rs @@ -17,7 +17,7 @@ use uucore::fs::display_permissions_unix; use uucore::libc::mode_t; #[cfg(not(windows))] use uucore::mode; -use uucore::{show_error, InvalidEncodingHandling}; +use uucore::{format_usage, show_error, InvalidEncodingHandling}; use walkdir::WalkDir; static ABOUT: &str = "Change the mode of each FILE to MODE. @@ -35,14 +35,10 @@ mod options { pub const FILE: &str = "FILE"; } -fn usage() -> String { - format!( - "{0} [OPTION]... MODE[,MODE]... FILE... -or: {0} [OPTION]... OCTAL-MODE FILE... -or: {0} [OPTION]... --reference=RFILE FILE...", - uucore::execution_phrase() - ) -} +const USAGE: &str = "\ + {} [OPTION]... MODE[,MODE]... FILE... + {} [OPTION]... OCTAL-MODE FILE... + {} [OPTION]... --reference=RFILE FILE..."; fn get_long_usage() -> String { String::from("Each MODE is of the form '[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+'.") @@ -58,13 +54,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // a possible MODE prefix '-' needs to be removed (e.g. "chmod -x FILE"). let mode_had_minus_prefix = mode::strip_minus_from_mode(&mut args); - let usage = usage(); let after_help = get_long_usage(); - let matches = uu_app() - .override_usage(&usage[..]) - .after_help(&after_help[..]) - .get_matches_from(args); + let matches = uu_app().after_help(&after_help[..]).get_matches_from(args); let changes = matches.is_present(options::CHANGES); let quiet = matches.is_present(options::QUIET); @@ -125,6 +117,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::CHANGES) diff --git a/src/uu/chown/src/chown.rs b/src/uu/chown/src/chown.rs index 06137e200..27a989847 100644 --- a/src/uu/chown/src/chown.rs +++ b/src/uu/chown/src/chown.rs @@ -9,6 +9,7 @@ use uucore::display::Quotable; pub use uucore::entries::{self, Group, Locate, Passwd}; +use uucore::format_usage; use uucore::perms::{chown_base, options, IfFrom}; use uucore::error::{FromIo, UResult, USimpleError}; @@ -20,12 +21,9 @@ use std::os::unix::fs::MetadataExt; static ABOUT: &str = "change file owner and group"; -fn get_usage() -> String { - format!( - "{0} [OPTION]... [OWNER][:[GROUP]] FILE...\n{0} [OPTION]... --reference=RFILE FILE...", - uucore::execution_phrase() - ) -} +const USAGE: &str = "\ + {} [OPTION]... [OWNER][:[GROUP]] FILE... + {} [OPTION]... --reference=RFILE FILE..."; fn parse_gid_uid_and_filter(matches: &ArgMatches) -> UResult<(Option, Option, IfFrom)> { let filter = if let Some(spec) = matches.value_of(options::FROM) { @@ -56,10 +54,8 @@ fn parse_gid_uid_and_filter(matches: &ArgMatches) -> UResult<(Option, Optio #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let usage = get_usage(); - chown_base( - uu_app().override_usage(&usage[..]), + uu_app(), args, options::ARG_OWNER, parse_gid_uid_and_filter, @@ -71,6 +67,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::verbosity::CHANGES) diff --git a/src/uu/chroot/src/chroot.rs b/src/uu/chroot/src/chroot.rs index f656ed77d..d18264566 100644 --- a/src/uu/chroot/src/chroot.rs +++ b/src/uu/chroot/src/chroot.rs @@ -17,10 +17,10 @@ use std::path::Path; use std::process::Command; use uucore::error::{set_exit_code, UResult}; use uucore::libc::{self, chroot, setgid, setgroups, setuid}; -use uucore::{entries, InvalidEncodingHandling}; +use uucore::{entries, format_usage, InvalidEncodingHandling}; static ABOUT: &str = "Run COMMAND with root directory set to NEWROOT."; -static SYNTAX: &str = "[OPTION]... NEWROOT [COMMAND [ARG]...]"; +static USAGE: &str = "{} [OPTION]... NEWROOT [COMMAND [ARG]...]"; mod options { pub const NEWROOT: &str = "newroot"; @@ -95,7 +95,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) - .override_usage(SYNTAX) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::NEWROOT) diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index ee47bfaa9..7f7ba9ed3 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -12,15 +12,15 @@ use std::io::{self, stdin, BufReader, Read}; use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult}; -use uucore::show; use uucore::InvalidEncodingHandling; +use uucore::{format_usage, show}; // NOTE: CRC_TABLE_LEN *must* be <= 256 as we cast 0..CRC_TABLE_LEN to u8 const CRC_TABLE_LEN: usize = 256; const CRC_TABLE: [u32; CRC_TABLE_LEN] = generate_crc_table(); const NAME: &str = "cksum"; -const SYNTAX: &str = "[OPTIONS] [FILE]..."; +const USAGE: &str = "{} [OPTIONS] [FILE]..."; const SUMMARY: &str = "Print CRC and size for each file"; const fn generate_crc_table() -> [u32; CRC_TABLE_LEN] { @@ -145,7 +145,7 @@ pub fn uu_app<'a>() -> App<'a> { .name(NAME) .version(crate_version!()) .about(SUMMARY) - .override_usage(SYNTAX) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::FILE) diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs index dd1281935..8064dd216 100644 --- a/src/uu/comm/src/comm.rs +++ b/src/uu/comm/src/comm.rs @@ -13,12 +13,13 @@ use std::io::{self, stdin, BufRead, BufReader, Stdin}; use std::path::Path; use uucore::error::FromIo; use uucore::error::UResult; -use uucore::InvalidEncodingHandling; +use uucore::{format_usage, InvalidEncodingHandling}; use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; static ABOUT: &str = "compare two sorted files line by line"; static LONG_HELP: &str = ""; +const USAGE: &str = "{} [OPTION]... FILE1 FILE2"; mod options { pub const COLUMN_1: &str = "1"; @@ -30,10 +31,6 @@ mod options { pub const FILE_2: &str = "FILE2"; } -fn usage() -> String { - format!("{} [OPTION]... FILE1 FILE2", uucore::execution_phrase()) -} - fn mkdelim(col: usize, opts: &ArgMatches) -> String { let mut s = String::new(); let delim = opts.value_of(options::DELIMITER).unwrap(); @@ -132,12 +129,11 @@ fn open_file(name: &str) -> io::Result { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let usage = usage(); let args = args .collect_str(InvalidEncodingHandling::ConvertLossy) .accept_any(); - let matches = uu_app().override_usage(&usage[..]).get_matches_from(args); + let matches = uu_app().get_matches_from(args); let filename1 = matches.value_of(options::FILE_1).unwrap(); let filename2 = matches.value_of(options::FILE_2).unwrap(); let mut f1 = open_file(filename1).map_err_context(|| filename1.to_string())?; @@ -152,6 +148,7 @@ pub fn uu_app<'a>() -> App<'a> { .version(crate_version!()) .about(ABOUT) .after_help(LONG_HELP) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::COLUMN_1) diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index 2ffa1e3ca..3673be931 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -19,6 +19,7 @@ extern crate quick_error; extern crate uucore; use uucore::display::Quotable; +use uucore::format_usage; use uucore::fs::FileInformation; #[cfg(windows)] use winapi::um::fileapi::CreateFileW; @@ -29,6 +30,8 @@ use std::borrow::Cow; use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; use filetime::FileTime; +#[cfg(unix)] +use libc::mkfifo; use quick_error::ResultExt; use std::collections::HashSet; use std::env; @@ -42,6 +45,10 @@ use std::fs::OpenOptions; use std::io; use std::io::{stdin, stdout, Write}; use std::mem; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; +#[cfg(unix)] +use std::os::unix::fs::{FileTypeExt, PermissionsExt}; #[cfg(target_os = "linux")] use std::os::unix::io::AsRawFd; #[cfg(windows)] @@ -54,9 +61,6 @@ use uucore::error::{set_exit_code, ExitCode, UError, UResult}; use uucore::fs::{canonicalize, MissingHandling, ResolveMode}; use walkdir::WalkDir; -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; - #[cfg(target_os = "linux")] ioctl!(write ficlone with 0x94, 9; std::os::raw::c_int); @@ -149,7 +153,7 @@ pub type Target = PathBuf; pub type TargetSlice = Path; /// Specifies whether when overwrite files -#[derive(Clone, Eq, PartialEq)] +#[derive(Clone, Copy, Eq, PartialEq)] pub enum ClobberMode { Force, RemoveDestination, @@ -157,7 +161,7 @@ pub enum ClobberMode { } /// Specifies whether when overwrite files -#[derive(Clone, Eq, PartialEq)] +#[derive(Clone, Copy, Eq, PartialEq)] pub enum OverwriteMode { /// [Default] Always overwrite existing files Clobber(ClobberMode), @@ -230,14 +234,10 @@ static ABOUT: &str = "Copy SOURCE to DEST, or multiple SOURCE(s) to DIRECTORY."; static LONG_HELP: &str = ""; static EXIT_ERR: i32 = 1; -fn usage() -> String { - format!( - "{0} [OPTION]... [-T] SOURCE DEST - {0} [OPTION]... SOURCE... DIRECTORY - {0} [OPTION]... -t DIRECTORY SOURCE...", - uucore::execution_phrase() - ) -} +const USAGE: &str = "\ + {} [OPTION]... [-T] SOURCE DEST + {} [OPTION]... SOURCE... DIRECTORY + {} [OPTION]... -t DIRECTORY SOURCE..."; // Argument constants mod options { @@ -297,9 +297,17 @@ static DEFAULT_ATTRIBUTES: &[Attribute] = &[ ]; pub fn uu_app<'a>() -> App<'a> { + const MODE_ARGS: &[&str] = &[ + options::LINK, + options::REFLINK, + options::SYMBOLIC_LINK, + options::ATTRIBUTES_ONLY, + options::COPY_CONTENTS, + ]; App::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg(Arg::new(options::TARGET_DIRECTORY) .short('t') @@ -316,17 +324,17 @@ pub fn uu_app<'a>() -> App<'a> { .arg(Arg::new(options::INTERACTIVE) .short('i') .long(options::INTERACTIVE) - .conflicts_with(options::NO_CLOBBER) + .overrides_with(options::NO_CLOBBER) .help("ask before overwriting files")) .arg(Arg::new(options::LINK) .short('l') .long(options::LINK) - .overrides_with(options::REFLINK) + .overrides_with_all(MODE_ARGS) .help("hard-link files instead of copying")) .arg(Arg::new(options::NO_CLOBBER) .short('n') .long(options::NO_CLOBBER) - .conflicts_with(options::INTERACTIVE) + .overrides_with(options::INTERACTIVE) .help("don't overwrite a file that already exists")) .arg(Arg::new(options::RECURSIVE) .short('r') @@ -346,8 +354,7 @@ pub fn uu_app<'a>() -> App<'a> { .arg(Arg::new(options::SYMBOLIC_LINK) .short('s') .long(options::SYMBOLIC_LINK) - .conflicts_with(options::LINK) - .overrides_with(options::REFLINK) + .overrides_with_all(MODE_ARGS) .help("make symbolic links instead of copying")) .arg(Arg::new(options::FORCE) .short('f') @@ -357,9 +364,9 @@ pub fn uu_app<'a>() -> App<'a> { Currently not implemented for Windows.")) .arg(Arg::new(options::REMOVE_DESTINATION) .long(options::REMOVE_DESTINATION) - .conflicts_with(options::FORCE) + .overrides_with(options::FORCE) .help("remove each existing destination file before attempting to open it \ - (contrast with --force). On Windows, current only works for writeable files.")) + (contrast with --force). On Windows, currently only works for writeable files.")) .arg(backup_control::arguments::backup()) .arg(backup_control::arguments::backup_no_args()) .arg(backup_control::arguments::suffix()) @@ -372,11 +379,11 @@ pub fn uu_app<'a>() -> App<'a> { .long(options::REFLINK) .takes_value(true) .value_name("WHEN") + .overrides_with_all(MODE_ARGS) .help("control clone/CoW copies. See below")) .arg(Arg::new(options::ATTRIBUTES_ONLY) .long(options::ATTRIBUTES_ONLY) - .conflicts_with(options::COPY_CONTENTS) - .overrides_with(options::REFLINK) + .overrides_with_all(MODE_ARGS) .help("Don't copy the file data, just the attributes")) .arg(Arg::new(options::PRESERVE) .long(options::PRESERVE) @@ -386,7 +393,7 @@ pub fn uu_app<'a>() -> App<'a> { .possible_values(PRESERVABLE_ATTRIBUTES) .min_values(0) .value_name("ATTR_LIST") - .conflicts_with_all(&[options::PRESERVE_DEFAULT_ATTRIBUTES, options::NO_PRESERVE]) + .overrides_with_all(&[options::ARCHIVE, options::PRESERVE_DEFAULT_ATTRIBUTES, options::NO_PRESERVE]) // -d sets this option // --archive sets this option .help("Preserve the specified attributes (default: mode, ownership (unix only), timestamps), \ @@ -394,13 +401,13 @@ pub fn uu_app<'a>() -> App<'a> { .arg(Arg::new(options::PRESERVE_DEFAULT_ATTRIBUTES) .short('p') .long(options::PRESERVE_DEFAULT_ATTRIBUTES) - .conflicts_with_all(&[options::PRESERVE, options::NO_PRESERVE, options::ARCHIVE]) + .overrides_with_all(&[options::PRESERVE, options::NO_PRESERVE, options::ARCHIVE]) .help("same as --preserve=mode,ownership(unix only),timestamps")) .arg(Arg::new(options::NO_PRESERVE) .long(options::NO_PRESERVE) .takes_value(true) .value_name("ATTR_LIST") - .conflicts_with_all(&[options::PRESERVE_DEFAULT_ATTRIBUTES, options::PRESERVE, options::ARCHIVE]) + .overrides_with_all(&[options::PRESERVE_DEFAULT_ATTRIBUTES, options::PRESERVE, options::ARCHIVE]) .help("don't preserve the specified attributes")) .arg(Arg::new(options::PARENTS) .long(options::PARENTS) @@ -409,18 +416,18 @@ pub fn uu_app<'a>() -> App<'a> { .arg(Arg::new(options::NO_DEREFERENCE) .short('P') .long(options::NO_DEREFERENCE) - .conflicts_with(options::DEREFERENCE) + .overrides_with(options::DEREFERENCE) // -d sets this option .help("never follow symbolic links in SOURCE")) .arg(Arg::new(options::DEREFERENCE) .short('L') .long(options::DEREFERENCE) - .conflicts_with(options::NO_DEREFERENCE) + .overrides_with(options::NO_DEREFERENCE) .help("always follow symbolic links in SOURCE")) .arg(Arg::new(options::ARCHIVE) .short('a') .long(options::ARCHIVE) - .conflicts_with_all(&[options::PRESERVE_DEFAULT_ATTRIBUTES, options::PRESERVE, options::NO_PRESERVE]) + .overrides_with_all(&[options::PRESERVE_DEFAULT_ATTRIBUTES, options::PRESERVE, options::NO_PRESERVE]) .help("Same as -dR --preserve=all")) .arg(Arg::new(options::NO_DEREFERENCE_PRESERVE_LINKS) .short('d') @@ -433,7 +440,7 @@ pub fn uu_app<'a>() -> App<'a> { // TODO: implement the following args .arg(Arg::new(options::COPY_CONTENTS) .long(options::COPY_CONTENTS) - .conflicts_with(options::ATTRIBUTES_ONLY) + .overrides_with(options::ATTRIBUTES_ONLY) .help("NotImplemented: copy contents of special files when recursive")) .arg(Arg::new(options::SPARSE) .long(options::SPARSE) @@ -456,14 +463,12 @@ pub fn uu_app<'a>() -> App<'a> { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let usage = usage(); let matches = uu_app() .after_help(&*format!( "{}\n{}", LONG_HELP, backup_control::BACKUP_CONTROL_LONG_HELP )) - .override_usage(&usage[..]) .get_matches_from(args); let options = Options::from_matches(&matches)?; @@ -969,6 +974,16 @@ fn copy_directory( return copy_file(root, target, options, symlinked_files); } + // check if root is a prefix of target + if path_has_prefix(target, root)? { + return Err(format!( + "cannot copy a directory, {}, into itself, {}", + root.quote(), + target.quote() + ) + .into()); + } + let current_dir = env::current_dir().unwrap_or_else(|e| crash!(1, "failed to get current directory {}", e)); @@ -1379,12 +1394,23 @@ fn copy_helper( let parent = dest.parent().unwrap_or(dest); fs::create_dir_all(parent)?; } - let is_symlink = fs::symlink_metadata(&source)?.file_type().is_symlink(); + + let file_type = fs::symlink_metadata(&source)?.file_type(); + let is_symlink = file_type.is_symlink(); + + #[cfg(unix)] + let is_fifo = file_type.is_fifo(); + #[cfg(not(unix))] + let is_fifo = false; + if source.as_os_str() == "/dev/null" { /* workaround a limitation of fs::copy * https://github.com/rust-lang/rust/issues/79390 */ File::create(dest).context(dest.display().to_string())?; + } else if is_fifo && options.recursive { + #[cfg(unix)] + copy_fifo(dest, options.overwrite)?; } else if is_symlink { copy_link(source, dest, symlinked_files)?; } else if options.reflink_mode != ReflinkMode::Never { @@ -1404,6 +1430,23 @@ fn copy_helper( Ok(()) } +// "Copies" a FIFO by creating a new one. This workaround is because Rust's +// built-in fs::copy does not handle FIFOs (see rust-lang/rust/issues/79390). +#[cfg(unix)] +fn copy_fifo(dest: &Path, overwrite: OverwriteMode) -> CopyResult<()> { + if dest.exists() { + overwrite.verify(dest)?; + fs::remove_file(&dest)?; + } + + let name = CString::new(dest.as_os_str().as_bytes()).unwrap(); + let err = unsafe { mkfifo(name.as_ptr(), 0o666) }; + if err == -1 { + return Err(format!("cannot create fifo {}: File exists", dest.quote()).into()); + } + Ok(()) +} + fn copy_link( source: &Path, dest: &Path, @@ -1487,7 +1530,6 @@ fn copy_on_write_macos( // Extract paths in a form suitable to be passed to a syscall. // The unwrap() is safe because they come from the command-line and so contain non nul // character. - use std::os::unix::ffi::OsStrExt; let src = CString::new(source.as_os_str().as_bytes()).unwrap(); let dst = CString::new(dest.as_os_str().as_bytes()).unwrap(); @@ -1573,6 +1615,13 @@ pub fn paths_refer_to_same_file(p1: &Path, p2: &Path) -> io::Result { Ok(pathbuf1 == pathbuf2) } +pub fn path_has_prefix(p1: &Path, p2: &Path) -> io::Result { + let pathbuf1 = canonicalize(p1, MissingHandling::Normal, ResolveMode::Logical)?; + let pathbuf2 = canonicalize(p2, MissingHandling::Normal, ResolveMode::Logical)?; + + Ok(pathbuf1.starts_with(pathbuf2)) +} + #[test] fn test_cp_localize_to_target() { assert!( diff --git a/src/uu/csplit/src/csplit.rs b/src/uu/csplit/src/csplit.rs index da0d24727..7793a8637 100644 --- a/src/uu/csplit/src/csplit.rs +++ b/src/uu/csplit/src/csplit.rs @@ -16,7 +16,7 @@ use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; use regex::Regex; use uucore::display::Quotable; use uucore::error::{FromIo, UResult}; -use uucore::InvalidEncodingHandling; +use uucore::{format_usage, InvalidEncodingHandling}; mod csplit_error; mod patterns; @@ -27,6 +27,7 @@ use crate::split_name::SplitName; static SUMMARY: &str = "split a file into sections determined by context lines"; static LONG_HELP: &str = "Output pieces of FILE separated by PATTERN(s) to files 'xx00', 'xx01', ..., and output byte counts of each piece to standard output."; +const USAGE: &str = "{} [OPTION]... FILE PATTERN..."; mod options { pub const SUFFIX_FORMAT: &str = "suffix-format"; @@ -40,13 +41,6 @@ mod options { pub const PATTERN: &str = "pattern"; } -fn usage() -> String { - format!( - "{0} [OPTION]... FILE PATTERN...", - uucore::execution_phrase() - ) -} - /// Command line options for csplit. pub struct CsplitOptions { split_name: crate::SplitName, @@ -719,12 +713,11 @@ mod tests { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let usage = usage(); let args = args .collect_str(InvalidEncodingHandling::Ignore) .accept_any(); - let matches = uu_app().override_usage(&usage[..]).get_matches_from(args); + let matches = uu_app().get_matches_from(args); // get the file to split let file_name = matches.value_of(options::FILE).unwrap(); @@ -757,6 +750,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(SUMMARY) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(options::SUFFIX_FORMAT) diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index fc8494965..ce82a2737 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -20,13 +20,13 @@ use uucore::error::{FromIo, UResult, USimpleError}; use self::searcher::Searcher; use uucore::ranges::Range; -use uucore::InvalidEncodingHandling; +use uucore::{format_usage, InvalidEncodingHandling}; mod searcher; static NAME: &str = "cut"; -static SYNTAX: &str = - "[-d] [-s] [-z] [--output-delimiter] ((-f|-b|-c) {{sequence}}) {{sourcefile}}+"; +static USAGE: &str = + "{} [-d] [-s] [-z] [--output-delimiter] ((-f|-b|-c) {{sequence}}) {{sourcefile}}+"; static SUMMARY: &str = "Prints specified byte or field columns from each line of stdin or the input files"; static LONG_HELP: &str = " @@ -537,7 +537,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .name(NAME) .version(crate_version!()) - .override_usage(SYNTAX) + .override_usage(format_usage(USAGE)) .about(SUMMARY) .after_help(LONG_HELP) .setting(AppSettings::InferLongArgs) diff --git a/src/uu/date/src/date.rs b/src/uu/date/src/date.rs index 5a88225bb..18e06aef4 100644 --- a/src/uu/date/src/date.rs +++ b/src/uu/date/src/date.rs @@ -21,7 +21,7 @@ use uucore::display::Quotable; #[cfg(not(any(target_os = "macos", target_os = "redox")))] use uucore::error::FromIo; use uucore::error::{UResult, USimpleError}; -use uucore::show_error; +use uucore::{format_usage, show_error}; #[cfg(windows)] use winapi::{ shared::minwindef::WORD, @@ -38,8 +38,10 @@ const MINUTE: &str = "minute"; const SECOND: &str = "second"; const NS: &str = "ns"; -const NAME: &str = "date"; const ABOUT: &str = "print or set the system date and time"; +const USAGE: &str = "\ + {} [OPTION]... [+FORMAT]... + {} [OPTION]... [MMDDhhmm[[CC]YY][.ss]]"; const OPT_DATE: &str = "date"; const OPT_FORMAT: &str = "format"; @@ -142,12 +144,7 @@ impl<'a> From<&'a str> for Rfc3339Format { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let syntax = format!( - "{0} [OPTION]... [+FORMAT]... - {0} [OPTION]... [MMDDhhmm[[CC]YY][.ss]]", - NAME - ); - let matches = uu_app().override_usage(&syntax[..]).get_matches_from(args); + let matches = uu_app().get_matches_from(args); let format = if let Some(form) = matches.value_of(OPT_FORMAT) { if !form.starts_with('+') { @@ -261,6 +258,7 @@ pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) + .override_usage(format_usage(USAGE)) .setting(AppSettings::InferLongArgs) .arg( Arg::new(OPT_DATE) diff --git a/src/uu/dd/src/blocks.rs b/src/uu/dd/src/blocks.rs new file mode 100644 index 000000000..331bad56b --- /dev/null +++ b/src/uu/dd/src/blocks.rs @@ -0,0 +1,423 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +// spell-checker:ignore datastructures rstat rposition cflags ctable + +use crate::conversion_tables::ConversionTable; +use crate::datastructures::InternalError; +use crate::progress::ReadStat; +use crate::Input; +use std::io::Read; + +const NEWLINE: u8 = b'\n'; +const SPACE: u8 = b' '; + +/// Split a slice into chunks, padding or truncating as necessary. +/// +/// The slice `buf` is split on newlines, then each block is resized +/// to `cbs` bytes, padding with spaces if necessary. This function +/// expects the input bytes to be ASCII-encoded. +/// +/// If `sync` is true and there has been at least one partial record +/// read from the input (as indicated in `rstat`), then leave an +/// all-spaces block at the end. Otherwise, remove the last block if +/// it is all spaces. +fn block(buf: &[u8], cbs: usize, sync: bool, rstat: &mut ReadStat) -> Vec> { + let mut blocks = buf + .split(|&e| e == NEWLINE) + .map(|split| split.to_vec()) + .fold(Vec::new(), |mut blocks, mut split| { + if split.len() > cbs { + rstat.records_truncated += 1; + } + split.resize(cbs, SPACE); + blocks.push(split); + + blocks + }); + + // If `sync` is true and there has been at least one partial + // record read from the input, then leave the all-spaces block at + // the end. Otherwise, remove it. + if let Some(last) = blocks.last() { + if (!sync || rstat.reads_partial == 0) && last.iter().all(|&e| e == SPACE) { + blocks.pop(); + } + } + + blocks +} + +/// Trims padding from each cbs-length partition of buf +/// as specified by conv=unblock and cbs=N +/// Expects ascii encoded data +fn unblock(buf: &[u8], cbs: usize) -> Vec { + buf.chunks(cbs).fold(Vec::new(), |mut acc, block| { + if let Some(last_char_idx) = block.iter().rposition(|&e| e != SPACE) { + // Include text up to last space. + acc.extend(&block[..=last_char_idx]); + } + + acc.push(NEWLINE); + acc + }) +} + +/// A helper for teasing out which options must be applied and in which order. +/// Some user options, such as the presence of conversion tables, will determine whether the input is assumed to be ascii. The parser sets the Input::non_ascii flag accordingly. +/// Examples: +/// - If conv=ebcdic or conv=ibm is specified then block, unblock or swab must be performed before the conversion happens since the source will start in ascii. +/// - If conv=ascii is specified then block, unblock or swab must be performed after the conversion since the source starts in ebcdic. +/// - If no conversion is specified then the source is assumed to be in ascii. +/// For more info see `info dd` +pub(crate) fn conv_block_unblock_helper( + mut buf: Vec, + i: &mut Input, + rstat: &mut ReadStat, +) -> Result, InternalError> { + // Local Predicate Fns ------------------------------------------------- + fn should_block_then_conv(i: &Input) -> bool { + !i.non_ascii && i.cflags.block.is_some() + } + fn should_conv_then_block(i: &Input) -> bool { + i.non_ascii && i.cflags.block.is_some() + } + fn should_unblock_then_conv(i: &Input) -> bool { + !i.non_ascii && i.cflags.unblock.is_some() + } + fn should_conv_then_unblock(i: &Input) -> bool { + i.non_ascii && i.cflags.unblock.is_some() + } + fn conv_only(i: &Input) -> bool { + i.cflags.ctable.is_some() && i.cflags.block.is_none() && i.cflags.unblock.is_none() + } + // Local Helper Fns ---------------------------------------------------- + fn apply_conversion(buf: &mut [u8], ct: &ConversionTable) { + for idx in 0..buf.len() { + buf[idx] = ct[buf[idx] as usize]; + } + } + // -------------------------------------------------------------------- + if conv_only(i) { + // no block/unblock + let ct = i.cflags.ctable.unwrap(); + apply_conversion(&mut buf, ct); + + Ok(buf) + } else if should_block_then_conv(i) { + // ascii input so perform the block first + let cbs = i.cflags.block.unwrap(); + + let mut blocks = block(&buf, cbs, i.cflags.sync.is_some(), rstat); + + if let Some(ct) = i.cflags.ctable { + for buf in &mut blocks { + apply_conversion(buf, ct); + } + } + + let blocks = blocks.into_iter().flatten().collect(); + + Ok(blocks) + } else if should_conv_then_block(i) { + // Non-ascii so perform the conversion first + let cbs = i.cflags.block.unwrap(); + + if let Some(ct) = i.cflags.ctable { + apply_conversion(&mut buf, ct); + } + + let blocks = block(&buf, cbs, i.cflags.sync.is_some(), rstat) + .into_iter() + .flatten() + .collect(); + + Ok(blocks) + } else if should_unblock_then_conv(i) { + // ascii input so perform the unblock first + let cbs = i.cflags.unblock.unwrap(); + + let mut buf = unblock(&buf, cbs); + + if let Some(ct) = i.cflags.ctable { + apply_conversion(&mut buf, ct); + } + + Ok(buf) + } else if should_conv_then_unblock(i) { + // Non-ascii input so perform the conversion first + let cbs = i.cflags.unblock.unwrap(); + + if let Some(ct) = i.cflags.ctable { + apply_conversion(&mut buf, ct); + } + + let buf = unblock(&buf, cbs); + + Ok(buf) + } else { + // The following error should not happen, as it results from + // insufficient command line data. This case should be caught + // by the parser before making it this far. + // Producing this error is an alternative to risking an unwrap call + // on 'cbs' if the required data is not provided. + Err(InternalError::InvalidConvBlockUnblockCase) + } +} + +#[cfg(test)] +mod tests { + + use crate::blocks::{block, unblock}; + use crate::progress::ReadStat; + + const NEWLINE: u8 = b'\n'; + const SPACE: u8 = b' '; + + #[test] + fn block_test_no_nl() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, 3u8]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]); + } + + #[test] + fn block_test_no_nl_short_record() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, 3u8]; + let res = block(&buf, 8, false, &mut rs); + + assert_eq!( + res, + vec![vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],] + ); + } + + #[test] + fn block_test_no_nl_trunc() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, 3u8, 4u8]; + let res = block(&buf, 4, false, &mut rs); + + // Commented section(s) should be truncated and appear for reference only. + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8 /*, 4u8*/],]); + assert_eq!(rs.records_truncated, 1); + } + + #[test] + fn block_test_nl_gt_cbs_trunc() { + let mut rs = ReadStat::default(); + let buf = [ + 0u8, 1u8, 2u8, 3u8, 4u8, NEWLINE, 0u8, 1u8, 2u8, 3u8, 4u8, NEWLINE, 5u8, 6u8, 7u8, 8u8, + ]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!( + res, + vec![ + // Commented section(s) should be truncated and appear for reference only. + vec![0u8, 1u8, 2u8, 3u8], + // vec![4u8, SPACE, SPACE, SPACE], + vec![0u8, 1u8, 2u8, 3u8], + // vec![4u8, SPACE, SPACE, SPACE], + vec![5u8, 6u8, 7u8, 8u8], + ] + ); + assert_eq!(rs.records_truncated, 2); + } + + #[test] + fn block_test_surrounded_nl() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE, 4u8, 5u8, 6u8, 7u8, 8u8]; + let res = block(&buf, 8, false, &mut rs); + + assert_eq!( + res, + vec![ + vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE], + vec![4u8, 5u8, 6u8, 7u8, 8u8, SPACE, SPACE, SPACE], + ] + ); + } + + #[test] + fn block_test_multiple_nl_same_cbs_block() { + let mut rs = ReadStat::default(); + let buf = [ + 0u8, 1u8, 2u8, 3u8, NEWLINE, 4u8, NEWLINE, 5u8, 6u8, 7u8, 8u8, 9u8, + ]; + let res = block(&buf, 8, false, &mut rs); + + assert_eq!( + res, + vec![ + vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE], + vec![4u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE], + vec![5u8, 6u8, 7u8, 8u8, 9u8, SPACE, SPACE, SPACE], + ] + ); + } + + #[test] + fn block_test_multiple_nl_diff_cbs_block() { + let mut rs = ReadStat::default(); + let buf = [ + 0u8, 1u8, 2u8, 3u8, NEWLINE, 4u8, 5u8, 6u8, 7u8, NEWLINE, 8u8, 9u8, + ]; + let res = block(&buf, 8, false, &mut rs); + + assert_eq!( + res, + vec![ + vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE], + vec![4u8, 5u8, 6u8, 7u8, SPACE, SPACE, SPACE, SPACE], + vec![8u8, 9u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE], + ] + ); + } + + #[test] + fn block_test_end_nl_diff_cbs_block() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]); + } + + #[test] + fn block_test_end_nl_same_cbs_block() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, NEWLINE]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, SPACE]]); + } + + #[test] + fn block_test_double_end_nl() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, NEWLINE, NEWLINE]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!( + res, + vec![vec![0u8, 1u8, 2u8, SPACE], vec![SPACE, SPACE, SPACE, SPACE],] + ); + } + + #[test] + fn block_test_start_nl() { + let mut rs = ReadStat::default(); + let buf = [NEWLINE, 0u8, 1u8, 2u8, 3u8]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!( + res, + vec![vec![SPACE, SPACE, SPACE, SPACE], vec![0u8, 1u8, 2u8, 3u8],] + ); + } + + #[test] + fn block_test_double_surrounded_nl_no_trunc() { + let mut rs = ReadStat::default(); + let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE, NEWLINE, 4u8, 5u8, 6u8, 7u8]; + let res = block(&buf, 8, false, &mut rs); + + assert_eq!( + res, + vec![ + vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE], + vec![SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE], + vec![4u8, 5u8, 6u8, 7u8, SPACE, SPACE, SPACE, SPACE], + ] + ); + } + + #[test] + fn block_test_double_surrounded_nl_double_trunc() { + let mut rs = ReadStat::default(); + let buf = [ + 0u8, 1u8, 2u8, 3u8, NEWLINE, NEWLINE, 4u8, 5u8, 6u8, 7u8, 8u8, + ]; + let res = block(&buf, 4, false, &mut rs); + + assert_eq!( + res, + vec![ + // Commented section(s) should be truncated and appear for reference only. + vec![0u8, 1u8, 2u8, 3u8], + vec![SPACE, SPACE, SPACE, SPACE], + vec![4u8, 5u8, 6u8, 7u8 /*, 8u8*/], + ] + ); + assert_eq!(rs.records_truncated, 1); + } + + #[test] + fn unblock_test_full_cbs() { + let buf = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8]; + let res = unblock(&buf, 8); + + assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, NEWLINE],); + } + + #[test] + fn unblock_test_all_space() { + let buf = [SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE]; + let res = unblock(&buf, 8); + + assert_eq!(res, vec![NEWLINE],); + } + + #[test] + fn unblock_test_decoy_spaces() { + let buf = [0u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, 7u8]; + let res = unblock(&buf, 8); + + assert_eq!( + res, + vec![0u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, 7u8, NEWLINE], + ); + } + + #[test] + fn unblock_test_strip_single_cbs() { + let buf = [0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE]; + let res = unblock(&buf, 8); + + assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, NEWLINE],); + } + + #[test] + fn unblock_test_strip_multi_cbs() { + let buf = vec![ + vec![0u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE], + vec![0u8, 1u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE], + vec![0u8, 1u8, 2u8, SPACE, SPACE, SPACE, SPACE, SPACE], + vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE], + ] + .into_iter() + .flatten() + .collect::>(); + + let res = unblock(&buf, 8); + + let exp = vec![ + vec![0u8, NEWLINE], + vec![0u8, 1u8, NEWLINE], + vec![0u8, 1u8, 2u8, NEWLINE], + vec![0u8, 1u8, 2u8, 3u8, NEWLINE], + ] + .into_iter() + .flatten() + .collect::>(); + + assert_eq!(res, exp); + } +} diff --git a/src/uu/dd/src/datastructures.rs b/src/uu/dd/src/datastructures.rs index 8380965a9..067058bbe 100644 --- a/src/uu/dd/src/datastructures.rs +++ b/src/uu/dd/src/datastructures.rs @@ -7,72 +7,11 @@ // spell-checker:ignore ctable, outfile use std::error::Error; -use std::time; use uucore::error::UError; use crate::conversion_tables::*; -pub struct ProgUpdate { - pub read_stat: ReadStat, - pub write_stat: WriteStat, - pub duration: time::Duration, -} - -impl ProgUpdate { - pub(crate) fn new( - read_stat: ReadStat, - write_stat: WriteStat, - duration: time::Duration, - ) -> Self { - Self { - read_stat, - write_stat, - duration, - } - } -} - -#[derive(Clone, Copy, Default)] -pub struct ReadStat { - pub reads_complete: u64, - pub reads_partial: u64, - pub records_truncated: u32, -} - -impl ReadStat { - /// Whether this counter has zero complete reads and zero partial reads. - pub(crate) fn is_empty(&self) -> bool { - self.reads_complete == 0 && self.reads_partial == 0 - } -} - -impl std::ops::AddAssign for ReadStat { - fn add_assign(&mut self, other: Self) { - *self = Self { - reads_complete: self.reads_complete + other.reads_complete, - reads_partial: self.reads_partial + other.reads_partial, - records_truncated: self.records_truncated + other.records_truncated, - } - } -} - -#[derive(Clone, Copy, Default)] -pub struct WriteStat { - pub writes_complete: u64, - pub writes_partial: u64, - pub bytes_total: u128, -} -impl std::ops::AddAssign for WriteStat { - fn add_assign(&mut self, other: Self) { - *self = Self { - writes_complete: self.writes_complete + other.writes_complete, - writes_partial: self.writes_partial + other.writes_partial, - bytes_total: self.bytes_total + other.bytes_total, - } - } -} - type Cbs = usize; /// Stores all Conv Flags that apply to the input @@ -138,23 +77,14 @@ pub struct OFlags { pub seek_bytes: bool, } -/// The value of the status cl-option. -/// Controls printing of transfer stats -#[derive(Copy, Clone, Debug, PartialEq)] -pub enum StatusLevel { - Progress, - Noxfer, - None, -} - /// The value of count=N /// Defaults to Reads(N) /// if iflag=count_bytes /// then becomes Bytes(N) #[derive(Debug, PartialEq)] pub enum CountType { - Reads(usize), - Bytes(usize), + Reads(u64), + Bytes(u64), } #[derive(Debug)] diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index b24e18049..d8bc3acd3 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -5,7 +5,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, btotal, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, outfile, parseargs, rlen, rmax, rposition, rremain, rsofar, rstat, sigusr, sigval, wlen, wstat seekable +// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, outfile, parseargs, rlen, rmax, rremain, rsofar, rstat, sigusr, wlen, wstat seekable mod datastructures; use datastructures::*; @@ -16,36 +16,32 @@ use parseargs::Matches; mod conversion_tables; use conversion_tables::*; +mod progress; +use progress::{gen_prog_updater, ProgUpdate, ReadStat, StatusLevel, WriteStat}; + +mod blocks; +use blocks::conv_block_unblock_helper; + use std::cmp; use std::convert::TryInto; use std::env; -#[cfg(target_os = "linux")] -use std::error::Error; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, Write}; #[cfg(target_os = "linux")] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use std::sync::mpsc; -#[cfg(target_os = "linux")] -use std::sync::{atomic::AtomicUsize, atomic::Ordering, Arc}; use std::thread; use std::time; -use byte_unit::Byte; use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; use gcd::Gcd; -#[cfg(target_os = "linux")] -use signal_hook::consts::signal; use uucore::display::Quotable; -use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::show_error; -use uucore::InvalidEncodingHandling; +use uucore::error::{FromIo, UResult}; +use uucore::{show_error, InvalidEncodingHandling}; const ABOUT: &str = "copy, and optionally convert, a file system resource"; const BUF_INIT_BYTE: u8 = 0xDD; -const NEWLINE: u8 = b'\n'; -const SPACE: u8 = b' '; struct Input { src: R, @@ -78,11 +74,13 @@ impl Input { }; if let Some(amt) = skip { - let num_bytes_read = i - .force_fill(amt.try_into().unwrap()) - .map_err_context(|| "failed to read input".to_string())?; - if num_bytes_read < amt { - show_error!("'standard input': cannot skip to specified offset"); + if let Err(e) = i.read_skip(amt) { + if let io::ErrorKind::UnexpectedEof = e.kind() { + show_error!("'standard input': cannot skip to specified offset"); + } else { + return io::Result::Err(e) + .map_err_context(|| "I/O error while skipping".to_string()); + } } } @@ -151,9 +149,6 @@ impl Input { }; if let Some(amt) = skip { - let amt: u64 = amt - .try_into() - .map_err(|_| USimpleError::new(1, "failed to parse seek amount"))?; src.seek(io::SeekFrom::Start(amt)) .map_err_context(|| "failed to seek in input file".to_string())?; } @@ -265,19 +260,18 @@ impl Input { }) } - /// Read the specified number of bytes from this reader. - /// - /// On success, this method returns the number of bytes read. If - /// this reader has fewer than `n` bytes available, then it reads - /// as many as possible. In that case, this method returns a - /// number less than `n`. - /// - /// # Errors - /// - /// If there is a problem reading. - fn force_fill(&mut self, n: u64) -> std::io::Result { - let mut buf = vec![]; - self.take(n).read_to_end(&mut buf) + /// Skips amount_to_read bytes from the Input by copying into a sink + fn read_skip(&mut self, amount_to_read: u64) -> std::io::Result<()> { + let copy_result = io::copy(&mut self.src.by_ref().take(amount_to_read), &mut io::sink()); + if let Ok(n) = copy_result { + if n != amount_to_read { + io::Result::Err(io::Error::new(io::ErrorKind::UnexpectedEof, "")) + } else { + Ok(()) + } + } else { + io::Result::Err(copy_result.unwrap_err()) + } } } @@ -304,8 +298,7 @@ impl OutputTrait for Output { // stdout is not seekable, so we just write null bytes. if let Some(amt) = seek { - let bytes = vec![b'\0'; amt]; - dst.write_all(&bytes) + io::copy(&mut io::repeat(0u8).take(amt as u64), &mut dst) .map_err_context(|| String::from("write error"))?; } @@ -351,8 +344,8 @@ where fn print_stats(&self, i: &Input, prog_update: &ProgUpdate) { match i.print_level { Some(StatusLevel::None) => {} - Some(StatusLevel::Noxfer) => print_io_lines(prog_update), - Some(StatusLevel::Progress) | None => print_transfer_stats(prog_update), + Some(StatusLevel::Noxfer) => prog_update.print_io_lines(), + Some(StatusLevel::Progress) | None => prog_update.print_transfer_stats(), } } @@ -521,10 +514,17 @@ impl OutputTrait for Output { let mut dst = open_dst(Path::new(&fname), &cflags, &oflags) .map_err_context(|| format!("failed to open {}", fname.quote()))?; - let i = seek.unwrap_or(0).try_into().unwrap(); + // Seek to the index in the output file, truncating if requested. + // + // Calling `set_len()` may result in an error (for + // example, when calling it on `/dev/null`), but we don't + // want to terminate the process when that happens. + // Instead, we suppress the error by calling + // `Result::ok()`. This matches the behavior of GNU `dd` + // when given the command-line argument `of=/dev/null`. + let i = seek.unwrap_or(0); if !cflags.notrunc { - dst.set_len(i) - .map_err_context(|| "failed to truncate output file".to_string())?; + dst.set_len(i).ok(); } dst.seek(io::SeekFrom::Start(i)) .map_err_context(|| "failed to seek in output file".to_string())?; @@ -589,146 +589,6 @@ impl Write for Output { } } -/// Splits the content of buf into cbs-length blocks -/// Appends padding as specified by conv=block and cbs=N -/// Expects ascii encoded data -fn block(buf: &[u8], cbs: usize, rstat: &mut ReadStat) -> Vec> { - let mut blocks = buf - .split(|&e| e == NEWLINE) - .map(|split| split.to_vec()) - .fold(Vec::new(), |mut blocks, mut split| { - if split.len() > cbs { - rstat.records_truncated += 1; - } - split.resize(cbs, SPACE); - blocks.push(split); - - blocks - }); - - if let Some(last) = blocks.last() { - if last.iter().all(|&e| e == SPACE) { - blocks.pop(); - } - } - - blocks -} - -/// Trims padding from each cbs-length partition of buf -/// as specified by conv=unblock and cbs=N -/// Expects ascii encoded data -fn unblock(buf: &[u8], cbs: usize) -> Vec { - buf.chunks(cbs).fold(Vec::new(), |mut acc, block| { - if let Some(last_char_idx) = block.iter().rposition(|&e| e != SPACE) { - // Include text up to last space. - acc.extend(&block[..=last_char_idx]); - } - - acc.push(NEWLINE); - acc - }) -} - -/// A helper for teasing out which options must be applied and in which order. -/// Some user options, such as the presence of conversion tables, will determine whether the input is assumed to be ascii. The parser sets the Input::non_ascii flag accordingly. -/// Examples: -/// - If conv=ebcdic or conv=ibm is specified then block, unblock or swab must be performed before the conversion happens since the source will start in ascii. -/// - If conv=ascii is specified then block, unblock or swab must be performed after the conversion since the source starts in ebcdic. -/// - If no conversion is specified then the source is assumed to be in ascii. -/// For more info see `info dd` -fn conv_block_unblock_helper( - mut buf: Vec, - i: &mut Input, - rstat: &mut ReadStat, -) -> Result, InternalError> { - // Local Predicate Fns ------------------------------------------------- - fn should_block_then_conv(i: &Input) -> bool { - !i.non_ascii && i.cflags.block.is_some() - } - fn should_conv_then_block(i: &Input) -> bool { - i.non_ascii && i.cflags.block.is_some() - } - fn should_unblock_then_conv(i: &Input) -> bool { - !i.non_ascii && i.cflags.unblock.is_some() - } - fn should_conv_then_unblock(i: &Input) -> bool { - i.non_ascii && i.cflags.unblock.is_some() - } - fn conv_only(i: &Input) -> bool { - i.cflags.ctable.is_some() && i.cflags.block.is_none() && i.cflags.unblock.is_none() - } - // Local Helper Fns ---------------------------------------------------- - fn apply_conversion(buf: &mut [u8], ct: &ConversionTable) { - for idx in 0..buf.len() { - buf[idx] = ct[buf[idx] as usize]; - } - } - // -------------------------------------------------------------------- - if conv_only(i) { - // no block/unblock - let ct = i.cflags.ctable.unwrap(); - apply_conversion(&mut buf, ct); - - Ok(buf) - } else if should_block_then_conv(i) { - // ascii input so perform the block first - let cbs = i.cflags.block.unwrap(); - - let mut blocks = block(&buf, cbs, rstat); - - if let Some(ct) = i.cflags.ctable { - for buf in &mut blocks { - apply_conversion(buf, ct); - } - } - - let blocks = blocks.into_iter().flatten().collect(); - - Ok(blocks) - } else if should_conv_then_block(i) { - // Non-ascii so perform the conversion first - let cbs = i.cflags.block.unwrap(); - - if let Some(ct) = i.cflags.ctable { - apply_conversion(&mut buf, ct); - } - - let blocks = block(&buf, cbs, rstat).into_iter().flatten().collect(); - - Ok(blocks) - } else if should_unblock_then_conv(i) { - // ascii input so perform the unblock first - let cbs = i.cflags.unblock.unwrap(); - - let mut buf = unblock(&buf, cbs); - - if let Some(ct) = i.cflags.ctable { - apply_conversion(&mut buf, ct); - } - - Ok(buf) - } else if should_conv_then_unblock(i) { - // Non-ascii input so perform the conversion first - let cbs = i.cflags.unblock.unwrap(); - - if let Some(ct) = i.cflags.ctable { - apply_conversion(&mut buf, ct); - } - - let buf = unblock(&buf, cbs); - - Ok(buf) - } else { - // The following error should not happen, as it results from - // insufficient command line data. This case should be caught - // by the parser before making it this far. - // Producing this error is an alternative to risking an unwrap call - // on 'cbs' if the required data is not provided. - Err(InternalError::InvalidConvBlockUnblockCase) - } -} - /// Read helper performs read operations common to all dd reads, and dispatches the buffer to relevant helper functions as dictated by the operations requested by the user. fn read_helper(i: &mut Input, bsize: usize) -> std::io::Result<(ReadStat, Vec)> { // Local Predicate Fns ----------------------------------------------- @@ -771,113 +631,6 @@ fn read_helper(i: &mut Input, bsize: usize) -> std::io::Result<(Read } } -// Print io lines of a status update: -// + records in -// + records out -fn print_io_lines(update: &ProgUpdate) { - eprintln!( - "{}+{} records in", - update.read_stat.reads_complete, update.read_stat.reads_partial - ); - if update.read_stat.records_truncated > 0 { - eprintln!("{} truncated records", update.read_stat.records_truncated); - } - eprintln!( - "{}+{} records out", - update.write_stat.writes_complete, update.write_stat.writes_partial - ); -} -// Print the progress line of a status update: -// bytes (, ) copied,