Merge branch 'master' into pr

.cirrus.yml

@@ -1,7 +1,14 @@
+env:
+  # Temporary workaround for error `error: sysinfo not supported on
+  # this platform` seen on FreeBSD platforms, affecting Rustup
+  #
+  # References: https://github.com/rust-lang/rustup/issues/2774
+  RUSTUP_IO_THREADS: 1
+
 task:
   name: stable x86_64-unknown-freebsd-12
   freebsd_instance:
-    image: freebsd-12-1-release-amd64
+    image: freebsd-12-2-release-amd64
   setup_script:
     - pkg install -y curl gmake
     - curl https://sh.rustup.rs -sSf --output rustup.sh

.github/workflows/CICD.yml

@@ -11,7 +11,7 @@ env:
   PROJECT_NAME: coreutils
   PROJECT_DESC: "Core universal (cross-platform) utilities"
   PROJECT_AUTH: "uutils"
-  RUST_MIN_SRV: "1.40.0" ## v1.40.0
+  RUST_MIN_SRV: "1.43.1" ## v1.43.0
   RUST_COV_SRV: "2020-08-01" ## (~v1.47.0) supported rust version for code coverage; (date required/used by 'coverage') ## !maint: refactor when code coverage support is included in the stable channel
 
 on: [push, pull_request]
@@ -235,6 +235,9 @@ jobs:
           arm-unknown-linux-gnueabihf) sudo apt-get -y update ; sudo apt-get -y install gcc-arm-linux-gnueabihf ;;
           aarch64-unknown-linux-gnu) sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu ;;
         esac
+        case '${{ matrix.job.os }}' in
+          macos-latest) brew install coreutils ;; # needed for testing
+        esac
     - name: Initialize workflow variables
       id: vars
       shell: bash
@@ -360,6 +363,10 @@ jobs:
         mkdir -p '${{ steps.vars.outputs.STAGING }}/dpkg'
     - name: rust toolchain ~ install
       uses: actions-rs/toolchain@v1
+      env:
+        # Override auto-detection of RAM for Rustc install.
+        # https://github.com/rust-lang/rustup/issues/2229#issuecomment-585855925
+        RUSTUP_UNPACK_RAM: "21474836480"
       with:
         toolchain: ${{ steps.vars.outputs.TOOLCHAIN }}
         target: ${{ matrix.job.target }}
@@ -486,6 +493,13 @@ jobs:
           - { os: windows-latest , features: windows }
     steps:
     - uses: actions/checkout@v1
+    - name: Install/setup prerequisites
+      shell: bash
+      run: |
+        ## install/setup prerequisites
+        case '${{ matrix.job.os }}' in
+          macos-latest) brew install coreutils ;; # needed for testing
+        esac
     # - name: Reattach HEAD ## may be needed for accurate code coverage info
     #   run: git checkout ${{ github.head_ref }}
     - name: Initialize workflow variables

.github/workflows/GNU.yml

@@ -12,16 +12,18 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: 'uutils'
-    - name: Chechout GNU coreutils
+    - name: Checkout GNU coreutils
       uses: actions/checkout@v2
       with:
         repository: 'coreutils/coreutils'
         path: 'gnu'
-    - name: Chechout GNU corelib
+        ref: v8.32
+    - name: Checkout GNU corelib
       uses: actions/checkout@v2
       with:
         repository: 'coreutils/gnulib'
         path: 'gnulib'
+        ref: 8e99f24c0931a38880c6ee9b8287c7da80b0036b
         fetch-depth: 0 # gnu gets upset if gnulib is a shallow checkout
     - name: Install `rust` toolchain
       uses: actions-rs/toolchain@v1
@@ -30,100 +32,43 @@ jobs:
         default: true
         profile: minimal # minimal component installation (ie, no documentation)
         components: rustfmt
-    - name: Build binaries
+    - name: Install deps
       shell: bash
       run: |
         sudo apt-get update
-        sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify python3-sphinx
-        pushd uutils
-        make PROFILE=release
-        BUILDDIR="$PWD/target/release/"
-        cp "${BUILDDIR}/install" "${BUILDDIR}/ginstall" # The GNU tests rename this script before running, to avoid confusion with the make target
-        # Create *sum binaries
-        for sum in b2sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum
-        do
-          sum_path="${BUILDDIR}/${sum}"
-          test -f "${sum_path}" || cp "${BUILDDIR}/hashsum" "${sum_path}"
-        done
-        test -f "${BUILDDIR}/[" || cp "${BUILDDIR}/test" "${BUILDDIR}/["
-        popd
-        GNULIB_SRCDIR="$PWD/gnulib"
-        pushd gnu/
-
-        # Any binaries that aren't built become `false` so their tests fail
-        for binary in $(./build-aux/gen-lists-of-programs.sh --list-progs)
-        do
-          bin_path="${BUILDDIR}/${binary}"
-          test -f "${bin_path}" || { echo "'${binary}' was not built with uutils, using the 'false' program"; cp "${BUILDDIR}/false" "${bin_path}"; }
-        done
-
-        ./bootstrap --gnulib-srcdir="$GNULIB_SRCDIR"
-        ./configure --quiet --disable-gcc-warnings
-        #Add timeout to to protect against hangs
-        sed -i 's|"\$@|/usr/bin/timeout 600 "\$@|' build-aux/test-driver
-        # Change the PATH in the Makefile to test the uutils coreutils instead of the GNU coreutils
-        sed -i "s/^[[:blank:]]*PATH=.*/  PATH='${BUILDDIR//\//\\/}\$(PATH_SEPARATOR)'\"\$\$PATH\" \\\/" Makefile
-        sed -i 's| tr | /usr/bin/tr |' tests/init.sh
-        make
-        # Generate the factor tests, so they can be fixed
-        for i in {00..36}
-        do
-          make tests/factor/t${i}.sh
-        done
-        grep -rl 'path_prepend_' tests/* | xargs sed -i 's|path_prepend_ ./src||'
-        sed -i -e 's|^seq |/usr/bin/seq |' -e 's|sha1sum |/usr/bin/sha1sum |' tests/factor/t*sh
-
-        # Remove tests checking for --version & --help
-        # Not really interesting for us and logs are too big
-        sed -i -e '/tests\/misc\/invalid-opt.pl/ D' \
-               -e '/tests\/misc\/help-version.sh/ D' \
-               -e '/tests\/misc\/help-version-getopt.sh/ D' \
-               Makefile
-
-        # Use the system coreutils where the test fails due to error in a util that is not the one being tested
-        sed -i 's|stat|/usr/bin/stat|' tests/chgrp/basic.sh tests/cp/existing-perm-dir.sh tests/touch/60-seconds.sh tests/misc/sort-compress-proc.sh
-        sed -i 's|ls -|/usr/bin/ls -|' tests/chgrp/posix-H.sh tests/chown/deref.sh tests/cp/same-file.sh tests/misc/mknod.sh tests/mv/part-symlink.sh tests/du/8gb.sh
-        sed -i 's|mkdir |/usr/bin/mkdir |' tests/cp/existing-perm-dir.sh tests/rm/empty-inacc.sh
-        sed -i 's|timeout \([[:digit:]]\)| /usr/bin/timeout \1|' tests/tail-2/inotify-rotate.sh tests/tail-2/inotify-dir-recreate.sh tests/tail-2/inotify-rotate-resources.sh tests/cp/parent-perm-race.sh tests/ls/infloop.sh tests/misc/sort-exit-early.sh tests/misc/sort-NaN-infloop.sh tests/misc/uniq-perf.sh tests/tail-2/inotify-only-regular.sh tests/tail-2/pipe-f2.sh tests/tail-2/retry.sh tests/tail-2/symlink.sh tests/tail-2/wait.sh tests/tail-2/pid.sh tests/dd/stats.sh tests/tail-2/follow-name.sh tests/misc/shuf.sh # Don't break the function called 'grep_timeout'
-        sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/mkdir/p-3.sh tests/tail-2/tail-n0f.sh tests/cp/fail-perm.sh tests/du/inaccessible-cwd.sh tests/mv/i-2.sh tests/chgrp/basic.sh tests/misc/shuf.sh
-        sed -i 's|sort |/usr/bin/sort |' tests/ls/hyperlink.sh tests/misc/test-N.sh
-        sed -i 's|split |/usr/bin/split |' tests/misc/factor-parallel.sh
-        sed -i 's|truncate |/usr/bin/truncate |' tests/split/fail.sh
-        sed -i 's|dd |/usr/bin/dd |' tests/du/8gb.sh tests/tail-2/big-4gb.sh tests/cp/fiemap-2.sh init.cfg
-        sed -i 's|id -|/usr/bin/id -|' tests/misc/runcon-no-reorder.sh
-        sed -i 's|touch |/usr/bin/touch |' tests/cp/preserve-link.sh tests/cp/reflink-perm.sh tests/ls/block-size.sh tests/ls/abmon-align.sh tests/ls/rt-1.sh tests/mv/update.sh tests/misc/ls-time.sh tests/misc/stat-nanoseconds.sh tests/misc/time-style.sh tests/misc/test-N.sh
-        sed -i 's|ln -|/usr/bin/ln -|' tests/cp/link-deref.sh
-        sed -i 's|printf |/usr/bin/printf |' tests/dd/ascii.sh
-        sed -i 's|cp |/usr/bin/cp |' tests/mv/hard-2.sh
-        sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh
-        sed -i 's|seq |/usr/bin/seq |' tests/misc/sort-discrim.sh
-
-        #Add specific timeout to tests that currently hang to limit time spent waiting
-        sed -i 's|seq \$|/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh
-        sed -i 's|cat |/usr/bin/timeout 0.1 cat |' tests/misc/cat-self.sh
-         
-
-        test -f "${BUILDDIR}/getlimits" || cp src/getlimits "${BUILDDIR}"
+        sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify python3-sphinx jq
+    - name: Build binaries
+      shell: bash
+      run: |
+       cd uutils
+       bash util/build-gnu.sh
     - name: Run GNU tests
       shell: bash
       run: |
-        BUILDDIR="${PWD}/uutils/target/release"
-        GNULIB_DIR="${PWD}/gnulib"
-        pushd gnu
-
-        timeout -sKILL 2h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no || : # Kill after 4 hours in case something gets stuck in make
+        bash uutils/util/run-gnu-test.sh
     - name: Extract tests info
       shell: bash
       run: |
-        if test -f gnu/tests/test-suite.log
+        LOG_FILE=gnu/tests/test-suite.log
+        if test -f "$LOG_FILE"
         then
-            TOTAL=$( grep "# TOTAL:" gnu/tests/test-suite.log|cut -d' ' -f2-)
-            PASS=$( grep "# PASS:" gnu/tests/test-suite.log|cut -d' ' -f2-)
-            SKIP=$( grep "# SKIP:" gnu/tests/test-suite.log|cut -d' ' -f2-)
-            FAIL=$( grep "# FAIL:" gnu/tests/test-suite.log|cut -d' ' -f2-)
-            XPASS=$( grep "# XPASS:" gnu/tests/test-suite.log|cut -d' ' -f2-)
-            ERROR=$( grep "# ERROR:" gnu/tests/test-suite.log|cut -d' ' -f2-)
-            echo "::warning ::GNU testsuite = $TOTAL / $PASS / $FAIL / $ERROR"
+            TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p"  "$LOG_FILE"|tr -d '\r'|head -n1)
+            PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
+            SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
+            FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
+            XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
+            ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1)
+            echo "::warning ::GNU testsuite = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR"
+            jq -n \
+                  --arg date "$(date --rfc-email)" \
+                  --arg sha "$GITHUB_SHA" \
+                  --arg total "$TOTAL" \
+                  --arg pass "$PASS" \
+                  --arg skip "$SKIP" \
+                  --arg fail "$FAIL" \
+                  --arg xpass "$XPASS" \
+                  --arg error "$ERROR" \
+                  '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > gnu-result.json
           else
             echo "::error ::Failed to get summary of test results"
           fi
@@ -132,3 +77,8 @@ jobs:
       with:
         name: test-report
         path: gnu/tests/**/*.log
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: gnu-result
+        path: gnu-result.json

.gitignore

@@ -12,3 +12,6 @@ target/
 Cargo.lock
 lib*.a
 /docs/_build
+*.iml
+### macOS ###
+.DS_Store

.travis.yml

@@ -1,72 +0,0 @@
-language: rust
-
-rust:
-    - stable
-    - beta
-
-os:
-    - linux
-    # - osx
-
-env:
-  # sphinx v1.8.0 is bugged & fails for linux builds; so, force specific `sphinx` version
-  global: FEATURES='' TEST_INSTALL='' SPHINX_VERSIONED='sphinx==1.7.8'
-
-matrix:
-  allow_failures:
-    - rust: beta
-    - rust: nightly
-  fast_finish: true
-  include:
-    - rust: 1.40.0
-      env: FEATURES=unix
-    # - rust: stable
-    #   os: linux
-    #   env: FEATURES=unix TEST_INSTALL=true
-    # - rust: stable
-    #   os: osx
-    #   env: FEATURES=macos TEST_INSTALL=true
-    - rust: nightly
-      os: linux
-      env: FEATURES=nightly,unix TEST_INSTALL=true
-    - rust: nightly
-      os: osx
-      env: FEATURES=nightly,macos TEST_INSTALL=true
-    - rust: nightly
-      os: linux
-      env: FEATURES=nightly,feat_os_unix_redox CC=x86_64-unknown-redox-gcc CARGO_ARGS='--no-default-features --target=x86_64-unknown-redox' REDOX=1
-
-cache:
-  directories:
-    - $HOME/.cargo
-
-sudo: true
-
-before_install:
-  - if [ $REDOX ]; then ./.travis/redox-toolchain.sh; fi
-
-install:
-  - if [ $TRAVIS_OS_NAME = linux ]; then sudo apt-get install python-pip && sudo pip install $SPHINX_VERSIONED; fi
-  - |
-    if [ $TRAVIS_OS_NAME = osx ]; then
-      brew update
-      brew upgrade python
-      pip3 install $SPHINX_VERSIONED
-    fi
-
-script:
-  - cargo build $CARGO_ARGS --features "$FEATURES"
-  - if [ ! $REDOX ]; then cargo test $CARGO_ARGS -p uucore -p coreutils --features "$FEATURES" --no-fail-fast; fi
-  - if [ -n "$TEST_INSTALL" ]; then mkdir installdir_test; DESTDIR=installdir_test make install; [ `ls installdir_test/usr/local/bin | wc -l` -gt 0 ]; fi
-
-addons:
-  apt:
-    packages:
-      - libssl-dev
-
-after_success: |
-  if [ "$TRAVIS_OS_NAME" = linux -a "$TRAVIS_RUST_VERSION" = stable ]; then
-    bash <(curl https://raw.githubusercontent.com/xd009642/tarpaulin/master/travis-install.sh)
-    cargo tarpaulin --out Xml
-    bash <(curl -s https://codecov.io/bash)
-  fi

CONTRIBUTING.md

@@ -22,6 +22,7 @@ search the issues to make sure no one else is working on it.
 1. Make sure that the code coverage is covering all of the cases, including errors.     
 1. The code must be clippy-warning-free and rustfmt-compliant.
 1. Don't hesitate to move common functions into uucore if they can be reused by other binaries.
+1. Unsafe code should be documented with Safety comments.
 
 ## Commit messages
 
@@ -69,10 +70,6 @@ lines for non-utility modules include:
 README: add help
 ```
 
-```
-travis: fix build
-```
-
 ```
 uucore: add new modules
 ```

Cargo.toml

@@ -326,6 +326,9 @@ wc       = { optional=true, version="0.0.6", package="uu_wc", path="src/uu/wc" }
 who      = { optional=true, version="0.0.6", package="uu_who", path="src/uu/who" }
 whoami   = { optional=true, version="0.0.6", package="uu_whoami", path="src/uu/whoami" }
 yes      = { optional=true, version="0.0.6", package="uu_yes", path="src/uu/yes" }
+
+factor_benches = { optional = true, version = "0.0.0", package = "uu_factor_benches", path = "tests/benches/factor" }
+
 #
 # * pinned transitive dependencies
 # Not needed for now. Keep as examples:
@@ -338,17 +341,15 @@ filetime = "0.2"
 glob = "0.3.0"
 libc = "0.2"
 nix = "0.20.0"
+pretty_assertions = "0.7.2"
 rand = "0.7"
 regex = "1.0"
 sha1 = { version="0.6", features=["std"] }
-## tempfile 3.2 depends on recent version of rand which depends on getrandom v0.2 which has compiler errors for MinRustV v1.32.0
-## min dep for tempfile = Rustc 1.40
-tempfile = "= 3.1.0"
+tempfile = "3.2.0"
 time = "0.1"
 unindent = "0.1"
 uucore = { version=">=0.0.8", package="uucore", path="src/uucore", features=["entries"] }
 walkdir = "2.2"
-tempdir = "0.3"
 
 [target.'cfg(unix)'.dev-dependencies]
 rust-users = { version="0.10", package="users" }

README.md

@@ -6,7 +6,6 @@
 [![LOC](https://tokei.rs/b1/github/uutils/coreutils?category=code)](https://github.com/Aaronepower/tokei)
 [![dependency status](https://deps.rs/repo/github/uutils/coreutils/status.svg)](https://deps.rs/repo/github/uutils/coreutils)
 
-[![Build Status](https://api.travis-ci.org/uutils/coreutils.svg?branch=master)](https://travis-ci.org/uutils/coreutils)
 [![Build Status (FreeBSD)](https://api.cirrus-ci.com/github/uutils/coreutils.svg)](https://cirrus-ci.com/github/uutils/coreutils/master)
 [![CodeCov](https://codecov.io/gh/uutils/coreutils/branch/master/graph/badge.svg)](https://codecov.io/gh/uutils/coreutils)
 
@@ -40,7 +39,7 @@ to compile anywhere, and this is as good a way as any to try and learn it.
 ### Rust Version
 
 uutils follows Rust's release channels and is tested against stable, beta and nightly.
-The current oldest supported version of the Rust compiler is `1.40.0`.
+The current oldest supported version of the Rust compiler is `1.43.1`.
 
 On both Windows and Redox, only the nightly version is tested currently.
 
@@ -93,7 +92,7 @@ $ cargo build --features "base32 cat echo rm" --no-default-features
 
 If you don't want to build the multicall binary and would prefer to build
 the utilities as individual binaries, that is also possible. Each utility
-is contained in it's own package within the main repository, named
+is contained in its own package within the main repository, named
 "uu_UTILNAME". To build individual utilities, use cargo to build just the
 specific packages (using the `--package` [aka `-p`] option). For example:
 
@@ -319,6 +318,16 @@ To pass an argument like "-v" to the busybox test runtime
 $ make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest
 ```
 
+## Comparing with GNU
+
+![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.png?raw=true)
+
+To run locally:
+```bash
+$ bash util/build-gnu.sh
+$ bash util/run-gnu-test.sh
+```
+
 ## Contribute
 
 To contribute to uutils, please see [CONTRIBUTING](CONTRIBUTING.md).
@@ -429,6 +438,7 @@ This is an auto-generated table showing which binaries compile for each target-t
 |windows-msvc|i686|y|y|y|y|y| | | | |y|y|y|y|y|y|y|y|y| |y|y|y| |y|y|y|y| |y|y|y|y| | |y| |y|y|y|y|y| | |y|y|y| |y| |y|y|y|y| | |y|y|y|y|y|y|y|y|y|y|y|y|y|y|y| | |y|y|y|y|y|y| |y|y|y|y|y| |y|y|y| |y| |y| |y|y|
 |windows-gnu|x86_64|y|y|y|y|y| | | | |y|y|y|y|y|y|y|y|y| |y|y|y| |y|y|y|y| |y|y|y|y| | |y| |y|y|y|y|y| | |y|y|y| |y| |y|y|y|y| | |y|y|y|y|y|y|y|y|y|y|y|y|y|y|y| | |y|y|y|y|y|y| |y|y|y|y|y|y|y|y|y| |y| |y| |y|y|
 |windows-msvc|x86_64|y|y|y|y|y| | | | |y|y|y|y|y|y|y|y|y| |y|y|y| |y|y|y|y| |y|y|y|y| | |y| |y|y|y|y|y| | |y|y|y| |y| |y|y|y|y| | |y|y|y|y|y|y|y|y|y|y|y|y|y|y|y| | |y|y|y|y|y|y| |y|y|y|y|y| |y|y|y| |y| |y| |y|y|
+|apple MacOS|aarch64|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y| |y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|
 |apple MacOS|x86_64|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y| |y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|
 |freebsd|x86_64|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|
 |netbsd|x86_64|y|y|y|y|y|y|y|y| |y|y|y|y|y|y| |y|y|y|y|y|y| |y|y|y|y|y|y|y|y|y| |y|y| |y|y|y|y|y|y|y|y|y|y|y|y| |y|y|y|y|y| |y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y|y| |y|y|y|y|y|y| |y|y|y| | |y| |y|y|

docs/compiles_table.py

@@ -26,6 +26,7 @@ TARGETS = [
     "x86_64-pc-windows-gnu",
     "x86_64-pc-windows-msvc",
     # Apple
+    "aarch64-apple-darwin",
     "x86_64-apple-darwin",
     "aarch64-apple-ios",
     "x86_64-apple-ios",
@@ -231,4 +232,4 @@ if __name__ == "__main__":
 
         prev_table, _, _ = load_csv(CACHE_PATH)
         new_table = merge_tables(prev_table, table)
-        save_csv(CACHE_PATH, new_table)
+        save_csv(CACHE_PATH, new_table)

src/uu/arch/Cargo.toml

@@ -16,6 +16,7 @@ path = "src/arch.rs"
 
 [dependencies]
 platform-info = "0.1"
+clap = "2.33"
 uucore = { version=">=0.0.8", package="uucore", path="../../uucore" }
 uucore_procs = { version=">=0.0.5", package="uucore_procs", path="../../uucore_procs" }
 

src/uu/arch/src/arch.rs

@@ -11,12 +11,19 @@ extern crate uucore;
 
 use platform_info::*;
 
-static SYNTAX: &str = "Display machine architecture";
+use clap::App;
+
+static VERSION: &str = env!("CARGO_PKG_VERSION");
+static ABOUT: &str = "Display machine architecture";
 static SUMMARY: &str = "Determine architecture name for current machine.";
-static LONG_HELP: &str = "";
 
 pub fn uumain(args: impl uucore::Args) -> i32 {
-    app!(SYNTAX, SUMMARY, LONG_HELP).parse(args.collect_str());
+    App::new(executable!())
+        .version(VERSION)
+        .about(ABOUT)
+        .after_help(SUMMARY)
+        .get_matches_from(args);
+
     let uts = return_if_err!(1, PlatformInfo::new());
     println!("{}", uts.machine().trim());
     0

src/uu/base32/Cargo.toml

@@ -15,6 +15,7 @@ edition = "2018"
 path = "src/base32.rs"
 
 [dependencies]
+clap = "2.33"
 uucore = { version=">=0.0.8", package="uucore", path="../../uucore", features = ["encoding"] }
 uucore_procs = { version=">=0.0.5", package="uucore_procs", path="../../uucore_procs" }
 

src/uu/base32/src/base32.rs

@@ -7,13 +7,14 @@
 
 #[macro_use]
 extern crate uucore;
+
+use std::io::{stdin, Read};
+
 use uucore::encoding::Format;
 
-mod base_common;
+pub mod base_common;
 
-static SYNTAX: &str = "[OPTION]... [FILE]";
-static SUMMARY: &str = "Base32 encode or decode FILE, or standard input, to standard output.";
-static LONG_HELP: &str = "
+static ABOUT: &str = "
  With no FILE, or when FILE is -, read standard input.
 
  The data are encoded as described for the base32 alphabet in RFC
@@ -22,13 +23,43 @@ static LONG_HELP: &str = "
  to attempt to recover from any other non-alphabet bytes in the
  encoded stream.
 ";
+static VERSION: &str = env!("CARGO_PKG_VERSION");
+
+static BASE_CMD_PARSE_ERROR: i32 = 1;
+
+fn get_usage() -> String {
+    format!("{0} [OPTION]... [FILE]", executable!())
+}
 
 pub fn uumain(args: impl uucore::Args) -> i32 {
-    base_common::execute(
-        args.collect_str(),
-        SYNTAX,
-        SUMMARY,
-        LONG_HELP,
-        Format::Base32,
-    )
+    let format = Format::Base32;
+    let usage = get_usage();
+    let name = executable!();
+
+    let config_result: Result<base_common::Config, String> =
+        base_common::parse_base_cmd_args(args, name, VERSION, ABOUT, &usage);
+
+    if config_result.is_err() {
+        match config_result {
+            Ok(_) => panic!(),
+            Err(s) => crash!(BASE_CMD_PARSE_ERROR, "{}", s),
+        }
+    }
+
+    // Create a reference to stdin so we can return a locked stdin from
+    // parse_base_cmd_args
+    let stdin_raw = stdin();
+    let config = config_result.unwrap();
+    let mut input: Box<dyn Read> = base_common::get_input(&config, &stdin_raw);
+
+    base_common::handle_input(
+        &mut input,
+        format,
+        config.wrap_cols,
+        config.ignore_garbage,
+        config.decode,
+        name,
+    );
+
+    0
 }

src/uu/base32/src/base_common.rs

@@ -7,73 +7,133 @@
 // For the full copyright and license information, please view the LICENSE file
 // that was distributed with this source code.
 
-use std::fs::File;
-use std::io::{stdin, stdout, BufReader, Read, Write};
-use std::path::Path;
+use std::io::{stdout, Read, Write};
 
 use uucore::encoding::{wrap_print, Data, Format};
+use uucore::InvalidEncodingHandling;
 
-pub fn execute(
-    args: Vec<String>,
-    syntax: &str,
-    summary: &str,
-    long_help: &str,
-    format: Format,
-) -> i32 {
-    let matches = app!(syntax, summary, long_help)
-        .optflag("d", "decode", "decode data")
-        .optflag(
-            "i",
-            "ignore-garbage",
-            "when decoding, ignore non-alphabetic characters",
-        )
-        .optopt(
-            "w",
-            "wrap",
-            "wrap encoded lines after COLS character (default 76, 0 to disable wrapping)",
-            "COLS",
-        )
-        .parse(args);
+use std::fs::File;
+use std::io::{BufReader, Stdin};
+use std::path::Path;
 
-    let line_wrap = matches.opt_str("wrap").map(|s| match s.parse() {
-        Ok(n) => n,
-        Err(e) => {
-            crash!(1, "invalid wrap size: ‘{}’: {}", s, e);
-        }
-    });
-    let ignore_garbage = matches.opt_present("ignore-garbage");
-    let decode = matches.opt_present("decode");
+use clap::{App, Arg};
 
-    if matches.free.len() > 1 {
-        show_usage_error!("extra operand ‘{}’", matches.free[0]);
-        return 1;
-    }
-
-    if matches.free.is_empty() || &matches.free[0][..] == "-" {
-        let stdin_raw = stdin();
-        handle_input(
-            &mut stdin_raw.lock(),
-            format,
-            line_wrap,
-            ignore_garbage,
-            decode,
-        );
-    } else {
-        let path = Path::new(matches.free[0].as_str());
-        let file_buf = safe_unwrap!(File::open(&path));
-        let mut input = BufReader::new(file_buf);
-        handle_input(&mut input, format, line_wrap, ignore_garbage, decode);
-    };
-
-    0
+// Config.
+pub struct Config {
+    pub decode: bool,
+    pub ignore_garbage: bool,
+    pub wrap_cols: Option<usize>,
+    pub to_read: Option<String>,
 }
 
-fn handle_input<R: Read>(
+pub mod options {
+    pub static DECODE: &str = "decode";
+    pub static WRAP: &str = "wrap";
+    pub static IGNORE_GARBAGE: &str = "ignore-garbage";
+    pub static FILE: &str = "file";
+}
+
+impl Config {
+    fn from(options: clap::ArgMatches) -> Result<Config, String> {
+        let file: Option<String> = match options.values_of(options::FILE) {
+            Some(mut values) => {
+                let name = values.next().unwrap();
+                if values.len() != 0 {
+                    return Err(format!("extra operand ‘{}’", name));
+                }
+
+                if name == "-" {
+                    None
+                } else {
+                    if !Path::exists(Path::new(name)) {
+                        return Err(format!("{}: No such file or directory", name));
+                    }
+                    Some(name.to_owned())
+                }
+            }
+            None => None,
+        };
+
+        let cols = match options.value_of(options::WRAP) {
+            Some(num) => match num.parse::<usize>() {
+                Ok(n) => Some(n),
+                Err(e) => {
+                    return Err(format!("Invalid wrap size: ‘{}’: {}", num, e));
+                }
+            },
+            None => None,
+        };
+
+        Ok(Config {
+            decode: options.is_present(options::DECODE),
+            ignore_garbage: options.is_present(options::IGNORE_GARBAGE),
+            wrap_cols: cols,
+            to_read: file,
+        })
+    }
+}
+
+pub fn parse_base_cmd_args(
+    args: impl uucore::Args,
+    name: &str,
+    version: &str,
+    about: &str,
+    usage: &str,
+) -> Result<Config, String> {
+    let app = App::new(name)
+        .version(version)
+        .about(about)
+        .usage(usage)
+        // Format arguments.
+        .arg(
+            Arg::with_name(options::DECODE)
+                .short("d")
+                .long(options::DECODE)
+                .help("decode data"),
+        )
+        .arg(
+            Arg::with_name(options::IGNORE_GARBAGE)
+                .short("i")
+                .long(options::IGNORE_GARBAGE)
+                .help("when decoding, ignore non-alphabetic characters"),
+        )
+        .arg(
+            Arg::with_name(options::WRAP)
+                .short("w")
+                .long(options::WRAP)
+                .takes_value(true)
+                .help(
+                    "wrap encoded lines after COLS character (default 76, 0 to disable wrapping)",
+                ),
+        )
+        // "multiple" arguments are used to check whether there is more than one
+        // file passed in.
+        .arg(Arg::with_name(options::FILE).index(1).multiple(true));
+    let arg_list = args
+        .collect_str(InvalidEncodingHandling::ConvertLossy)
+        .accept_any();
+    Config::from(app.get_matches_from(arg_list))
+}
+
+pub fn get_input<'a>(config: &Config, stdin_ref: &'a Stdin) -> Box<dyn Read + 'a> {
+    match &config.to_read {
+        Some(name) => {
+            let file_buf = safe_unwrap!(File::open(Path::new(name)));
+            Box::new(BufReader::new(file_buf)) // as Box<dyn Read>
+        }
+        None => {
+            Box::new(stdin_ref.lock()) // as Box<dyn Read>
+        }
+    }
+}
+
+pub fn handle_input<R: Read>(
     input: &mut R,
     format: Format,
     line_wrap: Option<usize>,
     ignore_garbage: bool,
     decode: bool,
+    name: &str,
 ) {
     let mut data = Data::new(input, format).ignore_garbage(ignore_garbage);
     if let Some(wrap) = line_wrap {
@@ -88,10 +148,14 @@ fn handle_input<R: Read>(
             Ok(s) => {
                 if stdout().write_all(&s).is_err() {
                     // on windows console, writing invalid utf8 returns an error
-                    crash!(1, "Cannot write non-utf8 data");
+                    eprintln!("{}: error: Cannot write non-utf8 data", name);
+                    exit!(1)
                 }
             }
-            Err(_) => crash!(1, "invalid input"),
+            Err(_) => {
+                eprintln!("{}: error: invalid input", name);
+                exit!(1)
+            }
         }
     }
 }

src/uu/base64/Cargo.toml

@@ -15,8 +15,10 @@ edition = "2018"
 path = "src/base64.rs"
 
 [dependencies]
+clap = "2.33"
 uucore = { version=">=0.0.8", package="uucore", path="../../uucore", features = ["encoding"] }
 uucore_procs = { version=">=0.0.5", package="uucore_procs", path="../../uucore_procs" }
+uu_base32 = { version=">=0.0.6", package="uu_base32", path="../base32"}
 
 [[bin]]
 name = "base64"

src/uu/base64/src/base64.rs

@@ -8,13 +8,14 @@
 
 #[macro_use]
 extern crate uucore;
+
+use uu_base32::base_common;
+
 use uucore::encoding::Format;
 
-mod base_common;
+use std::io::{stdin, Read};
 
-static SYNTAX: &str = "[OPTION]... [FILE]";
-static SUMMARY: &str = "Base64 encode or decode FILE, or standard input, to standard output.";
-static LONG_HELP: &str = "
+static ABOUT: &str = "
  With no FILE, or when FILE is -, read standard input.
 
  The data are encoded as described for the base64 alphabet in RFC
@@ -23,13 +24,42 @@ static LONG_HELP: &str = "
  to attempt to recover from any other non-alphabet bytes in the
  encoded stream.
 ";
+static VERSION: &str = env!("CARGO_PKG_VERSION");
+
+static BASE_CMD_PARSE_ERROR: i32 = 1;
+
+fn get_usage() -> String {
+    format!("{0} [OPTION]... [FILE]", executable!())
+}
 
 pub fn uumain(args: impl uucore::Args) -> i32 {
-    base_common::execute(
-        args.collect_str(),
-        SYNTAX,
-        SUMMARY,
-        LONG_HELP,
-        Format::Base64,
-    )
+    let format = Format::Base64;
+    let usage = get_usage();
+    let name = executable!();
+    let config_result: Result<base_common::Config, String> =
+        base_common::parse_base_cmd_args(args, name, VERSION, ABOUT, &usage);
+
+    if config_result.is_err() {
+        match config_result {
+            Ok(_) => panic!(),
+            Err(s) => crash!(BASE_CMD_PARSE_ERROR, "{}", s),
+        }
+    }
+
+    // Create a reference to stdin so we can return a locked stdin from
+    // parse_base_cmd_args
+    let stdin_raw = stdin();
+    let config = config_result.unwrap();
+    let mut input: Box<dyn Read> = base_common::get_input(&config, &stdin_raw);
+
+    base_common::handle_input(
+        &mut input,
+        format,
+        config.wrap_cols,
+        config.ignore_garbage,
+        config.decode,
+        name,
+    );
+
+    0
 }

src/uu/base64/src/base_common.rs

@@ -1,97 +0,0 @@
-// This file is part of the uutils coreutils package.
-//
-// (c) Jordy Dickinson <jordy.dickinson@gmail.com>
-// (c) Jian Zeng <anonymousknight96@gmail.com>
-// (c) Alex Lyon <arcterus@mail.com>
-//
-// For the full copyright and license information, please view the LICENSE file
-// that was distributed with this source code.
-
-use std::fs::File;
-use std::io::{stdin, stdout, BufReader, Read, Write};
-use std::path::Path;
-
-use uucore::encoding::{wrap_print, Data, Format};
-
-pub fn execute(
-    args: Vec<String>,
-    syntax: &str,
-    summary: &str,
-    long_help: &str,
-    format: Format,
-) -> i32 {
-    let matches = app!(syntax, summary, long_help)
-        .optflag("d", "decode", "decode data")
-        .optflag(
-            "i",
-            "ignore-garbage",
-            "when decoding, ignore non-alphabetic characters",
-        )
-        .optopt(
-            "w",
-            "wrap",
-            "wrap encoded lines after COLS character (default 76, 0 to disable wrapping)",
-            "COLS",
-        )
-        .parse(args);
-
-    let line_wrap = matches.opt_str("wrap").map(|s| match s.parse() {
-        Ok(n) => n,
-        Err(e) => {
-            crash!(1, "invalid wrap size: ‘{}’: {}", s, e);
-        }
-    });
-    let ignore_garbage = matches.opt_present("ignore-garbage");
-    let decode = matches.opt_present("decode");
-
-    if matches.free.len() > 1 {
-        show_usage_error!("extra operand ‘{}’", matches.free[0]);
-        return 1;
-    }
-
-    if matches.free.is_empty() || &matches.free[0][..] == "-" {
-        let stdin_raw = stdin();
-        handle_input(
-            &mut stdin_raw.lock(),
-            format,
-            line_wrap,
-            ignore_garbage,
-            decode,
-        );
-    } else {
-        let path = Path::new(matches.free[0].as_str());
-        let file_buf = safe_unwrap!(File::open(&path));
-        let mut input = BufReader::new(file_buf);
-        handle_input(&mut input, format, line_wrap, ignore_garbage, decode);
-    };
-
-    0
-}
-
-fn handle_input<R: Read>(
-    input: &mut R,
-    format: Format,
-    line_wrap: Option<usize>,
-    ignore_garbage: bool,
-    decode: bool,
-) {
-    let mut data = Data::new(input, format).ignore_garbage(ignore_garbage);
-    if let Some(wrap) = line_wrap {
-        data = data.line_wrap(wrap);
-    }
-
-    if !decode {
-        let encoded = data.encode();
-        wrap_print(&data, encoded);
-    } else {
-        match data.decode() {
-            Ok(s) => {
-                if stdout().write_all(&s).is_err() {
-                    // on windows console, writing invalid utf8 returns an error
-                    crash!(1, "Cannot write non-utf8 data");
-                }
-            }
-            Err(_) => crash!(1, "invalid input"),
-        }
-    }
-}

src/uu/basename/Cargo.toml

@@ -15,6 +15,7 @@ edition = "2018"
 path = "src/basename.rs"
 
 [dependencies]
+clap = "2.33.2"
 uucore = { version=">=0.0.8", package="uucore", path="../../uucore" }
 uucore_procs = { version=">=0.0.5", package="uucore_procs", path="../../uucore_procs" }
 

src/uu/basename/src/basename.rs

@@ -10,81 +10,106 @@
 #[macro_use]
 extern crate uucore;
 
+use clap::{App, Arg};
 use std::path::{is_separator, PathBuf};
+use uucore::InvalidEncodingHandling;
 
-static NAME: &str = "basename";
-static SYNTAX: &str = "NAME [SUFFIX]";
+static VERSION: &str = env!("CARGO_PKG_VERSION");
 static SUMMARY: &str = "Print NAME with any leading directory components removed
- If specified, also remove a trailing SUFFIX";
-static LONG_HELP: &str = "";
+If specified, also remove a trailing SUFFIX";
+
+fn get_usage() -> String {
+    format!(
+        "{0} NAME [SUFFIX]
+    {0} OPTION... NAME...",
+        executable!()
+    )
+}
+
+pub mod options {
+    pub static MULTIPLE: &str = "multiple";
+    pub static NAME: &str = "name";
+    pub static SUFFIX: &str = "suffix";
+    pub static ZERO: &str = "zero";
+}
 
 pub fn uumain(args: impl uucore::Args) -> i32 {
-    let args = args.collect_str();
-
+    let args = args
+        .collect_str(InvalidEncodingHandling::ConvertLossy)
+        .accept_any();
+    let usage = get_usage();
     //
     // Argument parsing
     //
-    let matches = app!(SYNTAX, SUMMARY, LONG_HELP)
-        .optflag(
-            "a",
-            "multiple",
-            "Support more than one argument. Treat every argument as a name.",
+    let matches = App::new(executable!())
+        .version(VERSION)
+        .about(SUMMARY)
+        .usage(&usage[..])
+        .arg(
+            Arg::with_name(options::MULTIPLE)
+                .short("a")
+                .long(options::MULTIPLE)
+                .help("support multiple arguments and treat each as a NAME"),
         )
-        .optopt(
-            "s",
-            "suffix",
-            "Remove a trailing suffix. This option implies the -a option.",
-            "SUFFIX",
+        .arg(Arg::with_name(options::NAME).multiple(true).hidden(true))
+        .arg(
+            Arg::with_name(options::SUFFIX)
+                .short("s")
+                .long(options::SUFFIX)
+                .value_name("SUFFIX")
+                .help("remove a trailing SUFFIX; implies -a"),
         )
-        .optflag(
-            "z",
-            "zero",
-            "Output a zero byte (ASCII NUL) at the end of each line, rather than a newline.",
+        .arg(
+            Arg::with_name(options::ZERO)
+                .short("z")
+                .long(options::ZERO)
+                .help("end each output line with NUL, not newline"),
         )
-        .parse(args);
+        .get_matches_from(args);
 
     // too few arguments
-    if matches.free.is_empty() {
+    if !matches.is_present(options::NAME) {
         crash!(
             1,
-            "{0}: {1}\nTry '{0} --help' for more information.",
-            NAME,
+            "{1}\nTry '{0} --help' for more information.",
+            executable!(),
             "missing operand"
         );
     }
-    let opt_s = matches.opt_present("s");
-    let opt_a = matches.opt_present("a");
-    let opt_z = matches.opt_present("z");
-    let multiple_paths = opt_s || opt_a;
+
+    let opt_suffix = matches.is_present(options::SUFFIX);
+    let opt_multiple = matches.is_present(options::MULTIPLE);
+    let opt_zero = matches.is_present(options::ZERO);
+    let multiple_paths = opt_suffix || opt_multiple;
     // too many arguments
-    if !multiple_paths && matches.free.len() > 2 {
+    if !multiple_paths && matches.occurrences_of(options::NAME) > 2 {
         crash!(
             1,
-            "{0}: extra operand '{1}'\nTry '{0} --help' for more information.",
-            NAME,
-            matches.free[2]
+            "extra operand '{1}'\nTry '{0} --help' for more information.",
+            executable!(),
+            matches.values_of(options::NAME).unwrap().nth(2).unwrap()
         );
     }
 
-    let suffix = if opt_s {
-        matches.opt_str("s").unwrap()
-    } else if !opt_a && matches.free.len() > 1 {
-        matches.free[1].clone()
+    let suffix = if opt_suffix {
+        matches.value_of(options::SUFFIX).unwrap()
+    } else if !opt_multiple && matches.occurrences_of(options::NAME) > 1 {
+        matches.values_of(options::NAME).unwrap().nth(1).unwrap()
     } else {
-        "".to_owned()
+        ""
     };
 
     //
     // Main Program Processing
     //
 
-    let paths = if multiple_paths {
-        &matches.free[..]
+    let paths: Vec<_> = if multiple_paths {
+        matches.values_of(options::NAME).unwrap().collect()
     } else {
-        &matches.free[0..1]
+        matches.values_of(options::NAME).unwrap().take(1).collect()
     };
 
-    let line_ending = if opt_z { "\0" } else { "\n" };
+    let line_ending = if opt_zero { "\0" } else { "\n" };
     for path in paths {
         print!("{}{}", basename(&path, &suffix), line_ending);
     }
@@ -111,6 +136,7 @@ fn basename(fullname: &str, suffix: &str) -> String {
     }
 }
 
+#[allow(clippy::manual_strip)] // can be replaced with strip_suffix once the minimum rust version is 1.45
 fn strip_suffix(name: &str, suffix: &str) -> String {
     if name == suffix {
         return name.to_owned();