Sort: Implement stable sort, ignore non-printing, month sort dedup, auto parallel sort through rayon, zero terminated sort, check silent (#2008)

Cargo.lock

@@ -1362,12 +1362,6 @@ dependencies = [
  "maybe-uninit",
 ]
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
 [[package]]
 name = "strsim"
 version = "0.8.0"
@@ -1522,17 +1516,6 @@ dependencies = [
  "serde_json",
 ]
 
-[[package]]
-name = "twox-hash"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59"
-dependencies = [
- "cfg-if 0.1.10",
- "rand 0.7.3",
- "static_assertions",
-]
-
 [[package]]
 name = "typenum"
 version = "1.13.0"
@@ -2301,10 +2284,11 @@ name = "uu_sort"
 version = "0.0.6"
 dependencies = [
  "clap",
+ "fnv",
  "itertools 0.8.2",
  "rand 0.7.3",
+ "rayon",
  "semver",
- "twox-hash",
  "uucore",
  "uucore_procs",
 ]

src/uu/sort/Cargo.toml

@@ -15,9 +15,10 @@ edition = "2018"
 path = "src/sort.rs"
 
 [dependencies]
+rayon = "1.5"
 rand = "0.7"
 clap = "2.33"
-twox-hash = "1.6.0"
+fnv = "1.0.7"
 itertools = "0.8.0"
 semver = "0.9.0"
 uucore = { version=">=0.0.8", package="uucore", path="../../uucore", features=["fs"] }

tests/by-util/test_sort.rs

@@ -1,44 +1,82 @@
 use crate::common::util::*;
 
+#[test]
+fn test_check_zero_terminated_failure() {
+    new_ucmd!()
+        .arg("-z")
+        .arg("-c")
+        .arg("zero-terminated.txt")
+        .fails()
+        .stdout_is("sort: disorder in line 0\n");
+}
+
+#[test]
+fn test_check_zero_terminated_success() {
+    new_ucmd!()
+        .arg("-z")
+        .arg("-c")
+        .arg("zero-terminated.expected")
+        .succeeds();
+}
+
+#[test]
+fn test_random_shuffle_len() {
+    // check whether output is the same length as the input
+    const FILE: &'static str = "default_unsorted_ints.expected";
+    let (at, _ucmd) = at_and_ucmd!();
+    let result = new_ucmd!().arg("-R").arg(FILE).run().stdout;
+    let expected = at.read(FILE);
+
+    assert_ne!(result, expected);
+    assert_eq!(result.len(), expected.len());
+}
+
+#[test]
+fn test_random_shuffle_contains_all_lines() {
+    // check whether lines of input are all in output
+    const FILE: &'static str = "default_unsorted_ints.expected";
+    let (at, _ucmd) = at_and_ucmd!();
+    let result = new_ucmd!().arg("-R").arg(FILE).run().stdout;
+    let expected = at.read(FILE);
+    let result_sorted = new_ucmd!().pipe_in(result.clone()).run().stdout;
+
+    assert_ne!(result, expected);
+    assert_eq!(result_sorted, expected);
+}
+
+#[test]
+fn test_random_shuffle_contains_two_runs_not_the_same() {
+    // check to verify that two random shuffles are not equal; this has the
+    // potential to fail in the unlikely event that random order is the same
+    // as the starting order, or if both random sorts end up having the same order.
+    const FILE: &'static str = "default_unsorted_ints.expected";
+    let (at, _ucmd) = at_and_ucmd!();
+    let result = new_ucmd!().arg("-R").arg(FILE).run().stdout;
+    let expected = at.read(FILE);
+    let unexpected = new_ucmd!().arg("-R").arg(FILE).run().stdout;
+
+    assert_ne!(result, expected);
+    assert_ne!(result, unexpected);
+}
+
 #[test]
 fn test_numeric_floats_and_ints() {
-    for numeric_sort_param in vec!["-n", "--numeric-sort"] {
-        let input = "1.444\n8.013\n1\n-8\n1.04\n-1";
-        new_ucmd!()
-            .arg(numeric_sort_param)
-            .pipe_in(input)
-            .succeeds()
-            .stdout_only("-8\n-1\n1\n1.04\n1.444\n8.013\n");
-    }
+    test_helper("numeric_floats_and_ints", "-n");
 }
 
 #[test]
 fn test_numeric_floats() {
-    for numeric_sort_param in vec!["-n", "--numeric-sort"] {
-        let input = "1.444\n8.013\n1.58590\n-8.90880\n1.040000000\n-.05";
-        new_ucmd!()
-            .arg(numeric_sort_param)
-            .pipe_in(input)
-            .succeeds()
-            .stdout_only("-8.90880\n-.05\n1.040000000\n1.444\n1.58590\n8.013\n");
-    }
+    test_helper("numeric_floats", "-n");
 }
 
 #[test]
 fn test_numeric_floats_with_nan() {
-    for numeric_sort_param in vec!["-n", "--numeric-sort"] {
-        let input = "1.444\n1.0/0.0\n1.58590\n-8.90880\n1.040000000\n-.05";
-        new_ucmd!()
-            .arg(numeric_sort_param)
-            .pipe_in(input)
-            .succeeds()
-            .stdout_only("-8.90880\n-.05\n1.0/0.0\n1.040000000\n1.444\n1.58590\n");
-    }
+    test_helper("numeric_floats_with_nan", "-n");
 }
 
 #[test]
 fn test_numeric_unfixed_floats() {
-    test_helper("numeric_fixed_floats", "-n");
+    test_helper("numeric_unfixed_floats", "-n");
 }
 
 #[test]
@@ -53,26 +91,12 @@ fn test_numeric_unsorted_ints() {
 
 #[test]
 fn test_human_block_sizes() {
-    for human_numeric_sort_param in vec!["-h", "--human-numeric-sort"] {
-        let input = "8981K\n909991M\n-8T\n21G\n0.8M";
-        new_ucmd!()
-            .arg(human_numeric_sort_param)
-            .pipe_in(input)
-            .succeeds()
-            .stdout_only("-8T\n0.8M\n8981K\n21G\n909991M\n");
-    }
+    test_helper("human_block_sizes", "-h");
 }
 
 #[test]
 fn test_month_default() {
-    for month_sort_param in vec!["-M", "--month-sort"] {
-        let input = "JAn\nMAY\n000may\nJun\nFeb";
-        new_ucmd!()
-            .arg(month_sort_param)
-            .pipe_in(input)
-            .succeeds()
-            .stdout_only("000may\nJAn\nFeb\nMAY\nJun\n");
-    }
+    test_helper("month_default", "-M");
 }
 
 #[test]
@@ -82,23 +106,12 @@ fn test_month_stable() {
 
 #[test]
 fn test_default_unsorted_ints() {
-    let input = "9\n1909888\n000\n1\n2";
-    new_ucmd!()
-        .pipe_in(input)
-        .succeeds()
-        .stdout_only("000\n1\n1909888\n2\n9\n");
+    test_helper("default_unsorted_ints", "");
 }
 
 #[test]
 fn test_numeric_unique_ints() {
-    for numeric_unique_sort_param in vec!["-nu"] {
-        let input = "9\n9\n8\n1\n";
-        new_ucmd!()
-            .arg(numeric_unique_sort_param)
-            .pipe_in(input)
-            .succeeds()
-            .stdout_only("1\n8\n9\n");
-    }
+    test_helper("numeric_unsorted_ints_unique", "-nu");
 }
 
 #[test]
@@ -116,6 +129,148 @@ fn test_dictionary_order() {
     test_helper("dictionary_order", "-d");
 }
 
+#[test]
+fn test_dictionary_order2() {
+    for non_dictionary_order2_param in vec!["-d"] {
+        new_ucmd!()
+            .pipe_in("a👦🏻aa	b\naaaa	b")
+            .arg(non_dictionary_order2_param)
+            .succeeds()
+            .stdout_only("a👦🏻aa	b\naaaa	b\n");
+    }
+}
+
+#[test]
+fn test_non_printing_chars() {
+    for non_printing_chars_param in vec!["-i"] {
+        new_ucmd!()
+            .pipe_in("a👦🏻aa	b\naaaa	b")
+            .arg(non_printing_chars_param)
+            .succeeds()
+            .stdout_only("aaaa	b\na👦🏻aa	b\n");
+    }
+}
+
+#[test]
+fn test_exponents_positive_general_fixed() {
+    for exponents_positive_general_param in vec!["-g"] {
+        new_ucmd!()
+            .pipe_in("100E6\n\n50e10\n+100000\n\n10000K78\n10E\n\n\n1000EDKLD\n\n\n100E6\n\n50e10\n+100000\n\n")
+            .arg(exponents_positive_general_param)
+            .succeeds()
+            .stdout_only("\n\n\n\n\n\n\n\n10000K78\n1000EDKLD\n10E\n+100000\n+100000\n100E6\n100E6\n50e10\n50e10\n");
+    }
+}
+
+#[test]
+fn test_exponents_positive_numeric() {
+    test_helper("exponents-positive-numeric", "-n");
+}
+
+#[test]
+fn test_months_dedup() {
+    test_helper("months-dedup", "-Mu");
+}
+
+#[test]
+fn test_mixed_floats_ints_chars_numeric() {
+    test_helper("mixed_floats_ints_chars_numeric", "-n");
+}
+
+#[test]
+fn test_mixed_floats_ints_chars_numeric_unique() {
+    test_helper("mixed_floats_ints_chars_numeric_unique", "-nu");
+}
+
+#[test]
+fn test_mixed_floats_ints_chars_numeric_reverse() {
+    test_helper("mixed_floats_ints_chars_numeric_unique_reverse", "-nur");
+}
+
+#[test]
+fn test_mixed_floats_ints_chars_numeric_stable() {
+    test_helper("mixed_floats_ints_chars_numeric_stable", "-ns");
+}
+
+#[test]
+fn test_numeric_floats_and_ints2() {
+    for numeric_sort_param in vec!["-n", "--numeric-sort"] {
+        let input = "1.444\n8.013\n1\n-8\n1.04\n-1";
+        new_ucmd!()
+            .arg(numeric_sort_param)
+            .pipe_in(input)
+            .succeeds()
+            .stdout_only("-8\n-1\n1\n1.04\n1.444\n8.013\n");
+    }
+}
+
+#[test]
+fn test_numeric_floats2() {
+    for numeric_sort_param in vec!["-n", "--numeric-sort"] {
+        let input = "1.444\n8.013\n1.58590\n-8.90880\n1.040000000\n-.05";
+        new_ucmd!()
+            .arg(numeric_sort_param)
+            .pipe_in(input)
+            .succeeds()
+            .stdout_only("-8.90880\n-.05\n1.040000000\n1.444\n1.58590\n8.013\n");
+    }
+}
+
+#[test]
+fn test_numeric_floats_with_nan2() {
+    test_helper("numeric-floats-with-nan2", "-n");
+}
+
+#[test]
+fn test_human_block_sizes2() {
+    for human_numeric_sort_param in vec!["-h", "--human-numeric-sort"] {
+        let input = "8981K\n909991M\n-8T\n21G\n0.8M";
+        new_ucmd!()
+            .arg(human_numeric_sort_param)
+            .pipe_in(input)
+            .succeeds()
+            .stdout_only("-8T\n0.8M\n8981K\n21G\n909991M\n");
+    }
+}
+
+#[test]
+fn test_month_default2() {
+    for month_sort_param in vec!["-M", "--month-sort"] {
+        let input = "JAn\nMAY\n000may\nJun\nFeb";
+        new_ucmd!()
+            .arg(month_sort_param)
+            .pipe_in(input)
+            .succeeds()
+            .stdout_only("000may\nJAn\nFeb\nMAY\nJun\n");
+    }
+}
+
+#[test]
+fn test_default_unsorted_ints2() {
+    let input = "9\n1909888\n000\n1\n2";
+    new_ucmd!()
+        .pipe_in(input)
+        .succeeds()
+        .stdout_only("000\n1\n1909888\n2\n9\n");
+}
+
+#[test]
+fn test_numeric_unique_ints2() {
+    for numeric_unique_sort_param in vec!["-nu"] {
+        let input = "9\n9\n8\n1\n";
+        new_ucmd!()
+            .arg(numeric_unique_sort_param)
+            .pipe_in(input)
+            .succeeds()
+            .stdout_only("1\n8\n9\n");
+    }
+}
+
+#[test]
+fn test_zero_terminated() {
+    test_helper("zero-terminated", "-z");
+}
+
 #[test]
 fn test_multiple_files() {
     new_ucmd!()
@@ -192,6 +347,15 @@ fn test_check() {
         .stdout_is("");
 }
 
+#[test]
+fn test_check_silent() {
+    new_ucmd!()
+        .arg("-C")
+        .arg("check_fail.txt")
+        .fails()
+        .stdout_is("");
+}
+
 fn test_helper(file_name: &str, args: &str) {
     new_ucmd!()
         .arg(args)

tests/fixtures/sort/exponents-positive-general.expected

@@ -0,0 +1,12 @@
+
+
+
+
+
+
+10E
+1000EDKLD
+10000K78
++100000
+100E6
+50e10

tests/fixtures/sort/exponents-positive-general.txt

@@ -0,0 +1,12 @@
+10000K78
+10E
+
+
+1000EDKLD
+
+
+100E6
+
+50e10
++100000
+

tests/fixtures/sort/exponents-positive-numeric.expected

@@ -0,0 +1,12 @@
+
+
+
+
+
+
++100000
+10E
+50e10
+100E6
+1000EDKLD
+10000K78

tests/fixtures/sort/exponents-positive-numeric.txt

@@ -0,0 +1,12 @@
+10000K78
+10E
+
+
+1000EDKLD
+
+
+100E6
+
+50e10
++100000
+

tests/fixtures/sort/human-mixed-inputs-reverse.expected

@@ -0,0 +1,37 @@
+.2T
+2G
+100M
+7800900K
+51887300-
+1890777
+56908-90078
+6780.0009866
+6780.000986
+789----009999   90-0   90-0
+1
+0001
+apr
+MAY
+JUNNNN
+JAN
+AUG 
+APR 
+0000000
+00
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+-1.4

tests/fixtures/sort/human-mixed-inputs-reverse.txt

@@ -0,0 +1,37 @@
+JAN
+
+0000000
+
+00
+
+0001
+
+1
+
+-1.4
+
+JUNNNN
+AUG 
+
+apr
+
+APR 
+
+
+MAY
+1890777
+
+56908-90078
+
+51887300-
+
+6780.0009866
+
+789----009999   90-0   90-0
+
+6780.000986
+
+100M
+7800900K
+2G
+.2T

tests/fixtures/sort/human-mixed-inputs-stable.expected

@@ -0,0 +1,37 @@
+-1.4
+JAN
+
+0000000
+
+00
+
+
+
+
+JUNNNN
+AUG 
+
+apr
+
+APR 
+
+
+MAY
+
+
+
+
+
+
+0001
+1
+789----009999   90-0   90-0
+6780.000986
+6780.0009866
+56908-90078
+1890777
+51887300-
+7800900K
+100M
+2G
+.2T

tests/fixtures/sort/human-mixed-inputs-stable.txt

@@ -0,0 +1,37 @@
+JAN
+
+0000000
+
+00
+
+0001
+
+1
+
+-1.4
+
+JUNNNN
+AUG 
+
+apr
+
+APR 
+
+
+MAY
+1890777
+
+56908-90078
+
+51887300-
+
+6780.0009866
+
+789----009999   90-0   90-0
+
+6780.000986
+
+100M
+7800900K
+2G
+.2T

tests/fixtures/sort/human-mixed-inputs-unique.expected

@@ -0,0 +1,13 @@
+-1.4
+JAN
+0001
+789----009999   90-0   90-0
+6780.000986
+6780.0009866
+56908-90078
+1890777
+51887300-
+7800900K
+100M
+2G
+.2T

tests/fixtures/sort/human-mixed-inputs-unique.txt

@@ -0,0 +1,37 @@
+JAN
+
+0000000
+
+00
+
+0001
+
+1
+
+-1.4
+
+JUNNNN
+AUG 
+
+apr
+
+APR 
+
+
+MAY
+1890777
+
+56908-90078
+
+51887300-
+
+6780.0009866
+
+789----009999   90-0   90-0
+
+6780.000986
+
+100M
+7800900K
+2G
+.2T

tests/fixtures/sort/human-mixed-inputs.expected

@@ -0,0 +1,37 @@
+-1.4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+00
+0000000
+APR 
+AUG 
+JAN
+JUNNNN
+MAY
+apr
+0001
+1
+789----009999   90-0   90-0
+6780.000986
+6780.0009866
+56908-90078
+1890777
+51887300-
+7800900K
+100M
+2G
+.2T

tests/fixtures/sort/human-mixed-inputs.txt

@@ -0,0 +1,46 @@
+JAN
+
+0000000
+
+00
+
+0001
+
+1
+
+-1.4
+
+JUNNNN
+AUG 
+
+apr
+
+APR 
+
+
+MAY
+1890777
+
+56908-90078
+
+51887300-
+
+6780.0009866
+
+789----009999   90-0   90-0
+
+6780.000986
+
+1M
+10M
+100M
+1000M
+10000M
+
+7800900K
+780090K
+78009K
+7800K
+780K
+2G
+.2T

tests/fixtures/sort/mixed_floats_ints_chars_numeric.expected

@@ -0,0 +1,30 @@
+-2028789030
+-896689
+-8.90880
+-1
+-.05
+
+
+
+
+
+
+
+
+000
+CARAvan
+00000001
+1
+1.040000000
+1.444
+1.58590
+8.013
+45
+46.89
+               4567.
+				37800
+576,446.88800000
+576,446.890
+4798908.340000000000
+4798908.45
+4798908.8909800

tests/fixtures/sort/mixed_floats_ints_chars_numeric.txt

@@ -0,0 +1,30 @@
+576,446.890
+576,446.88800000
+
+
+               4567.
+45
+46.89
+-1
+1
+00000001
+4798908.340000000000
+4798908.45
+4798908.8909800
+
+
+				37800
+
+-2028789030
+-896689
+CARAvan
+
+-8.90880
+-.05
+1.444
+1.58590
+1.040000000
+
+8.013
+
+000

tests/fixtures/sort/mixed_floats_ints_chars_numeric_reverse.expected

@@ -0,0 +1,30 @@
+4798908.8909800
+4798908.45
+4798908.340000000000
+576,446.890
+576,446.88800000
+				37800
+               4567.
+46.89
+45
+8.013
+1.58590
+1.444
+1.040000000
+1
+00000001
+CARAvan
+000
+
+
+
+
+
+
+
+
+-.05
+-1
+-8.90880
+-896689
+-2028789030

tests/fixtures/sort/mixed_floats_ints_chars_numeric_reverse_stable.expected

@@ -0,0 +1,30 @@
+4798908.8909800
+4798908.45
+4798908.340000000000
+576,446.890
+576,446.88800000
+				37800
+               4567.
+46.89
+45
+8.013
+1.58590
+1.444
+1.040000000
+1
+00000001
+
+
+
+
+
+CARAvan
+
+
+
+000
+-.05
+-1
+-8.90880
+-896689
+-2028789030