coreutils/.github/workflows/fuzzing.yml

name: Fuzzing

# spell-checker:ignore (people) taiki-e
# spell-checker:ignore (misc) fuzzer PIPESTATUS

env:
  CARGO_INCREMENTAL: "0"

on:
  pull_request:
  push:
    branches:
      - '*'

permissions:
  contents: read # to fetch code (actions/checkout)

# End the current execution if there is a new changeset in the PR.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  uufuzz-examples:
    name: Build and test uufuzz examples
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - name: Build uufuzz library
      run: |
        cd fuzz/uufuzz
        cargo build --release
    - name: Run uufuzz tests
      run: |
        cd fuzz/uufuzz
        cargo test --lib
    - name: Build and run uufuzz examples
      run: |
        cd fuzz/uufuzz
        echo "Building all examples..."
        cargo build --examples --release

        # Run all examples except integration_testing (which has FD issues in CI)
        for example in examples/*.rs; do
          example_name=$(basename "$example" .rs)
          if [ "$example_name" != "integration_testing" ]; then
            cargo run --example "$example_name" --release
          fi
        done

  fuzz-build:
    name: Build the fuzzers
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - name: Install `cargo-fuzz`
      uses: taiki-e/install-action@v2
      with:
        tool: cargo-fuzz
    - name: Emulate a nightly toolchain
      run: |
        echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}"
    - name: Run `cargo-fuzz build`
      # Force the correct target
      # https://github.com/rust-fuzz/cargo-fuzz/issues/398
      run: cargo fuzz build --target host-tuple

  fuzz-run:
    needs: fuzz-build
    name: Fuzz (${{ matrix.group.name }})
    runs-on: ubuntu-latest
    timeout-minutes: 15
    env:
      RUN_FOR: 60
    strategy:
      matrix:
        group:
          - name: smoke
            targets: |
              fuzz_test:true
              fuzz_date:true
              fuzz_expr:true
              fuzz_printf:true
              fuzz_echo:true
          - name: parsers
            targets: |
              fuzz_parse_glob:true
              fuzz_parse_size:false
              fuzz_parse_time:false
              fuzz_seq_parse_number:false
          - name: commands-a
            targets: |
              fuzz_seq:false
              fuzz_sort:false
              fuzz_wc:false
              fuzz_cut:false
              fuzz_split:false
          - name: commands-b
            targets: |
              fuzz_tr:false
              fuzz_env:false
              fuzz_cksum:false
              fuzz_non_utf8_paths:true
              fuzz_dirname:true

    steps:
    - uses: actions/checkout@v7.0.1
      with:
        persist-credentials: false
    - name: Install `cargo-fuzz`
      uses: taiki-e/install-action@v2
      with:
        tool: cargo-fuzz
    - name: Emulate a nightly toolchain
      run: |
        echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}"
    - name: Run ${{ matrix.group.name }} fuzz targets
      shell: bash
      run: |
        set -o pipefail
        mkdir -p fuzz/stats
        REQUIRED_TARGET_FAILED=0

        while IFS=: read -r TARGET SHOULD_PASS; do
          [ -n "$TARGET" ] || continue

          STATS_FILE="fuzz/stats/${TARGET}.txt"
          # Force the correct target
          # https://github.com/rust-fuzz/cargo-fuzz/issues/398
          set +e
          cargo fuzz run --target host-tuple "$TARGET" -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE"
          FUZZ_STATUS=${PIPESTATUS[0]}
          set -e

          # Save should_pass value to file for summary job to use
          echo "$SHOULD_PASS" > "fuzz/stats/${TARGET}.should_pass"

          # Print stats to job output for immediate visibility
          echo "----------------------------------------"
          echo "FUZZING STATISTICS FOR $TARGET"
          echo "----------------------------------------"
          echo "Runs:           $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
          echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec"
          echo "New Units:      $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
          echo "Expected:       $SHOULD_PASS"
          echo "Exit code:      $FUZZ_STATUS"
          if grep -q "SUMMARY: " "$STATS_FILE"; then
            echo "Status:         $(grep "SUMMARY: " "$STATS_FILE" | head -1)"
          else
            echo "Status:         Completed"
          fi
          echo "----------------------------------------"

          # Add summary to GitHub step summary
          echo "### Fuzzing Results for $TARGET" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY

          if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
            echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
          fi

          if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
            echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY
          fi

          if grep -q "stat::new_units_added" "$STATS_FILE"; then
            echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
          fi

          echo "| Should pass | $SHOULD_PASS |" >> $GITHUB_STEP_SUMMARY
          echo "| Exit code | $FUZZ_STATUS |" >> $GITHUB_STEP_SUMMARY

          if grep -q "SUMMARY: " "$STATS_FILE"; then
            echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY
          else
            echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY
          fi

          echo "" >> $GITHUB_STEP_SUMMARY

          if [ "$SHOULD_PASS" = "true" ] && [ "$FUZZ_STATUS" -ne 0 ]; then
            REQUIRED_TARGET_FAILED=1
          fi
        done <<< "${{ matrix.group.targets }}"

        exit "$REQUIRED_TARGET_FAILED"
    - name: Upload Stats
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: fuzz-stats-${{ matrix.group.name }}
        path: |
          fuzz/stats/*.txt
          fuzz/stats/*.should_pass
        retention-days: 5
  fuzz-summary:
    needs: fuzz-run
    name: Fuzzing Summary
    runs-on: ubuntu-latest
    if: always()
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          persist-credentials: false
      - name: Download all stats
        uses: actions/download-artifact@v8
        with:
          path: fuzz/stats-artifacts
          pattern: fuzz-stats-*
          merge-multiple: true
      - name: Prepare stats directory
        run: |
          mkdir -p fuzz/stats
          # Debug: List content of stats-artifacts directory
          echo "Contents of stats-artifacts directory:"
          find fuzz/stats-artifacts -type f | sort

          # Extract files from the artifact directories - handle nested directories
          find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \;
          find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \;

          # Debug information
          echo "Contents of stats directory after extraction:"
          ls -la fuzz/stats/
          echo "Contents of should_pass files (if any):"
          cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found"
      - name: Generate Summary
        run: |
          echo "# Fuzzing Summary" > fuzzing_summary.md
          echo "" >> fuzzing_summary.md
          echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md
          echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md

          TOTAL_RUNS=0
          TOTAL_NEW_UNITS=0

          for stat_file in fuzz/stats/*.txt; do
            TARGET=$(basename "$stat_file" .txt)
            SHOULD_PASS_FILE="${stat_file%.*}.should_pass"

            # Get expected status
            if [ -f "$SHOULD_PASS_FILE" ]; then
              EXPECTED=$(cat "$SHOULD_PASS_FILE")
            else
              EXPECTED="unknown"
            fi

            # Extract runs
            if grep -q "stat::number_of_executed_units" "$stat_file"; then
              RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}')
              TOTAL_RUNS=$((TOTAL_RUNS + RUNS))
            else
              RUNS="unknown"
            fi

            # Extract execution rate
            if grep -q "stat::average_exec_per_sec" "$stat_file"; then
              EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}')
            else
              EXEC_RATE="unknown"
            fi

            # Extract new units added
            if grep -q "stat::new_units_added" "$stat_file"; then
              NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}')
              if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then
                TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS))
              fi
            else
              NEW_UNITS="unknown"
            fi

            # Extract status
            if grep -q "SUMMARY: " "$stat_file"; then
              STATUS=$(grep "SUMMARY: " "$stat_file" | head -1)
            else
              STATUS="Completed"
            fi

            echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md
          done

          echo "" >> fuzzing_summary.md
          echo "## Overall Statistics" >> fuzzing_summary.md
          echo "" >> fuzzing_summary.md
          echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md
          echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md
          echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md

          # Add count by expected status
          echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md
          echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md

          # Write to GitHub step summary
          cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY
      - name: Show Summary
        run: |
          cat fuzzing_summary.md
      - name: Upload Summary
        uses: actions/upload-artifact@v7
        with:
          name: fuzzing-summary
          path: fuzzing_summary.md
          retention-days: 5