You've already forked uutils.github.io
mirror of
https://github.com/uutils/uutils.github.io.git
synced 2026-06-10 16:12:28 -07:00
Add uutils grep to the playground
grep ships as its own standalone WASM module (not part of the coreutils multicall binary) and depends on the Oniguruma C library, so the website workflow now checks out uutils/grep, installs the WASI SDK to provide a wasm sysroot for the onig_sys C sources, builds grep.wasm, and appends "grep" to the generated command list. The terminal loads grep.wasm as an optional second module and dispatches grep directly (argv[0] = "grep"). grep defaults to --color=never since the WASI shim reports stdout as a TTY, which would otherwise emit match- highlight escapes that corrupt piped output; explicit --color is honored. Adds a grep emoji example and grep WASM integration tests.
This commit is contained in:
@@ -43,6 +43,13 @@ jobs:
|
||||
path: './findutils'
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Checkout Grep Repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: uutils/grep
|
||||
path: './grep'
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install `rust` toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
@@ -174,6 +181,39 @@ jobs:
|
||||
echo "site build: ${site_short} (${site_date})"
|
||||
fi
|
||||
|
||||
- name: Build grep WASM binary
|
||||
run: |
|
||||
# uutils grep is a standalone binary (not part of the coreutils
|
||||
# multicall) and depends on the Oniguruma C library (onig_sys), so the
|
||||
# WASM build needs a WASI sysroot to compile the bundled C sources.
|
||||
WASI_SDK_VERSION=25
|
||||
WASI_SDK_DIR="wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux"
|
||||
curl -sL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/${WASI_SDK_DIR}.tar.gz" | tar xz
|
||||
export WASI_SDK_PATH="$PWD/${WASI_SDK_DIR}"
|
||||
export CC_wasm32_wasip1="$WASI_SDK_PATH/bin/clang"
|
||||
export CFLAGS_wasm32_wasip1="--sysroot=$WASI_SDK_PATH/share/wasi-sysroot"
|
||||
cd grep
|
||||
cargo build --release --target wasm32-wasip1 --bin grep
|
||||
if [ -f target/wasm32-wasip1/release/grep.wasm ]; then
|
||||
mkdir -p ../uutils.github.io/static/wasm
|
||||
cp target/wasm32-wasip1/release/grep.wasm ../uutils.github.io/static/wasm/grep.wasm
|
||||
# Optimize WASM size if wasm-opt is available
|
||||
if command -v wasm-opt &> /dev/null; then
|
||||
wasm-opt -Oz ../uutils.github.io/static/wasm/grep.wasm -o ../uutils.github.io/static/wasm/grep.wasm
|
||||
fi
|
||||
echo "grep WASM binary size: $(du -h ../uutils.github.io/static/wasm/grep.wasm | cut -f1)"
|
||||
# Advertise grep in the playground's command list. grep ships as its
|
||||
# own WASM module, so it isn't picked up by the coreutils feat_wasm
|
||||
# scan above; append it to the generated list here.
|
||||
commands_js=../uutils.github.io/static/wasm/commands.js
|
||||
if [ -f "$commands_js" ]; then
|
||||
existing=$(sed -n 's/^const WASM_COMMANDS = \[\(.*\)\];$/\1/p' "$commands_js")
|
||||
echo "const WASM_COMMANDS = [${existing}, \"grep\"];" > "$commands_js"
|
||||
else
|
||||
echo 'const WASM_COMMANDS = ["grep"];' > "$commands_js"
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Run Zola
|
||||
uses: shalzz/zola-deploy-action@v0.22.1
|
||||
env:
|
||||
|
||||
@@ -88,6 +88,7 @@ Click an example to run it in the terminal:
|
||||
<button class="playground-example">printf '🐑\n🐑\n🐑\n🐑\n🐑\n' | nl</button>
|
||||
<button class="playground-example">echo '🍎,🍌,🍒,🥝' | cut -d🍌 -f2</button>
|
||||
<button class="playground-example">printf '🍒 cherry\n🍎 apple\n🍌 banana\n' | sort -k2</button>
|
||||
<button class="playground-example">printf '🍎 apple\n🍌 banana\n🍒 cherry\n🥝 kiwi\n' | grep 🍌</button>
|
||||
<button class="playground-example">sort -n < numbers.txt | head -3</button>
|
||||
<button class="playground-example">date</button>
|
||||
<button class="playground-example">uname -a</button>
|
||||
|
||||
+73
-12
@@ -14,6 +14,9 @@ if (typeof SharedArrayBuffer === "undefined") {
|
||||
}
|
||||
|
||||
const WASM_URL = "/wasm/uutils.wasm";
|
||||
// grep ships as its own standalone WASM module (it is not part of the
|
||||
// coreutils multicall binary), loaded lazily alongside it.
|
||||
const GREP_WASM_URL = "/wasm/grep.wasm";
|
||||
const XTERM_CSS = "https://cdn.jsdelivr.net/npm/@xterm/xterm@5.5.0/css/xterm.min.css";
|
||||
const XTERM_CSS_INTEGRITY = "sha384-tStR1zLfWgsiXCF3IgfB3lBa8KmBe/lG287CL9WCeKgQYcp1bjb4/+mwN6oti4Co";
|
||||
const XTERM_JS = "https://cdn.jsdelivr.net/npm/@xterm/xterm@5.5.0/lib/xterm.min.js";
|
||||
@@ -48,6 +51,7 @@ const FALLBACK_COMMANDS = [
|
||||
"sha1sum", "sha224sum", "sha256sum", "sha384sum", "sha512sum",
|
||||
"shred", "shuf", "sleep", "sum", "tee", "true", "truncate",
|
||||
"uname", "unexpand", "uniq", "unlink", "vdir", "wc",
|
||||
"grep",
|
||||
];
|
||||
const AVAILABLE_COMMANDS =
|
||||
(typeof WASM_COMMANDS !== "undefined" && Array.isArray(WASM_COMMANDS) && WASM_COMMANDS.length > 0)
|
||||
@@ -63,6 +67,7 @@ const LOCALE_SHORTCUTS = {
|
||||
};
|
||||
|
||||
let wasmModule = null;
|
||||
let grepModule = null; // standalone grep WASM module (null if unavailable)
|
||||
let wasiShim = null;
|
||||
let terminal = null;
|
||||
let inputBuffer = "";
|
||||
@@ -111,35 +116,68 @@ async function loadWasiShim() {
|
||||
return wasiShim;
|
||||
}
|
||||
|
||||
async function loadWasm() {
|
||||
if (wasmModule) return wasmModule;
|
||||
const response = await fetch(WASM_URL);
|
||||
/**
|
||||
* Fetch and compile a WASM module from the given URL.
|
||||
* Returns { module, size } where size is the downloaded byte length (0 if the
|
||||
* server didn't send a content-length header).
|
||||
*/
|
||||
async function compileWasmModule(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch WASM binary: ${response.status}`);
|
||||
}
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength) wasmSize = parseInt(contentLength, 10);
|
||||
const size = contentLength ? parseInt(contentLength, 10) : 0;
|
||||
// compileStreaming requires application/wasm content-type; fall back if not set.
|
||||
// Clone the response so the fallback path can read the body without re-fetching.
|
||||
const cloned = response.clone();
|
||||
let module;
|
||||
try {
|
||||
if (WebAssembly.compileStreaming) {
|
||||
wasmModule = await WebAssembly.compileStreaming(response);
|
||||
module = await WebAssembly.compileStreaming(response);
|
||||
} else {
|
||||
wasmModule = await WebAssembly.compile(await response.arrayBuffer());
|
||||
module = await WebAssembly.compile(await response.arrayBuffer());
|
||||
}
|
||||
} catch (e) {
|
||||
// Some servers don't set proper MIME type, compile from the cloned response
|
||||
console.warn("WASM compileStreaming failed, falling back to arrayBuffer:", e.message);
|
||||
wasmModule = await WebAssembly.compile(await cloned.arrayBuffer());
|
||||
module = await WebAssembly.compile(await cloned.arrayBuffer());
|
||||
}
|
||||
return { module, size };
|
||||
}
|
||||
|
||||
async function loadWasm() {
|
||||
if (wasmModule) return wasmModule;
|
||||
const { module, size } = await compileWasmModule(WASM_URL);
|
||||
wasmModule = module;
|
||||
wasmSize = size;
|
||||
return wasmModule;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the standalone grep WASM module. grep is optional: if the binary isn't
|
||||
* present (e.g. local dev without a CI build), this resolves to null and grep
|
||||
* commands report that they're unavailable rather than breaking the terminal.
|
||||
*/
|
||||
async function loadGrepWasm() {
|
||||
if (grepModule) return grepModule;
|
||||
try {
|
||||
const { module, size } = await compileWasmModule(GREP_WASM_URL);
|
||||
grepModule = module;
|
||||
wasmSize += size;
|
||||
} catch (e) {
|
||||
console.warn("grep WASM unavailable:", e.message);
|
||||
grepModule = null;
|
||||
}
|
||||
return grepModule;
|
||||
}
|
||||
|
||||
async function initWasm() {
|
||||
if (wasmReady) return;
|
||||
try {
|
||||
await Promise.all([loadWasiShim(), loadWasm()]);
|
||||
// grep is optional and loadGrepWasm swallows its own errors, so it never
|
||||
// blocks the coreutils module from becoming ready.
|
||||
await Promise.all([loadWasiShim(), loadWasm(), loadGrepWasm()]);
|
||||
wasmReady = true;
|
||||
} catch (e) {
|
||||
// Will fall back to JS implementations
|
||||
@@ -218,8 +256,9 @@ function lookupDir(path) {
|
||||
* Run a single uutils command via the WASM module using browser_wasi_shim.
|
||||
* Returns { stdout: string, stderr: string, exitCode: number }
|
||||
*/
|
||||
async function runCommand(argv, stdinData = "") {
|
||||
async function runCommand(argv, stdinData = "", module = wasmModule) {
|
||||
if (!wasmReady) throw new Error("WASM not loaded");
|
||||
if (!module) throw new Error("WASM module not loaded");
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
@@ -306,7 +345,7 @@ async function runCommand(argv, stdinData = "") {
|
||||
|
||||
let exitCode = 0;
|
||||
try {
|
||||
const result = await WebAssembly.instantiate(wasmModule, {
|
||||
const result = await WebAssembly.instantiate(module, {
|
||||
wasi_snapshot_preview1: wasi.wasiImport,
|
||||
});
|
||||
// instantiate(Module) returns Instance; instantiate(Buffer) returns {instance}
|
||||
@@ -453,6 +492,7 @@ async function executeCommandLine(line) {
|
||||
" echo '5 3 1 4 2' | fmt -w1 | sort -n\n" +
|
||||
" wc -l fruits.txt\n" +
|
||||
" seq 1 10 | factor\n" +
|
||||
" grep -i alice names.txt\n" +
|
||||
" basename /usr/local/bin/rustc\n" +
|
||||
" date\n" +
|
||||
" uname -a\n"
|
||||
@@ -526,6 +566,13 @@ async function executeCommandLine(line) {
|
||||
return `uutils: command not found: ${cmd}\nType 'help' for available commands.\n`;
|
||||
}
|
||||
|
||||
// grep is a separate WASM module rather than part of the coreutils
|
||||
// multicall binary.
|
||||
const isGrep = cmd === "grep";
|
||||
if (isGrep && !grepModule) {
|
||||
return "grep is not available in this build.\n";
|
||||
}
|
||||
|
||||
try {
|
||||
// Resolve relative paths using the virtual cwd
|
||||
const resolvedArgs = args.map((arg, i) => {
|
||||
@@ -539,8 +586,21 @@ async function executeCommandLine(line) {
|
||||
if (!hasPathArg && cwd && ["ls", "dir"].includes(cmd)) {
|
||||
resolvedArgs.push(cwd);
|
||||
}
|
||||
const wasmArgs = ["coreutils", ...resolvedArgs];
|
||||
const result = await runCommand(wasmArgs, stdinData);
|
||||
// grep is invoked directly (argv[0] = "grep"); coreutils utilities go
|
||||
// through the multicall dispatcher (argv = ["coreutils", <util>, ...]).
|
||||
let dispatchArgs = resolvedArgs;
|
||||
if (isGrep) {
|
||||
// browser_wasi_shim reports stdout as a TTY, so grep would emit GNU
|
||||
// match-highlight escape codes by default. That looks fine in the
|
||||
// terminal but corrupts piped/redirected output (e.g. `grep x | wc`),
|
||||
// so default to no color unless the user asks for it explicitly.
|
||||
const hasColorFlag = resolvedArgs.some(a => a === "--color" || a.startsWith("--color="));
|
||||
dispatchArgs = hasColorFlag
|
||||
? resolvedArgs
|
||||
: [resolvedArgs[0], "--color=never", ...resolvedArgs.slice(1)];
|
||||
}
|
||||
const wasmArgs = isGrep ? dispatchArgs : ["coreutils", ...resolvedArgs];
|
||||
const result = await runCommand(wasmArgs, stdinData, isGrep ? grepModule : wasmModule);
|
||||
if (result.stderr) {
|
||||
return result.stderr + result.stdout;
|
||||
}
|
||||
@@ -864,6 +924,7 @@ window._uutilsTestInternals = {
|
||||
get locale() { return currentLocale; },
|
||||
set locale(v) { currentLocale = v; },
|
||||
get wasmReady() { return wasmReady; },
|
||||
get grepReady() { return grepModule !== null; },
|
||||
initWasm,
|
||||
LOCALE_SHORTCUTS,
|
||||
SAMPLE_FILES,
|
||||
|
||||
@@ -450,6 +450,28 @@ async function runTests() {
|
||||
"w\nx\n");
|
||||
}
|
||||
|
||||
// ===== grep (standalone WASM module) =====
|
||||
if (T.grepReady) {
|
||||
section("grep WASM module");
|
||||
|
||||
assert("grep in AVAILABLE_COMMANDS",
|
||||
AVAILABLE_COMMANDS.includes("grep"), true);
|
||||
|
||||
await assertAsync("grep from file",
|
||||
executeCommandLine("grep Alice names.txt"),
|
||||
"Alice\n");
|
||||
|
||||
await assertAsync("grep -i from stdin",
|
||||
executeCommandLine("echo 'APPLE\npear\nApricot' | grep -i ap"),
|
||||
"APPLE\nApricot\n");
|
||||
|
||||
await assertAsync("grep regex via pipe",
|
||||
executeCommandLine("echo 'foo1\nbar\nfoo2' | grep 'foo[0-9]'"),
|
||||
"foo1\nfoo2\n");
|
||||
} else {
|
||||
section("grep WASM module (SKIPPED - grep.wasm not loaded)");
|
||||
}
|
||||
|
||||
} else {
|
||||
section("l10n WASM integration (SKIPPED - WASM not loaded)");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user