mirror of
https://github.com/uutils/uutils.github.io.git
synced 2026-10-16 03:24:02 -07:00
playground: ship uutils findutils (find) alongside grep
Generalize the single-module grep handling into a registry of standalone WASM modules so additional uutils tools can drop in. Add `find`, built from uutils/findutils to wasm32-wasip1 in CI, with graceful degradation when the binary is absent. Only `find` is shipped: xargs/locate/updatedb need process spawning or an on-disk database, neither of which works in the browser WASI sandbox. Add an emoji `find . -name '*.md'` example backed by fruit-emoji files in the virtual FS to show off Unicode/multibyte filename matching, plus tests.
This commit is contained in:
1 parent
281bf18b68
commit
81f75aeb0c
4 files changed
+145
-33
No files matched your search
@@ -227,6 +227,58 @@ jobs:
|
||||
echo "grep WASM build: ${grep_short} (${grep_date})"
|
||||
fi
|
||||
|
||||
- name: Build find WASM binary
|
||||
run: |
|
||||
# uutils findutils ships `find` as a standalone binary (not part of the
|
||||
# coreutils multicall) and, like grep, depends on the Oniguruma C
|
||||
# library (onig_sys), so the WASM build needs a WASI sysroot to compile
|
||||
# the bundled C sources. Reuse the WASI SDK fetched by the grep step if
|
||||
# it's still around, otherwise download it.
|
||||
WASI_SDK_VERSION=25
|
||||
WASI_SDK_DIR="wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux"
|
||||
if [ ! -d "$WASI_SDK_DIR" ]; then
|
||||
curl -sL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/${WASI_SDK_DIR}.tar.gz" | tar xz
|
||||
fi
|
||||
export WASI_SDK_PATH="$PWD/${WASI_SDK_DIR}"
|
||||
export CC_wasm32_wasip1="$WASI_SDK_PATH/bin/clang"
|
||||
export CFLAGS_wasm32_wasip1="--sysroot=$WASI_SDK_PATH/share/wasi-sysroot"
|
||||
cd findutils
|
||||
# Only `find` is shipped: xargs/locate/updatedb need process spawning or
|
||||
# an on-disk database, neither of which works in the browser WASI sandbox.
|
||||
cargo build --release --target wasm32-wasip1 --bin find
|
||||
if [ -f target/wasm32-wasip1/release/find.wasm ]; then
|
||||
mkdir -p ../uutils.github.io/static/wasm
|
||||
cp target/wasm32-wasip1/release/find.wasm ../uutils.github.io/static/wasm/find.wasm
|
||||
# Optimize WASM size if wasm-opt is available
|
||||
if command -v wasm-opt &> /dev/null; then
|
||||
wasm-opt -Oz ../uutils.github.io/static/wasm/find.wasm -o ../uutils.github.io/static/wasm/find.wasm
|
||||
fi
|
||||
echo "find WASM binary size: $(du -h ../uutils.github.io/static/wasm/find.wasm | cut -f1)"
|
||||
# Advertise find in the playground's command list. find ships as its
|
||||
# own WASM module, so it isn't picked up by the coreutils feat_wasm
|
||||
# scan above; append it to the generated list here.
|
||||
commands_js=../uutils.github.io/static/wasm/commands.js
|
||||
if [ -f "$commands_js" ]; then
|
||||
existing=$(sed -n 's/^const WASM_COMMANDS = \[\(.*\)\];$/\1/p' "$commands_js")
|
||||
echo "const WASM_COMMANDS = [${existing}, \"find\"];" > "$commands_js"
|
||||
else
|
||||
echo 'const WASM_COMMANDS = ["find"];' > "$commands_js"
|
||||
fi
|
||||
# Record the findutils commit used to build its WASM module so the
|
||||
# playground can show it alongside the coreutils build.
|
||||
find_hash=$(git rev-parse HEAD)
|
||||
find_short=$(git rev-parse --short HEAD)
|
||||
find_date=$(git show -s --format=%cI HEAD)
|
||||
{
|
||||
echo "const UUTILS_FINDUTILS_VERSION = {"
|
||||
echo " commit: \"${find_hash}\","
|
||||
echo " short: \"${find_short}\","
|
||||
echo " date: \"${find_date}\""
|
||||
echo "};"
|
||||
} >> ../uutils.github.io/static/wasm/version.js
|
||||
echo "find WASM build: ${find_short} (${find_date})"
|
||||
fi
|
||||
|
||||
- name: Run Zola
|
||||
uses: shalzz/zola-deploy-action@v0.22.1
|
||||
env:
|
||||
|
||||
@@ -61,6 +61,12 @@ template = "page.html"
|
||||
parts.push('grep <a href="' + grepUrl + '"><code>' +
|
||||
UUTILS_GREP_VERSION.short + '</code></a> (' + grepDate + ')');
|
||||
}
|
||||
if (typeof UUTILS_FINDUTILS_VERSION !== "undefined") {
|
||||
var findDate = UUTILS_FINDUTILS_VERSION.date.split("T")[0];
|
||||
var findUrl = "https://github.com/uutils/findutils/commit/" + UUTILS_FINDUTILS_VERSION.commit;
|
||||
parts.push('findutils <a href="' + findUrl + '"><code>' +
|
||||
UUTILS_FINDUTILS_VERSION.short + '</code></a> (' + findDate + ')');
|
||||
}
|
||||
if (typeof SITE_VERSION !== "undefined") {
|
||||
var siteDate = SITE_VERSION.date.split("T")[0];
|
||||
var siteUrl = "https://github.com/uutils/uutils.github.io/commit/" + SITE_VERSION.commit;
|
||||
@@ -95,6 +101,7 @@ Click an example to run it in the terminal:
|
||||
<button class="playground-example">echo '🍎,🍌,🍒,🥝' | cut -d🍌 -f2</button>
|
||||
<button class="playground-example">printf '🍒 cherry\n🍎 apple\n🍌 banana\n' | sort -k2</button>
|
||||
<button class="playground-example">printf '🍎 apple\n🍌 banana\n🍒 cherry\n🥝 kiwi\n' | grep 🍌</button>
|
||||
<button class="playground-example">find . -name '*.md'</button>
|
||||
<button class="playground-example">sort -n < numbers.txt | head -3</button>
|
||||
<button class="playground-example">date</button>
|
||||
<button class="playground-example">uname -a</button>
|
||||
|
||||
+58
-33
@@ -14,9 +14,14 @@ if (typeof SharedArrayBuffer === "undefined") {
|
||||
}
|
||||
|
||||
const WASM_URL = "/wasm/uutils.wasm";
|
||||
// grep ships as its own standalone WASM module (it is not part of the
|
||||
// coreutils multicall binary), loaded lazily alongside it.
|
||||
const GREP_WASM_URL = "/wasm/grep.wasm";
|
||||
// Some utilities ship as their own standalone WASM modules rather than as part
|
||||
// of the coreutils multicall binary (grep lives in uutils/grep, find in
|
||||
// uutils/findutils). They are loaded lazily alongside the multicall module and
|
||||
// each is optional — see loadStandaloneWasm.
|
||||
const STANDALONE_WASM_URLS = {
|
||||
grep: "/wasm/grep.wasm",
|
||||
find: "/wasm/find.wasm",
|
||||
};
|
||||
const XTERM_CSS = "https://cdn.jsdelivr.net/npm/@xterm/xterm@5.5.0/css/xterm.min.css";
|
||||
const XTERM_CSS_INTEGRITY = "sha384-tStR1zLfWgsiXCF3IgfB3lBa8KmBe/lG287CL9WCeKgQYcp1bjb4/+mwN6oti4Co";
|
||||
const XTERM_JS = "https://cdn.jsdelivr.net/npm/@xterm/xterm@5.5.0/lib/xterm.min.js";
|
||||
@@ -34,6 +39,11 @@ const SAMPLE_FILES = {
|
||||
"fruits.txt": "banana\napple\ncherry\ndate\napple\nbanana\ncherry\napple\n",
|
||||
"csv.txt": "name,age,city\nAlice,30,Paris\nBob,25,London\nCharlie,35,Berlin\nDiana,28,Tokyo\n",
|
||||
"words.txt": "hello world\nfoo bar baz\nthe quick brown fox\njumps over the lazy dog\n",
|
||||
// Emoji-named files so `find` has something fun (and Unicode!) to match.
|
||||
"🍎.md": "# Apple\n",
|
||||
"🍌.md": "# Banana\n",
|
||||
"🍒.md": "# Cherry\n",
|
||||
"🥝.md": "# Kiwi\n",
|
||||
};
|
||||
|
||||
// Commands available in the feat_wasm build.
|
||||
@@ -51,7 +61,7 @@ const FALLBACK_COMMANDS = [
|
||||
"sha1sum", "sha224sum", "sha256sum", "sha384sum", "sha512sum",
|
||||
"shred", "shuf", "sleep", "sum", "tee", "true", "truncate",
|
||||
"uname", "unexpand", "uniq", "unlink", "vdir", "wc",
|
||||
"grep",
|
||||
"grep", "find",
|
||||
];
|
||||
const AVAILABLE_COMMANDS =
|
||||
(typeof WASM_COMMANDS !== "undefined" && Array.isArray(WASM_COMMANDS) && WASM_COMMANDS.length > 0)
|
||||
@@ -67,7 +77,9 @@ const LOCALE_SHORTCUTS = {
|
||||
};
|
||||
|
||||
let wasmModule = null;
|
||||
let grepModule = null; // standalone grep WASM module (null if unavailable)
|
||||
// Compiled standalone modules, keyed by command name (e.g. "grep", "find").
|
||||
// A key is present only once its module has loaded successfully.
|
||||
const standaloneModules = {};
|
||||
let wasiShim = null;
|
||||
let terminal = null;
|
||||
let inputBuffer = "";
|
||||
@@ -155,29 +167,32 @@ async function loadWasm() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the standalone grep WASM module. grep is optional: if the binary isn't
|
||||
* present (e.g. local dev without a CI build), this resolves to null and grep
|
||||
* commands report that they're unavailable rather than breaking the terminal.
|
||||
* Load the optional standalone WASM modules (grep, find, …). Each is optional:
|
||||
* if a binary isn't present (e.g. local dev without a CI build), its error is
|
||||
* swallowed and the corresponding command reports that it's unavailable rather
|
||||
* than breaking the terminal or blocking the coreutils module.
|
||||
*/
|
||||
async function loadGrepWasm() {
|
||||
if (grepModule) return grepModule;
|
||||
try {
|
||||
const { module, size } = await compileWasmModule(GREP_WASM_URL);
|
||||
grepModule = module;
|
||||
wasmSize += size;
|
||||
} catch (e) {
|
||||
console.warn("grep WASM unavailable:", e.message);
|
||||
grepModule = null;
|
||||
}
|
||||
return grepModule;
|
||||
async function loadStandaloneWasm() {
|
||||
await Promise.all(
|
||||
Object.entries(STANDALONE_WASM_URLS).map(async ([cmd, url]) => {
|
||||
if (standaloneModules[cmd]) return;
|
||||
try {
|
||||
const { module, size } = await compileWasmModule(url);
|
||||
standaloneModules[cmd] = module;
|
||||
wasmSize += size;
|
||||
} catch (e) {
|
||||
console.warn(`${cmd} WASM unavailable:`, e.message);
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async function initWasm() {
|
||||
if (wasmReady) return;
|
||||
try {
|
||||
// grep is optional and loadGrepWasm swallows its own errors, so it never
|
||||
// blocks the coreutils module from becoming ready.
|
||||
await Promise.all([loadWasiShim(), loadWasm(), loadGrepWasm()]);
|
||||
// The standalone modules are optional and loadStandaloneWasm swallows its
|
||||
// own errors, so they never block the coreutils module from becoming ready.
|
||||
await Promise.all([loadWasiShim(), loadWasm(), loadStandaloneWasm()]);
|
||||
wasmReady = true;
|
||||
} catch (e) {
|
||||
// Will fall back to JS implementations
|
||||
@@ -493,6 +508,7 @@ async function executeCommandLine(line) {
|
||||
" wc -l fruits.txt\n" +
|
||||
" seq 1 10 | factor\n" +
|
||||
" grep -i alice names.txt\n" +
|
||||
" find . -name '*.md'\n" +
|
||||
" basename /usr/local/bin/rustc\n" +
|
||||
" date\n" +
|
||||
" uname -a\n"
|
||||
@@ -566,11 +582,11 @@ async function executeCommandLine(line) {
|
||||
return `uutils: command not found: ${cmd}\nType 'help' for available commands.\n`;
|
||||
}
|
||||
|
||||
// grep is a separate WASM module rather than part of the coreutils
|
||||
// multicall binary.
|
||||
const isGrep = cmd === "grep";
|
||||
if (isGrep && !grepModule) {
|
||||
return "grep is not available in this build.\n";
|
||||
// Some utilities (grep, find) are separate WASM modules rather than part
|
||||
// of the coreutils multicall binary.
|
||||
const isStandalone = cmd in STANDALONE_WASM_URLS;
|
||||
if (isStandalone && !standaloneModules[cmd]) {
|
||||
return `${cmd} is not available in this build.\n`;
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -586,10 +602,18 @@ async function executeCommandLine(line) {
|
||||
if (!hasPathArg && cwd && ["ls", "dir"].includes(cmd)) {
|
||||
resolvedArgs.push(cwd);
|
||||
}
|
||||
// grep is invoked directly (argv[0] = "grep"); coreutils utilities go
|
||||
// through the multicall dispatcher (argv = ["coreutils", <util>, ...]).
|
||||
// find takes its starting paths *before* the expression. When the user
|
||||
// gives none (e.g. `find -type f`), GNU find defaults to "."; mirror that
|
||||
// but use the virtual cwd so `cd subdir; find` searches the right place.
|
||||
if (cmd === "find") {
|
||||
const hasStartPath = resolvedArgs.length > 1 && !resolvedArgs[1].startsWith("-");
|
||||
if (!hasStartPath) resolvedArgs.splice(1, 0, cwd || ".");
|
||||
}
|
||||
// Standalone utilities are invoked directly (argv[0] = the command name);
|
||||
// coreutils utilities go through the multicall dispatcher
|
||||
// (argv = ["coreutils", <util>, ...]).
|
||||
let dispatchArgs = resolvedArgs;
|
||||
if (isGrep) {
|
||||
if (cmd === "grep") {
|
||||
// browser_wasi_shim reports stdout as a TTY, so grep would emit GNU
|
||||
// match-highlight escape codes by default. That looks fine in the
|
||||
// terminal but corrupts piped/redirected output (e.g. `grep x | wc`),
|
||||
@@ -599,8 +623,8 @@ async function executeCommandLine(line) {
|
||||
? resolvedArgs
|
||||
: [resolvedArgs[0], "--color=never", ...resolvedArgs.slice(1)];
|
||||
}
|
||||
const wasmArgs = isGrep ? dispatchArgs : ["coreutils", ...resolvedArgs];
|
||||
const result = await runCommand(wasmArgs, stdinData, isGrep ? grepModule : wasmModule);
|
||||
const wasmArgs = isStandalone ? dispatchArgs : ["coreutils", ...resolvedArgs];
|
||||
const result = await runCommand(wasmArgs, stdinData, isStandalone ? standaloneModules[cmd] : wasmModule);
|
||||
if (result.stderr) {
|
||||
return result.stderr + result.stdout;
|
||||
}
|
||||
@@ -924,7 +948,8 @@ window._uutilsTestInternals = {
|
||||
get locale() { return currentLocale; },
|
||||
set locale(v) { currentLocale = v; },
|
||||
get wasmReady() { return wasmReady; },
|
||||
get grepReady() { return grepModule !== null; },
|
||||
get grepReady() { return !!standaloneModules.grep; },
|
||||
get findReady() { return !!standaloneModules.find; },
|
||||
initWasm,
|
||||
LOCALE_SHORTCUTS,
|
||||
SAMPLE_FILES,
|
||||
|
||||
@@ -472,6 +472,34 @@ async function runTests() {
|
||||
section("grep WASM module (SKIPPED - grep.wasm not loaded)");
|
||||
}
|
||||
|
||||
// ===== find (standalone WASM module) =====
|
||||
if (T.findReady) {
|
||||
section("find WASM module");
|
||||
|
||||
assert("find in AVAILABLE_COMMANDS",
|
||||
AVAILABLE_COMMANDS.includes("find"), true);
|
||||
|
||||
await assertAsync("find by exact name",
|
||||
executeCommandLine("find . -name names.txt"),
|
||||
"./names.txt\n");
|
||||
|
||||
await assertAsync("find with no matches",
|
||||
executeCommandLine("find . -name '*.csv'"),
|
||||
"");
|
||||
|
||||
// Exercises directory traversal plus a pipe into a coreutils utility.
|
||||
await assertAsync("find piped into sort",
|
||||
executeCommandLine("find . -type f -name '*.txt' | sort"),
|
||||
"./csv.txt\n./fruits.txt\n./names.txt\n./numbers.txt\n./words.txt\n");
|
||||
|
||||
// find matches (and returns) emoji / multibyte filenames.
|
||||
await assertAsync("find emoji-named files",
|
||||
executeCommandLine("find . -name '*.md' | sort"),
|
||||
"./🍌.md\n./🍎.md\n./🍒.md\n./🥝.md\n");
|
||||
} else {
|
||||
section("find WASM module (SKIPPED - find.wasm not loaded)");
|
||||
}
|
||||
|
||||
} else {
|
||||
section("l10n WASM integration (SKIPPED - WASM not loaded)");
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user