From 8e269eaaa2fb44d1448ea12069aca1007cc77f3d Mon Sep 17 00:00:00 2001 From: Joel Winarske Date: Thu, 18 Jun 2026 08:02:13 -0700 Subject: [PATCH] feat(cross): add matrix render command and CI cross workflow emb matrix renders a GitHub Actions {"include":[...]} matrix from the cross: blocks of one or more manifests: one cell per (manifest x target x supported host), each carrying the emb cross args, provider/triple, host runner (container for hosts without a hosted runner), preflight tools, and the sysroot_key/build_key for actions/cache. The cross.yaml workflow renders the matrix, then validates every cell with emb cross --dry-run (hermetic: no download/mount/ssh) as the PR gate - ubuntu directly, fedora in a container. An opt-in build job runs the real cross build keyed on the cache keys. Signed-off-by: Joel Winarske --- .github/workflows/cross.yaml | 127 +++++++++++ lib/src/command_runner.dart | 1 + lib/src/commands/commands.dart | 1 + lib/src/commands/matrix_command.dart | 233 +++++++++++++++++++++ test/src/commands/matrix_command_test.dart | 140 +++++++++++++ 5 files changed, 502 insertions(+) create mode 100644 .github/workflows/cross.yaml create mode 100644 lib/src/commands/matrix_command.dart create mode 100644 test/src/commands/matrix_command_test.dart diff --git a/.github/workflows/cross.yaml b/.github/workflows/cross.yaml new file mode 100644 index 0000000..31f163c --- /dev/null +++ b/.github/workflows/cross.yaml @@ -0,0 +1,127 @@ +name: cross + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + paths: + - ".github/workflows/cross.yaml" + - "lib/src/cross/**" + - "lib/src/commands/cross_command.dart" + - "lib/src/commands/matrix_command.dart" + - "examples/cross/**" + - "pubspec.yaml" + push: + branches: + - main + paths: + - ".github/workflows/cross.yaml" + - "lib/src/cross/**" + - "lib/src/commands/cross_command.dart" + - "lib/src/commands/matrix_command.dart" + - "examples/cross/**" + - "pubspec.yaml" + workflow_dispatch: + inputs: + build: + description: "Also run real cross builds (L2 — network + privileges)." + type: boolean + default: false + +jobs: + # ── Stage 1: render the matrix from the example manifests ──────────────── + # Side-effect free (parse only): one cell per (manifest × target × supported + # host). Hosts without a GitHub runner (fedora) ride an ubuntu-latest host in + # a container image — the matrix carries that as `container`. + plan: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v5 + - uses: dart-lang/setup-dart@v1 + with: + sdk: "stable" + - run: dart pub get + - name: 🧮 Render cross matrix + id: gen + run: | + dart run emb_cli:emb matrix examples/cross -o matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" + - name: 📋 Show matrix + run: cat matrix.json + + # ── Stage 2: validate every cell (the PR gate) ────────────────────────── + # `--dry-run` resolves the plan with no download / mount / ssh, so each board + # validates on every host it declares — ubuntu directly, fedora in a + # container. This promotes the doc's L0/L1 surface from "parses" to "the real + # CLI plans this board end-to-end on this host". + validate: + needs: plan + runs-on: ${{ matrix.runs_on }} + container: ${{ matrix.container || '' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + # The fedora container is minimal: give setup-dart its extract tools and + # the packagekit_dart build hook a `git`/`cmake` so it fails *cleanly* + # (no C++ compiler installed → configure exits non-zero, the hook no-ops) + # rather than throwing on a missing executable. The dry-run itself never + # touches PackageKit. + - name: 🧱 Bootstrap container + if: ${{ matrix.container != '' }} + run: dnf -y install git tar xz unzip findutils which cmake ninja-build + - uses: actions/checkout@v5 + - uses: dart-lang/setup-dart@v1 + with: + sdk: "stable" + - run: dart pub get + - name: 🔎 Plan ${{ matrix.id }} / ${{ matrix.target }} on ${{ matrix.host }} (${{ matrix.provider }}) + run: dart run emb_cli:emb cross ${{ matrix.args }} --dry-run + + # ── Stage 3 (opt-in): real cross build ────────────────────────────────── + # Manual only (workflow_dispatch + build=true). Per docs/cross_integration.md + # §13, full builds are L2: they need network and host privileges (image + # loop-mount, qemu-static apt chroot, SDK install). The `yocto-recipe` / + # device-rsync providers additionally need state no hosted runner has — point + # `runs-on` at a self-hosted runner for those, or filter them out of the set. + build: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.build }} + needs: plan + runs-on: ${{ matrix.runs_on }} + container: ${{ matrix.container || '' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + FLUTTER_WORKSPACE: ${{ github.workspace }}/.ws + steps: + - name: 🧱 Bootstrap container + if: ${{ matrix.container != '' }} + run: dnf -y install git tar xz unzip findutils which cmake ninja-build + - uses: actions/checkout@v5 + - uses: dart-lang/setup-dart@v1 + with: + sdk: "stable" + - run: dart pub get + # The toolchain + sysroot extraction is the expensive, content-addressed + # artifact — keyed on the cell's sysroot_key so cpu-only variants of one + # board (rpi4/rpi5 on one image) restore the same cache. + - name: ♻️ Restore toolchain + sysroot + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.ws/.config/flutter_workspace + key: cross-${{ matrix.provider }}-${{ matrix.sysroot_key }} + - name: 🧰 Install preflight tools (apt) + if: ${{ matrix.container == '' && matrix.preflight != '' }} + run: | + sudo apt-get update + sudo apt-get install -y ${{ matrix.preflight }} qemu-user-static binfmt-support + - name: 🧰 Install preflight tools (dnf) + if: ${{ matrix.container != '' && matrix.preflight != '' }} + run: dnf -y install ${{ matrix.preflight }} qemu-user-static + - name: 🏗️ Build ${{ matrix.id }} / ${{ matrix.target }} on ${{ matrix.host }} (${{ matrix.provider }}) + run: dart run emb_cli:emb cross ${{ matrix.args }} --build diff --git a/lib/src/command_runner.dart b/lib/src/command_runner.dart index d9727e2..933541b 100644 --- a/lib/src/command_runner.dart +++ b/lib/src/command_runner.dart @@ -47,6 +47,7 @@ class EmbCliCommandRunner extends CompletionCommandRunner { addCommand(BundleCommand(logger: _logger)); addCommand(BuildCommand(logger: _logger)); addCommand(CrossCommand(logger: _logger)); + addCommand(MatrixCommand(logger: _logger)); addCommand(EnvCommand(logger: _logger)); addCommand(UpdateCommand(logger: _logger, pubUpdater: _pubUpdater)); } diff --git a/lib/src/commands/commands.dart b/lib/src/commands/commands.dart index 79820f7..2b4dc23 100644 --- a/lib/src/commands/commands.dart +++ b/lib/src/commands/commands.dart @@ -7,6 +7,7 @@ export 'doctor_command.dart'; export 'engine_command.dart'; export 'env_command.dart'; export 'flutter_command.dart'; +export 'matrix_command.dart'; export 'setup_command.dart'; export 'sync_command.dart'; export 'update_command.dart'; diff --git a/lib/src/commands/matrix_command.dart b/lib/src/commands/matrix_command.dart new file mode 100644 index 0000000..646084e --- /dev/null +++ b/lib/src/commands/matrix_command.dart @@ -0,0 +1,233 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:emb_cli/src/cross/cross_keys.dart'; +import 'package:emb_cli/src/cross/cross_provider.dart'; +import 'package:emb_cli/src/cross/cross_target.dart'; +import 'package:emb_cli/src/host/host_info.dart'; +import 'package:emb_cli/src/manifest/manifest_loader.dart'; +import 'package:emb_cli/src/workspace/workspace.dart'; +import 'package:mason_logger/mason_logger.dart'; +import 'package:path/path.dart' as p; + +/// {@template matrix_command} +/// `emb matrix ...` — render a CI build matrix from the `cross:` +/// blocks of one or more manifests. +/// +/// Emits GitHub Actions `{"include":[...]}` JSON: one cell per +/// (manifest × target × supported host). Each cell carries the `emb cross` +/// args to run, the provider/triple, the host runner (`runs_on`, +/// optionally `container`), the provider's preflight tools, and the +/// content-addressed `sysroot_key`/`build_key` for `actions/cache`. +/// +/// Side-effect free — it only parses manifests (no download/mount/ssh), so it +/// runs on any host. Pair each cell with `emb cross --dry-run` as a +/// per-cell PR gate, and (opt-in) `emb cross --build` for the real +/// cross build keyed on the cache keys. +/// {@endtemplate} +class MatrixCommand extends Command { + /// {@macro matrix_command} + MatrixCommand({ + required Logger logger, + HostInfo? host, + ManifestLoader loader = const ManifestLoader(), + }) : _logger = logger, + _host = host, + _loader = loader { + argParser + ..addMultiOption( + 'host', + help: + 'Restrict to these host types (intersected with each ' + "manifest's supported_host_types). Default: every supported host.", + ) + ..addOption('target', help: 'Only emit the cells for this target name.') + ..addOption( + 'output', + abbr: 'o', + help: + 'Write the JSON to this file instead of stdout (robust against ' + 'banners polluting a captured stdout).', + ) + ..addFlag( + 'pretty', + help: 'Indent the JSON (stdout only).', + negatable: false, + ); + } + + final Logger _logger; + final HostInfo? _host; + final ManifestLoader _loader; + + @override + String get name => 'matrix'; + + @override + String get description => + 'Render a CI build matrix from manifest cross: blocks.'; + + @override + Future run() async { + final args = argResults!; + if (args.rest.isEmpty) { + _logger.err( + 'Usage: emb matrix ... [--host h] [--target t] [-o file]', + ); + return ExitCode.usage.code; + } + + final hostFilter = args['host'] as List; + final targetFilter = args['target'] as String?; + final host = _host ?? HostInfo.detect(); + final workspace = Workspace.resolve(); + + final include = >[]; + for (final file in _collect(args.rest)) { + final manifest = _loader.loadManifestFile(file); + if (manifest == null) { + _logger.warn('skip ${file.path}: not a manifest'); + continue; + } + final crossMap = manifest.raw['cross']; + if (crossMap is! Map) continue; // not a cross manifest — silently skip + + // Hosts: the manifest's declared support, narrowed by --host. Empty + // supported_host_types falls back to ubuntu so a terse manifest emits. + final supported = manifest.supportedHostTypes.isEmpty + ? const ['ubuntu'] + : manifest.supportedHostTypes; + final hosts = hostFilter.isEmpty + ? supported + : hostFilter.where(supported.contains).toList(); + if (hosts.isEmpty) continue; + + for (final entry in _targets(crossMap)) { + if (targetFilter != null && entry.name != targetFilter) continue; + final CrossTarget target; + try { + target = CrossTarget.fromMap(entry.merged); + // fromMap throws ArgumentError on an unknown/missing provider token. + // ignore: avoid_catching_errors + } on ArgumentError catch (e) { + _logger.warn('skip ${file.path}#${entry.name}: ${e.message}'); + continue; + } + final provider = CrossProvider.forTarget( + target, + workspace: workspace, + host: host, + ); + // `default` is the single-target shape (no cross.targets); it needs no + // --target. A named target does. + final crossArgs = entry.name == 'default' + ? file.path + : '${file.path} --target ${entry.name}'; + + for (final h in hosts) { + final runner = _runner(h); + include.add({ + 'id': manifest.id, + 'manifest': file.path, + 'target': entry.name, + 'host': h, + 'runs_on': runner.runsOn, + if (runner.container != null) 'container': runner.container, + 'provider': target.provider.token, + 'triple': target.triple ?? '', + 'preflight': provider.preflightTools.join(' '), + 'sysroot_key': sysrootKey(target), + 'build_key': buildKey(target), + 'args': crossArgs, + }); + } + } + } + + final payload = {'include': include}; + final outPath = args['output'] as String?; + if (outPath != null) { + File(outPath).writeAsStringSync(jsonEncode(payload)); + final n = include.length; + _logger.info('Wrote $outPath ($n cell${n == 1 ? "" : "s"}).'); + } else { + _logger.info( + (args['pretty'] as bool) + ? const JsonEncoder.withIndent(' ').convert(payload) + : jsonEncode(payload), + ); + } + return ExitCode.success.code; + } + + /// Expand a `cross:` map into its targets. A `cross.targets` map yields one + /// entry per named target (shared fields merged under each override); a flat + /// `cross:` yields a single synthetic `default` target. + List<({String name, Map merged})> _targets( + Map crossMap, + ) { + final targets = crossMap['targets']; + if (targets is Map && targets.isNotEmpty) { + return [ + for (final e in targets.entries) + if (e.value is Map) + ( + name: e.key.toString(), + merged: { + for (final s in crossMap.entries) + if (s.key != 'targets') s.key: s.value, + ...e.value as Map, + }, + ), + ]; + } + return [(name: 'default', merged: Map.from(crossMap))]; + } + + /// Gather manifest files from the given paths: a file is taken as-is; a + /// directory contributes its `emb.yaml` (if any) plus every `*.emb.yaml` + /// (sorted). Deduplicated by absolute path. + List _collect(List paths) { + final seen = {}; + final files = []; + void add(File f) { + if (seen.add(p.absolute(f.path))) files.add(f); + } + + for (final path in paths) { + final type = FileSystemEntity.typeSync(path); + if (type == FileSystemEntityType.file) { + add(File(path)); + } else if (type == FileSystemEntityType.directory) { + final embYaml = File(p.join(path, 'emb.yaml')); + if (embYaml.existsSync()) add(embYaml); + Directory(path) + .listSync() + .whereType() + .where((f) => f.path.endsWith('.emb.yaml')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)) + ..forEach(add); + } else { + _logger.warn('skip: no such path $path'); + } + } + return files; + } + + /// Map a host type to a GitHub Actions runner. Distros without a hosted + /// runner ride an `ubuntu-latest` host inside a container image. + ({String runsOn, String? container}) _runner(String hostType) { + if (hostType == 'fedora') { + return (runsOn: 'ubuntu-latest', container: 'fedora:41'); + } + if (hostType == 'darwin' || hostType == 'macos') { + return (runsOn: 'macos-latest', container: null); + } + if (hostType == 'windows') { + return (runsOn: 'windows-latest', container: null); + } + return (runsOn: 'ubuntu-latest', container: null); // ubuntu/debian/default + } +} diff --git a/test/src/commands/matrix_command_test.dart b/test/src/commands/matrix_command_test.dart new file mode 100644 index 0000000..7010b57 --- /dev/null +++ b/test/src/commands/matrix_command_test.dart @@ -0,0 +1,140 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:emb_cli/src/commands/matrix_command.dart'; +import 'package:emb_cli/src/host/host_info.dart'; +import 'package:mason_logger/mason_logger.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +const _host = HostInfo( + os: HostOs.linux, + machineArch: 'x86_64', + archAliases: {'x86_64', 'x64', 'amd64'}, + hostType: 'fedora', + versionId: '43', +); + +const _singleTarget = ''' +id: ivi-homescreen +type: app +supported_archs: [arm64] +supported_host_types: [ubuntu, fedora] +cross: + provider: arm-gnu + triple: aarch64-none-linux-gnu + toolchain_version: 12.3.rel1 + image_url: https://example.com/os.img.xz + cpu_flags: [-mcpu=cortex-a76] +'''; + +const _multiTarget = ''' +id: ivi-homescreen +type: app +supported_host_types: [ubuntu] +cross: + provider: arm-gnu + toolchain_version: 12.3.rel1 + targets: + rpi5: + image_url: https://example.com/raspios.img.xz + cpu_flags: [-mcpu=cortex-a76] + rpi4: + image_url: https://example.com/raspios.img.xz + cpu_flags: [-mcpu=cortex-a72] +'''; + +void main() { + late Directory tmp; + setUp(() => tmp = Directory.systemTemp.createTempSync('emb_matrix_')); + tearDown(() => tmp.deleteSync(recursive: true)); + + Future run(List args) { + final runner = CommandRunner('emb', 'test') + ..addCommand(MatrixCommand(logger: Logger(), host: _host)); + return runner.run(args); + } + + File write(String name, String yaml) => + File(p.join(tmp.path, name))..writeAsStringSync(yaml); + + /// Render to a file (robust capture) and return its `include` list. + Future>> render(List args) async { + final outFile = p.join(tmp.path, 'matrix.json'); + final code = await run(['matrix', ...args, '-o', outFile]); + expect(code, ExitCode.success.code); + final decoded = jsonDecode(File(outFile).readAsStringSync()) as Map; + return (decoded['include'] as List).cast>(); + } + + test('usage error with no path argument', () async { + expect(await run(['matrix']), ExitCode.usage.code); + }); + + test('a single-target manifest fans out over its supported hosts', () async { + final f = write('pi5.emb.yaml', _singleTarget); + final include = await render([f.path]); + + expect(include, hasLength(2)); // ubuntu + fedora + final ubuntu = include.firstWhere((e) => e['host'] == 'ubuntu'); + expect(ubuntu['provider'], 'arm-gnu'); + expect(ubuntu['triple'], 'aarch64-none-linux-gnu'); + expect(ubuntu['target'], 'default'); + expect(ubuntu['runs_on'], 'ubuntu-latest'); + expect(ubuntu['args'], f.path); // no --target for a flat cross block + expect(ubuntu['preflight'], 'tar xz rsync'); // arm-gnu preflight tools + expect(ubuntu['sysroot_key'], isNotEmpty); + expect(ubuntu['build_key'], isNotEmpty); + + final fedora = include.firstWhere((e) => e['host'] == 'fedora'); + expect(fedora['runs_on'], 'ubuntu-latest'); + expect(fedora['container'], 'fedora:41'); // no hosted fedora runner + }); + + test( + 'cpu-only variants share a sysroot_key but differ on build_key', + () async { + final f = write('rpi.emb.yaml', _multiTarget); + final include = await render([f.path]); + + expect(include, hasLength(2)); // rpi5, rpi4 (one host: ubuntu) + final rpi5 = include.firstWhere((e) => e['target'] == 'rpi5'); + final rpi4 = include.firstWhere((e) => e['target'] == 'rpi4'); + expect(rpi5['args'], '${f.path} --target rpi5'); + // Same image → shared extraction; only cpu tuning differs. + expect(rpi5['sysroot_key'], rpi4['sysroot_key']); + expect(rpi5['build_key'], isNot(rpi4['build_key'])); + }, + ); + + test( + '--host narrows to the intersection with supported_host_types', + () async { + final f = write('pi5.emb.yaml', _singleTarget); + final include = await render([f.path, '--host', 'ubuntu']); + expect(include, hasLength(1)); + expect(include.single['host'], 'ubuntu'); + }, + ); + + test('--target emits only the named target', () async { + final f = write('rpi.emb.yaml', _multiTarget); + final include = await render([f.path, '--target', 'rpi4']); + expect(include, hasLength(1)); + expect(include.single['target'], 'rpi4'); + }); + + test('a directory globs *.emb.yaml', () async { + write('pi5.emb.yaml', _singleTarget); + write('rpi.emb.yaml', _multiTarget); + final include = await render([tmp.path, '--host', 'ubuntu']); + expect(include, hasLength(3)); // pi5 default + rpi5 + rpi4 + }); + + test('a manifest without a cross: block is skipped', () async { + final f = write('plain.emb.yaml', 'id: plain\ntype: app\n'); + final include = await render([f.path]); + expect(include, isEmpty); + }); +}