feat(cross): add matrix render command and CI cross workflow

emb matrix renders a GitHub Actions {"include":[...]} matrix from the
cross: blocks of one or more manifests: one cell per (manifest x target x
supported host), each carrying the emb cross args, provider/triple, host
runner (container for hosts without a hosted runner), preflight tools, and
the sysroot_key/build_key for actions/cache.

The cross.yaml workflow renders the matrix, then validates every cell with
emb cross --dry-run (hermetic: no download/mount/ssh) as the PR gate -
ubuntu directly, fedora in a container. An opt-in build job runs the real
cross build keyed on the cache keys.

Signed-off-by: Joel Winarske <joel.winarske@gmail.com>
This commit is contained in:
Joel Winarske
2026-06-18 08:02:13 -07:00
parent abebccd437
commit 8e269eaaa2
5 changed files with 502 additions and 0 deletions
+127
View File
@@ -0,0 +1,127 @@
name: cross
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
paths:
- ".github/workflows/cross.yaml"
- "lib/src/cross/**"
- "lib/src/commands/cross_command.dart"
- "lib/src/commands/matrix_command.dart"
- "examples/cross/**"
- "pubspec.yaml"
push:
branches:
- main
paths:
- ".github/workflows/cross.yaml"
- "lib/src/cross/**"
- "lib/src/commands/cross_command.dart"
- "lib/src/commands/matrix_command.dart"
- "examples/cross/**"
- "pubspec.yaml"
workflow_dispatch:
inputs:
build:
description: "Also run real cross builds (L2 — network + privileges)."
type: boolean
default: false
jobs:
# ── Stage 1: render the matrix from the example manifests ────────────────
# Side-effect free (parse only): one cell per (manifest × target × supported
# host). Hosts without a GitHub runner (fedora) ride an ubuntu-latest host in
# a container image — the matrix carries that as `container`.
plan:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.gen.outputs.matrix }}
steps:
- uses: actions/checkout@v5
- uses: dart-lang/setup-dart@v1
with:
sdk: "stable"
- run: dart pub get
- name: 🧮 Render cross matrix
id: gen
run: |
dart run emb_cli:emb matrix examples/cross -o matrix.json
echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"
- name: 📋 Show matrix
run: cat matrix.json
# ── Stage 2: validate every cell (the PR gate) ──────────────────────────
# `--dry-run` resolves the plan with no download / mount / ssh, so each board
# validates on every host it declares — ubuntu directly, fedora in a
# container. This promotes the doc's L0/L1 surface from "parses" to "the real
# CLI plans this board end-to-end on this host".
validate:
needs: plan
runs-on: ${{ matrix.runs_on }}
container: ${{ matrix.container || '' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
steps:
# The fedora container is minimal: give setup-dart its extract tools and
# the packagekit_dart build hook a `git`/`cmake` so it fails *cleanly*
# (no C++ compiler installed → configure exits non-zero, the hook no-ops)
# rather than throwing on a missing executable. The dry-run itself never
# touches PackageKit.
- name: 🧱 Bootstrap container
if: ${{ matrix.container != '' }}
run: dnf -y install git tar xz unzip findutils which cmake ninja-build
- uses: actions/checkout@v5
- uses: dart-lang/setup-dart@v1
with:
sdk: "stable"
- run: dart pub get
- name: 🔎 Plan ${{ matrix.id }} / ${{ matrix.target }} on ${{ matrix.host }} (${{ matrix.provider }})
run: dart run emb_cli:emb cross ${{ matrix.args }} --dry-run
# ── Stage 3 (opt-in): real cross build ──────────────────────────────────
# Manual only (workflow_dispatch + build=true). Per docs/cross_integration.md
# §13, full builds are L2: they need network and host privileges (image
# loop-mount, qemu-static apt chroot, SDK install). The `yocto-recipe` /
# device-rsync providers additionally need state no hosted runner has — point
# `runs-on` at a self-hosted runner for those, or filter them out of the set.
build:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.build }}
needs: plan
runs-on: ${{ matrix.runs_on }}
container: ${{ matrix.container || '' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
env:
FLUTTER_WORKSPACE: ${{ github.workspace }}/.ws
steps:
- name: 🧱 Bootstrap container
if: ${{ matrix.container != '' }}
run: dnf -y install git tar xz unzip findutils which cmake ninja-build
- uses: actions/checkout@v5
- uses: dart-lang/setup-dart@v1
with:
sdk: "stable"
- run: dart pub get
# The toolchain + sysroot extraction is the expensive, content-addressed
# artifact — keyed on the cell's sysroot_key so cpu-only variants of one
# board (rpi4/rpi5 on one image) restore the same cache.
- name: ♻️ Restore toolchain + sysroot
uses: actions/cache@v4
with:
path: ${{ github.workspace }}/.ws/.config/flutter_workspace
key: cross-${{ matrix.provider }}-${{ matrix.sysroot_key }}
- name: 🧰 Install preflight tools (apt)
if: ${{ matrix.container == '' && matrix.preflight != '' }}
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.preflight }} qemu-user-static binfmt-support
- name: 🧰 Install preflight tools (dnf)
if: ${{ matrix.container != '' && matrix.preflight != '' }}
run: dnf -y install ${{ matrix.preflight }} qemu-user-static
- name: 🏗️ Build ${{ matrix.id }} / ${{ matrix.target }} on ${{ matrix.host }} (${{ matrix.provider }})
run: dart run emb_cli:emb cross ${{ matrix.args }} --build
+1
View File
@@ -47,6 +47,7 @@ class EmbCliCommandRunner extends CompletionCommandRunner<int> {
addCommand(BundleCommand(logger: _logger));
addCommand(BuildCommand(logger: _logger));
addCommand(CrossCommand(logger: _logger));
addCommand(MatrixCommand(logger: _logger));
addCommand(EnvCommand(logger: _logger));
addCommand(UpdateCommand(logger: _logger, pubUpdater: _pubUpdater));
}
+1
View File
@@ -7,6 +7,7 @@ export 'doctor_command.dart';
export 'engine_command.dart';
export 'env_command.dart';
export 'flutter_command.dart';
export 'matrix_command.dart';
export 'setup_command.dart';
export 'sync_command.dart';
export 'update_command.dart';
+233
View File
@@ -0,0 +1,233 @@
import 'dart:convert';
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:emb_cli/src/cross/cross_keys.dart';
import 'package:emb_cli/src/cross/cross_provider.dart';
import 'package:emb_cli/src/cross/cross_target.dart';
import 'package:emb_cli/src/host/host_info.dart';
import 'package:emb_cli/src/manifest/manifest_loader.dart';
import 'package:emb_cli/src/workspace/workspace.dart';
import 'package:mason_logger/mason_logger.dart';
import 'package:path/path.dart' as p;
/// {@template matrix_command}
/// `emb matrix <manifest|dir>...` — render a CI build matrix from the `cross:`
/// blocks of one or more manifests.
///
/// Emits GitHub Actions `{"include":[...]}` JSON: one cell per
/// (manifest × target × supported host). Each cell carries the `emb cross`
/// args to run, the provider/triple, the host runner (`runs_on`,
/// optionally `container`), the provider's preflight tools, and the
/// content-addressed `sysroot_key`/`build_key` for `actions/cache`.
///
/// Side-effect free — it only parses manifests (no download/mount/ssh), so it
/// runs on any host. Pair each cell with `emb cross <args> --dry-run` as a
/// per-cell PR gate, and (opt-in) `emb cross <args> --build` for the real
/// cross build keyed on the cache keys.
/// {@endtemplate}
class MatrixCommand extends Command<int> {
/// {@macro matrix_command}
MatrixCommand({
required Logger logger,
HostInfo? host,
ManifestLoader loader = const ManifestLoader(),
}) : _logger = logger,
_host = host,
_loader = loader {
argParser
..addMultiOption(
'host',
help:
'Restrict to these host types (intersected with each '
"manifest's supported_host_types). Default: every supported host.",
)
..addOption('target', help: 'Only emit the cells for this target name.')
..addOption(
'output',
abbr: 'o',
help:
'Write the JSON to this file instead of stdout (robust against '
'banners polluting a captured stdout).',
)
..addFlag(
'pretty',
help: 'Indent the JSON (stdout only).',
negatable: false,
);
}
final Logger _logger;
final HostInfo? _host;
final ManifestLoader _loader;
@override
String get name => 'matrix';
@override
String get description =>
'Render a CI build matrix from manifest cross: blocks.';
@override
Future<int> run() async {
final args = argResults!;
if (args.rest.isEmpty) {
_logger.err(
'Usage: emb matrix <manifest|dir>... [--host h] [--target t] [-o file]',
);
return ExitCode.usage.code;
}
final hostFilter = args['host'] as List<String>;
final targetFilter = args['target'] as String?;
final host = _host ?? HostInfo.detect();
final workspace = Workspace.resolve();
final include = <Map<String, dynamic>>[];
for (final file in _collect(args.rest)) {
final manifest = _loader.loadManifestFile(file);
if (manifest == null) {
_logger.warn('skip ${file.path}: not a manifest');
continue;
}
final crossMap = manifest.raw['cross'];
if (crossMap is! Map) continue; // not a cross manifest — silently skip
// Hosts: the manifest's declared support, narrowed by --host. Empty
// supported_host_types falls back to ubuntu so a terse manifest emits.
final supported = manifest.supportedHostTypes.isEmpty
? const ['ubuntu']
: manifest.supportedHostTypes;
final hosts = hostFilter.isEmpty
? supported
: hostFilter.where(supported.contains).toList();
if (hosts.isEmpty) continue;
for (final entry in _targets(crossMap)) {
if (targetFilter != null && entry.name != targetFilter) continue;
final CrossTarget target;
try {
target = CrossTarget.fromMap(entry.merged);
// fromMap throws ArgumentError on an unknown/missing provider token.
// ignore: avoid_catching_errors
} on ArgumentError catch (e) {
_logger.warn('skip ${file.path}#${entry.name}: ${e.message}');
continue;
}
final provider = CrossProvider.forTarget(
target,
workspace: workspace,
host: host,
);
// `default` is the single-target shape (no cross.targets); it needs no
// --target. A named target does.
final crossArgs = entry.name == 'default'
? file.path
: '${file.path} --target ${entry.name}';
for (final h in hosts) {
final runner = _runner(h);
include.add({
'id': manifest.id,
'manifest': file.path,
'target': entry.name,
'host': h,
'runs_on': runner.runsOn,
if (runner.container != null) 'container': runner.container,
'provider': target.provider.token,
'triple': target.triple ?? '',
'preflight': provider.preflightTools.join(' '),
'sysroot_key': sysrootKey(target),
'build_key': buildKey(target),
'args': crossArgs,
});
}
}
}
final payload = {'include': include};
final outPath = args['output'] as String?;
if (outPath != null) {
File(outPath).writeAsStringSync(jsonEncode(payload));
final n = include.length;
_logger.info('Wrote $outPath ($n cell${n == 1 ? "" : "s"}).');
} else {
_logger.info(
(args['pretty'] as bool)
? const JsonEncoder.withIndent(' ').convert(payload)
: jsonEncode(payload),
);
}
return ExitCode.success.code;
}
/// Expand a `cross:` map into its targets. A `cross.targets` map yields one
/// entry per named target (shared fields merged under each override); a flat
/// `cross:` yields a single synthetic `default` target.
List<({String name, Map<dynamic, dynamic> merged})> _targets(
Map<dynamic, dynamic> crossMap,
) {
final targets = crossMap['targets'];
if (targets is Map && targets.isNotEmpty) {
return [
for (final e in targets.entries)
if (e.value is Map)
(
name: e.key.toString(),
merged: <dynamic, dynamic>{
for (final s in crossMap.entries)
if (s.key != 'targets') s.key: s.value,
...e.value as Map,
},
),
];
}
return [(name: 'default', merged: Map<dynamic, dynamic>.from(crossMap))];
}
/// Gather manifest files from the given paths: a file is taken as-is; a
/// directory contributes its `emb.yaml` (if any) plus every `*.emb.yaml`
/// (sorted). Deduplicated by absolute path.
List<File> _collect(List<String> paths) {
final seen = <String>{};
final files = <File>[];
void add(File f) {
if (seen.add(p.absolute(f.path))) files.add(f);
}
for (final path in paths) {
final type = FileSystemEntity.typeSync(path);
if (type == FileSystemEntityType.file) {
add(File(path));
} else if (type == FileSystemEntityType.directory) {
final embYaml = File(p.join(path, 'emb.yaml'));
if (embYaml.existsSync()) add(embYaml);
Directory(path)
.listSync()
.whereType<File>()
.where((f) => f.path.endsWith('.emb.yaml'))
.toList()
..sort((a, b) => a.path.compareTo(b.path))
..forEach(add);
} else {
_logger.warn('skip: no such path $path');
}
}
return files;
}
/// Map a host type to a GitHub Actions runner. Distros without a hosted
/// runner ride an `ubuntu-latest` host inside a container image.
({String runsOn, String? container}) _runner(String hostType) {
if (hostType == 'fedora') {
return (runsOn: 'ubuntu-latest', container: 'fedora:41');
}
if (hostType == 'darwin' || hostType == 'macos') {
return (runsOn: 'macos-latest', container: null);
}
if (hostType == 'windows') {
return (runsOn: 'windows-latest', container: null);
}
return (runsOn: 'ubuntu-latest', container: null); // ubuntu/debian/default
}
}
+140
View File
@@ -0,0 +1,140 @@
import 'dart:convert';
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:emb_cli/src/commands/matrix_command.dart';
import 'package:emb_cli/src/host/host_info.dart';
import 'package:mason_logger/mason_logger.dart';
import 'package:path/path.dart' as p;
import 'package:test/test.dart';
const _host = HostInfo(
os: HostOs.linux,
machineArch: 'x86_64',
archAliases: {'x86_64', 'x64', 'amd64'},
hostType: 'fedora',
versionId: '43',
);
const _singleTarget = '''
id: ivi-homescreen
type: app
supported_archs: [arm64]
supported_host_types: [ubuntu, fedora]
cross:
provider: arm-gnu
triple: aarch64-none-linux-gnu
toolchain_version: 12.3.rel1
image_url: https://example.com/os.img.xz
cpu_flags: [-mcpu=cortex-a76]
''';
const _multiTarget = '''
id: ivi-homescreen
type: app
supported_host_types: [ubuntu]
cross:
provider: arm-gnu
toolchain_version: 12.3.rel1
targets:
rpi5:
image_url: https://example.com/raspios.img.xz
cpu_flags: [-mcpu=cortex-a76]
rpi4:
image_url: https://example.com/raspios.img.xz
cpu_flags: [-mcpu=cortex-a72]
''';
void main() {
late Directory tmp;
setUp(() => tmp = Directory.systemTemp.createTempSync('emb_matrix_'));
tearDown(() => tmp.deleteSync(recursive: true));
Future<int?> run(List<String> args) {
final runner = CommandRunner<int>('emb', 'test')
..addCommand(MatrixCommand(logger: Logger(), host: _host));
return runner.run(args);
}
File write(String name, String yaml) =>
File(p.join(tmp.path, name))..writeAsStringSync(yaml);
/// Render to a file (robust capture) and return its `include` list.
Future<List<Map<String, dynamic>>> render(List<String> args) async {
final outFile = p.join(tmp.path, 'matrix.json');
final code = await run(['matrix', ...args, '-o', outFile]);
expect(code, ExitCode.success.code);
final decoded = jsonDecode(File(outFile).readAsStringSync()) as Map;
return (decoded['include'] as List).cast<Map<String, dynamic>>();
}
test('usage error with no path argument', () async {
expect(await run(['matrix']), ExitCode.usage.code);
});
test('a single-target manifest fans out over its supported hosts', () async {
final f = write('pi5.emb.yaml', _singleTarget);
final include = await render([f.path]);
expect(include, hasLength(2)); // ubuntu + fedora
final ubuntu = include.firstWhere((e) => e['host'] == 'ubuntu');
expect(ubuntu['provider'], 'arm-gnu');
expect(ubuntu['triple'], 'aarch64-none-linux-gnu');
expect(ubuntu['target'], 'default');
expect(ubuntu['runs_on'], 'ubuntu-latest');
expect(ubuntu['args'], f.path); // no --target for a flat cross block
expect(ubuntu['preflight'], 'tar xz rsync'); // arm-gnu preflight tools
expect(ubuntu['sysroot_key'], isNotEmpty);
expect(ubuntu['build_key'], isNotEmpty);
final fedora = include.firstWhere((e) => e['host'] == 'fedora');
expect(fedora['runs_on'], 'ubuntu-latest');
expect(fedora['container'], 'fedora:41'); // no hosted fedora runner
});
test(
'cpu-only variants share a sysroot_key but differ on build_key',
() async {
final f = write('rpi.emb.yaml', _multiTarget);
final include = await render([f.path]);
expect(include, hasLength(2)); // rpi5, rpi4 (one host: ubuntu)
final rpi5 = include.firstWhere((e) => e['target'] == 'rpi5');
final rpi4 = include.firstWhere((e) => e['target'] == 'rpi4');
expect(rpi5['args'], '${f.path} --target rpi5');
// Same image → shared extraction; only cpu tuning differs.
expect(rpi5['sysroot_key'], rpi4['sysroot_key']);
expect(rpi5['build_key'], isNot(rpi4['build_key']));
},
);
test(
'--host narrows to the intersection with supported_host_types',
() async {
final f = write('pi5.emb.yaml', _singleTarget);
final include = await render([f.path, '--host', 'ubuntu']);
expect(include, hasLength(1));
expect(include.single['host'], 'ubuntu');
},
);
test('--target emits only the named target', () async {
final f = write('rpi.emb.yaml', _multiTarget);
final include = await render([f.path, '--target', 'rpi4']);
expect(include, hasLength(1));
expect(include.single['target'], 'rpi4');
});
test('a directory globs *.emb.yaml', () async {
write('pi5.emb.yaml', _singleTarget);
write('rpi.emb.yaml', _multiTarget);
final include = await render([tmp.path, '--host', 'ubuntu']);
expect(include, hasLength(3)); // pi5 default + rpi5 + rpi4
});
test('a manifest without a cross: block is skipped', () async {
final f = write('plain.emb.yaml', 'id: plain\ntype: app\n');
final include = await render([f.path]);
expect(include, isEmpty);
});
}