From 3db0af7a3cb07b93d01f238bdafaa22be22f5793 Mon Sep 17 00:00:00 2001 From: Alexander Kiselev Date: Sun, 25 Jan 2026 04:47:22 -0800 Subject: [PATCH] feat: Add comprehensive E2E test suite and fix CLI argument conflicts Add E2E test infrastructure: - DaemonTestHarness for managing daemon lifecycle in tests - Test fixtures and helpers in tests/common/ - Sample binary fixture for integration tests Add test coverage: - command_tests.rs: version, doctor, config commands - project_tests.rs: project create/list/info/delete, import, analyze - daemon_tests.rs: daemon start/status/ping/stop/clear-cache - query_tests.rs: function list, strings, memory, decompile, xref - unimplemented_tests.rs: 39 tests for graceful error messages Fix CLI bugs: - DisasmArgs: rename count to num_instructions (--instructions/-n) to avoid conflict with QueryOptions.count - GraphExportArgs: add unique arg id for format positional to avoid conflict with QueryOptions.format Co-Authored-By: Claude Opus 4.5 --- Cargo.lock | 2 + Cargo.toml | 6 + src/cli.rs | 7 +- tests/README.md | 192 +++++++++++++++++++++++ tests/command_tests.rs | 78 ++++++++++ tests/common/README.md | 163 +++++++++++++++++++ tests/common/mod.rs | 205 ++++++++++++++++++++++++ tests/daemon_tests.rs | 128 +++++++++++++++ tests/e2e.rs | 272 ++++---------------------------- tests/project_tests.rs | 171 ++++++++++++++++++++ tests/query_tests.rs | 292 +++++++++++++++++++++++++++++++++++ tests/unimplemented_tests.rs | 86 +++++++++++ 12 files changed, 1362 insertions(+), 240 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/command_tests.rs create mode 100644 tests/common/README.md create mode 100644 tests/common/mod.rs create mode 100644 tests/daemon_tests.rs create mode 100644 tests/project_tests.rs create mode 100644 tests/query_tests.rs create mode 100644 tests/unimplemented_tests.rs diff --git a/Cargo.lock b/Cargo.lock index efcdf52..aa4b2f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -795,6 +795,7 @@ dependencies = [ "lazy_static", "log", "md5", + "once_cell", "pest", "pest_derive", "predicates", @@ -813,6 +814,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "uuid", "walkdir", "which", "zip", diff --git a/Cargo.toml b/Cargo.toml index 19600f5..66f6bfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,12 @@ assert_cmd = "2.0" predicates = "3.0" tempfile = "3.8" serial_test = "3.0" +uuid = { version = "1.6", features = ["v4"] } +once_cell = "1.19" + +[lib] +name = "ghidra_cli" +path = "src/lib.rs" [[bin]] name = "ghidra" diff --git a/src/cli.rs b/src/cli.rs index 2446125..558c9b2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -512,6 +512,8 @@ pub struct GraphFunctionArgs { #[derive(Args, Clone, Serialize, Deserialize, Debug)] pub struct GraphExportArgs { + /// Export format (e.g., dot, json) + #[arg(id = "export_format")] pub format: String, #[command(flatten)] pub options: QueryOptions, @@ -527,8 +529,9 @@ pub struct DecompileArgs { #[derive(Args, Clone, Serialize, Deserialize, Debug)] pub struct DisasmArgs { pub address: String, - #[arg(long)] - pub count: Option, + /// Number of instructions to disassemble + #[arg(long = "instructions", short = 'n')] + pub num_instructions: Option, #[command(flatten)] pub options: QueryOptions, } diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..48793e5 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,192 @@ +# E2E Test Suite + +Comprehensive end-to-end test coverage for ghidra-cli commands. + +## Architecture + +Tests are organized by functional area into separate files: + +``` +tests/ +├── common/ +│ └── mod.rs # DaemonTestHarness, fixtures, helpers +├── daemon_tests.rs # Daemon lifecycle: start/stop/restart/status/ping +├── project_tests.rs # Project: create/list/delete/info +├── query_tests.rs # Function/strings/memory/xref/dump queries +├── command_tests.rs # Basic commands: version/doctor/config/init +├── unimplemented_tests.rs # Graceful error tests for stub commands +└── e2e.rs # Lightweight smoke test +``` + +## Per-Suite Daemon Lifecycle + +Tests requiring daemon interaction use `DaemonTestHarness` from `common/mod.rs`. Each test suite starts its own daemon instance to amortize 5-30s startup overhead across all tests in that file. + +**Why per-suite instead of per-test**: Starting a daemon for every test would add 5-30 minutes to CI time for 60+ tests. Per-suite daemons run tests serially within the suite but allow parallel execution across different test files. + +**Why not shared global daemon**: State leakage between suites causes flaky tests and debugging nightmares. Each suite gets isolation. + +## Data Flow + +``` +Test Suite Start + | + v +DaemonTestHarness::new() + | + +---> Start daemon process + +---> Wait for IPC socket + +---> Verify with ping + | + v +Run tests (serial within suite) + | + v +DaemonTestHarness::drop() + | + +---> Send shutdown command + +---> Wait for process exit + +---> Cleanup socket file +``` + +## Running Tests + +Run all tests: +```bash +cargo test +``` + +Run specific test suite: +```bash +cargo test --test daemon_tests +cargo test --test query_tests +cargo test --test command_tests +``` + +Run single test: +```bash +cargo test --test query_tests test_function_list +``` + +Skip daemon tests (faster): +```bash +cargo test --test command_tests +cargo test --test project_tests --lib +``` + +## Test Requirements + +### Ghidra Installation + +Tests check for Ghidra availability using `skip_if_no_ghidra!()` macro. Tests skip with clear message if `ghidra doctor` fails. + +### Test Fixtures + +Sample binary fixture required: `tests/fixtures/sample_binary` + +Build fixture: +```bash +rustc --edition 2021 -o tests/fixtures/sample_binary tests/fixtures/sample_binary.rs +``` + +Fixture contains functions: add, multiply, factorial, fibonacci, process_string, xor_encrypt, simple_hash, init_data, main + +## Adding New Tests + +### Non-Daemon Tests + +Add to appropriate file (`command_tests.rs`, `project_tests.rs`): + +```rust +#[test] +fn test_my_command() { + skip_if_no_ghidra!(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("my-command") + .assert() + .success(); +} +``` + +### Daemon-Dependent Tests + +Add to `daemon_tests.rs` or `query_tests.rs`: + +```rust +#[test] +#[serial] +fn test_my_query() { + skip_if_no_ghidra!(); + + let harness = &*HARNESS; // Shared daemon instance + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("my-query") + .arg("--project").arg(TEST_PROJECT) + .arg("--program").arg(TEST_PROGRAM) + .assert() + .success(); +} +``` + +Mark with `#[serial]` to prevent daemon state races within suite. + +## Invariants + +- Daemon must be fully started before any query test runs +- Each test suite gets its own daemon instance (no sharing) +- Socket files must be cleaned up even on test failure (Drop impl) +- Tests must not assume specific function addresses (use name-based lookups) + +## Troubleshooting + +### "Daemon failed to start within 120s timeout" + +Ghidra cold start can be slow. Ensure: +- Ghidra installation is valid (`ghidra doctor`) +- Sufficient disk space for temporary Ghidra project +- Not running on extremely constrained CI resources + +### "Test fixture not found" + +Compile sample_binary: +```bash +rustc --edition 2021 -o tests/fixtures/sample_binary tests/fixtures/sample_binary.rs +``` + +### "Socket already in use" / "Address in use" + +Each test suite generates UUID-based socket path to prevent collisions. This error suggests: +- Previous test run leaked daemon process (kill manually) +- Filesystem issue preventing socket cleanup + +Find leaked processes: +```bash +ps aux | grep ghidra +kill +``` + +### Tests hang or timeout + +- Check if Ghidra daemon is stuck (check process list) +- Verify network/IPC permissions for Unix sockets +- Increase timeout in test (daemon startup can vary) + +### Import/analysis takes too long + +Tests use 300s timeout for import/analyze operations. On slow systems: +- Run fewer parallel test suites +- Ensure Ghidra has adequate heap memory +- Check disk I/O performance + +## Tradeoffs + +**Per-suite vs per-test daemon**: Chose speed over maximum isolation. Tests within suite are serial, but suite-to-suite parallelism maintained. + +**UUID socket paths vs fixed paths**: Chose reliability over simplicity. Guarantees uniqueness even with PID wrap on long-running CI. + +**Testing unimplemented commands**: Adds maintenance burden but documents gaps and ensures graceful failures for stub commands. diff --git a/tests/command_tests.rs b/tests/command_tests.rs new file mode 100644 index 0000000..147b542 --- /dev/null +++ b/tests/command_tests.rs @@ -0,0 +1,78 @@ +//! Tests for basic CLI commands that don't require daemon. + +use assert_cmd::Command; +use predicates::prelude::*; + +mod common; + +#[test] +fn test_version() { + Command::cargo_bin("ghidra") + .unwrap() + .arg("version") + .assert() + .success() + .stdout(predicate::str::contains("ghidra-cli")); +} + +#[test] +fn test_doctor() { + Command::cargo_bin("ghidra") + .unwrap() + .arg("doctor") + .assert() + .success() + .stdout(predicate::str::contains("Ghidra CLI Doctor")); +} + +#[test] +fn test_config_list() { + Command::cargo_bin("ghidra") + .unwrap() + .arg("config") + .arg("list") + .assert() + .success() + .stdout(predicate::str::contains("ghidra_install_dir")); +} + +#[test] +fn test_config_get() { + Command::cargo_bin("ghidra") + .unwrap() + .arg("config") + .arg("get") + .arg("ghidra_install_dir") + .assert() + .success(); +} + +#[test] +fn test_config_set() { + let temp = tempfile::tempdir().unwrap(); + let config_path = temp.path().join("config.yaml"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_CONFIG", &config_path) + .arg("config") + .arg("set") + .arg("default_output_format") + .arg("json") + .assert() + .success(); +} + +#[test] +fn test_config_reset() { + let temp = tempfile::tempdir().unwrap(); + let config_path = temp.path().join("config.yaml"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_CONFIG", &config_path) + .arg("config") + .arg("reset") + .assert() + .success(); +} diff --git a/tests/common/README.md b/tests/common/README.md new file mode 100644 index 0000000..747a9d4 --- /dev/null +++ b/tests/common/README.md @@ -0,0 +1,163 @@ +# Common Test Utilities + +Shared infrastructure for E2E tests. + +## DaemonTestHarness + +Manages daemon lifecycle for test suites requiring Ghidra daemon interaction. + +### Usage + +```rust +use common::{DaemonTestHarness, ensure_test_project}; + +const TEST_PROJECT: &str = "my-test"; +const TEST_PROGRAM: &str = "sample_binary"; + +#[test] +#[serial] +fn test_with_daemon() { + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + + let harness = DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon"); + + let mut client = harness.client().unwrap(); + // Use client for IPC calls + + // Daemon automatically shuts down when harness drops +} +``` + +### Shared Daemon Pattern + +For multiple tests in same suite: + +```rust +use once_cell::sync::Lazy; + +static HARNESS: Lazy = Lazy::new(|| { + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon") +}); + +#[test] +#[serial] +fn test_one() { + let harness = &*HARNESS; + // Use harness +} + +#[test] +#[serial] +fn test_two() { + let harness = &*HARNESS; + // Same daemon instance +} +``` + +All tests using shared daemon must be marked `#[serial]` to prevent state races. + +### Why Runtime Field Exists + +`DaemonTestHarness` contains a `tokio::runtime::Runtime` field to: + +1. **Prevent panic-during-panic**: Creating Runtime during Drop panic unwinding causes abort. Pre-created runtime allows safe cleanup. +2. **Amortize overhead**: Runtime creation takes ~10ms. Reusing across all async operations saves time. + +### Socket Path Isolation + +Each harness instance generates UUID-based Unix socket path: +``` +/tmp/ghidra-test-.sock +``` + +UUID prevents collisions: +- Between parallel test suites +- Across test runs on long-running CI (PID can wrap) + +### Cleanup Guarantees + +Drop implementation ensures best-effort cleanup: +1. Send shutdown via IPC (ignores errors) +2. Wait up to 5s for graceful exit +3. Kill process if still running +4. Remove socket file + +Accepts minor leak risk on panic-during-panic (rare edge case). + +## Fixtures + +### fixture_binary() + +Returns path to compiled sample_binary fixture. + +```rust +let binary = fixture_binary(); +assert!(binary.exists()); +``` + +Binary must be compiled before tests: +```bash +rustc --edition 2021 -o tests/fixtures/sample_binary tests/fixtures/sample_binary.rs +``` + +### ensure_test_project() + +Idempotent project setup using `Once::call_once`. Imports and analyzes sample_binary if needed. + +```rust +ensure_test_project("my-project", "sample_binary"); +// Second call does nothing - project already exists +``` + +Handles "already exists" errors gracefully. Safe to call from multiple tests. + +## skip_if_no_ghidra! Macro + +Tests should call this macro to skip gracefully when Ghidra unavailable: + +```rust +#[test] +fn test_something() { + skip_if_no_ghidra!(); + + // Test code runs only if ghidra doctor succeeds +} +``` + +Runs `ghidra doctor` and returns early if fails. Prints message: "Skipping test: Ghidra not available" + +## Design Decisions + +### Exponential Backoff Parameters + +`wait_for_ready()` uses: +- Initial delay: 100ms (responsive for fast starts) +- Multiplier: 2x +- Max attempts: 12 +- Total timeout: 120s + +Covers 100ms to ~200s range. Typical fast start exits in <5s. + +### ChildGuard Pattern + +`DaemonTestHarness::new()` uses ChildGuard to prevent daemon process leaks: + +```rust +struct ChildGuard(Option); +impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + } + } +} +``` + +If `wait_for_ready()` returns early due to error, ChildGuard ensures daemon process is killed. Without this, failed initialization leaks processes. + +### Why 5s Shutdown Timeout + +Most daemons shut down in <1s. 5s allows graceful cleanup without blocking tests indefinitely. If daemon hangs, hard kill prevents test suite deadlock. diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..405d818 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,205 @@ +//! Common test utilities for E2E tests. + +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::sync::Once; +use std::time::Duration; + +/// Get path to the sample_binary test fixture. +pub fn fixture_binary() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join("sample_binary") +} + +/// Ensure test project exists with analyzed sample binary. +/// Uses Once::call_once for idempotent setup across multiple tests. +pub fn ensure_test_project(project: &str, program: &str) { + static SETUP: Once = Once::new(); + SETUP.call_once(|| { + let binary = fixture_binary(); + if !binary.exists() { + panic!( + "Test fixture not found: {:?}\nRun: rustc --edition 2021 -o tests/fixtures/sample_binary tests/fixtures/sample_binary.rs", + binary + ); + } + + eprintln!("=== Setting up test project (import + analyze) ==="); + + let mut cmd = assert_cmd::Command::cargo_bin("ghidra").expect("Failed to find ghidra binary"); + let result = cmd + .arg("import") + .arg(binary.to_str().unwrap()) + .arg("--project") + .arg(project) + .arg("--program") + .arg(program) + .timeout(std::time::Duration::from_secs(300)) + .output() + .expect("Failed to run import command"); + + if !result.status.success() { + let stderr = String::from_utf8_lossy(&result.stderr); + let stdout = String::from_utf8_lossy(&result.stdout); + eprintln!("Import stdout: {}", stdout); + eprintln!("Import stderr: {}", stderr); + if !stderr.contains("already exists") && !stdout.contains("already exists") { + eprintln!("Warning: Import may have failed, but continuing..."); + } + } else { + eprintln!("Binary imported successfully"); + } + + eprintln!("=== Test project setup complete ==="); + }); +} + +/// Test harness that manages daemon lifecycle for a test suite. +pub struct DaemonTestHarness { + child: Child, + socket_path: PathBuf, + project: String, + // Runtime field prevents panic-during-panic in Drop (cannot create Runtime during panic unwinding) + // and amortizes Runtime creation overhead across all async operations in this harness. + runtime: tokio::runtime::Runtime, +} + +impl DaemonTestHarness { + /// Start daemon for testing. Blocks until daemon is ready or timeout. + pub fn new(project: &str, program: &str) -> Result { + let socket_path = get_unique_socket_path(); + + let mut cmd = Command::new(env!("CARGO_BIN_EXE_ghidra")); + cmd.env("GHIDRA_CLI_SOCKET", &socket_path) + .arg("daemon") + .arg("start") + .arg("--foreground") + .arg("--project") + .arg(project); + + let child = cmd.spawn().context("Failed to spawn daemon")?; + + // ChildGuard ensures daemon process is killed if wait_for_ready() returns early due to error. + // Without this, failed initialization would leak daemon processes. + struct ChildGuard(Option); + impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + } + } + } + let mut guard = ChildGuard(Some(child)); + + let runtime = tokio::runtime::Runtime::new() + .context("Failed to create tokio runtime")?; + + let mut harness = Self { + child: guard.0.take().unwrap(), + socket_path, + project: project.to_string(), + runtime, + }; + + // 120s timeout: Ghidra cold start can be slow on constrained CI environments. + // Covers worst case without causing flaky tests. + harness.wait_for_ready(Duration::from_secs(120))?; + + Ok(harness) + } + + /// Wait for daemon to be ready using exponential backoff. + fn wait_for_ready(&mut self, timeout: Duration) -> Result<()> { + let start = std::time::Instant::now(); + // Exponential backoff: 100ms initial (responsive for fast starts), 2x multiplier, 12 max attempts. + // Covers 100ms to ~200s range; total max wait ~409s but typical fast start exits in <5s. + let mut delay = Duration::from_millis(100); + let max_attempts = 12; + + for attempt in 0..max_attempts { + if start.elapsed() > timeout { + anyhow::bail!("Daemon failed to start within {}s timeout", timeout.as_secs()); + } + + std::thread::sleep(delay); + + if let Ok(mut client) = self.client() { + match self.runtime.block_on(client.ping()) { + Ok(true) => return Ok(()), + Ok(false) => {}, + Err(e) => { + if attempt == max_attempts - 1 { + anyhow::bail!("Connection error during ping: {}", e); + } + } + } + } + + delay = delay.saturating_mul(2); + } + + anyhow::bail!("Daemon failed to respond after {} attempts", max_attempts) + } + + /// Get async IPC client connected to daemon. + pub fn client(&self) -> Result { + self.runtime.block_on(async { + ghidra_cli::ipc::client::DaemonClient::connect().await + }) + } + + /// Get socket path for this daemon instance. + pub fn socket_path(&self) -> &PathBuf { + &self.socket_path + } + + /// Get project name. + pub fn project(&self) -> &str { + &self.project + } +} + +impl Drop for DaemonTestHarness { + fn drop(&mut self) { + if let Ok(mut client) = self.client() { + let _ = self.runtime.block_on(client.shutdown()); + } + + // 5s wait before kill: allows graceful shutdown to complete. + // Most daemons shut down in <1s; 5s handles slow cleanup without blocking tests indefinitely. + let timeout = Duration::from_secs(5); + let start = std::time::Instant::now(); + + while start.elapsed() < timeout { + if let Ok(Some(_)) = self.child.try_wait() { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + + let _ = self.child.kill(); + let _ = std::fs::remove_file(&self.socket_path); + } +} + +/// Generate unique socket path for test isolation. +/// +/// UUID guarantees uniqueness across parallel test suites and long-running CI (PID can wrap). +fn get_unique_socket_path() -> PathBuf { + std::env::temp_dir().join(format!("ghidra-test-{}.sock", uuid::Uuid::new_v4())) +} + +/// Skip test if Ghidra is not available. +#[macro_export] +macro_rules! skip_if_no_ghidra { + () => { + let doctor = assert_cmd::Command::cargo_bin("ghidra").unwrap().arg("doctor").output(); + if doctor.is_err() || !doctor.unwrap().status.success() { + eprintln!("Skipping test: Ghidra not available"); + return; + } + }; +} diff --git a/tests/daemon_tests.rs b/tests/daemon_tests.rs new file mode 100644 index 0000000..5765cee --- /dev/null +++ b/tests/daemon_tests.rs @@ -0,0 +1,128 @@ +//! Tests for daemon lifecycle commands. + +use assert_cmd::Command; +use predicates::prelude::*; +use serial_test::serial; + +#[macro_use] +mod common; +use common::{ensure_test_project, DaemonTestHarness}; + +const TEST_PROJECT: &str = "daemon-test"; +const TEST_PROGRAM: &str = "sample_binary"; + +#[test] +#[serial] +fn test_daemon_start() { + + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + + let harness = DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("status") + .assert() + .success(); + + drop(harness); +} + +#[test] +#[serial] +fn test_daemon_status() { + + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + + let harness = DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("status") + .assert() + .success() + .stdout(predicate::str::contains("running")); + + drop(harness); +} + +#[test] +#[serial] +fn test_daemon_ping() { + + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + + let harness = DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("ping") + .assert() + .success(); + + drop(harness); +} + +#[test] +#[serial] +fn test_daemon_clear_cache() { + + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + + let harness = DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("clear-cache") + .assert() + .success(); + + drop(harness); +} + +#[test] +#[serial] +fn test_daemon_lifecycle() { + + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + + let harness = DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM) + .expect("Failed to start daemon"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("status") + .assert() + .success() + .stdout(predicate::str::contains("running")); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("ping") + .assert() + .success(); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("daemon") + .arg("stop") + .assert() + .success(); +} diff --git a/tests/e2e.rs b/tests/e2e.rs index e616165..cf89bbf 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1,245 +1,41 @@ -//! End-to-end tests for ghidra-cli +//! End-to-end smoke tests for ghidra-cli //! -//! These tests require a working Ghidra installation. The test project -//! is set up automatically on first run. +//! This is a lightweight smoke test that verifies basic CLI functionality. +//! Comprehensive test coverage is in: +//! - command_tests.rs (version, doctor, config) +//! - project_tests.rs (project management, import, analyze) +//! - daemon_tests.rs (daemon lifecycle) +//! - query_tests.rs (function, strings, memory, decompile, dump) +//! - unimplemented_tests.rs (graceful error messages) use assert_cmd::Command; use predicates::prelude::*; -use serial_test::serial; -use std::path::PathBuf; -use std::sync::Once; -static SETUP: Once = Once::new(); -static PROJECT_NAME: &str = "e2e-test"; -static PROGRAM_NAME: &str = "sample_binary"; +mod common; -/// Get the path to the test fixture binary -fn fixture_binary() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests") - .join("fixtures") - .join("sample_binary") -} - -/// Ensure the test project is set up (import + analyze the sample binary). -/// This runs only once per test run, regardless of how many tests call it. -fn ensure_project_setup() { - SETUP.call_once(|| { - let binary = fixture_binary(); - if !binary.exists() { - panic!( - "Test fixture not found: {:?}\nRun: rustc --edition 2021 -o tests/fixtures/sample_binary tests/fixtures/sample_binary.rs", - binary - ); - } - - eprintln!("=== Setting up E2E test project (import + analyze) ==="); - - // Import the binary - let mut cmd = Command::cargo_bin("ghidra").expect("Failed to find ghidra binary"); - let result = cmd - .arg("import") - .arg(binary.to_str().unwrap()) - .arg("--project") - .arg(PROJECT_NAME) - .arg("--program") - .arg(PROGRAM_NAME) - .timeout(std::time::Duration::from_secs(300)) - .output() - .expect("Failed to run import command"); - - if !result.status.success() { - let stderr = String::from_utf8_lossy(&result.stderr); - let stdout = String::from_utf8_lossy(&result.stdout); - eprintln!("Import stdout: {}", stdout); - eprintln!("Import stderr: {}", stderr); - // Don't panic - project might already exist - if !stderr.contains("already exists") && !stdout.contains("already exists") { - eprintln!("Warning: Import may have failed, but continuing..."); - } - } else { - eprintln!("Binary imported successfully"); - } - - eprintln!("=== E2E test project setup complete ==="); - }); -} - -mod e2e_tests { - use super::*; - - /// Test that doctor command works - #[test] - fn test_doctor() { - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("doctor") - .assert() - .success() - .stdout(predicate::str::contains("Ghidra CLI Doctor")); - } - - /// Test version command - #[test] - fn test_version() { - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("version") - .assert() - .success() - .stdout(predicate::str::contains("ghidra-cli")); - } - - /// Test config list command - #[test] - fn test_config_list() { - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("config") - .arg("list") - .assert() - .success() - .stdout(predicate::str::contains("ghidra_install_dir")); - } - - /// Test import command with sample binary - #[test] - #[serial] - fn test_import_binary() { - let binary = fixture_binary(); - if !binary.exists() { - panic!( - "Test fixture not found. Run: rustc --edition 2021 -o tests/fixtures/sample_binary tests/fixtures/sample_binary.rs" - ); - } - - // Use a unique project name for this test - let project = format!("e2e-import-{}", std::process::id()); - - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("import") - .arg(binary.to_str().unwrap()) - .arg("--project") - .arg(&project) - .arg("--program") - .arg("sample_binary") - .timeout(std::time::Duration::from_secs(300)) - .assert() - .success() - .stdout(predicate::str::contains("Successfully imported")); - } - - /// Test function list command on pre-analyzed binary - /// NOTE: This test requires the daemon to be running. Skipped pending daemon E2E test infrastructure. - #[test] - #[serial] - #[ignore = "Requires daemon to be running. Run with --ignored to include daemon tests."] - fn test_function_list() { - ensure_project_setup(); - - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("function") - .arg("list") - .arg("--project") - .arg(PROJECT_NAME) - .arg("--program") - .arg(PROGRAM_NAME) - .arg("--limit") - .arg("100") - .timeout(std::time::Duration::from_secs(300)) - .assert() - .success() - // Check for our known exported functions - .stdout(predicate::str::contains("main")) - .stdout( - predicate::str::contains("fibonacci").or(predicate::str::contains("factorial")), - ); - } - - /// Test decompile command - /// NOTE: This test requires the daemon to be running. - #[test] - #[serial] - #[ignore = "Requires daemon to be running. Run with --ignored to include daemon tests."] - fn test_decompile() { - ensure_project_setup(); - - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("decompile") - .arg("main") // Decompile main function - .arg("--project") - .arg(PROJECT_NAME) - .arg("--program") - .arg(PROGRAM_NAME) - .timeout(std::time::Duration::from_secs(300)) - .assert() - .success() - // Should contain decompiled C code - .stdout(predicate::str::contains("void").or(predicate::str::contains("int"))); - } - - /// Test strings command - /// NOTE: This test requires the daemon to be running. - #[test] - #[serial] - #[ignore = "Requires daemon to be running. Run with --ignored to include daemon tests."] - fn test_strings() { - ensure_project_setup(); - - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("strings") - .arg("list") - .arg("--project") - .arg(PROJECT_NAME) - .arg("--program") - .arg(PROGRAM_NAME) - .arg("--limit") - .arg("100") // Increase limit to find our test strings - .timeout(std::time::Duration::from_secs(300)) - .assert() - .success() - // Check for strings that exist in a typical ELF binary - // (libc symbols are reliably present) - .stdout(predicate::str::contains("address")) - .stdout(predicate::str::contains("value")); - } - - /// Test memory map command - /// NOTE: This test requires the daemon to be running. - #[test] - #[serial] - #[ignore = "Requires daemon to be running. Run with --ignored to include daemon tests."] - fn test_memory_map() { - ensure_project_setup(); - - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("memory") - .arg("map") - .arg("--project") - .arg(PROJECT_NAME) - .arg("--program") - .arg(PROGRAM_NAME) - .timeout(std::time::Duration::from_secs(300)) - .assert() - .success() - // Should show memory sections - .stdout(predicate::str::contains(".text").or(predicate::str::contains("r"))); - } - - /// Test summary command - /// NOTE: This test requires the daemon to be running. - #[test] - #[serial] - #[ignore = "Requires daemon to be running. Run with --ignored to include daemon tests."] - fn test_summary() { - ensure_project_setup(); - - let mut cmd = Command::cargo_bin("ghidra").unwrap(); - cmd.arg("summary") - .arg("--project") - .arg(PROJECT_NAME) - .arg("--program") - .arg(PROGRAM_NAME) - .timeout(std::time::Duration::from_secs(300)) - .assert() - .success() - .stdout(predicate::str::contains("Program Summary")); - } +/// Smoke test - verifies basic CLI commands work +#[test] +fn test_smoke() { + // Version command should always work + Command::cargo_bin("ghidra") + .unwrap() + .arg("version") + .assert() + .success() + .stdout(predicate::str::contains("ghidra-cli")); + + // Doctor command verifies installation + Command::cargo_bin("ghidra") + .unwrap() + .arg("doctor") + .assert() + .success(); + + // Config list should work + Command::cargo_bin("ghidra") + .unwrap() + .arg("config") + .arg("list") + .assert() + .success(); } diff --git a/tests/project_tests.rs b/tests/project_tests.rs new file mode 100644 index 0000000..dced1ce --- /dev/null +++ b/tests/project_tests.rs @@ -0,0 +1,171 @@ +//! Tests for project management commands. + +use assert_cmd::Command; +use predicates::prelude::*; +use serial_test::serial; + +mod common; + +/// Generate unique project name for test isolation. +/// UUID prevents collisions in parallel CI runs. +fn unique_project_name(prefix: &str) -> String { + format!("test-{}-{}", prefix, uuid::Uuid::new_v4()) +} + +#[test] +fn test_project_create() { + let project = unique_project_name("create"); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("create") + .arg(&project) + .assert() + .success() + .stdout(predicate::str::contains("Created project")); + + // Cleanup + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("delete") + .arg(&project) + .assert() + .success(); +} + +#[test] +fn test_project_list() { + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("list") + .assert() + .success(); +} + +#[test] +fn test_project_info() { + let project = unique_project_name("info"); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("create") + .arg(&project) + .assert() + .success(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("info") + .arg(&project) + .assert() + .success(); + + // Cleanup + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("delete") + .arg(&project) + .assert() + .success(); +} + +#[test] +fn test_project_lifecycle() { + let project = unique_project_name("lifecycle"); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("create") + .arg(&project) + .assert() + .success(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("list") + .assert() + .success() + .stdout(predicate::str::contains(&project)); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("delete") + .arg(&project) + .assert() + .success(); +} + +#[test] +#[serial] +fn test_import_binary() { + let project = unique_project_name("import"); + let binary = common::fixture_binary(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("import") + .arg(binary.to_str().unwrap()) + .arg("--project") + .arg(&project) + .arg("--program") + .arg("sample_binary") + .timeout(std::time::Duration::from_secs(300)) + .assert() + .success() + .stdout(predicate::str::contains("Successfully imported")); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("delete") + .arg(&project) + .assert() + .success(); +} + +#[test] +#[serial] +fn test_analyze_program() { + let project = unique_project_name("analyze"); + let binary = common::fixture_binary(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("import") + .arg(binary.to_str().unwrap()) + .arg("--project") + .arg(&project) + .arg("--program") + .arg("sample_binary") + .timeout(std::time::Duration::from_secs(300)) + .assert() + .success(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("analyze") + .arg("--project") + .arg(&project) + .arg("--program") + .arg("sample_binary") + .timeout(std::time::Duration::from_secs(300)) + .assert() + .success(); + + Command::cargo_bin("ghidra") + .unwrap() + .arg("project") + .arg("delete") + .arg(&project) + .assert() + .success(); +} diff --git a/tests/query_tests.rs b/tests/query_tests.rs new file mode 100644 index 0000000..471537e --- /dev/null +++ b/tests/query_tests.rs @@ -0,0 +1,292 @@ +//! Tests for query commands that require daemon. + +use assert_cmd::Command; +use once_cell::sync::Lazy; +use predicates::prelude::*; +use serial_test::serial; + +#[macro_use] +mod common; +use common::{ensure_test_project, DaemonTestHarness}; + +const TEST_PROJECT: &str = "query-test"; +const TEST_PROGRAM: &str = "sample_binary"; + +static HARNESS: Lazy = Lazy::new(|| { + ensure_test_project(TEST_PROJECT, TEST_PROGRAM); + DaemonTestHarness::new(TEST_PROJECT, TEST_PROGRAM).expect("Failed to start daemon") +}); + +#[test] +#[serial] +fn test_function_list() { + let harness = &*HARNESS; + + let output = Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("function") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success() + .get_output() + .stdout + .clone(); + + let stdout = String::from_utf8_lossy(&output); + assert!(stdout.contains("main")); + assert!(stdout.contains("fibonacci") || stdout.contains("factorial")); +} + +#[test] +#[serial] +fn test_function_list_limit() { + let harness = &*HARNESS; + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("function") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .arg("--limit") + .arg("5") + .assert() + .success(); +} + +#[test] +#[serial] +fn test_function_list_filter() { + let harness = &*HARNESS; + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("function") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .arg("--filter") + .arg("main") + .assert() + .success() + .stdout(predicate::str::contains("main")); +} + +#[test] +#[serial] +fn test_strings_list() { + let harness = &*HARNESS; + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("strings") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .arg("--limit") + .arg("100") + .assert() + .success() + .stdout(predicate::str::contains("address")); +} + +#[test] +#[serial] +fn test_memory_map() { + let harness = &*HARNESS; + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("memory") + .arg("map") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success() + .stdout(predicate::str::contains(".text").or(predicate::str::contains("r"))); +} + +#[test] +#[serial] +fn test_summary() { + let harness = &*HARNESS; + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("summary") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success() + .stdout(predicate::str::contains("Program Summary")); +} + +#[test] +#[serial] +fn test_decompile_by_name() { + let harness = &*HARNESS; + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("decompile") + .arg("main") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success() + .stdout(predicate::str::contains("void").or(predicate::str::contains("int"))); +} + +#[test] +#[serial] +fn test_decompile_by_address() { + let harness = &*HARNESS; + + let output = Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("function") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .arg("--format") + .arg("json") + .assert() + .success() + .get_output() + .stdout + .clone(); + + let stdout = String::from_utf8_lossy(&output); + let functions: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let main_addr = functions + .as_array() + .and_then(|arr| arr.iter().find(|f| f["name"].as_str() == Some("main"))) + .and_then(|f| f["address"].as_str()) + .expect("Could not find main function address"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("decompile") + .arg(main_addr) + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success(); +} + +#[test] +#[serial] +fn test_xref_to() { + let harness = &*HARNESS; + + let output = Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("function") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .arg("--format") + .arg("json") + .assert() + .success() + .get_output() + .stdout + .clone(); + + let stdout = String::from_utf8_lossy(&output); + let functions: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let main_addr = functions + .as_array() + .and_then(|arr| arr.iter().find(|f| f["name"].as_str() == Some("main"))) + .and_then(|f| f["address"].as_str()) + .expect("Could not find main function address"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("xref") + .arg("to") + .arg(main_addr) + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success(); +} + +#[test] +#[serial] +fn test_xref_from() { + let harness = &*HARNESS; + + let output = Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("function") + .arg("list") + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .arg("--format") + .arg("json") + .assert() + .success() + .get_output() + .stdout + .clone(); + + let stdout = String::from_utf8_lossy(&output); + let functions: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let main_addr = functions + .as_array() + .and_then(|arr| arr.iter().find(|f| f["name"].as_str() == Some("main"))) + .and_then(|f| f["address"].as_str()) + .expect("Could not find main function address"); + + Command::cargo_bin("ghidra") + .unwrap() + .env("GHIDRA_CLI_SOCKET", harness.socket_path()) + .arg("xref") + .arg("from") + .arg(main_addr) + .arg("--project") + .arg(TEST_PROJECT) + .arg("--program") + .arg(TEST_PROGRAM) + .assert() + .success(); +} diff --git a/tests/unimplemented_tests.rs b/tests/unimplemented_tests.rs new file mode 100644 index 0000000..485ebe9 --- /dev/null +++ b/tests/unimplemented_tests.rs @@ -0,0 +1,86 @@ +//! Tests for unimplemented commands to ensure graceful error messages. +//! +//! These tests verify that unimplemented commands print a helpful message +//! instead of crashing or panicking. +//! +//! NOTE: Current CLI outputs to stdout with exit 0. This should eventually +//! be changed to stderr with exit 1 for proper error handling. + +use assert_cmd::Command; +use predicates::prelude::*; + +// Macro reduces boilerplate for unimplemented command tests. +macro_rules! test_unimplemented { + ($name:ident, $($arg:expr),*) => { + #[test] + fn $name() { + Command::cargo_bin("ghidra").unwrap() + $(.arg($arg))* + .assert() + .success() // CLI currently exits 0 for unimplemented + .stdout(predicate::str::contains("not yet implemented") + .or(predicate::str::contains("Command not yet implemented"))); + } + }; +} + +// Program commands (use --program flag) +test_unimplemented!(test_program_close, "program", "close", "--program", "test"); +test_unimplemented!(test_program_delete, "program", "delete", "--program", "test"); +test_unimplemented!(test_program_info, "program", "info", "--program", "test"); +test_unimplemented!(test_program_export, "program", "export", "--program", "test", "json"); + +// Symbol commands (use positional args) +test_unimplemented!(test_symbol_list, "symbol", "list"); +test_unimplemented!(test_symbol_get, "symbol", "get", "0x1000"); +test_unimplemented!(test_symbol_create, "symbol", "create", "0x1000", "test_sym"); +test_unimplemented!(test_symbol_delete, "symbol", "delete", "test_sym"); +test_unimplemented!(test_symbol_rename, "symbol", "rename", "test_sym", "new_sym"); + +// Type commands (use positional args) +test_unimplemented!(test_type_list, "type", "list"); +test_unimplemented!(test_type_get, "type", "get", "int"); +test_unimplemented!(test_type_create, "type", "create", "my_struct"); +test_unimplemented!(test_type_apply, "type", "apply", "0x1000", "int"); + +// Comment commands (use positional args) +test_unimplemented!(test_comment_list, "comment", "list"); +test_unimplemented!(test_comment_get, "comment", "get", "0x1000"); +test_unimplemented!(test_comment_set, "comment", "set", "0x1000", "test"); +test_unimplemented!(test_comment_delete, "comment", "delete", "0x1000"); + +// Find commands +test_unimplemented!(test_find_string, "find", "string", "test"); +test_unimplemented!(test_find_bytes, "find", "bytes", "deadbeef"); +test_unimplemented!(test_find_function, "find", "function", "test"); +test_unimplemented!(test_find_calls, "find", "calls", "test"); +test_unimplemented!(test_find_crypto, "find", "crypto"); +test_unimplemented!(test_find_interesting, "find", "interesting"); + +// Graph commands +test_unimplemented!(test_graph_calls, "graph", "calls"); +test_unimplemented!(test_graph_callers, "graph", "callers", "main"); +test_unimplemented!(test_graph_callees, "graph", "callees", "main"); +test_unimplemented!(test_graph_export, "graph", "export", "dot"); + +// Diff commands +test_unimplemented!(test_diff_programs, "diff", "programs", "prog1", "prog2"); +test_unimplemented!(test_diff_functions, "diff", "functions"); + +// Patch commands +test_unimplemented!(test_patch_bytes, "patch", "bytes", "0x1000", "deadbeef"); +test_unimplemented!(test_patch_nop, "patch", "nop", "0x1000"); +test_unimplemented!(test_patch_export, "patch", "export", "--output", "test.bin"); + +// Script commands +test_unimplemented!(test_script_run, "script", "run", "test.py"); +test_unimplemented!(test_script_python, "script", "python", "test.py"); +test_unimplemented!(test_script_java, "script", "java", "test.java"); +test_unimplemented!(test_script_list, "script", "list"); + +// Disasm command +test_unimplemented!(test_disasm, "disasm", "0x1000"); + +// Other commands +test_unimplemented!(test_batch, "batch", "test.txt"); +test_unimplemented!(test_stats, "stats");