diff --git a/README.md b/README.md index fe4ccfc..6b2e884 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,18 @@ distrobox enter --no-workdir captioneer-dev -- bash -lc ' Run this from the repository root. The build and run helpers resolve that path automatically. +### Manual desktop smoke test + +`scripts/smoke/desktop-smoke.sh` boots a nested headless Wayland desktop (sway) inside the `captioneer-dev` Distrobox, routes synthesized `espeak-ng` speech through a temporary PipeWire null sink, runs the real `build/captioneer-desktop` binary against it, and fails on ERROR diagnostics, an unclean shutdown, or an empty caption transcript. + +```bash +./scripts/download-models.sh # once, if models/ is missing +./scripts/distrobox/build.sh # produce build/captioneer-desktop +./scripts/smoke/desktop-smoke.sh +``` + +Exit `0` means pass; a report is written under `build/smoke//report.txt` together with the application logs and caption transcripts. The harness installs `sway` and `espeak-ng` into the Distrobox on demand and restores the previous default sink when it finishes. If VAD recognition of synthetic speech proves flaky, rerun with `CAPTIONEER_SMOKE_MODE=fixed`. This test is manual-only and intentionally not part of CI: it needs downloaded models, a nested compositor, and live audio routing. + ## Gitea Actions and releases The `Tests` workflow runs the normal and GTK-tagged Go tests, `go vet`, and ShellCheck for every pull request and branch push. To make it mandatory for pull requests, open the repository's **Settings → Branches**, edit the protection rule for `master`, enable required status checks, and select **Tests / Go and GTK tests** after its first run. Workflow files can report the check, but the branch-protection rule is what prevents merging when it fails. diff --git a/plan_smoke_test.md b/plan_smoke_test.md new file mode 100644 index 0000000..10dbe5e --- /dev/null +++ b/plan_smoke_test.md @@ -0,0 +1,122 @@ +--- +name: Desktop smoke harness +overview: Add a Distrobox-based nested Wayland desktop smoke test that boots a headless compositor, plays synthesized speech into a temporary PipeWire null sink, runs captioneer-desktop, and fails on errors or missing captions — fast enough for manual runs, realistic enough to catch integration breakages. +todos: + - id: smoke-lib + content: Add scripts/smoke/lib.sh (null sink, espeak fixture, wait helpers, cleanup traps) + status: completed + - id: smoke-script + content: Add scripts/smoke/desktop-smoke.sh orchestrating labwc + captioneer-desktop + assertions + status: completed + - id: smoke-deps + content: On-demand Distrobox package install for labwc/espeak-ng (and ffmpeg if needed) + status: completed + - id: smoke-docs-ci + content: Document in README; ShellCheck the new scripts from scripts/ci/test.sh + status: in_progress +isProject: false +--- + +# Desktop smoke harness + +## Approach (chosen default) + +**Not a full QEMU VM.** A nested Wayland compositor inside the existing Distrobox is the better tradeoff: boots in seconds, still exercises GTK, Layer Shell, Pulse capture, model worker, and caption logging. Full VMs add minutes of boot/image management without much extra signal for this app. + +```mermaid +flowchart TD + host[Host PipeWire] --> nullSink[temp null sink] + script[desktop-smoke.sh] --> box[captioneer-dev Distrobox] + box --> labwc[labwc headless Wayland] + box --> app[captioneer-desktop] + nullSink --> play[paplay espeak WAV] + nullSink --> app + app --> log[stderr diagnostics] + app --> transcript[caption-log dir] + script --> assert[pass/fail report] + log --> assert + transcript --> assert +``` + +## What you will run + +```bash +./scripts/download-models.sh # once, if models/ missing +./scripts/distrobox/build.sh # ensure build/captioneer-desktop exists +./scripts/smoke/desktop-smoke.sh +``` + +Exit `0` = pass. Prints a short report under `build/smoke/`. + +## Harness layout + +| Path | Role | +| --- | --- | +| [`scripts/smoke/desktop-smoke.sh`](scripts/smoke/desktop-smoke.sh) | Orchestrator (manual entrypoint) | +| [`scripts/smoke/lib.sh`](scripts/smoke/lib.sh) | Shared helpers: null sink, espeak fixture, wait/assert | +| Extend [`scripts/distrobox/common.sh`](scripts/distrobox/common.sh) | Add smoke packages: `labwc`, `espeak-ng`, `ffmpeg` (or keep install-on-demand in smoke script to avoid bloating every build) | + +**Package strategy:** install smoke deps on demand inside `desktop-smoke.sh` via `sudo dnf install -y` in the Distrobox (same pattern as `ensure_dependencies`), so normal `build.sh` / `run.sh` stay lean. + +## Test sequence + +1. **Preflight** + - Require Distrobox, `build/captioneer-desktop`, `models/parakeet-tdt-v2/`, `models/silero_vad_v5.onnx` + - Create work dir `build/smoke//` + +2. **Audio sandbox (host-visible via Distrobox Pulse integration)** + - `pactl load-module module-null-sink sink_name=captioneer-smoke` + - Save/restore previous default sink + - `pactl set-default-sink captioneer-smoke` + - Synthesize a short clear English line with `espeak-ng` → WAV in the work dir + - Loop-play with `paplay` for ~15–20s so VAD/fixed mode has time to finalize + +3. **Nested desktop** + - Start `labwc` (wlroots, Layer Shell capable) with `WLR_BACKENDS=headless WLR_LIBINPUT_NO_DEVICES=1` + - Export `WAYLAND_DISPLAY` to that socket; `GSK_RENDERER=cairo` + +4. **Launch Captioneer** + ```bash + ./build/captioneer-desktop \ + --no-config \ + --start-captions=true \ + --output=both \ + --stderr-diagnostics=true \ + --caption-log=true \ + --caption-log-dir="$workdir/transcripts" \ + --models-dir=models \ + --mode=vad \ + --model-auto-restart=false + ``` + - Redirect stderr to `$workdir/captioneer.stderr.log` + - Present main window first (already true in current [`src/gui/app.go`](src/gui/app.go)), so compositor + GUI come up before capture + +5. **Settle + stop** + - Wait until stderr contains `Model worker ready` (timeout ~60s) + - Keep playing speech ~15s more + - Send clean shutdown (`gdbus` `app.exit` or SIGINT) + - Wait for exit; unload null sink; kill labwc + +6. **Assertions (fail hard)** + - Process exited 0 (or SIGINT-clean) + - stderr has `Model worker ready` and `Capturing from:` + - No `ERROR` lines in stderr (allow listed WARNINGs only if needed, e.g. always-on-top on Wayland) + - At least one non-empty caption log file under the smoke transcript dir **or**, if recognition is too flaky on synthetic speech, fall back to requiring `INFO [lifecycle]` start/stop without ERROR and treat caption text as a soft check printed in the report + +**Caption reliability default:** require caption log content. If espeak+VAD proves flaky in practice, switch the harness to `--mode=fixed --chunk-duration=1s` (still in the same script) before softening the assertion — fixed mode is more predictable with continuous playback. + +## Report + +Write `$workdir/report.txt` with: compositor, sink name, key log greps, transcript excerpt, pass/fail. Always print the report path on exit. + +## Docs / CI wiring + +- Short “Manual desktop smoke” subsection in [`README.md`](README.md) Development section +- Add `scripts/smoke/desktop-smoke.sh` + `lib.sh` to ShellCheck in [`scripts/ci/test.sh`](scripts/ci/test.sh) +- **Do not** add this to Gitea CI yet (needs models + nested compositor + audio; too heavy/flaky for every PR). Manual-only until it proves stable. + +## Out of scope + +- Full QEMU/libvirt guest image +- Automating CTRL+drag overlay positioning +- Pixel screenshot OCR of the overlay bubble diff --git a/scripts/ci/test.sh b/scripts/ci/test.sh index 081cb12..6a20634 100755 --- a/scripts/ci/test.sh +++ b/scripts/ci/test.sh @@ -16,4 +16,6 @@ shellcheck \ scripts/distrobox/run.sh \ scripts/ci/test.sh \ scripts/ci/package-release.sh \ - scripts/ci/publish-release.sh + scripts/ci/publish-release.sh \ + scripts/smoke/lib.sh \ + scripts/smoke/desktop-smoke.sh diff --git a/scripts/smoke/desktop-smoke.sh b/scripts/smoke/desktop-smoke.sh new file mode 100755 index 0000000..ba6d1b9 --- /dev/null +++ b/scripts/smoke/desktop-smoke.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# Manual desktop smoke test. +# +# Boots a headless Wayland compositor (sway) inside the captioneer-dev +# Distrobox, routes synthesized speech through a temporary PipeWire null +# sink, runs the real captioneer-desktop binary against it, and fails on +# diagnostics errors or missing captions. +# +# Usage (from the repository root, on the host): +# ./scripts/download-models.sh # once +# ./scripts/distrobox/build.sh # produce build/captioneer-desktop +# ./scripts/smoke/desktop-smoke.sh +# +# Environment: +# CAPTIONEER_SMOKE_MODE vad (default) or fixed +# CAPTIONEER_SMOKE_SPEECH_S seconds of speech to play (default 20) +set -Eeuo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +repo_root="$(cd -- "$script_dir/../.." && pwd -P)" + +# --------------------------------------------------------------------------- +# Host phase: prepare the Distrobox and re-execute this script inside it. +# --------------------------------------------------------------------------- +if [[ -z "${CAPTIONEER_SMOKE_INNER:-}" && -z "${CONTAINER_ID:-}" ]]; then + # shellcheck source=scripts/distrobox/common.sh + source "$repo_root/scripts/distrobox/common.sh" + prepare_distrobox + + [[ -x "$repo_root/build/captioneer-desktop" ]] \ + || die "build/captioneer-desktop is missing; run ./scripts/distrobox/build.sh first" + [[ -d "$repo_root/models/parakeet-tdt-v2" ]] \ + || die "models/parakeet-tdt-v2 is missing; run ./scripts/download-models.sh first" + [[ -f "$repo_root/models/silero_vad_v5.onnx" ]] \ + || die "models/silero_vad_v5.onnx is missing; run ./scripts/download-models.sh first" + + # shellcheck disable=SC2016 # Positional parameters expand inside the container. + # shellcheck disable=SC2154 # container_name is defined by common.sh above. + exec distrobox enter --no-workdir --name "$container_name" -- \ + bash -lc 'CAPTIONEER_SMOKE_INNER=1 exec bash "$1"' captioneer-smoke \ + "$repo_root/scripts/smoke/desktop-smoke.sh" +fi + +# --------------------------------------------------------------------------- +# Inner phase: runs inside the Distrobox. +# --------------------------------------------------------------------------- +# shellcheck source=scripts/smoke/lib.sh +source "$script_dir/lib.sh" + +smoke_mode="${CAPTIONEER_SMOKE_MODE:-vad}" +speech_seconds="${CAPTIONEER_SMOKE_SPEECH_S:-20}" + +# On-demand smoke dependencies; kept out of common.sh so normal builds stay lean. +smoke_packages=(sway espeak-ng) +missing_packages=() +for package in "${smoke_packages[@]}"; do + rpm -q "$package" >/dev/null 2>&1 || missing_packages+=("$package") +done +if (( ${#missing_packages[@]} > 0 )); then + smoke_log "installing smoke dependencies: ${missing_packages[*]}" + sudo dnf install -y "${missing_packages[@]}" +fi + +cd -- "$repo_root" +workdir="$repo_root/build/smoke/$(date +%Y%m%d-%H%M%S)" +mkdir -p "$workdir/transcripts" +smoke_log "work directory: $workdir" + +trap smoke_cleanup EXIT + +smoke_setup_audio "$workdir" +wayland_socket="$(smoke_start_compositor "$workdir")" +smoke_log "nested compositor ready on $wayland_socket" + +# --------------------------------------------------------------------------- +# Launch the desktop application. +# --------------------------------------------------------------------------- +env \ + WAYLAND_DISPLAY="$wayland_socket" \ + GDK_BACKEND=wayland \ + GSK_RENDERER=cairo \ + ./build/captioneer-desktop \ + --no-config \ + --start-captions=true \ + --output=both \ + --stderr-diagnostics=true \ + --caption-log=true \ + --caption-log-dir="$workdir/transcripts" \ + --models-dir=models \ + --mode="$smoke_mode" \ + --model-auto-restart=false \ + > "$workdir/captioneer.stdout.log" \ + 2> "$workdir/captioneer.stderr.log" & +SMOKE_APP_PID=$! +smoke_log "captioneer-desktop started (pid $SMOKE_APP_PID, mode $smoke_mode)" + +stderr_log="$workdir/captioneer.stderr.log" + +if ! smoke_wait_for_line "$stderr_log" "Model worker ready" 60; then + smoke_fail "model worker did not become ready within 60s" +fi +if ! smoke_wait_for_line "$stderr_log" "Capturing from:" 30; then + smoke_fail "audio capture did not start within 30s" +fi + +if (( ${#SMOKE_FAILURES[@]} == 0 )); then + smoke_log "playing synthesized speech for ${speech_seconds}s" + smoke_play_speech "$workdir" "$speech_seconds" + wait "$SMOKE_PLAYER_PID" 2>/dev/null || true + SMOKE_PLAYER_PID="" + # Allow VAD finalization (500ms pause) and transcript flush to settle. + sleep 4 +fi + +smoke_stop_app +app_exit=0 +if smoke_wait_for_exit "$SMOKE_APP_PID" 20; then + wait "$SMOKE_APP_PID" 2>/dev/null || app_exit=$? +else + smoke_fail "application did not exit within 20s of SIGINT" + kill -KILL "$SMOKE_APP_PID" 2>/dev/null || true + app_exit=137 +fi +SMOKE_APP_PID="" + +# --------------------------------------------------------------------------- +# Assertions. +# --------------------------------------------------------------------------- +if (( app_exit == 0 )); then + smoke_pass "clean shutdown (exit 0)" +else + smoke_fail "application exited with status $app_exit" +fi + +if grep -q "Model worker ready" "$stderr_log"; then + smoke_pass "model worker started" +fi +if grep -q "Capturing from:" "$stderr_log"; then + smoke_pass "audio capture attached to a monitor source" +fi + +if grep -E " ERROR " "$stderr_log" > "$workdir/errors.txt"; then + smoke_fail "diagnostics contain ERROR lines (see errors.txt)" +else + smoke_pass "no ERROR diagnostics" +fi + +mapfile -t caption_files < <(find "$workdir/transcripts" -type f -size +0c 2>/dev/null) +if (( ${#caption_files[@]} > 0 )); then + smoke_pass "caption log contains recognized speech" +else + smoke_fail "no captions were recognized; inspect $workdir and consider CAPTIONEER_SMOKE_MODE=fixed" +fi + +# --------------------------------------------------------------------------- +# Report. +# --------------------------------------------------------------------------- +report="$workdir/report.txt" +{ + printf 'Captioneer desktop smoke report\n' + printf 'Date: %s\n' "$(date --rfc-3339=seconds)" + printf 'Mode: %s\n' "$smoke_mode" + printf 'Compositor: sway headless (%s)\n' "$wayland_socket" + printf 'Sink: %s\n' "$SMOKE_SINK_NAME" + printf 'App exit: %s\n' "$app_exit" + printf '\n--- Key diagnostics ---\n' + grep -E " (ERROR|WARNING) |Model worker ready|Capturing from:|Shutting down" \ + "$stderr_log" 2>/dev/null || printf '(none)\n' + printf '\n--- Caption transcript excerpt ---\n' + if (( ${#caption_files[@]} > 0 )); then + head -n 10 "${caption_files[@]}" 2>/dev/null + else + printf '(empty)\n' + fi + printf '\n--- Result ---\n' + if (( ${#SMOKE_FAILURES[@]} == 0 )); then + printf 'PASS\n' + else + printf 'FAIL:\n' + printf ' - %s\n' "${SMOKE_FAILURES[@]}" + fi +} > "$report" + +cat "$report" +smoke_log "full report: $report" + +if (( ${#SMOKE_FAILURES[@]} > 0 )); then + exit 1 +fi +exit 0 diff --git a/scripts/smoke/lib.sh b/scripts/smoke/lib.sh new file mode 100755 index 0000000..dce26bd --- /dev/null +++ b/scripts/smoke/lib.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Shared helpers for the manual desktop smoke test. Sourced by +# scripts/smoke/desktop-smoke.sh inside the captioneer-dev Distrobox. +set -Eeuo pipefail + +SMOKE_SINK_NAME="captioneer-smoke" +SMOKE_SINK_MODULE="" +SMOKE_PREVIOUS_SINK="" +SMOKE_COMPOSITOR_PID="" +SMOKE_APP_PID="" +SMOKE_PLAYER_PID="" +SMOKE_FAILURES=() + +smoke_log() { + printf 'smoke: %s\n' "$*" >&2 +} + +smoke_die() { + printf 'smoke: FATAL: %s\n' "$*" >&2 + exit 1 +} + +smoke_fail() { + SMOKE_FAILURES+=("$1") + smoke_log "FAIL: $1" +} + +smoke_pass() { + smoke_log "ok: $1" +} + +# smoke_wait_for_line FILE PATTERN TIMEOUT_SECONDS +smoke_wait_for_line() { + local file=$1 pattern=$2 timeout=$3 waited=0 + until grep -q -- "$pattern" "$file" 2>/dev/null; do + if (( waited >= timeout )); then + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + return 0 +} + +# smoke_wait_for_exit PID TIMEOUT_SECONDS; returns 1 on timeout +smoke_wait_for_exit() { + local pid=$1 timeout=$2 waited=0 + while kill -0 "$pid" 2>/dev/null; do + if (( waited >= timeout )); then + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + return 0 +} + +smoke_setup_audio() { + local workdir=$1 + command -v pactl >/dev/null 2>&1 || smoke_die "pactl is required" + command -v paplay >/dev/null 2>&1 || smoke_die "paplay is required" + SMOKE_PREVIOUS_SINK="$(pactl get-default-sink 2>/dev/null || true)" + SMOKE_SINK_MODULE="$(pactl load-module module-null-sink \ + "sink_name=$SMOKE_SINK_NAME" \ + "sink_properties=device.description=Captioneer-Smoke")" + pactl set-default-sink "$SMOKE_SINK_NAME" + smoke_log "null sink $SMOKE_SINK_NAME loaded (module $SMOKE_SINK_MODULE, previous default: ${SMOKE_PREVIOUS_SINK:-none})" + + command -v espeak-ng >/dev/null 2>&1 || smoke_die "espeak-ng is required" + espeak-ng -v en-us -s 140 -w "$workdir/speech.wav" \ + "The quick brown fox jumps over the lazy dog. Live captions should appear for this sentence." + [[ -s "$workdir/speech.wav" ]] || smoke_die "espeak-ng produced no audio fixture" +} + +smoke_teardown_audio() { + if [[ -n "$SMOKE_PLAYER_PID" ]] && kill -0 "$SMOKE_PLAYER_PID" 2>/dev/null; then + kill "$SMOKE_PLAYER_PID" 2>/dev/null || true + fi + if [[ -n "$SMOKE_PREVIOUS_SINK" ]]; then + pactl set-default-sink "$SMOKE_PREVIOUS_SINK" 2>/dev/null || true + fi + if [[ -n "$SMOKE_SINK_MODULE" ]]; then + pactl unload-module "$SMOKE_SINK_MODULE" 2>/dev/null || true + SMOKE_SINK_MODULE="" + fi +} + +# smoke_play_speech WORKDIR SECONDS: loop the fixture into the null sink. +smoke_play_speech() { + local workdir=$1 seconds=$2 + ( + end=$((SECONDS + seconds)) + while (( SECONDS < end )); do + paplay --device="$SMOKE_SINK_NAME" "$workdir/speech.wav" 2>/dev/null || sleep 1 + done + ) & + SMOKE_PLAYER_PID=$! +} + +# smoke_start_compositor WORKDIR: starts headless sway; prints new socket name. +# sway is wlroots-based (Layer Shell capable) and, unlike labwc, can disable +# Xwayland, whose sockets are not creatable inside a rootless Distrobox. +smoke_start_compositor() { + local workdir=$1 candidate socket="" waited=0 + command -v sway >/dev/null 2>&1 || smoke_die "sway is required" + local before="$workdir/sockets-before.txt" + find "$XDG_RUNTIME_DIR" -maxdepth 1 -name 'wayland-[0-9]*' ! -name '*.lock' \ + -printf '%f\n' 2>/dev/null | sort > "$before" + + printf 'xwayland disable\n' > "$workdir/sway-config" + WLR_BACKENDS=headless \ + WLR_LIBINPUT_NO_DEVICES=1 \ + WLR_RENDERER=pixman \ + sway -c "$workdir/sway-config" > "$workdir/compositor.log" 2>&1 & + SMOKE_COMPOSITOR_PID=$! + + while (( waited < 15 )); do + while IFS= read -r candidate; do + socket="$candidate" + done < <(find "$XDG_RUNTIME_DIR" -maxdepth 1 -name 'wayland-[0-9]*' ! -name '*.lock' \ + -printf '%f\n' 2>/dev/null | sort | comm -13 "$before" -) + if [[ -n "$socket" ]]; then + printf '%s\n' "$socket" + return 0 + fi + kill -0 "$SMOKE_COMPOSITOR_PID" 2>/dev/null || smoke_die "sway exited early; see $workdir/compositor.log" + sleep 1 + waited=$((waited + 1)) + done + smoke_die "sway did not create a Wayland socket; see $workdir/compositor.log" +} + +smoke_teardown_compositor() { + if [[ -n "$SMOKE_COMPOSITOR_PID" ]] && kill -0 "$SMOKE_COMPOSITOR_PID" 2>/dev/null; then + kill "$SMOKE_COMPOSITOR_PID" 2>/dev/null || true + fi +} + +smoke_stop_app() { + if [[ -n "$SMOKE_APP_PID" ]] && kill -0 "$SMOKE_APP_PID" 2>/dev/null; then + kill -INT "$SMOKE_APP_PID" 2>/dev/null || true + fi +} + +smoke_cleanup() { + smoke_stop_app + if [[ -n "$SMOKE_APP_PID" ]]; then + smoke_wait_for_exit "$SMOKE_APP_PID" 10 || kill -KILL "$SMOKE_APP_PID" 2>/dev/null || true + fi + smoke_teardown_audio + smoke_teardown_compositor +}