docs: add manual desktop smoke test instructions to README and update CI test script

- Introduce a new section in README detailing the manual desktop smoke test process for the application. - Include commands for setting up and running the smoke test, along with expected outcomes and logging details. - Update the CI test script to include the new smoke test script in the shellcheck validation process.
2026-07-17 20:07:38 +03:00
parent 5b4a0d0785
commit d6fda83397
5 changed files with 479 additions and 1 deletions
@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+# Manual desktop smoke test.
+#
+# Boots a headless Wayland compositor (sway) inside the captioneer-dev
+# Distrobox, routes synthesized speech through a temporary PipeWire null
+# sink, runs the real captioneer-desktop binary against it, and fails on
+# diagnostics errors or missing captions.
+#
+# Usage (from the repository root, on the host):
+#   ./scripts/download-models.sh        # once
+#   ./scripts/distrobox/build.sh        # produce build/captioneer-desktop
+#   ./scripts/smoke/desktop-smoke.sh
+#
+# Environment:
+#   CAPTIONEER_SMOKE_MODE      vad (default) or fixed
+#   CAPTIONEER_SMOKE_SPEECH_S  seconds of speech to play (default 20)
+set -Eeuo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+repo_root="$(cd -- "$script_dir/../.." && pwd -P)"
+
+# ---------------------------------------------------------------------------
+# Host phase: prepare the Distrobox and re-execute this script inside it.
+# ---------------------------------------------------------------------------
+if [[ -z "${CAPTIONEER_SMOKE_INNER:-}" && -z "${CONTAINER_ID:-}" ]]; then
+  # shellcheck source=scripts/distrobox/common.sh
+  source "$repo_root/scripts/distrobox/common.sh"
+  prepare_distrobox
+
+  [[ -x "$repo_root/build/captioneer-desktop" ]] \
+    || die "build/captioneer-desktop is missing; run ./scripts/distrobox/build.sh first"
+  [[ -d "$repo_root/models/parakeet-tdt-v2" ]] \
+    || die "models/parakeet-tdt-v2 is missing; run ./scripts/download-models.sh first"
+  [[ -f "$repo_root/models/silero_vad_v5.onnx" ]] \
+    || die "models/silero_vad_v5.onnx is missing; run ./scripts/download-models.sh first"
+
+  # shellcheck disable=SC2016 # Positional parameters expand inside the container.
+  # shellcheck disable=SC2154 # container_name is defined by common.sh above.
+  exec distrobox enter --no-workdir --name "$container_name" -- \
+    bash -lc 'CAPTIONEER_SMOKE_INNER=1 exec bash "$1"' captioneer-smoke \
+    "$repo_root/scripts/smoke/desktop-smoke.sh"
+fi
+
+# ---------------------------------------------------------------------------
+# Inner phase: runs inside the Distrobox.
+# ---------------------------------------------------------------------------
+# shellcheck source=scripts/smoke/lib.sh
+source "$script_dir/lib.sh"
+
+smoke_mode="${CAPTIONEER_SMOKE_MODE:-vad}"
+speech_seconds="${CAPTIONEER_SMOKE_SPEECH_S:-20}"
+
+# On-demand smoke dependencies; kept out of common.sh so normal builds stay lean.
+smoke_packages=(sway espeak-ng)
+missing_packages=()
+for package in "${smoke_packages[@]}"; do
+  rpm -q "$package" >/dev/null 2>&1 || missing_packages+=("$package")
+done
+if (( ${#missing_packages[@]} > 0 )); then
+  smoke_log "installing smoke dependencies: ${missing_packages[*]}"
+  sudo dnf install -y "${missing_packages[@]}"
+fi
+
+cd -- "$repo_root"
+workdir="$repo_root/build/smoke/$(date +%Y%m%d-%H%M%S)"
+mkdir -p "$workdir/transcripts"
+smoke_log "work directory: $workdir"
+
+trap smoke_cleanup EXIT
+
+smoke_setup_audio "$workdir"
+wayland_socket="$(smoke_start_compositor "$workdir")"
+smoke_log "nested compositor ready on $wayland_socket"
+
+# ---------------------------------------------------------------------------
+# Launch the desktop application.
+# ---------------------------------------------------------------------------
+env \
+  WAYLAND_DISPLAY="$wayland_socket" \
+  GDK_BACKEND=wayland \
+  GSK_RENDERER=cairo \
+  ./build/captioneer-desktop \
+  --no-config \
+  --start-captions=true \
+  --output=both \
+  --stderr-diagnostics=true \
+  --caption-log=true \
+  --caption-log-dir="$workdir/transcripts" \
+  --models-dir=models \
+  --mode="$smoke_mode" \
+  --model-auto-restart=false \
+  > "$workdir/captioneer.stdout.log" \
+  2> "$workdir/captioneer.stderr.log" &
+SMOKE_APP_PID=$!
+smoke_log "captioneer-desktop started (pid $SMOKE_APP_PID, mode $smoke_mode)"
+
+stderr_log="$workdir/captioneer.stderr.log"
+
+if ! smoke_wait_for_line "$stderr_log" "Model worker ready" 60; then
+  smoke_fail "model worker did not become ready within 60s"
+fi
+if ! smoke_wait_for_line "$stderr_log" "Capturing from:" 30; then
+  smoke_fail "audio capture did not start within 30s"
+fi
+
+if (( ${#SMOKE_FAILURES[@]} == 0 )); then
+  smoke_log "playing synthesized speech for ${speech_seconds}s"
+  smoke_play_speech "$workdir" "$speech_seconds"
+  wait "$SMOKE_PLAYER_PID" 2>/dev/null || true
+  SMOKE_PLAYER_PID=""
+  # Allow VAD finalization (500ms pause) and transcript flush to settle.
+  sleep 4
+fi
+
+smoke_stop_app
+app_exit=0
+if smoke_wait_for_exit "$SMOKE_APP_PID" 20; then
+  wait "$SMOKE_APP_PID" 2>/dev/null || app_exit=$?
+else
+  smoke_fail "application did not exit within 20s of SIGINT"
+  kill -KILL "$SMOKE_APP_PID" 2>/dev/null || true
+  app_exit=137
+fi
+SMOKE_APP_PID=""
+
+# ---------------------------------------------------------------------------
+# Assertions.
+# ---------------------------------------------------------------------------
+if (( app_exit == 0 )); then
+  smoke_pass "clean shutdown (exit 0)"
+else
+  smoke_fail "application exited with status $app_exit"
+fi
+
+if grep -q "Model worker ready" "$stderr_log"; then
+  smoke_pass "model worker started"
+fi
+if grep -q "Capturing from:" "$stderr_log"; then
+  smoke_pass "audio capture attached to a monitor source"
+fi
+
+if grep -E " ERROR " "$stderr_log" > "$workdir/errors.txt"; then
+  smoke_fail "diagnostics contain ERROR lines (see errors.txt)"
+else
+  smoke_pass "no ERROR diagnostics"
+fi
+
+mapfile -t caption_files < <(find "$workdir/transcripts" -type f -size +0c 2>/dev/null)
+if (( ${#caption_files[@]} > 0 )); then
+  smoke_pass "caption log contains recognized speech"
+else
+  smoke_fail "no captions were recognized; inspect $workdir and consider CAPTIONEER_SMOKE_MODE=fixed"
+fi
+
+# ---------------------------------------------------------------------------
+# Report.
+# ---------------------------------------------------------------------------
+report="$workdir/report.txt"
+{
+  printf 'Captioneer desktop smoke report\n'
+  printf 'Date:        %s\n' "$(date --rfc-3339=seconds)"
+  printf 'Mode:        %s\n' "$smoke_mode"
+  printf 'Compositor:  sway headless (%s)\n' "$wayland_socket"
+  printf 'Sink:        %s\n' "$SMOKE_SINK_NAME"
+  printf 'App exit:    %s\n' "$app_exit"
+  printf '\n--- Key diagnostics ---\n'
+  grep -E " (ERROR|WARNING) |Model worker ready|Capturing from:|Shutting down" \
+    "$stderr_log" 2>/dev/null || printf '(none)\n'
+  printf '\n--- Caption transcript excerpt ---\n'
+  if (( ${#caption_files[@]} > 0 )); then
+    head -n 10 "${caption_files[@]}" 2>/dev/null
+  else
+    printf '(empty)\n'
+  fi
+  printf '\n--- Result ---\n'
+  if (( ${#SMOKE_FAILURES[@]} == 0 )); then
+    printf 'PASS\n'
+  else
+    printf 'FAIL:\n'
+    printf '  - %s\n' "${SMOKE_FAILURES[@]}"
+  fi
+} > "$report"
+
+cat "$report"
+smoke_log "full report: $report"
+
+if (( ${#SMOKE_FAILURES[@]} > 0 )); then
+  exit 1
+fi
+exit 0
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+# Shared helpers for the manual desktop smoke test. Sourced by
+# scripts/smoke/desktop-smoke.sh inside the captioneer-dev Distrobox.
+set -Eeuo pipefail
+
+SMOKE_SINK_NAME="captioneer-smoke"
+SMOKE_SINK_MODULE=""
+SMOKE_PREVIOUS_SINK=""
+SMOKE_COMPOSITOR_PID=""
+SMOKE_APP_PID=""
+SMOKE_PLAYER_PID=""
+SMOKE_FAILURES=()
+
+smoke_log() {
+  printf 'smoke: %s\n' "$*" >&2
+}
+
+smoke_die() {
+  printf 'smoke: FATAL: %s\n' "$*" >&2
+  exit 1
+}
+
+smoke_fail() {
+  SMOKE_FAILURES+=("$1")
+  smoke_log "FAIL: $1"
+}
+
+smoke_pass() {
+  smoke_log "ok: $1"
+}
+
+# smoke_wait_for_line FILE PATTERN TIMEOUT_SECONDS
+smoke_wait_for_line() {
+  local file=$1 pattern=$2 timeout=$3 waited=0
+  until grep -q -- "$pattern" "$file" 2>/dev/null; do
+    if (( waited >= timeout )); then
+      return 1
+    fi
+    sleep 1
+    waited=$((waited + 1))
+  done
+  return 0
+}
+
+# smoke_wait_for_exit PID TIMEOUT_SECONDS; returns 1 on timeout
+smoke_wait_for_exit() {
+  local pid=$1 timeout=$2 waited=0
+  while kill -0 "$pid" 2>/dev/null; do
+    if (( waited >= timeout )); then
+      return 1
+    fi
+    sleep 1
+    waited=$((waited + 1))
+  done
+  return 0
+}
+
+smoke_setup_audio() {
+  local workdir=$1
+  command -v pactl >/dev/null 2>&1 || smoke_die "pactl is required"
+  command -v paplay >/dev/null 2>&1 || smoke_die "paplay is required"
+  SMOKE_PREVIOUS_SINK="$(pactl get-default-sink 2>/dev/null || true)"
+  SMOKE_SINK_MODULE="$(pactl load-module module-null-sink \
+    "sink_name=$SMOKE_SINK_NAME" \
+    "sink_properties=device.description=Captioneer-Smoke")"
+  pactl set-default-sink "$SMOKE_SINK_NAME"
+  smoke_log "null sink $SMOKE_SINK_NAME loaded (module $SMOKE_SINK_MODULE, previous default: ${SMOKE_PREVIOUS_SINK:-none})"
+
+  command -v espeak-ng >/dev/null 2>&1 || smoke_die "espeak-ng is required"
+  espeak-ng -v en-us -s 140 -w "$workdir/speech.wav" \
+    "The quick brown fox jumps over the lazy dog. Live captions should appear for this sentence."
+  [[ -s "$workdir/speech.wav" ]] || smoke_die "espeak-ng produced no audio fixture"
+}
+
+smoke_teardown_audio() {
+  if [[ -n "$SMOKE_PLAYER_PID" ]] && kill -0 "$SMOKE_PLAYER_PID" 2>/dev/null; then
+    kill "$SMOKE_PLAYER_PID" 2>/dev/null || true
+  fi
+  if [[ -n "$SMOKE_PREVIOUS_SINK" ]]; then
+    pactl set-default-sink "$SMOKE_PREVIOUS_SINK" 2>/dev/null || true
+  fi
+  if [[ -n "$SMOKE_SINK_MODULE" ]]; then
+    pactl unload-module "$SMOKE_SINK_MODULE" 2>/dev/null || true
+    SMOKE_SINK_MODULE=""
+  fi
+}
+
+# smoke_play_speech WORKDIR SECONDS: loop the fixture into the null sink.
+smoke_play_speech() {
+  local workdir=$1 seconds=$2
+  (
+    end=$((SECONDS + seconds))
+    while (( SECONDS < end )); do
+      paplay --device="$SMOKE_SINK_NAME" "$workdir/speech.wav" 2>/dev/null || sleep 1
+    done
+  ) &
+  SMOKE_PLAYER_PID=$!
+}
+
+# smoke_start_compositor WORKDIR: starts headless sway; prints new socket name.
+# sway is wlroots-based (Layer Shell capable) and, unlike labwc, can disable
+# Xwayland, whose sockets are not creatable inside a rootless Distrobox.
+smoke_start_compositor() {
+  local workdir=$1 candidate socket="" waited=0
+  command -v sway >/dev/null 2>&1 || smoke_die "sway is required"
+  local before="$workdir/sockets-before.txt"
+  find "$XDG_RUNTIME_DIR" -maxdepth 1 -name 'wayland-[0-9]*' ! -name '*.lock' \
+    -printf '%f\n' 2>/dev/null | sort > "$before"
+
+  printf 'xwayland disable\n' > "$workdir/sway-config"
+  WLR_BACKENDS=headless \
+    WLR_LIBINPUT_NO_DEVICES=1 \
+    WLR_RENDERER=pixman \
+    sway -c "$workdir/sway-config" > "$workdir/compositor.log" 2>&1 &
+  SMOKE_COMPOSITOR_PID=$!
+
+  while (( waited < 15 )); do
+    while IFS= read -r candidate; do
+      socket="$candidate"
+    done < <(find "$XDG_RUNTIME_DIR" -maxdepth 1 -name 'wayland-[0-9]*' ! -name '*.lock' \
+      -printf '%f\n' 2>/dev/null | sort | comm -13 "$before" -)
+    if [[ -n "$socket" ]]; then
+      printf '%s\n' "$socket"
+      return 0
+    fi
+    kill -0 "$SMOKE_COMPOSITOR_PID" 2>/dev/null || smoke_die "sway exited early; see $workdir/compositor.log"
+    sleep 1
+    waited=$((waited + 1))
+  done
+  smoke_die "sway did not create a Wayland socket; see $workdir/compositor.log"
+}
+
+smoke_teardown_compositor() {
+  if [[ -n "$SMOKE_COMPOSITOR_PID" ]] && kill -0 "$SMOKE_COMPOSITOR_PID" 2>/dev/null; then
+    kill "$SMOKE_COMPOSITOR_PID" 2>/dev/null || true
+  fi
+}
+
+smoke_stop_app() {
+  if [[ -n "$SMOKE_APP_PID" ]] && kill -0 "$SMOKE_APP_PID" 2>/dev/null; then
+    kill -INT "$SMOKE_APP_PID" 2>/dev/null || true
+  fi
+}
+
+smoke_cleanup() {
+  smoke_stop_app
+  if [[ -n "$SMOKE_APP_PID" ]]; then
+    smoke_wait_for_exit "$SMOKE_APP_PID" 10 || kill -KILL "$SMOKE_APP_PID" 2>/dev/null || true
+  fi
+  smoke_teardown_audio
+  smoke_teardown_compositor
+}