kato
d6fda83397
Tests / Go and GTK tests (push) Failing after 1m57s
- Introduce a new section in README detailing the manual desktop smoke test process for the application. - Include commands for setting up and running the smoke test, along with expected outcomes and logging details. - Update the CI test script to include the new smoke test script in the shellcheck validation process.
191 lines
6.6 KiB
Bash
Executable File
191 lines
6.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Manual desktop smoke test.
|
|
#
|
|
# Boots a headless Wayland compositor (sway) inside the captioneer-dev
|
|
# Distrobox, routes synthesized speech through a temporary PipeWire null
|
|
# sink, runs the real captioneer-desktop binary against it, and fails on
|
|
# diagnostics errors or missing captions.
|
|
#
|
|
# Usage (from the repository root, on the host):
|
|
# ./scripts/download-models.sh # once
|
|
# ./scripts/distrobox/build.sh # produce build/captioneer-desktop
|
|
# ./scripts/smoke/desktop-smoke.sh
|
|
#
|
|
# Environment:
|
|
# CAPTIONEER_SMOKE_MODE vad (default) or fixed
|
|
# CAPTIONEER_SMOKE_SPEECH_S seconds of speech to play (default 20)
|
|
set -Eeuo pipefail
|
|
|
|
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
|
repo_root="$(cd -- "$script_dir/../.." && pwd -P)"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Host phase: prepare the Distrobox and re-execute this script inside it.
|
|
# ---------------------------------------------------------------------------
|
|
if [[ -z "${CAPTIONEER_SMOKE_INNER:-}" && -z "${CONTAINER_ID:-}" ]]; then
|
|
# shellcheck source=scripts/distrobox/common.sh
|
|
source "$repo_root/scripts/distrobox/common.sh"
|
|
prepare_distrobox
|
|
|
|
[[ -x "$repo_root/build/captioneer-desktop" ]] \
|
|
|| die "build/captioneer-desktop is missing; run ./scripts/distrobox/build.sh first"
|
|
[[ -d "$repo_root/models/parakeet-tdt-v2" ]] \
|
|
|| die "models/parakeet-tdt-v2 is missing; run ./scripts/download-models.sh first"
|
|
[[ -f "$repo_root/models/silero_vad_v5.onnx" ]] \
|
|
|| die "models/silero_vad_v5.onnx is missing; run ./scripts/download-models.sh first"
|
|
|
|
# shellcheck disable=SC2016 # Positional parameters expand inside the container.
|
|
# shellcheck disable=SC2154 # container_name is defined by common.sh above.
|
|
exec distrobox enter --no-workdir --name "$container_name" -- \
|
|
bash -lc 'CAPTIONEER_SMOKE_INNER=1 exec bash "$1"' captioneer-smoke \
|
|
"$repo_root/scripts/smoke/desktop-smoke.sh"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Inner phase: runs inside the Distrobox.
|
|
# ---------------------------------------------------------------------------
|
|
# shellcheck source=scripts/smoke/lib.sh
|
|
source "$script_dir/lib.sh"
|
|
|
|
smoke_mode="${CAPTIONEER_SMOKE_MODE:-vad}"
|
|
speech_seconds="${CAPTIONEER_SMOKE_SPEECH_S:-20}"
|
|
|
|
# On-demand smoke dependencies; kept out of common.sh so normal builds stay lean.
|
|
smoke_packages=(sway espeak-ng)
|
|
missing_packages=()
|
|
for package in "${smoke_packages[@]}"; do
|
|
rpm -q "$package" >/dev/null 2>&1 || missing_packages+=("$package")
|
|
done
|
|
if (( ${#missing_packages[@]} > 0 )); then
|
|
smoke_log "installing smoke dependencies: ${missing_packages[*]}"
|
|
sudo dnf install -y "${missing_packages[@]}"
|
|
fi
|
|
|
|
cd -- "$repo_root"
|
|
workdir="$repo_root/build/smoke/$(date +%Y%m%d-%H%M%S)"
|
|
mkdir -p "$workdir/transcripts"
|
|
smoke_log "work directory: $workdir"
|
|
|
|
trap smoke_cleanup EXIT
|
|
|
|
smoke_setup_audio "$workdir"
|
|
wayland_socket="$(smoke_start_compositor "$workdir")"
|
|
smoke_log "nested compositor ready on $wayland_socket"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Launch the desktop application.
|
|
# ---------------------------------------------------------------------------
|
|
env \
|
|
WAYLAND_DISPLAY="$wayland_socket" \
|
|
GDK_BACKEND=wayland \
|
|
GSK_RENDERER=cairo \
|
|
./build/captioneer-desktop \
|
|
--no-config \
|
|
--start-captions=true \
|
|
--output=both \
|
|
--stderr-diagnostics=true \
|
|
--caption-log=true \
|
|
--caption-log-dir="$workdir/transcripts" \
|
|
--models-dir=models \
|
|
--mode="$smoke_mode" \
|
|
--model-auto-restart=false \
|
|
> "$workdir/captioneer.stdout.log" \
|
|
2> "$workdir/captioneer.stderr.log" &
|
|
SMOKE_APP_PID=$!
|
|
smoke_log "captioneer-desktop started (pid $SMOKE_APP_PID, mode $smoke_mode)"
|
|
|
|
stderr_log="$workdir/captioneer.stderr.log"
|
|
|
|
if ! smoke_wait_for_line "$stderr_log" "Model worker ready" 60; then
|
|
smoke_fail "model worker did not become ready within 60s"
|
|
fi
|
|
if ! smoke_wait_for_line "$stderr_log" "Capturing from:" 30; then
|
|
smoke_fail "audio capture did not start within 30s"
|
|
fi
|
|
|
|
if (( ${#SMOKE_FAILURES[@]} == 0 )); then
|
|
smoke_log "playing synthesized speech for ${speech_seconds}s"
|
|
smoke_play_speech "$workdir" "$speech_seconds"
|
|
wait "$SMOKE_PLAYER_PID" 2>/dev/null || true
|
|
SMOKE_PLAYER_PID=""
|
|
# Allow VAD finalization (500ms pause) and transcript flush to settle.
|
|
sleep 4
|
|
fi
|
|
|
|
smoke_stop_app
|
|
app_exit=0
|
|
if smoke_wait_for_exit "$SMOKE_APP_PID" 20; then
|
|
wait "$SMOKE_APP_PID" 2>/dev/null || app_exit=$?
|
|
else
|
|
smoke_fail "application did not exit within 20s of SIGINT"
|
|
kill -KILL "$SMOKE_APP_PID" 2>/dev/null || true
|
|
app_exit=137
|
|
fi
|
|
SMOKE_APP_PID=""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Assertions.
|
|
# ---------------------------------------------------------------------------
|
|
if (( app_exit == 0 )); then
|
|
smoke_pass "clean shutdown (exit 0)"
|
|
else
|
|
smoke_fail "application exited with status $app_exit"
|
|
fi
|
|
|
|
if grep -q "Model worker ready" "$stderr_log"; then
|
|
smoke_pass "model worker started"
|
|
fi
|
|
if grep -q "Capturing from:" "$stderr_log"; then
|
|
smoke_pass "audio capture attached to a monitor source"
|
|
fi
|
|
|
|
if grep -E " ERROR " "$stderr_log" > "$workdir/errors.txt"; then
|
|
smoke_fail "diagnostics contain ERROR lines (see errors.txt)"
|
|
else
|
|
smoke_pass "no ERROR diagnostics"
|
|
fi
|
|
|
|
mapfile -t caption_files < <(find "$workdir/transcripts" -type f -size +0c 2>/dev/null)
|
|
if (( ${#caption_files[@]} > 0 )); then
|
|
smoke_pass "caption log contains recognized speech"
|
|
else
|
|
smoke_fail "no captions were recognized; inspect $workdir and consider CAPTIONEER_SMOKE_MODE=fixed"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Report.
|
|
# ---------------------------------------------------------------------------
|
|
report="$workdir/report.txt"
|
|
{
|
|
printf 'Captioneer desktop smoke report\n'
|
|
printf 'Date: %s\n' "$(date --rfc-3339=seconds)"
|
|
printf 'Mode: %s\n' "$smoke_mode"
|
|
printf 'Compositor: sway headless (%s)\n' "$wayland_socket"
|
|
printf 'Sink: %s\n' "$SMOKE_SINK_NAME"
|
|
printf 'App exit: %s\n' "$app_exit"
|
|
printf '\n--- Key diagnostics ---\n'
|
|
grep -E " (ERROR|WARNING) |Model worker ready|Capturing from:|Shutting down" \
|
|
"$stderr_log" 2>/dev/null || printf '(none)\n'
|
|
printf '\n--- Caption transcript excerpt ---\n'
|
|
if (( ${#caption_files[@]} > 0 )); then
|
|
head -n 10 "${caption_files[@]}" 2>/dev/null
|
|
else
|
|
printf '(empty)\n'
|
|
fi
|
|
printf '\n--- Result ---\n'
|
|
if (( ${#SMOKE_FAILURES[@]} == 0 )); then
|
|
printf 'PASS\n'
|
|
else
|
|
printf 'FAIL:\n'
|
|
printf ' - %s\n' "${SMOKE_FAILURES[@]}"
|
|
fi
|
|
} > "$report"
|
|
|
|
cat "$report"
|
|
smoke_log "full report: $report"
|
|
|
|
if (( ${#SMOKE_FAILURES[@]} > 0 )); then
|
|
exit 1
|
|
fi
|
|
exit 0
|