feat: implement isolated model worker with auto-restart and hot-reload
Introduce a supervised child process for capture, VAD, and recognition, enabling automatic restart on crash/hang and hot-reload via SIGHUP. Add `--model-auto-restart`, `--model-timeout`, and `--model-shutdown-timeout` options. Refactor `Run()` with lifecycle hooks for proper process group cleanup. Update README with new behavior and CLI flags.
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"tea.chunkbyte.com/kato/captioneer/src/config"
|
||||
)
|
||||
|
||||
func TestWorkerHealthDetectsStartupAndOperationTimeouts(t *testing.T) {
|
||||
started := time.Unix(100, 0)
|
||||
health := workerHealth{startedAt: started, lastSeen: started}
|
||||
|
||||
if err := health.timeoutError(started.Add(time.Second), 2*time.Second); err != nil {
|
||||
t.Fatalf("healthy startup reported a timeout: %v", err)
|
||||
}
|
||||
if err := health.timeoutError(started.Add(2*time.Second), 2*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "did not become ready") {
|
||||
t.Fatalf("startup timeout = %v", err)
|
||||
}
|
||||
|
||||
health.observe(messageReady, started.Add(2*time.Second))
|
||||
health.observe(messageBusy, started.Add(3*time.Second))
|
||||
if err := health.timeoutError(started.Add(4*time.Second), 2*time.Second); err != nil {
|
||||
t.Fatalf("healthy operation reported a timeout: %v", err)
|
||||
}
|
||||
if err := health.timeoutError(started.Add(5*time.Second), 2*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "operation exceeded") {
|
||||
t.Fatalf("operation timeout = %v", err)
|
||||
}
|
||||
|
||||
health.observe(messageIdle, started.Add(5*time.Second))
|
||||
health.observe(messageHeartbeat, started.Add(6*time.Second))
|
||||
if err := health.timeoutError(started.Add(7*time.Second), 2*time.Second); err != nil {
|
||||
t.Fatalf("idle worker reported a timeout: %v", err)
|
||||
}
|
||||
if err := health.timeoutError(started.Add(8*time.Second), 2*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "heartbeat was silent") {
|
||||
t.Fatalf("heartbeat timeout = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerSettingsRoundTrip(t *testing.T) {
|
||||
want := config.Settings{
|
||||
Mode: config.ModeVAD,
|
||||
ChunkDuration: time.Second,
|
||||
ModelsDir: "models with spaces",
|
||||
Threads: 3,
|
||||
PreviewThresholdDB: -42,
|
||||
ModelAutoRestart: true,
|
||||
ModelTimeout: 12 * time.Second,
|
||||
ModelShutdownTimeout: 2 * time.Second,
|
||||
}
|
||||
encoded, err := encodeWorkerSettings(want)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := decodeWorkerSettings(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("settings round trip = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user