docs: update README with VAD, new threshold option, and silence handling

Update README to clarify VAD mode behavior: provisional caption only after Silero detects speech, Parakeet idle during silence. Document new --vad-threshold option to control Silero speech confidence. Add troubleshooting tip for music/noise false detections.
2026-07-17 10:35:59 +03:00
parent b0487be39e
commit 55fbd357b0
8 changed files with 226 additions and 41 deletions
@@ -20,6 +20,7 @@ type Event struct {
 type Transcriber interface {
 	Decode(samples []float32) string
 	AcceptVAD(samples []float32)
+	SpeechActive() bool
 	FlushVAD()
 	NextSpeechSegment() (start int, samples []float32, ok bool)
 }
@@ -3,22 +3,30 @@ package captions

 import (
 	"strings"
+	"time"

 	"tea.chunkbyte.com/kato/captioneer/src/audio"
 	"tea.chunkbyte.com/kato/captioneer/src/config"
 )

+// Provisional recognition uses only a recent window. Re-decoding an entire
+// long utterance every refresh grows increasingly expensive, while the UI only
+// presents the latest provisional lines. The final event still decodes the
+// complete VAD segment.
+const maxPreviewDuration = 10 * time.Second
+
 type Processor struct {
 	settings    config.Settings
 	transcriber Transcriber
 	emit        func(Event) error

-	fixedSamples   []float32
-	previewSamples []float32
-	previewActive  bool
-	previewStart   int
-	nextPreviewAt  int
-	totalSamples   int
+	fixedSamples    []float32
+	previewSamples  []float32
+	previewActive   bool
+	previewStart    int
+	nextPreviewAt   int
+	previewReceived int
+	totalSamples    int
 }

 func New(settings config.Settings, transcriber Transcriber, emit func(Event) error) *Processor {
@@ -43,19 +51,26 @@ func (p *Processor) Accept(samples []float32) error {
 func (p *Processor) Flush() error {
 	if p.settings.Mode == config.ModeFixed {
 		if len(p.fixedSamples) > 0 {
-			return p.emitFinal(p.totalSamples, p.totalSamples+len(p.fixedSamples), p.fixedSamples)
+			return p.emitFixedFinal(p.totalSamples, p.totalSamples+len(p.fixedSamples), p.fixedSamples)
 		}
 		return nil
 	}
 	p.transcriber.FlushVAD()
-	return p.drainVAD()
+	if err := p.drainVAD(); err != nil {
+		return err
+	}
+	if p.previewActive {
+		p.resetPreview()
+		return p.emit(Event{Kind: Hide})
+	}
+	return nil
 }

 func (p *Processor) acceptFixed(samples []float32) error {
 	p.fixedSamples = append(p.fixedSamples, samples...)
 	chunkSize := audio.DurationSamples(p.settings.ChunkDuration)
 	for len(p.fixedSamples) >= chunkSize {
-		if err := p.emitFinal(p.totalSamples, p.totalSamples+chunkSize, p.fixedSamples[:chunkSize]); err != nil {
+		if err := p.emitFixedFinal(p.totalSamples, p.totalSamples+chunkSize, p.fixedSamples[:chunkSize]); err != nil {
 			return err
 		}
 		p.fixedSamples = p.fixedSamples[chunkSize:]
@@ -68,20 +83,33 @@ func (p *Processor) acceptVAD(samples []float32) error {
 	p.totalSamples += len(samples)
 	p.transcriber.AcceptVAD(samples)

-	if p.previewActive || audio.RMSDBFS(samples) >= p.settings.PreviewThresholdDB {
+	speechActive := p.transcriber.SpeechActive()
+	if speechActive && (p.previewActive || audio.RMSDBFS(samples) >= p.settings.PreviewThresholdDB) {
 		if !p.previewActive {
 			p.previewActive = true
 			p.previewStart = p.totalSamples - len(samples)
 		}
 		p.previewSamples = append(p.previewSamples, samples...)
-		if len(p.previewSamples) >= p.nextPreviewAt {
+		p.previewReceived += len(samples)
+		p.trimPreviewWindow()
+		if p.previewReceived >= p.nextPreviewAt {
 			if err := p.emitProvisional(); err != nil {
 				return err
 			}
 			p.nextPreviewAt += audio.DurationSamples(p.settings.ChunkDuration)
 		}
 	}
-	return p.drainVAD()
+	if err := p.drainVAD(); err != nil {
+		return err
+	}
+	// A short VAD false-positive can become active without meeting the minimum
+	// speech duration, so no final segment is produced. Clear its provisional
+	// text instead of leaving a stale caption on screen.
+	if !speechActive && p.previewActive {
+		p.resetPreview()
+		return p.emit(Event{Kind: Hide})
+	}
+	return nil
 }

 func (p *Processor) drainVAD() error {
@@ -93,13 +121,28 @@ func (p *Processor) drainVAD() error {
 		if err := p.emitFinal(start, start+len(samples), samples); err != nil {
 			return err
 		}
-		p.previewSamples = nil
-		p.previewActive = false
-		p.previewStart = 0
-		p.nextPreviewAt = audio.DurationSamples(p.settings.ChunkDuration)
+		p.resetPreview()
 	}
 }

+func (p *Processor) trimPreviewWindow() {
+	maxSamples := audio.DurationSamples(maxPreviewDuration)
+	if len(p.previewSamples) <= maxSamples {
+		return
+	}
+	dropped := len(p.previewSamples) - maxSamples
+	p.previewSamples = p.previewSamples[dropped:]
+	p.previewStart += dropped
+}
+
+func (p *Processor) resetPreview() {
+	p.previewSamples = nil
+	p.previewActive = false
+	p.previewStart = 0
+	p.previewReceived = 0
+	p.nextPreviewAt = audio.DurationSamples(p.settings.ChunkDuration)
+}
+
 func (p *Processor) emitProvisional() error {
 	text := strings.TrimSpace(p.transcriber.Decode(p.previewSamples))
 	if text == "" {
@@ -125,3 +168,10 @@ func (p *Processor) emitFinal(start, end int, samples []float32) error {
 		EndedAt:   audio.SamplesDuration(end),
 	})
 }
+
+func (p *Processor) emitFixedFinal(start, end int, samples []float32) error {
+	if audio.RMSDBFS(samples) < p.settings.PreviewThresholdDB {
+		return p.emit(Event{Kind: Hide})
+	}
+	return p.emitFinal(start, end, samples)
+}
@@ -14,13 +14,17 @@ type fakeSegment struct {
 }

 type fakeTranscriber struct {
-	text        string
-	decodeTexts []string
-	segments    []fakeSegment
-	flushed     bool
+	text         string
+	decodeTexts  []string
+	segments     []fakeSegment
+	flushed      bool
+	speechActive bool
+	decodeSizes  []int
+	acceptedVAD  int
 }

-func (f *fakeTranscriber) Decode([]float32) string {
+func (f *fakeTranscriber) Decode(samples []float32) string {
+	f.decodeSizes = append(f.decodeSizes, len(samples))
 	if len(f.decodeTexts) == 0 {
 		return f.text
 	}
@@ -28,8 +32,9 @@ func (f *fakeTranscriber) Decode([]float32) string {
 	f.decodeTexts = f.decodeTexts[1:]
 	return text
 }
-func (f *fakeTranscriber) AcceptVAD([]float32) {}
-func (f *fakeTranscriber) FlushVAD()           { f.flushed = true }
+func (f *fakeTranscriber) AcceptVAD(samples []float32) { f.acceptedVAD += len(samples) }
+func (f *fakeTranscriber) SpeechActive() bool          { return f.speechActive }
+func (f *fakeTranscriber) FlushVAD()                   { f.flushed = true }
 func (f *fakeTranscriber) NextSpeechSegment() (int, []float32, bool) {
 	if len(f.segments) == 0 {
 		return 0, nil, false
@@ -42,12 +47,14 @@ func (f *fakeTranscriber) NextSpeechSegment() (int, []float32, bool) {
 func TestFixedModeEmitsFinalCaption(t *testing.T) {
 	transcriber := &fakeTranscriber{text: "hello"}
 	var events []Event
-	processor := New(config.Settings{Mode: config.ModeFixed, ChunkDuration: time.Second}, transcriber, func(event Event) error {
+	processor := New(config.Settings{
+		Mode: config.ModeFixed, ChunkDuration: time.Second, PreviewThresholdDB: -45,
+	}, transcriber, func(event Event) error {
 		events = append(events, event)
 		return nil
 	})

-	if err := processor.Accept(make([]float32, audio.SampleRate)); err != nil {
+	if err := processor.Accept(constantSamples(audio.SampleRate, 0.5)); err != nil {
 		t.Fatal(err)
 	}
 	if len(events) != 1 || events[0].Kind != Final || events[0].Text != "hello" {
@@ -61,12 +68,14 @@ func TestFixedModeEmitsFinalCaption(t *testing.T) {
 func TestFixedModeFlushesFinalPartialChunkWithExactTimestamps(t *testing.T) {
 	transcriber := &fakeTranscriber{decodeTexts: []string{"first", "partial"}}
 	var events []Event
-	processor := New(config.Settings{Mode: config.ModeFixed, ChunkDuration: time.Second}, transcriber, func(event Event) error {
+	processor := New(config.Settings{
+		Mode: config.ModeFixed, ChunkDuration: time.Second, PreviewThresholdDB: -45,
+	}, transcriber, func(event Event) error {
 		events = append(events, event)
 		return nil
 	})

-	if err := processor.Accept(make([]float32, audio.SampleRate+audio.SampleRate/2)); err != nil {
+	if err := processor.Accept(constantSamples(audio.SampleRate+audio.SampleRate/2, 0.5)); err != nil {
 		t.Fatal(err)
 	}
 	if err := processor.Flush(); err != nil {
@@ -81,7 +90,7 @@ func TestFixedModeFlushesFinalPartialChunkWithExactTimestamps(t *testing.T) {
 }

 func TestVADModeEmitsProvisionalThenFinal(t *testing.T) {
-	transcriber := &fakeTranscriber{text: "speech"}
+	transcriber := &fakeTranscriber{text: "speech", speechActive: true}
 	var events []Event
 	processor := New(config.Settings{
 		Mode:               config.ModeVAD,
@@ -92,14 +101,12 @@ func TestVADModeEmitsProvisionalThenFinal(t *testing.T) {
 		return nil
 	})

-	loud := make([]float32, audio.SampleRate)
-	for i := range loud {
-		loud[i] = 0.5
-	}
+	loud := constantSamples(audio.SampleRate, 0.5)
 	if err := processor.Accept(loud); err != nil {
 		t.Fatal(err)
 	}
 	transcriber.segments = []fakeSegment{{start: 0, samples: loud}}
+	transcriber.speechActive = false
 	if err := processor.Accept(make([]float32, audio.SamplesPerPacket())); err != nil {
 		t.Fatal(err)
 	}
@@ -112,11 +119,13 @@ func TestVADModeEmitsProvisionalThenFinal(t *testing.T) {
 func TestEmptyFinalEmitsHide(t *testing.T) {
 	transcriber := &fakeTranscriber{text: " \n\t "}
 	var event Event
-	processor := New(config.Settings{Mode: config.ModeFixed, ChunkDuration: time.Second}, transcriber, func(got Event) error {
+	processor := New(config.Settings{
+		Mode: config.ModeFixed, ChunkDuration: time.Second, PreviewThresholdDB: -45,
+	}, transcriber, func(got Event) error {
 		event = got
 		return nil
 	})
-	if err := processor.Accept(make([]float32, audio.SampleRate)); err != nil {
+	if err := processor.Accept(constantSamples(audio.SampleRate, 0.5)); err != nil {
 		t.Fatal(err)
 	}
 	if event.Kind != Hide {
@@ -124,6 +133,101 @@ func TestEmptyFinalEmitsHide(t *testing.T) {
 	}
 }

+func TestFixedModeSilenceStaysOutOfRecognizer(t *testing.T) {
+	transcriber := &fakeTranscriber{text: "hallucinated caption"}
+	var events []Event
+	processor := New(config.Settings{
+		Mode:               config.ModeFixed,
+		ChunkDuration:      time.Second,
+		PreviewThresholdDB: -45,
+	}, transcriber, func(event Event) error {
+		events = append(events, event)
+		return nil
+	})
+
+	if err := processor.Accept(make([]float32, audio.SampleRate)); err != nil {
+		t.Fatal(err)
+	}
+	if len(transcriber.decodeSizes) != 0 {
+		t.Fatalf("silent audio reached recognizer: decode sizes = %v", transcriber.decodeSizes)
+	}
+	if len(events) != 1 || events[0].Kind != Hide {
+		t.Fatalf("silent chunk events = %#v, want one Hide", events)
+	}
+}
+
+func TestVADModeDoesNotPreviewLoudNonSpeech(t *testing.T) {
+	transcriber := &fakeTranscriber{text: "music hallucination"}
+	var events []Event
+	processor := New(config.Settings{
+		Mode:               config.ModeVAD,
+		ChunkDuration:      time.Second,
+		PreviewThresholdDB: -45,
+	}, transcriber, func(event Event) error {
+		events = append(events, event)
+		return nil
+	})
+
+	loudMusic := constantSamples(audio.SampleRate, 0.5)
+	if err := processor.Accept(loudMusic); err != nil {
+		t.Fatal(err)
+	}
+	if transcriber.acceptedVAD != len(loudMusic) {
+		t.Fatalf("VAD received %d samples, want %d", transcriber.acceptedVAD, len(loudMusic))
+	}
+	if len(transcriber.decodeSizes) != 0 || len(events) != 0 {
+		t.Fatalf("non-speech reached recognizer: decodes=%v events=%#v", transcriber.decodeSizes, events)
+	}
+}
+
+func TestVADModeClearsRejectedShortSpeechPreview(t *testing.T) {
+	transcriber := &fakeTranscriber{text: "draft", speechActive: true}
+	var events []Event
+	processor := New(config.Settings{
+		Mode:               config.ModeVAD,
+		ChunkDuration:      time.Second,
+		PreviewThresholdDB: -45,
+	}, transcriber, func(event Event) error {
+		events = append(events, event)
+		return nil
+	})
+
+	if err := processor.Accept(constantSamples(audio.SampleRate, 0.5)); err != nil {
+		t.Fatal(err)
+	}
+	transcriber.speechActive = false
+	if err := processor.Accept(make([]float32, audio.SamplesPerPacket())); err != nil {
+		t.Fatal(err)
+	}
+	if len(events) != 2 || events[0].Kind != Provisional || events[1].Kind != Hide {
+		t.Fatalf("events = %#v, want Provisional then Hide", events)
+	}
+}
+
+func TestVADPreviewRecognitionWindowIsBounded(t *testing.T) {
+	transcriber := &fakeTranscriber{text: "draft", speechActive: true}
+	processor := New(config.Settings{
+		Mode:               config.ModeVAD,
+		ChunkDuration:      time.Second,
+		PreviewThresholdDB: -45,
+	}, transcriber, func(Event) error { return nil })
+
+	for range 15 {
+		if err := processor.Accept(constantSamples(audio.SampleRate, 0.5)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	maxSamples := audio.DurationSamples(maxPreviewDuration)
+	for _, size := range transcriber.decodeSizes {
+		if size > maxSamples {
+			t.Fatalf("preview decode used %d samples, limit is %d", size, maxSamples)
+		}
+	}
+	if len(transcriber.decodeSizes) != 15 {
+		t.Fatalf("preview decode count = %d, want 15", len(transcriber.decodeSizes))
+	}
+}
+
 func TestFlushFlushesVAD(t *testing.T) {
 	transcriber := &fakeTranscriber{}
 	processor := New(config.Settings{Mode: config.ModeVAD, ChunkDuration: time.Second}, transcriber, func(Event) error { return nil })
@@ -134,3 +238,11 @@ func TestFlushFlushesVAD(t *testing.T) {
 		t.Fatal("VAD was not flushed")
 	}
 }
+
+func constantSamples(count int, value float32) []float32 {
+	samples := make([]float32, count)
+	for i := range samples {
+		samples[i] = value
+	}
+	return samples
+}