diff --git a/.gitignore b/.gitignore index 3156a19..f50ac56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ # Downloaded local speech-recognition models. models/parakeet-tdt-v2/ models/silero_vad_v5.onnx + +# Locally built executable. +/captioneer diff --git a/README.md b/README.md index ff84a9c..f9daa59 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# audio-listen +# Captioneer Local, live English captions for the audio playing through your default PulseAudio/PipeWire output device. It captures that output's monitor source, runs the Parakeet TDT v2 model locally, and writes captions to the terminal. @@ -24,7 +24,7 @@ Clone the project, then download the required models once: ```bash git clone -cd audio-listen +cd captioneer ./scripts/download-models.sh ``` @@ -35,20 +35,20 @@ The download installs the Parakeet TDT v2 int8 model and Silero VAD into `models Use VAD mode for normal live captions. It shows a temporary `…` caption while speech is active, then prints a timestamped final caption after 0.5 seconds of silence: ```bash -go run . --mode=vad +go run ./src --mode=vad ``` Use fixed mode to decode exact time slices, even through silence: ```bash -go run . --mode=fixed --chunk-duration=1s +go run ./src --mode=fixed --chunk-duration=1s ``` Build a reusable executable instead of using `go run`: ```bash -go build -o audio-listen . -./audio-listen --mode=vad +go build -o captioneer ./src +./captioneer --mode=vad ``` Press Ctrl+C to stop. Pending audio is flushed before exit. @@ -70,17 +70,28 @@ Examples: ```bash # Faster fixed chunks -go run . --mode=fixed --chunk-duration=500ms +go run ./src --mode=fixed --chunk-duration=500ms # Fewer VAD preview redraws and four CPU threads -go run . --mode=vad --chunk-duration=2s --threads=4 +go run ./src --mode=vad --chunk-duration=2s --threads=4 # Store or reuse models outside this repository -go run . --mode=vad --models-dir=/path/to/models +go run ./src --mode=vad --models-dir=/path/to/models ``` There are no environment variables to configure. All supported configuration is supplied with command-line options. +## Source layout + +All Go code lives in `src/`. It remains one executable package, split into focused files: + +- `main.go` coordinates startup and shutdown. +- `config.go` defines and validates command-line options. +- `capture.go` reads the default system-audio monitor. +- `audio.go` contains PCM conversion and audio calculations. +- `models.go` validates and initializes Parakeet and Silero. +- `captions.go` handles fixed chunks, VAD segments, and terminal output. + ## Output and troubleshooting Final captions go to standard output in this form: @@ -92,7 +103,7 @@ Final captions go to standard output in this form: Startup messages and overload warnings go to standard error, so you can save final captions separately: ```bash -go run . --mode=vad > captions.txt +go run ./src --mode=vad > captions.txt ``` - **Missing model files:** run `./scripts/download-models.sh` again. @@ -104,5 +115,5 @@ go run . --mode=vad > captions.txt ```bash go test ./... -go build ./... +go build -o captioneer ./src ``` diff --git a/go.mod b/go.mod index bbec2ec..9784944 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module audio-meter +module tea.chunkbyte.com/kato/captioneer go 1.26.2 diff --git a/main.go b/main.go deleted file mode 100644 index ec5e7df..0000000 --- a/main.go +++ /dev/null @@ -1,270 +0,0 @@ -package main - -import ( - "context" - "encoding/binary" - "errors" - "flag" - "fmt" - "io" - "log" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strings" - "sync" - "syscall" - "time" - - sherpa "github.com/k2-fsa/sherpa-onnx-go-linux" -) - -const ( - sampleRate = 16000 - channels = 1 - bytesPerSample = 2 - packetDuration = 100 * time.Millisecond - defaultModelsDir = "models" -) - -type settings struct { - mode string - chunkDuration time.Duration - modelsDir string - threads int - previewThresholdDB float64 -} - -func main() { - cfg := parseFlags() - if err := validateSettings(cfg); err != nil { - log.Fatal(err) - } - if err := validatePrograms(); err != nil { - log.Fatal(err) - } - paths, err := validateModels(cfg.modelsDir, cfg.mode) - if err != nil { - log.Fatal(err) - } - - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - - recognizer := newRecognizer(paths, cfg.threads) - defer sherpa.DeleteOfflineRecognizer(recognizer) - - var vad *sherpa.VoiceActivityDetector - if cfg.mode == "vad" { - vad = newVAD(paths.vad, cfg.threads) - defer sherpa.DeleteVoiceActivityDetector(vad) - } - - monitorSource, err := defaultMonitorSource(ctx) - if err != nil { - log.Fatalf("find default output monitor: %v", err) - } - fmt.Fprintf(os.Stderr, "Capturing from: %s\n", monitorSource) - fmt.Fprintln(os.Stderr, "Press Ctrl+C to stop.") - - packets, waitCapture, err := capturePackets(ctx, monitorSource) - if err != nil { - log.Fatalf("start system-audio capture: %v", err) - } - - processor := newCaptionProcessor(cfg, recognizer, vad, os.Stdout) - for packet := range packets { - processor.accept(packet) - } - processor.flush() - if err := waitCapture(); err != nil && ctx.Err() == nil { - log.Fatalf("audio capture stopped unexpectedly: %v", err) - } -} - -func parseFlags() settings { - var cfg settings - flag.StringVar(&cfg.mode, "mode", "", "caption mode: vad or fixed (required)") - flag.DurationVar(&cfg.chunkDuration, "chunk-duration", time.Second, "fixed chunk size or VAD preview refresh interval") - flag.StringVar(&cfg.modelsDir, "models-dir", defaultModelsDir, "directory containing downloaded Sherpa models") - flag.IntVar(&cfg.threads, "threads", 2, "CPU threads used by recognition and VAD") - flag.Float64Var(&cfg.previewThresholdDB, "preview-threshold-dbfs", -45, "RMS dBFS threshold used to begin VAD previews") - flag.Parse() - return cfg -} - -func validateSettings(cfg settings) error { - if cfg.mode != "fixed" && cfg.mode != "vad" { - return errors.New("--mode is required and must be either fixed or vad") - } - if cfg.chunkDuration <= 0 { - return errors.New("--chunk-duration must be positive") - } - if cfg.threads <= 0 { - return errors.New("--threads must be positive") - } - return nil -} - -func validatePrograms() error { - for _, program := range []string{"pactl", "parec"} { - if _, err := exec.LookPath(program); err != nil { - return fmt.Errorf("%s was not found in PATH: %w", program, err) - } - } - return nil -} - -type modelPaths struct { - encoder string - decoder string - joiner string - tokens string - vad string -} - -func validateModels(modelsDir, mode string) (modelPaths, error) { - parakeetDir := filepath.Join(modelsDir, "parakeet-tdt-v2") - paths := modelPaths{ - encoder: filepath.Join(parakeetDir, "encoder.int8.onnx"), - decoder: filepath.Join(parakeetDir, "decoder.int8.onnx"), - joiner: filepath.Join(parakeetDir, "joiner.int8.onnx"), - tokens: filepath.Join(parakeetDir, "tokens.txt"), - vad: filepath.Join(modelsDir, "silero_vad_v5.onnx"), - } - required := []string{paths.encoder, paths.decoder, paths.joiner, paths.tokens} - if mode == "vad" { - required = append(required, paths.vad) - } - var missing []string - for _, path := range required { - info, err := os.Stat(path) - if err != nil || info.IsDir() { - missing = append(missing, path) - } - } - if len(missing) > 0 { - return modelPaths{}, fmt.Errorf("missing required model files:\n %s\nRun ./scripts/download-models.sh", strings.Join(missing, "\n ")) - } - return paths, nil -} - -func newRecognizer(paths modelPaths, threads int) *sherpa.OfflineRecognizer { - recognizer := sherpa.NewOfflineRecognizer(&sherpa.OfflineRecognizerConfig{ - FeatConfig: sherpa.FeatureConfig{SampleRate: sampleRate, FeatureDim: 80}, - ModelConfig: sherpa.OfflineModelConfig{ - Transducer: sherpa.OfflineTransducerModelConfig{ - Encoder: paths.encoder, - Decoder: paths.decoder, - Joiner: paths.joiner, - }, - Tokens: paths.tokens, - NumThreads: threads, - Provider: "cpu", - ModelType: "nemo_transducer", - }, - DecodingMethod: "greedy_search", - }) - if recognizer == nil { - log.Fatal("create Parakeet recognizer: Sherpa returned nil") - } - return recognizer -} - -func newVAD(model string, threads int) *sherpa.VoiceActivityDetector { - vad := sherpa.NewVoiceActivityDetector(&sherpa.VadModelConfig{ - SileroVad: sherpa.SileroVadModelConfig{ - Model: model, - Threshold: 0.5, - MinSilenceDuration: 0.5, - MinSpeechDuration: 0.25, - WindowSize: 512, - MaxSpeechDuration: 30, - }, - SampleRate: sampleRate, - NumThreads: threads, - Provider: "cpu", - }, 60) - if vad == nil { - log.Fatal("create Silero VAD: Sherpa returned nil") - } - return vad -} - -func defaultMonitorSource(ctx context.Context) (string, error) { - output, err := exec.CommandContext(ctx, "pactl", "get-default-sink").Output() - if err != nil { - return "", err - } - return strings.TrimSpace(string(output)) + ".monitor", nil -} - -func capturePackets(ctx context.Context, monitorSource string) (<-chan []float32, func() error, error) { - cmd := exec.CommandContext(ctx, "parec", - "--device="+monitorSource, - "--format=s16le", - fmt.Sprintf("--rate=%d", sampleRate), - fmt.Sprintf("--channels=%d", channels), - "--raw", - "--latency-msec=50", - ) - cmd.Stderr = os.Stderr - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, nil, err - } - if err := cmd.Start(); err != nil { - return nil, nil, err - } - - packets := make(chan []float32, 50) // Five seconds at 100 ms per packet. - var waitOnce sync.Once - var waitErr error - wait := func() error { - waitOnce.Do(func() { waitErr = cmd.Wait() }) - return waitErr - } - - go func() { - defer close(packets) - packetBytes := make([]byte, samplesPerPacket()*bytesPerSample) - for { - n, readErr := io.ReadFull(stdout, packetBytes) - if n > 0 { - packet := pcm16LEToFloat32(packetBytes[:n]) - select { - case packets <- packet: - default: - // Keep the newest audio so captions recover quickly after overload. - select { - case <-packets: - default: - } - select { - case packets <- packet: - default: - } - fmt.Fprintln(os.Stderr, "warning: caption processing overloaded; dropped 100 ms of oldest audio") - } - } - if readErr != nil { - if ctx.Err() == nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { - fmt.Fprintf(os.Stderr, "audio capture read error: %v\n", readErr) - } - return - } - } - }() - return packets, wait, nil -} - -func samplesPerPacket() int { return int(sampleRate * packetDuration / time.Second) } - -func pcm16LEToFloat32(pcm []byte) []float32 { - samples := make([]float32, len(pcm)/bytesPerSample) - for i := range samples { - samples[i] = float32(int16(binary.LittleEndian.Uint16(pcm[i*2:]))) / 32768 - } - return samples -} diff --git a/src/audio.go b/src/audio.go new file mode 100644 index 0000000..e1fa135 --- /dev/null +++ b/src/audio.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/binary" + "fmt" + "math" + "time" +) + +const ( + sampleRate = 16000 + channels = 1 + bytesPerSample = 2 + packetDuration = 100 * time.Millisecond +) + +func samplesPerPacket() int { + return durationSamples(packetDuration) +} + +func pcm16LEToFloat32(pcm []byte) []float32 { + samples := make([]float32, len(pcm)/bytesPerSample) + for i := range samples { + samples[i] = float32(int16(binary.LittleEndian.Uint16(pcm[i*2:]))) / 32768 + } + return samples +} + +func durationSamples(duration time.Duration) int { + return int(duration * sampleRate / time.Second) +} + +func rmsDBFS(samples []float32) float64 { + if len(samples) == 0 { + return math.Inf(-1) + } + + var sum float64 + for _, sample := range samples { + sum += float64(sample * sample) + } + rms := math.Sqrt(sum / float64(len(samples))) + if rms == 0 { + return math.Inf(-1) + } + return 20 * math.Log10(rms) +} + +func formatAudioTime(samples int) string { + return fmt.Sprintf("%.2fs", float64(samples)/sampleRate) +} diff --git a/main_test.go b/src/audio_test.go similarity index 60% rename from main_test.go rename to src/audio_test.go index 930005d..7d89719 100644 --- a/main_test.go +++ b/src/audio_test.go @@ -3,7 +3,6 @@ package main import ( "encoding/binary" "math" - "strings" "testing" "time" ) @@ -39,22 +38,3 @@ func TestRMSDBFS(t *testing.T) { t.Fatalf("silence = %f dBFS, want -Inf", got) } } - -func TestValidateSettings(t *testing.T) { - if err := validateSettings(settings{mode: "fixed", chunkDuration: time.Second, threads: 1}); err != nil { - t.Fatalf("valid settings failed: %v", err) - } - if err := validateSettings(settings{mode: "", chunkDuration: time.Second, threads: 1}); err == nil { - t.Fatal("missing mode unexpectedly passed") - } -} - -func TestValidateModelsExplainsHowToInstallThem(t *testing.T) { - _, err := validateModels(t.TempDir(), "vad") - if err == nil { - t.Fatal("missing models unexpectedly passed validation") - } - if !strings.Contains(err.Error(), "./scripts/download-models.sh") { - t.Fatalf("missing-model error did not provide setup command: %v", err) - } -} diff --git a/caption.go b/src/captions.go similarity index 85% rename from caption.go rename to src/captions.go index 69e6fdd..0fefea8 100644 --- a/caption.go +++ b/src/captions.go @@ -3,9 +3,7 @@ package main import ( "fmt" "io" - "math" "strings" - "time" sherpa "github.com/k2-fsa/sherpa-onnx-go-linux" ) @@ -129,26 +127,3 @@ func (p *captionProcessor) decode(samples []float32) string { p.recognizer.Decode(stream) return strings.TrimSpace(stream.GetResult().Text) } - -func durationSamples(duration time.Duration) int { - return int(duration * sampleRate / time.Second) -} - -func rmsDBFS(samples []float32) float64 { - if len(samples) == 0 { - return math.Inf(-1) - } - var sum float64 - for _, sample := range samples { - sum += float64(sample * sample) - } - rms := math.Sqrt(sum / float64(len(samples))) - if rms == 0 { - return math.Inf(-1) - } - return 20 * math.Log10(rms) -} - -func formatAudioTime(samples int) string { - return fmt.Sprintf("%.2fs", float64(samples)/sampleRate) -} diff --git a/src/capture.go b/src/capture.go new file mode 100644 index 0000000..beb66bc --- /dev/null +++ b/src/capture.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" +) + +func validatePrograms() error { + for _, program := range []string{"pactl", "parec"} { + if _, err := exec.LookPath(program); err != nil { + return fmt.Errorf("%s was not found in PATH: %w", program, err) + } + } + return nil +} + +func defaultMonitorSource(ctx context.Context) (string, error) { + output, err := exec.CommandContext(ctx, "pactl", "get-default-sink").Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(output)) + ".monitor", nil +} + +func capturePackets(ctx context.Context, monitorSource string) (<-chan []float32, func() error, error) { + cmd := exec.CommandContext(ctx, "parec", + "--device="+monitorSource, + "--format=s16le", + fmt.Sprintf("--rate=%d", sampleRate), + fmt.Sprintf("--channels=%d", channels), + "--raw", + "--latency-msec=50", + ) + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + if err := cmd.Start(); err != nil { + return nil, nil, err + } + + packets := make(chan []float32, 50) // Five seconds at 100 ms per packet. + var waitOnce sync.Once + var waitErr error + wait := func() error { + waitOnce.Do(func() { waitErr = cmd.Wait() }) + return waitErr + } + + go func() { + defer close(packets) + packetBytes := make([]byte, samplesPerPacket()*bytesPerSample) + for { + n, readErr := io.ReadFull(stdout, packetBytes) + if n > 0 { + enqueueLatest(packets, pcm16LEToFloat32(packetBytes[:n])) + } + if readErr != nil { + if ctx.Err() == nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { + fmt.Fprintf(os.Stderr, "audio capture read error: %v\n", readErr) + } + return + } + } + }() + return packets, wait, nil +} + +func enqueueLatest(packets chan []float32, packet []float32) { + select { + case packets <- packet: + return + default: + } + + // Keep the newest audio so captions recover quickly after overload. + select { + case <-packets: + default: + } + select { + case packets <- packet: + default: + } + fmt.Fprintln(os.Stderr, "warning: caption processing overloaded; dropped 100 ms of oldest audio") +} diff --git a/src/config.go b/src/config.go new file mode 100644 index 0000000..b9b3939 --- /dev/null +++ b/src/config.go @@ -0,0 +1,41 @@ +package main + +import ( + "errors" + "flag" + "time" +) + +const defaultModelsDir = "models" + +type settings struct { + mode string + chunkDuration time.Duration + modelsDir string + threads int + previewThresholdDB float64 +} + +func parseFlags() settings { + var cfg settings + flag.StringVar(&cfg.mode, "mode", "", "caption mode: vad or fixed (required)") + flag.DurationVar(&cfg.chunkDuration, "chunk-duration", time.Second, "fixed chunk size or VAD preview refresh interval") + flag.StringVar(&cfg.modelsDir, "models-dir", defaultModelsDir, "directory containing downloaded Sherpa models") + flag.IntVar(&cfg.threads, "threads", 2, "CPU threads used by recognition and VAD") + flag.Float64Var(&cfg.previewThresholdDB, "preview-threshold-dbfs", -45, "RMS dBFS threshold used to begin VAD previews") + flag.Parse() + return cfg +} + +func validateSettings(cfg settings) error { + if cfg.mode != "fixed" && cfg.mode != "vad" { + return errors.New("--mode is required and must be either fixed or vad") + } + if cfg.chunkDuration <= 0 { + return errors.New("--chunk-duration must be positive") + } + if cfg.threads <= 0 { + return errors.New("--threads must be positive") + } + return nil +} diff --git a/src/config_test.go b/src/config_test.go new file mode 100644 index 0000000..351c699 --- /dev/null +++ b/src/config_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestValidateSettings(t *testing.T) { + if err := validateSettings(settings{mode: "fixed", chunkDuration: time.Second, threads: 1}); err != nil { + t.Fatalf("valid settings failed: %v", err) + } + if err := validateSettings(settings{mode: "", chunkDuration: time.Second, threads: 1}); err == nil { + t.Fatal("missing mode unexpectedly passed") + } +} + +func TestValidateModelsExplainsHowToInstallThem(t *testing.T) { + _, err := validateModels(t.TempDir(), "vad") + if err == nil { + t.Fatal("missing models unexpectedly passed validation") + } + if !strings.Contains(err.Error(), "./scripts/download-models.sh") { + t.Fatalf("missing-model error did not provide setup command: %v", err) + } +} diff --git a/src/main.go b/src/main.go new file mode 100644 index 0000000..914c44b --- /dev/null +++ b/src/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + sherpa "github.com/k2-fsa/sherpa-onnx-go-linux" +) + +func main() { + cfg := parseFlags() + if err := validateSettings(cfg); err != nil { + log.Fatal(err) + } + if err := validatePrograms(); err != nil { + log.Fatal(err) + } + paths, err := validateModels(cfg.modelsDir, cfg.mode) + if err != nil { + log.Fatal(err) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + recognizer := newRecognizer(paths, cfg.threads) + defer sherpa.DeleteOfflineRecognizer(recognizer) + + var vad *sherpa.VoiceActivityDetector + if cfg.mode == "vad" { + vad = newVAD(paths.vad, cfg.threads) + defer sherpa.DeleteVoiceActivityDetector(vad) + } + + monitorSource, err := defaultMonitorSource(ctx) + if err != nil { + log.Fatalf("find default output monitor: %v", err) + } + fmt.Fprintf(os.Stderr, "Capturing from: %s\n", monitorSource) + fmt.Fprintln(os.Stderr, "Press Ctrl+C to stop.") + + packets, waitCapture, err := capturePackets(ctx, monitorSource) + if err != nil { + log.Fatalf("start system-audio capture: %v", err) + } + + processor := newCaptionProcessor(cfg, recognizer, vad, os.Stdout) + for packet := range packets { + processor.accept(packet) + } + processor.flush() + if err := waitCapture(); err != nil && ctx.Err() == nil { + log.Fatalf("audio capture stopped unexpectedly: %v", err) + } +} diff --git a/src/models.go b/src/models.go new file mode 100644 index 0000000..a76ca74 --- /dev/null +++ b/src/models.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + sherpa "github.com/k2-fsa/sherpa-onnx-go-linux" +) + +type modelPaths struct { + encoder string + decoder string + joiner string + tokens string + vad string +} + +func validateModels(modelsDir, mode string) (modelPaths, error) { + parakeetDir := filepath.Join(modelsDir, "parakeet-tdt-v2") + paths := modelPaths{ + encoder: filepath.Join(parakeetDir, "encoder.int8.onnx"), + decoder: filepath.Join(parakeetDir, "decoder.int8.onnx"), + joiner: filepath.Join(parakeetDir, "joiner.int8.onnx"), + tokens: filepath.Join(parakeetDir, "tokens.txt"), + vad: filepath.Join(modelsDir, "silero_vad_v5.onnx"), + } + + required := []string{paths.encoder, paths.decoder, paths.joiner, paths.tokens} + if mode == "vad" { + required = append(required, paths.vad) + } + + var missing []string + for _, path := range required { + info, err := os.Stat(path) + if err != nil || info.IsDir() { + missing = append(missing, path) + } + } + if len(missing) > 0 { + return modelPaths{}, fmt.Errorf("missing required model files:\n %s\nRun ./scripts/download-models.sh", strings.Join(missing, "\n ")) + } + return paths, nil +} + +func newRecognizer(paths modelPaths, threads int) *sherpa.OfflineRecognizer { + recognizer := sherpa.NewOfflineRecognizer(&sherpa.OfflineRecognizerConfig{ + FeatConfig: sherpa.FeatureConfig{SampleRate: sampleRate, FeatureDim: 80}, + ModelConfig: sherpa.OfflineModelConfig{ + Transducer: sherpa.OfflineTransducerModelConfig{ + Encoder: paths.encoder, + Decoder: paths.decoder, + Joiner: paths.joiner, + }, + Tokens: paths.tokens, + NumThreads: threads, + Provider: "cpu", + ModelType: "nemo_transducer", + }, + DecodingMethod: "greedy_search", + }) + if recognizer == nil { + log.Fatal("create Parakeet recognizer: Sherpa returned nil") + } + return recognizer +} + +func newVAD(model string, threads int) *sherpa.VoiceActivityDetector { + vad := sherpa.NewVoiceActivityDetector(&sherpa.VadModelConfig{ + SileroVad: sherpa.SileroVadModelConfig{ + Model: model, + Threshold: 0.5, + MinSilenceDuration: 0.5, + MinSpeechDuration: 0.25, + WindowSize: 512, + MaxSpeechDuration: 30, + }, + SampleRate: sampleRate, + NumThreads: threads, + Provider: "cpu", + }, 60) + if vad == nil { + log.Fatal("create Silero VAD: Sherpa returned nil") + } + return vad +}