refactor: move source files to src/ directory and rename project to Captioneer
Restructure Go source code into a dedicated `src/` directory, rename the project from "audio-listen" to "Captioneer", and update README accordingly. Add `.gitignore` entry for the built binary `captioneer`.
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
sampleRate = 16000
|
||||
channels = 1
|
||||
bytesPerSample = 2
|
||||
packetDuration = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
func samplesPerPacket() int {
|
||||
return durationSamples(packetDuration)
|
||||
}
|
||||
|
||||
func pcm16LEToFloat32(pcm []byte) []float32 {
|
||||
samples := make([]float32, len(pcm)/bytesPerSample)
|
||||
for i := range samples {
|
||||
samples[i] = float32(int16(binary.LittleEndian.Uint16(pcm[i*2:]))) / 32768
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
func durationSamples(duration time.Duration) int {
|
||||
return int(duration * sampleRate / time.Second)
|
||||
}
|
||||
|
||||
func rmsDBFS(samples []float32) float64 {
|
||||
if len(samples) == 0 {
|
||||
return math.Inf(-1)
|
||||
}
|
||||
|
||||
var sum float64
|
||||
for _, sample := range samples {
|
||||
sum += float64(sample * sample)
|
||||
}
|
||||
rms := math.Sqrt(sum / float64(len(samples)))
|
||||
if rms == 0 {
|
||||
return math.Inf(-1)
|
||||
}
|
||||
return 20 * math.Log10(rms)
|
||||
}
|
||||
|
||||
func formatAudioTime(samples int) string {
|
||||
return fmt.Sprintf("%.2fs", float64(samples)/sampleRate)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPCM16LEToFloat32(t *testing.T) {
|
||||
pcm := make([]byte, 6)
|
||||
binary.LittleEndian.PutUint16(pcm[0:], uint16(0))
|
||||
binary.LittleEndian.PutUint16(pcm[2:], uint16(32767))
|
||||
binary.LittleEndian.PutUint16(pcm[4:], uint16(32768))
|
||||
got := pcm16LEToFloat32(pcm)
|
||||
want := []float32{0, 32767.0 / 32768.0, -1}
|
||||
for i := range want {
|
||||
if math.Abs(float64(got[i]-want[i])) > 0.00001 {
|
||||
t.Fatalf("sample %d = %f, want %f", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurationSamples(t *testing.T) {
|
||||
if got := durationSamples(time.Second); got != sampleRate {
|
||||
t.Fatalf("one second = %d samples, want %d", got, sampleRate)
|
||||
}
|
||||
if got := durationSamples(1500 * time.Millisecond); got != 24000 {
|
||||
t.Fatalf("1.5 seconds = %d samples, want 24000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRMSDBFS(t *testing.T) {
|
||||
if got := rmsDBFS([]float32{1, -1}); math.Abs(got) > 0.00001 {
|
||||
t.Fatalf("full-scale RMS = %f dBFS, want 0", got)
|
||||
}
|
||||
if got := rmsDBFS([]float32{0, 0}); !math.IsInf(got, -1) {
|
||||
t.Fatalf("silence = %f dBFS, want -Inf", got)
|
||||
}
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
sherpa "github.com/k2-fsa/sherpa-onnx-go-linux"
|
||||
)
|
||||
|
||||
type captionProcessor struct {
|
||||
cfg settings
|
||||
recognizer *sherpa.OfflineRecognizer
|
||||
vad *sherpa.VoiceActivityDetector
|
||||
output io.Writer
|
||||
|
||||
fixedSamples []float32
|
||||
previewSamples []float32
|
||||
previewActive bool
|
||||
previewVisible bool
|
||||
nextPreviewAt int
|
||||
totalSamples int
|
||||
}
|
||||
|
||||
func newCaptionProcessor(cfg settings, recognizer *sherpa.OfflineRecognizer, vad *sherpa.VoiceActivityDetector, output io.Writer) *captionProcessor {
|
||||
return &captionProcessor{
|
||||
cfg: cfg,
|
||||
recognizer: recognizer,
|
||||
vad: vad,
|
||||
output: output,
|
||||
nextPreviewAt: durationSamples(cfg.chunkDuration),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *captionProcessor) accept(samples []float32) {
|
||||
if len(samples) == 0 {
|
||||
return
|
||||
}
|
||||
if p.cfg.mode == "fixed" {
|
||||
p.acceptFixed(samples)
|
||||
return
|
||||
}
|
||||
p.acceptVAD(samples)
|
||||
}
|
||||
|
||||
func (p *captionProcessor) acceptFixed(samples []float32) {
|
||||
p.fixedSamples = append(p.fixedSamples, samples...)
|
||||
chunkSize := durationSamples(p.cfg.chunkDuration)
|
||||
for len(p.fixedSamples) >= chunkSize {
|
||||
p.emitCommitted(p.totalSamples, p.totalSamples+chunkSize, p.fixedSamples[:chunkSize])
|
||||
p.fixedSamples = p.fixedSamples[chunkSize:]
|
||||
p.totalSamples += chunkSize
|
||||
}
|
||||
}
|
||||
|
||||
func (p *captionProcessor) acceptVAD(samples []float32) {
|
||||
p.totalSamples += len(samples)
|
||||
p.vad.AcceptWaveform(samples)
|
||||
|
||||
if p.previewActive || rmsDBFS(samples) >= p.cfg.previewThresholdDB {
|
||||
p.previewActive = true
|
||||
p.previewSamples = append(p.previewSamples, samples...)
|
||||
if len(p.previewSamples) >= p.nextPreviewAt {
|
||||
p.emitPreview(p.previewSamples)
|
||||
p.nextPreviewAt += durationSamples(p.cfg.chunkDuration)
|
||||
}
|
||||
}
|
||||
p.drainVAD()
|
||||
}
|
||||
|
||||
func (p *captionProcessor) drainVAD() {
|
||||
for !p.vad.IsEmpty() {
|
||||
segment := p.vad.Front()
|
||||
p.vad.Pop()
|
||||
p.clearPreview()
|
||||
p.emitCommitted(segment.Start, segment.Start+len(segment.Samples), segment.Samples)
|
||||
p.previewSamples = nil
|
||||
p.previewActive = false
|
||||
p.nextPreviewAt = durationSamples(p.cfg.chunkDuration)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *captionProcessor) flush() {
|
||||
if p.cfg.mode == "fixed" {
|
||||
if len(p.fixedSamples) > 0 {
|
||||
p.emitCommitted(p.totalSamples, p.totalSamples+len(p.fixedSamples), p.fixedSamples)
|
||||
}
|
||||
return
|
||||
}
|
||||
p.vad.Flush()
|
||||
p.drainVAD()
|
||||
p.clearPreview()
|
||||
}
|
||||
|
||||
func (p *captionProcessor) emitPreview(samples []float32) {
|
||||
text := p.decode(samples)
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(p.output, "\r\033[2K… %s", text)
|
||||
p.previewVisible = true
|
||||
}
|
||||
|
||||
func (p *captionProcessor) emitCommitted(start, end int, samples []float32) {
|
||||
text := p.decode(samples)
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
p.clearPreview()
|
||||
fmt.Fprintf(p.output, "[%s-%s] %s\n", formatAudioTime(start), formatAudioTime(end), text)
|
||||
}
|
||||
|
||||
func (p *captionProcessor) clearPreview() {
|
||||
if p.previewVisible {
|
||||
fmt.Fprint(p.output, "\r\033[2K")
|
||||
p.previewVisible = false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *captionProcessor) decode(samples []float32) string {
|
||||
if len(samples) == 0 {
|
||||
return ""
|
||||
}
|
||||
stream := sherpa.NewOfflineStream(p.recognizer)
|
||||
defer sherpa.DeleteOfflineStream(stream)
|
||||
stream.AcceptWaveform(sampleRate, samples)
|
||||
p.recognizer.Decode(stream)
|
||||
return strings.TrimSpace(stream.GetResult().Text)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func validatePrograms() error {
|
||||
for _, program := range []string{"pactl", "parec"} {
|
||||
if _, err := exec.LookPath(program); err != nil {
|
||||
return fmt.Errorf("%s was not found in PATH: %w", program, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func defaultMonitorSource(ctx context.Context) (string, error) {
|
||||
output, err := exec.CommandContext(ctx, "pactl", "get-default-sink").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(output)) + ".monitor", nil
|
||||
}
|
||||
|
||||
func capturePackets(ctx context.Context, monitorSource string) (<-chan []float32, func() error, error) {
|
||||
cmd := exec.CommandContext(ctx, "parec",
|
||||
"--device="+monitorSource,
|
||||
"--format=s16le",
|
||||
fmt.Sprintf("--rate=%d", sampleRate),
|
||||
fmt.Sprintf("--channels=%d", channels),
|
||||
"--raw",
|
||||
"--latency-msec=50",
|
||||
)
|
||||
cmd.Stderr = os.Stderr
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
packets := make(chan []float32, 50) // Five seconds at 100 ms per packet.
|
||||
var waitOnce sync.Once
|
||||
var waitErr error
|
||||
wait := func() error {
|
||||
waitOnce.Do(func() { waitErr = cmd.Wait() })
|
||||
return waitErr
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(packets)
|
||||
packetBytes := make([]byte, samplesPerPacket()*bytesPerSample)
|
||||
for {
|
||||
n, readErr := io.ReadFull(stdout, packetBytes)
|
||||
if n > 0 {
|
||||
enqueueLatest(packets, pcm16LEToFloat32(packetBytes[:n]))
|
||||
}
|
||||
if readErr != nil {
|
||||
if ctx.Err() == nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) {
|
||||
fmt.Fprintf(os.Stderr, "audio capture read error: %v\n", readErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return packets, wait, nil
|
||||
}
|
||||
|
||||
func enqueueLatest(packets chan []float32, packet []float32) {
|
||||
select {
|
||||
case packets <- packet:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Keep the newest audio so captions recover quickly after overload.
|
||||
select {
|
||||
case <-packets:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case packets <- packet:
|
||||
default:
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "warning: caption processing overloaded; dropped 100 ms of oldest audio")
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultModelsDir = "models"
|
||||
|
||||
type settings struct {
|
||||
mode string
|
||||
chunkDuration time.Duration
|
||||
modelsDir string
|
||||
threads int
|
||||
previewThresholdDB float64
|
||||
}
|
||||
|
||||
func parseFlags() settings {
|
||||
var cfg settings
|
||||
flag.StringVar(&cfg.mode, "mode", "", "caption mode: vad or fixed (required)")
|
||||
flag.DurationVar(&cfg.chunkDuration, "chunk-duration", time.Second, "fixed chunk size or VAD preview refresh interval")
|
||||
flag.StringVar(&cfg.modelsDir, "models-dir", defaultModelsDir, "directory containing downloaded Sherpa models")
|
||||
flag.IntVar(&cfg.threads, "threads", 2, "CPU threads used by recognition and VAD")
|
||||
flag.Float64Var(&cfg.previewThresholdDB, "preview-threshold-dbfs", -45, "RMS dBFS threshold used to begin VAD previews")
|
||||
flag.Parse()
|
||||
return cfg
|
||||
}
|
||||
|
||||
func validateSettings(cfg settings) error {
|
||||
if cfg.mode != "fixed" && cfg.mode != "vad" {
|
||||
return errors.New("--mode is required and must be either fixed or vad")
|
||||
}
|
||||
if cfg.chunkDuration <= 0 {
|
||||
return errors.New("--chunk-duration must be positive")
|
||||
}
|
||||
if cfg.threads <= 0 {
|
||||
return errors.New("--threads must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateSettings(t *testing.T) {
|
||||
if err := validateSettings(settings{mode: "fixed", chunkDuration: time.Second, threads: 1}); err != nil {
|
||||
t.Fatalf("valid settings failed: %v", err)
|
||||
}
|
||||
if err := validateSettings(settings{mode: "", chunkDuration: time.Second, threads: 1}); err == nil {
|
||||
t.Fatal("missing mode unexpectedly passed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateModelsExplainsHowToInstallThem(t *testing.T) {
|
||||
_, err := validateModels(t.TempDir(), "vad")
|
||||
if err == nil {
|
||||
t.Fatal("missing models unexpectedly passed validation")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "./scripts/download-models.sh") {
|
||||
t.Fatalf("missing-model error did not provide setup command: %v", err)
|
||||
}
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
sherpa "github.com/k2-fsa/sherpa-onnx-go-linux"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := parseFlags()
|
||||
if err := validateSettings(cfg); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := validatePrograms(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
paths, err := validateModels(cfg.modelsDir, cfg.mode)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
recognizer := newRecognizer(paths, cfg.threads)
|
||||
defer sherpa.DeleteOfflineRecognizer(recognizer)
|
||||
|
||||
var vad *sherpa.VoiceActivityDetector
|
||||
if cfg.mode == "vad" {
|
||||
vad = newVAD(paths.vad, cfg.threads)
|
||||
defer sherpa.DeleteVoiceActivityDetector(vad)
|
||||
}
|
||||
|
||||
monitorSource, err := defaultMonitorSource(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("find default output monitor: %v", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "Capturing from: %s\n", monitorSource)
|
||||
fmt.Fprintln(os.Stderr, "Press Ctrl+C to stop.")
|
||||
|
||||
packets, waitCapture, err := capturePackets(ctx, monitorSource)
|
||||
if err != nil {
|
||||
log.Fatalf("start system-audio capture: %v", err)
|
||||
}
|
||||
|
||||
processor := newCaptionProcessor(cfg, recognizer, vad, os.Stdout)
|
||||
for packet := range packets {
|
||||
processor.accept(packet)
|
||||
}
|
||||
processor.flush()
|
||||
if err := waitCapture(); err != nil && ctx.Err() == nil {
|
||||
log.Fatalf("audio capture stopped unexpectedly: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
sherpa "github.com/k2-fsa/sherpa-onnx-go-linux"
|
||||
)
|
||||
|
||||
type modelPaths struct {
|
||||
encoder string
|
||||
decoder string
|
||||
joiner string
|
||||
tokens string
|
||||
vad string
|
||||
}
|
||||
|
||||
func validateModels(modelsDir, mode string) (modelPaths, error) {
|
||||
parakeetDir := filepath.Join(modelsDir, "parakeet-tdt-v2")
|
||||
paths := modelPaths{
|
||||
encoder: filepath.Join(parakeetDir, "encoder.int8.onnx"),
|
||||
decoder: filepath.Join(parakeetDir, "decoder.int8.onnx"),
|
||||
joiner: filepath.Join(parakeetDir, "joiner.int8.onnx"),
|
||||
tokens: filepath.Join(parakeetDir, "tokens.txt"),
|
||||
vad: filepath.Join(modelsDir, "silero_vad_v5.onnx"),
|
||||
}
|
||||
|
||||
required := []string{paths.encoder, paths.decoder, paths.joiner, paths.tokens}
|
||||
if mode == "vad" {
|
||||
required = append(required, paths.vad)
|
||||
}
|
||||
|
||||
var missing []string
|
||||
for _, path := range required {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.IsDir() {
|
||||
missing = append(missing, path)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return modelPaths{}, fmt.Errorf("missing required model files:\n %s\nRun ./scripts/download-models.sh", strings.Join(missing, "\n "))
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func newRecognizer(paths modelPaths, threads int) *sherpa.OfflineRecognizer {
|
||||
recognizer := sherpa.NewOfflineRecognizer(&sherpa.OfflineRecognizerConfig{
|
||||
FeatConfig: sherpa.FeatureConfig{SampleRate: sampleRate, FeatureDim: 80},
|
||||
ModelConfig: sherpa.OfflineModelConfig{
|
||||
Transducer: sherpa.OfflineTransducerModelConfig{
|
||||
Encoder: paths.encoder,
|
||||
Decoder: paths.decoder,
|
||||
Joiner: paths.joiner,
|
||||
},
|
||||
Tokens: paths.tokens,
|
||||
NumThreads: threads,
|
||||
Provider: "cpu",
|
||||
ModelType: "nemo_transducer",
|
||||
},
|
||||
DecodingMethod: "greedy_search",
|
||||
})
|
||||
if recognizer == nil {
|
||||
log.Fatal("create Parakeet recognizer: Sherpa returned nil")
|
||||
}
|
||||
return recognizer
|
||||
}
|
||||
|
||||
func newVAD(model string, threads int) *sherpa.VoiceActivityDetector {
|
||||
vad := sherpa.NewVoiceActivityDetector(&sherpa.VadModelConfig{
|
||||
SileroVad: sherpa.SileroVadModelConfig{
|
||||
Model: model,
|
||||
Threshold: 0.5,
|
||||
MinSilenceDuration: 0.5,
|
||||
MinSpeechDuration: 0.25,
|
||||
WindowSize: 512,
|
||||
MaxSpeechDuration: 30,
|
||||
},
|
||||
SampleRate: sampleRate,
|
||||
NumThreads: threads,
|
||||
Provider: "cpu",
|
||||
}, 60)
|
||||
if vad == nil {
|
||||
log.Fatal("create Silero VAD: Sherpa returned nil")
|
||||
}
|
||||
return vad
|
||||
}
|
||||
Reference in New Issue
Block a user