kato/Captioneer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

feat(overlay): improve provisional text display and line splitting

Browse Source
This commit is contained in:
kato
2026-07-16 19:27:39 +03:00
parent 2f5ca41ac3
commit 13ddebbf98
4 changed files with 217 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/output/overlay/overlay_gtk.go
+35 -14
View File
@@ -383,8 +383,10 @@ static CaptioneerOverlay *captioneer_overlay_new(
// view state, rather than GTK ellipsizing, enforces its six-entry limit.
gtk_label_set_lines(GTK_LABEL(overlay->final_label), -1);
gtk_label_set_ellipsize(GTK_LABEL(overlay->final_label), PANGO_ELLIPSIZE_NONE);
gtk_label_set_lines(GTK_LABEL(overlay->preview_label), 2);
gtk_label_set_ellipsize(GTK_LABEL(overlay->preview_label), PANGO_ELLIPSIZE_END);
// Provisional text is already split and bounded by the Go view state. Let
// GTK allocate all of those lines instead of replacing overflow with "…".
gtk_label_set_lines(GTK_LABEL(overlay->preview_label), -1);
gtk_label_set_ellipsize(GTK_LABEL(overlay->preview_label), PANGO_ELLIPSIZE_NONE);
gtk_widget_add_css_class(overlay->final_label, "captioneer-final");
gtk_widget_add_css_class(overlay->preview_label, "captioneer-preview");
gtk_window_set_child(overlay->window, box);
@@ -471,15 +473,21 @@ import (
"tea.chunkbyte.com/kato/captioneer/src/config"
)
const finalFadeRefreshInterval = 200 * time.Millisecond
const (
finalFadeRefreshInterval = 200 * time.Millisecond
averageGlyphWidthFactor = 0.58
minimumLineCharacters = 20
maximumLineCharacters = 64
)
type Sink struct {
native *C.CaptioneerOverlay
finalTimeout time.Duration
finalTimer *time.Timer
state viewState
mu sync.RWMutex
closed bool
native *C.CaptioneerOverlay
finalTimeout time.Duration
finalTimer *time.Timer
lineCharacters int
state viewState
mu sync.RWMutex
closed bool
}
func New(settings config.OverlaySettings) (*Sink, string, error) {
@@ -512,7 +520,11 @@ func New(settings config.OverlaySettings) (*Sink, string, error) {
}
return nil, warning, errors.New(message)
}
return &Sink{native: native, finalTimeout: settings.FinalTimeout}, warning, nil
return &Sink{
native: native,
finalTimeout: settings.FinalTimeout,
lineCharacters: overlayLineCharacters(settings),
}, warning, nil
}
func (s *Sink) Publish(event captions.Event) error {
@@ -522,17 +534,26 @@ func (s *Sink) Publish(event captions.Event) error {
return errors.New("overlay is closed")
}
now := time.Now()
s.state.Apply(event, now, s.finalTimeout)
if event.Kind == captions.Final || event.Kind == captions.Hide {
s.stopFinalTimer()
}
s.state.Apply(event, now, s.finalTimeout, s.lineCharacters)
if event.Kind == captions.Final {
s.stopFinalTimer()
s.scheduleFinalRefresh(now)
}
s.postState()
return nil
}
func overlayLineCharacters(settings config.OverlaySettings) int {
characters := int(float64(settings.MaxWidth) / (float64(settings.FontSize) * averageGlyphWidthFactor))
if characters < minimumLineCharacters {
return minimumLineCharacters
}
if characters > maximumLineCharacters {
return maximumLineCharacters
}
return characters
}
func (s *Sink) refreshFinals() {
s.mu.Lock()
defer s.mu.Unlock()
src/output/overlay/state.go
+113 -11
View File
@@ -3,16 +3,20 @@ package overlay
import (
"fmt"
"html"
"math"
"strings"
"time"
"unicode/utf8"
"tea.chunkbyte.com/kato/captioneer/src/captions"
)
const (
maxVisibleFinalLines = 6
maxVisiblePreviewLines = 3
oldestLineBrightness = 0.65
defaultLineFadeDuration = 3 * time.Second
readingWordsPerSecond = 3.0
)
type finalLine struct {
@@ -26,26 +30,39 @@ type viewState struct {
Provisional string
}
func (s *viewState) Apply(event captions.Event, now time.Time, finalTimeout time.Duration) {
func (s *viewState) Apply(event captions.Event, now time.Time, finalTimeout time.Duration, maxLineCharacters int) {
switch event.Kind {
case captions.Provisional:
s.Provisional = strings.TrimSpace(event.Text)
chunks := splitCaptionText(event.Text, maxLineCharacters)
if len(chunks) > maxVisiblePreviewLines {
chunks = chunks[len(chunks)-maxVisiblePreviewLines:]
}
s.Provisional = strings.Join(chunks, "\n")
case captions.Final:
text := strings.TrimSpace(event.Text)
if text == "" {
chunks := splitCaptionText(event.Text, maxLineCharacters)
if len(chunks) == 0 {
return
}
line := finalLine{CreatedAt: now, Text: text}
if finalTimeout > 0 {
line.ExpiresAt = now.Add(finalTimeout)
if len(chunks) > maxVisibleFinalLines {
chunks = chunks[len(chunks)-maxVisibleFinalLines:]
}
expiresAt := now
for _, text := range chunks {
line := finalLine{CreatedAt: now, Text: text}
if finalTimeout > 0 {
expiresAt = expiresAt.Add(readingLifetime(text, finalTimeout))
line.ExpiresAt = expiresAt
}
s.Finals = append(s.Finals, line)
}
s.Finals = append(s.Finals, line)
if len(s.Finals) > maxVisibleFinalLines {
s.Finals = s.Finals[len(s.Finals)-maxVisibleFinalLines:]
}
s.Provisional = ""
case captions.Hide:
*s = viewState{}
// A blank recognition result must not erase completed captions before
// their individual reading lifetimes have elapsed.
s.Provisional = ""
}
}
@@ -84,7 +101,11 @@ func (s viewState) FinalMarkup(now time.Time, lineLifetime time.Duration) string
if i > 0 {
markup.WriteByte('\n')
}
brightness := lineBrightness(now.Sub(line.CreatedAt), fadeDuration)
lineFadeDuration := fadeDuration
if !line.ExpiresAt.IsZero() {
lineFadeDuration = line.ExpiresAt.Sub(line.CreatedAt)
}
brightness := lineBrightness(now.Sub(line.CreatedAt), lineFadeDuration)
channel := int(brightness*255 + 0.5)
fmt.Fprintf(&markup, `<span foreground="#%02X%02X%02X">`, channel, channel, channel)
markup.WriteString(html.EscapeString(line.Text))
@@ -99,13 +120,94 @@ func (s viewState) NeedsFadeRefresh(now time.Time, lineLifetime time.Duration) b
fadeDuration = defaultLineFadeDuration
}
for _, line := range s.Finals {
if now.Sub(line.CreatedAt) < fadeDuration {
lineFadeDuration := fadeDuration
if !line.ExpiresAt.IsZero() {
lineFadeDuration = line.ExpiresAt.Sub(line.CreatedAt)
}
if now.Sub(line.CreatedAt) < lineFadeDuration {
return true
}
}
return false
}
func splitCaptionText(text string, maxCharacters int) []string {
if maxCharacters < 1 {
maxCharacters = 1
}
words := strings.Fields(text)
if len(words) == 0 {
return nil
}
var chunks []string
var current strings.Builder
currentCharacters := 0
flush := func() {
if currentCharacters == 0 {
return
}
chunks = append(chunks, current.String())
current.Reset()
currentCharacters = 0
}
for _, word := range words {
wordCharacters := utf8.RuneCountInString(word)
if wordCharacters > maxCharacters {
flush()
runes := []rune(word)
for len(runes) > maxCharacters {
chunks = append(chunks, string(runes[:maxCharacters]))
runes = runes[maxCharacters:]
}
if len(runes) > 0 {
current.WriteString(string(runes))
currentCharacters = len(runes)
}
} else {
separatorCharacters := 0
if currentCharacters > 0 {
separatorCharacters = 1
}
if currentCharacters+separatorCharacters+wordCharacters > maxCharacters {
flush()
separatorCharacters = 0
}
if separatorCharacters > 0 {
current.WriteByte(' ')
currentCharacters++
}
current.WriteString(word)
currentCharacters += wordCharacters
}
if endsSentence(word) {
flush()
}
}
flush()
return chunks
}
func endsSentence(word string) bool {
trimmed := strings.TrimRight(word, `"'”’)]}`)
return strings.HasSuffix(trimmed, ".") || strings.HasSuffix(trimmed, "!") || strings.HasSuffix(trimmed, "?")
}
func readingLifetime(text string, minimum time.Duration) time.Duration {
if minimum <= 0 {
return 0
}
words := len(strings.Fields(text))
readingSeconds := math.Ceil(float64(words) / readingWordsPerSecond)
readingTime := time.Duration(readingSeconds * float64(time.Second))
if readingTime < minimum {
return minimum
}
return readingTime
}
func (s viewState) NextFinalExpiry() time.Time {
if len(s.Finals) == 0 {
return time.Time{}
src/output/overlay/state_test.go
+66 -13
View File
@@ -7,29 +7,31 @@ import (
"tea.chunkbyte.com/kato/captioneer/src/captions"
)
const testLineCharacters = 80
func TestViewStateKeepsCompletedLinesAboveNewProvisional(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
state.Apply(captions.Event{Kind: captions.Provisional, Text: " first draft "}, now, 4*time.Second)
state.Apply(captions.Event{Kind: captions.Final, Text: " first final "}, now, 4*time.Second)
state.Apply(captions.Event{Kind: captions.Final, Text: " second final "}, now, 4*time.Second)
state.Apply(captions.Event{Kind: captions.Provisional, Text: " next draft "}, now, 4*time.Second)
state.Apply(captions.Event{Kind: captions.Provisional, Text: " first draft "}, now, 4*time.Second, testLineCharacters)
state.Apply(captions.Event{Kind: captions.Final, Text: " first final "}, now, 4*time.Second, testLineCharacters)
state.Apply(captions.Event{Kind: captions.Final, Text: " second final "}, now, 4*time.Second, testLineCharacters)
state.Apply(captions.Event{Kind: captions.Provisional, Text: " next draft "}, now, 4*time.Second, testLineCharacters)
if state.FinalText() != "first final\nsecond final" || state.Provisional != "next draft" {
t.Fatalf("unexpected rows: %#v", state)
}
state.Apply(captions.Event{Kind: captions.Hide}, now, 4*time.Second)
if state.FinalText() != "" || state.Provisional != "" {
t.Fatalf("hide did not clear rows: %#v", state)
state.Apply(captions.Event{Kind: captions.Hide}, now, 4*time.Second, testLineCharacters)
if state.FinalText() != "first final\nsecond final" || state.Provisional != "" {
t.Fatalf("blank result erased completed captions: %#v", state)
}
}
func TestViewStateExpiresEachCompletedLineAfterItsOwnLifetime(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
state.Apply(captions.Event{Kind: captions.Final, Text: "first"}, now, 3*time.Second)
state.Apply(captions.Event{Kind: captions.Final, Text: "second"}, now.Add(time.Second), 3*time.Second)
state.Apply(captions.Event{Kind: captions.Provisional, Text: "draft"}, now, 3*time.Second)
state.Apply(captions.Event{Kind: captions.Final, Text: "first"}, now, 3*time.Second, testLineCharacters)
state.Apply(captions.Event{Kind: captions.Final, Text: "second"}, now.Add(time.Second), 3*time.Second, testLineCharacters)
state.Apply(captions.Event{Kind: captions.Provisional, Text: "draft"}, now, 3*time.Second, testLineCharacters)
if state.ExpireFinal(now.Add(2999 * time.Millisecond)) {
t.Fatal("completed line expired early")
@@ -46,7 +48,7 @@ func TestViewStateLimitsVisibleCompletedLines(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
for i := 0; i <= maxVisibleFinalLines; i++ {
state.Apply(captions.Event{Kind: captions.Final, Text: string(rune('a' + i))}, now, 3*time.Second)
state.Apply(captions.Event{Kind: captions.Final, Text: string(rune('a' + i))}, now, 3*time.Second, testLineCharacters)
}
if len(state.Finals) != maxVisibleFinalLines || state.FinalText() != "b\nc\nd\ne\nf\ng" {
t.Fatalf("unexpected completed-line limit: %#v", state)
@@ -56,7 +58,7 @@ func TestViewStateLimitsVisibleCompletedLines(t *testing.T) {
func TestViewStateZeroTimeoutDoesNotExpire(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
state.Apply(captions.Event{Kind: captions.Final, Text: "final"}, now, 0)
state.Apply(captions.Event{Kind: captions.Final, Text: "final"}, now, 0, testLineCharacters)
if state.ExpireFinal(now.Add(24 * time.Hour)) {
t.Fatal("zero-timeout caption expired")
}
@@ -65,7 +67,7 @@ func TestViewStateZeroTimeoutDoesNotExpire(t *testing.T) {
func TestViewStateFinalMarkupFadesAndEscapesText(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
state.Apply(captions.Event{Kind: captions.Final, Text: "<hello & goodbye>"}, now, 3*time.Second)
state.Apply(captions.Event{Kind: captions.Final, Text: "<hello & goodbye>"}, now, 3*time.Second, testLineCharacters)
if got := state.FinalMarkup(now, 3*time.Second); got != `<span foreground="#FFFFFF">&lt;hello &amp; goodbye&gt;</span>` {
t.Fatalf("new line markup = %q", got)
@@ -74,3 +76,54 @@ func TestViewStateFinalMarkupFadesAndEscapesText(t *testing.T) {
t.Fatalf("old line markup = %q", got)
}
}
func TestSplitCaptionTextUsesSentencesAndWordBoundaries(t *testing.T) {
got := splitCaptionText("Hello there. This sentence should wrap cleanly at words.", 18)
want := []string{"Hello there.", "This sentence", "should wrap", "cleanly at words."}
if len(got) != len(want) {
t.Fatalf("chunks = %#v, want %#v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("chunks = %#v, want %#v", got, want)
}
}
}
func TestProvisionalCaptionKeepsOnlyLatestDigestibleChunks(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
state.Apply(captions.Event{Kind: captions.Provisional, Text: "One. Two. Three. Four."}, now, 3*time.Second, testLineCharacters)
if state.Provisional != "Two.\nThree.\nFour." {
t.Fatalf("provisional text = %q", state.Provisional)
}
}
func TestReadingLifetimeGrowsWithWordCount(t *testing.T) {
text := "one two three four five six seven eight nine ten eleven twelve"
if got := readingLifetime(text, 3*time.Second); got != 4*time.Second {
t.Fatalf("reading lifetime = %s, want 4s", got)
}
if got := readingLifetime("short caption", 3*time.Second); got != 3*time.Second {
t.Fatalf("short reading lifetime = %s, want 3s", got)
}
}
func TestFinalizedChunksExpireInReadingOrder(t *testing.T) {
now := time.Unix(100, 0)
var state viewState
state.Apply(captions.Event{Kind: captions.Final, Text: "First sentence. Second sentence."}, now, 3*time.Second, testLineCharacters)
if len(state.Finals) != 2 {
t.Fatalf("final chunks = %#v", state.Finals)
}
if state.Finals[0].ExpiresAt != now.Add(3*time.Second) {
t.Fatalf("first expiry = %s", state.Finals[0].ExpiresAt)
}
if state.Finals[1].ExpiresAt != now.Add(6*time.Second) {
t.Fatalf("second expiry = %s", state.Finals[1].ExpiresAt)
}
if !state.ExpireFinal(now.Add(3*time.Second)) || state.FinalText() != "Second sentence." {
t.Fatalf("unexpected state after first reading window: %#v", state)
}
}