feat: implement configurable background jobs and toggle flags

Introduce environment variables to globally and individually control background jobs:
- `WARPBOX_JOBS_ENABLED` to toggle all background workers.
- `WARPBOX_CLEANUP_ENABLED` to toggle the expired box cleanup job.
- `WARPBOX_THUMBNAIL_ENABLED` to toggle the thumbnail generation job.

Refactor background tasks into a dedicated `backend/libs/jobs` package, allowing jobs to be registered, scheduled, and conditionally run based on the new configuration flags. Additionally, update the default maximum upload size in `.env.example` to 16GB and document the new settings in the README.
This commit is contained in:
2026-05-29 22:25:59 +03:00
parent 720b45a9a6
commit 74ede000b4
11 changed files with 431 additions and 275 deletions

View File

@@ -11,38 +11,44 @@ import (
)
type Config struct {
AppName string
Environment string
Addr string
BaseURL string
DataDir string
AdminToken string
StaticDir string
TemplateDir string
ReadTimeout time.Duration
WriteTimeout time.Duration
IdleTimeout time.Duration
CleanupEvery time.Duration
ThumbnailEvery time.Duration
MaxUploadSize int64
AppName string
Environment string
Addr string
BaseURL string
DataDir string
AdminToken string
StaticDir string
TemplateDir string
ReadTimeout time.Duration
WriteTimeout time.Duration
IdleTimeout time.Duration
JobsEnabled bool
CleanupEnabled bool
CleanupEvery time.Duration
ThumbnailEnabled bool
ThumbnailEvery time.Duration
MaxUploadSize int64
}
func Load() (Config, error) {
cfg := Config{
AppName: envString("WARPBOX_APP_NAME", "warpbox.dev"),
Environment: envString("WARPBOX_ENV", "development"),
Addr: envString("WARPBOX_ADDR", ":8080"),
BaseURL: strings.TrimRight(envString("WARPBOX_BASE_URL", "http://localhost:8080"), "/"),
DataDir: envString("WARPBOX_DATA_DIR", defaultPath("data")),
AdminToken: envString("WARPBOX_ADMIN_TOKEN", ""),
StaticDir: envString("WARPBOX_STATIC_DIR", defaultPath("static")),
TemplateDir: envString("WARPBOX_TEMPLATE_DIR", defaultPath("templates")),
ReadTimeout: envDuration("WARPBOX_READ_TIMEOUT", 15*time.Second),
WriteTimeout: envDuration("WARPBOX_WRITE_TIMEOUT", 60*time.Second),
IdleTimeout: envDuration("WARPBOX_IDLE_TIMEOUT", 120*time.Second),
CleanupEvery: envDuration("WARPBOX_CLEANUP_EVERY", time.Hour),
ThumbnailEvery: envDuration("WARPBOX_THUMBNAIL_EVERY", time.Minute),
MaxUploadSize: envMegabytes("WARPBOX_MAX_UPLOAD_SIZE_MB", 2048), // 2 GiB default.
AppName: envString("WARPBOX_APP_NAME", "warpbox.dev"),
Environment: envString("WARPBOX_ENV", "development"),
Addr: envString("WARPBOX_ADDR", ":8080"),
BaseURL: strings.TrimRight(envString("WARPBOX_BASE_URL", "http://localhost:8080"), "/"),
DataDir: envString("WARPBOX_DATA_DIR", defaultPath("data")),
AdminToken: envString("WARPBOX_ADMIN_TOKEN", ""),
StaticDir: envString("WARPBOX_STATIC_DIR", defaultPath("static")),
TemplateDir: envString("WARPBOX_TEMPLATE_DIR", defaultPath("templates")),
ReadTimeout: envDuration("WARPBOX_READ_TIMEOUT", 15*time.Second),
WriteTimeout: envDuration("WARPBOX_WRITE_TIMEOUT", 60*time.Second),
IdleTimeout: envDuration("WARPBOX_IDLE_TIMEOUT", 120*time.Second),
JobsEnabled: envBool("WARPBOX_JOBS_ENABLED", true),
CleanupEnabled: envBool("WARPBOX_CLEANUP_ENABLED", true),
CleanupEvery: envDuration("WARPBOX_CLEANUP_EVERY", time.Hour),
ThumbnailEnabled: envBool("WARPBOX_THUMBNAIL_ENABLED", true),
ThumbnailEvery: envDuration("WARPBOX_THUMBNAIL_EVERY", time.Minute),
MaxUploadSize: envMegabytes("WARPBOX_MAX_UPLOAD_SIZE_MB", 2048), // 2 GiB default.
}
if cfg.BaseURL == "" {
@@ -90,6 +96,19 @@ func envDuration(key string, fallback time.Duration) time.Duration {
return parsed
}
func envBool(key string, fallback bool) bool {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
parsed, err := strconv.ParseBool(value)
if err != nil {
return fallback
}
return parsed
}
func envMegabytes(key string, fallback float64) int64 {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {

View File

@@ -32,3 +32,20 @@ func TestParseMegabytesRejectsInvalidValues(t *testing.T) {
}
}
}
func TestEnvBool(t *testing.T) {
t.Setenv("WARPBOX_TEST_BOOL", "false")
if got := envBool("WARPBOX_TEST_BOOL", true); got {
t.Fatalf("envBool() = true, want false")
}
t.Setenv("WARPBOX_TEST_BOOL", "1")
if got := envBool("WARPBOX_TEST_BOOL", false); !got {
t.Fatalf("envBool() = false, want true")
}
t.Setenv("WARPBOX_TEST_BOOL", "not-a-bool")
if got := envBool("WARPBOX_TEST_BOOL", true); !got {
t.Fatalf("envBool() did not fall back to true")
}
}

View File

@@ -3,10 +3,10 @@ package httpserver
import (
"log/slog"
"net/http"
"time"
"warpbox.dev/backend/libs/config"
"warpbox.dev/backend/libs/handlers"
"warpbox.dev/backend/libs/jobs"
"warpbox.dev/backend/libs/middleware"
"warpbox.dev/backend/libs/services"
"warpbox.dev/backend/libs/web"
@@ -22,8 +22,7 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
if err != nil {
return nil, err
}
stopCleanup := startCleanup(uploadService, cfg.CleanupEvery, logger)
stopThumbnails := startThumbnails(uploadService, cfg.ThumbnailEvery, logger)
stopJobs := jobs.StartAll(cfg, logger, uploadService)
app := handlers.NewApp(cfg, logger, renderer, uploadService)
router := http.NewServeMux()
@@ -46,8 +45,7 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
IdleTimeout: cfg.IdleTimeout,
}
server.RegisterOnShutdown(func() {
stopCleanup()
stopThumbnails()
stopJobs()
if err := uploadService.Close(); err != nil {
logger.Error("failed to close upload service", "source", "shutdown", "severity", "error", "error", err.Error())
}
@@ -55,72 +53,3 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
return server, nil
}
func startCleanup(uploadService *services.UploadService, interval time.Duration, logger *slog.Logger) func() {
if interval <= 0 {
return func() {}
}
stop := make(chan struct{})
go func() {
if cleaned, err := uploadService.CleanupExpired(); err != nil {
logger.Warn("initial cleanup failed", "source", "housekeeping", "severity", "warn", "code", 4201, "error", err.Error())
} else if cleaned > 0 {
logger.Info("initial cleanup complete", "source", "housekeeping", "severity", "user_activity", "code", 2202, "cleaned", cleaned)
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if _, err := uploadService.CleanupExpired(); err != nil {
logger.Warn("scheduled cleanup failed", "source", "housekeeping", "severity", "warn", "code", 4202, "error", err.Error())
}
case <-stop:
return
}
}
}()
return func() {
close(stop)
}
}
func startThumbnails(uploadService *services.UploadService, interval time.Duration, logger *slog.Logger) func() {
if interval <= 0 {
return func() {}
}
stop := make(chan struct{})
run := func(source string) {
result, err := uploadService.GenerateMissingThumbnails()
if err != nil {
logger.Warn("thumbnail job failed", "source", "thumbnail", "severity", "warn", "code", 4203, "error", err.Error())
return
}
if result.Generated > 0 || result.Failed > 0 {
logger.Info("thumbnail job run", "source", source, "severity", "user_activity", "code", 2204, "generated", result.Generated, "failed", result.Failed)
}
}
go func() {
run("thumbnail")
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
run("thumbnail")
case <-stop:
return
}
}
}()
return func() {
close(stop)
}
}

View File

@@ -0,0 +1,57 @@
package jobs
import (
"log/slog"
"time"
"warpbox.dev/backend/libs/config"
"warpbox.dev/backend/libs/services"
)
func newCleanupJob(cfg config.Config, logger *slog.Logger, uploadService *services.UploadService) job {
return job{
name: "cleanup",
enabled: cfg.CleanupEnabled,
interval: cfg.CleanupEvery,
run: func() {
cleaned, err := cleanupUnavailableBoxes(uploadService, logger)
if err != nil {
logger.Warn("cleanup job failed", "source", "housekeeping", "severity", "warn", "code", 4202, "error", err.Error())
return
}
if cleaned > 0 {
logger.Info("cleanup job complete", "source", "housekeeping", "severity", "user_activity", "code", 2202, "cleaned", cleaned)
}
},
}
}
func cleanupUnavailableBoxes(uploadService *services.UploadService, logger *slog.Logger) (int, error) {
boxes, err := uploadService.ListBoxes(0)
if err != nil {
return 0, err
}
now := time.Now().UTC()
cleaned := 0
for _, box := range boxes {
if !shouldDeleteBox(box, now) {
continue
}
if err := uploadService.DeleteBoxWithSource(box.ID, "housekeeping"); err != nil {
return cleaned, err
}
cleaned++
}
if cleaned > 0 {
logger.Info("unavailable boxes cleaned", "source", "housekeeping", "severity", "user_activity", "code", 2201, "cleaned", cleaned)
}
return cleaned, nil
}
func shouldDeleteBox(box services.Box, now time.Time) bool {
if !box.ExpiresAt.After(now) {
return true
}
return box.MaxDownloads > 0 && box.DownloadCount >= box.MaxDownloads
}

View File

@@ -0,0 +1,50 @@
package jobs
import (
"testing"
"time"
"warpbox.dev/backend/libs/services"
)
func TestShouldDeleteBox(t *testing.T) {
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
tests := map[string]struct {
box services.Box
want bool
}{
"expired": {
box: services.Box{ExpiresAt: now.Add(-time.Second)},
want: true,
},
"expires now": {
box: services.Box{ExpiresAt: now},
want: true,
},
"download limit reached": {
box: services.Box{ExpiresAt: now.Add(time.Hour), MaxDownloads: 3, DownloadCount: 3},
want: true,
},
"download limit exceeded": {
box: services.Box{ExpiresAt: now.Add(time.Hour), MaxDownloads: 3, DownloadCount: 4},
want: true,
},
"active unlimited": {
box: services.Box{ExpiresAt: now.Add(time.Hour)},
want: false,
},
"active under limit": {
box: services.Box{ExpiresAt: now.Add(time.Hour), MaxDownloads: 3, DownloadCount: 2},
want: false,
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
if got := shouldDeleteBox(tt.box, now); got != tt.want {
t.Fatalf("shouldDeleteBox() = %v, want %v", got, tt.want)
}
})
}
}

75
backend/libs/jobs/jobs.go Normal file
View File

@@ -0,0 +1,75 @@
package jobs
import (
"log/slog"
"sync"
"time"
"warpbox.dev/backend/libs/config"
"warpbox.dev/backend/libs/services"
)
type job struct {
name string
enabled bool
interval time.Duration
run func()
}
func StartAll(cfg config.Config, logger *slog.Logger, uploadService *services.UploadService) func() {
if !cfg.JobsEnabled {
logger.Info("background jobs disabled", "source", "jobs", "severity", "dev")
return func() {}
}
stops := []func(){
start(newCleanupJob(cfg, logger, uploadService), logger),
start(newThumbnailsJob(cfg, logger, uploadService), logger),
}
var once sync.Once
return func() {
once.Do(func() {
for _, stop := range stops {
stop()
}
})
}
}
func start(j job, logger *slog.Logger) func() {
if !j.enabled {
logger.Info("background job disabled", "source", "jobs", "severity", "dev", "job", j.name)
return func() {}
}
if j.interval <= 0 {
logger.Info("background job disabled by interval", "source", "jobs", "severity", "dev", "job", j.name, "interval", j.interval.String())
return func() {}
}
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
j.run()
ticker := time.NewTicker(j.interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
j.run()
case <-stop:
return
}
}
}()
var once sync.Once
return func() {
once.Do(func() {
close(stop)
<-done
})
}
}

View File

@@ -0,0 +1,161 @@
package jobs
import (
"image"
_ "image/gif"
"image/jpeg"
_ "image/jpeg"
_ "image/png"
"log/slog"
"os"
"os/exec"
"strings"
"time"
_ "golang.org/x/image/webp"
"warpbox.dev/backend/libs/config"
"warpbox.dev/backend/libs/services"
)
type thumbnailJobResult struct {
Scanned int
Generated int
Failed int
}
func newThumbnailsJob(cfg config.Config, logger *slog.Logger, uploadService *services.UploadService) job {
return job{
name: "thumbnail",
enabled: cfg.ThumbnailEnabled,
interval: cfg.ThumbnailEvery,
run: func() {
result, err := generateMissingThumbnails(uploadService, logger)
if err != nil {
logger.Warn("thumbnail job failed", "source", "thumbnail", "severity", "warn", "code", 4203, "error", err.Error())
return
}
if result.Generated > 0 || result.Failed > 0 {
logger.Info("thumbnail job complete", "source", "thumbnail", "severity", "user_activity", "code", 2204, "generated", result.Generated, "failed", result.Failed)
}
},
}
}
func generateMissingThumbnails(uploadService *services.UploadService, logger *slog.Logger) (thumbnailJobResult, error) {
boxes, err := uploadService.ListBoxes(0)
if err != nil {
return thumbnailJobResult{}, err
}
var result thumbnailJobResult
now := time.Now().UTC()
for _, box := range boxes {
if !box.ExpiresAt.After(now) {
continue
}
changed := false
for i := range box.Files {
file := &box.Files[i]
if file.Thumbnail != "" || !needsThumbnail(*file) {
continue
}
result.Scanned++
thumbnail, err := generateThumbnail(uploadService, box, *file)
if err != nil {
logger.Warn("thumbnail generation failed", "source", "thumbnail", "severity", "warn", "code", 4101, "file_id", file.ID, "error", err.Error())
result.Failed++
continue
}
if thumbnail == "" {
result.Failed++
continue
}
file.Thumbnail = thumbnail
changed = true
result.Generated++
}
if changed {
if err := uploadService.SaveBox(box); err != nil {
return result, err
}
}
}
return result, nil
}
func needsThumbnail(file services.File) bool {
return file.PreviewKind == "image" || file.PreviewKind == "video"
}
func generateThumbnail(uploadService *services.UploadService, box services.Box, file services.File) (string, error) {
thumbnailName := "@thumb@" + file.ID + ".jpg"
thumbnailPath := uploadService.ThumbnailPath(box, services.File{Thumbnail: thumbnailName})
sourcePath := uploadService.FilePath(box, file)
switch {
case strings.HasPrefix(file.ContentType, "image/"):
return thumbnailName, createImageThumbnail(sourcePath, thumbnailPath)
case strings.HasPrefix(file.ContentType, "video/"):
return thumbnailName, createVideoThumbnail(sourcePath, thumbnailPath)
default:
return "", nil
}
}
func createImageThumbnail(sourcePath, targetPath string) error {
source, err := os.Open(sourcePath)
if err != nil {
return err
}
defer source.Close()
img, _, err := image.Decode(source)
if err != nil {
return err
}
thumb := resizeNearest(img, 360, 240)
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return err
}
defer target.Close()
return jpeg.Encode(target, thumb, &jpeg.Options{Quality: 82})
}
func createVideoThumbnail(sourcePath, targetPath string) error {
return exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourcePath, "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run()
}
func resizeNearest(src image.Image, maxWidth, maxHeight int) *image.RGBA {
bounds := src.Bounds()
width := bounds.Dx()
height := bounds.Dy()
if width <= 0 || height <= 0 {
return image.NewRGBA(image.Rect(0, 0, 1, 1))
}
scale := min(float64(maxWidth)/float64(width), float64(maxHeight)/float64(height))
if scale > 1 {
scale = 1
}
targetWidth := max(1, int(float64(width)*scale))
targetHeight := max(1, int(float64(height)*scale))
dst := image.NewRGBA(image.Rect(0, 0, targetWidth, targetHeight))
for y := 0; y < targetHeight; y++ {
for x := 0; x < targetWidth; x++ {
srcX := bounds.Min.X + int(float64(x)/scale)
srcY := bounds.Min.Y + int(float64(y)/scale)
dst.Set(x, y, src.At(srcX, srcY))
}
}
return dst
}

View File

@@ -9,23 +9,15 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"image"
_ "image/gif"
"image/jpeg"
_ "image/jpeg"
_ "image/png"
"io"
"log/slog"
"mime/multipart"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"go.etcd.io/bbolt"
_ "golang.org/x/image/webp"
"warpbox.dev/backend/libs/helpers"
)
@@ -96,12 +88,6 @@ type AdminStats struct {
TotalSizeLabel string
}
type ThumbnailJobResult struct {
Scanned int
Generated int
Failed int
}
type AdminBox struct {
ID string
CreatedAt time.Time
@@ -229,7 +215,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
})
}
if err := s.saveBox(box); err != nil {
if err := s.SaveBox(box); err != nil {
return UploadResult{}, err
}
@@ -338,6 +324,10 @@ func (s *UploadService) AdminBoxes(limit int) ([]AdminBox, error) {
}
func (s *UploadService) DeleteBox(boxID string) error {
return s.DeleteBoxWithSource(boxID, "admin")
}
func (s *UploadService) DeleteBoxWithSource(boxID, source string) error {
if err := s.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket(boxesBucket).Delete([]byte(boxID))
}); err != nil {
@@ -346,86 +336,10 @@ func (s *UploadService) DeleteBox(boxID string) error {
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
return err
}
s.logger.Info("box deleted", "source", "admin", "severity", "user_activity", "code", 2101, "box_id", boxID)
s.logger.Info("box deleted", "source", source, "severity", "user_activity", "code", 2101, "box_id", boxID)
return nil
}
func (s *UploadService) CleanupExpired() (int, error) {
boxes, err := s.ListBoxes(0)
if err != nil {
return 0, err
}
now := time.Now().UTC()
cleaned := 0
for _, box := range boxes {
if box.ExpiresAt.After(now) {
continue
}
if err := s.DeleteBox(box.ID); err != nil {
return cleaned, err
}
cleaned++
}
if cleaned > 0 {
s.logger.Info("expired boxes cleaned", "source", "housekeeping", "severity", "user_activity", "code", 2201, "cleaned", cleaned)
}
return cleaned, nil
}
func (s *UploadService) GenerateMissingThumbnails() (ThumbnailJobResult, error) {
boxes, err := s.ListBoxes(0)
if err != nil {
return ThumbnailJobResult{}, err
}
var result ThumbnailJobResult
for _, box := range boxes {
if time.Now().UTC().After(box.ExpiresAt) {
continue
}
changed := false
for i := range box.Files {
file := &box.Files[i]
if file.Thumbnail != "" || !needsThumbnail(*file) {
continue
}
result.Scanned++
path := s.FilePath(box, *file)
thumbnail := s.generateThumbnail(box.ID, file.ID, path, file.ContentType)
if thumbnail == "" {
result.Failed++
continue
}
file.Thumbnail = thumbnail
changed = true
result.Generated++
}
if changed {
if err := s.saveBox(box); err != nil {
return result, err
}
}
}
if result.Generated > 0 || result.Failed > 0 {
s.logger.Info("thumbnail job complete",
"source", "thumbnail",
"severity", "user_activity",
"code", 2203,
"scanned", result.Scanned,
"generated", result.Generated,
"failed", result.Failed,
)
}
return result, nil
}
func (s *UploadService) FindFile(box Box, fileID string) (File, error) {
for _, file := range box.Files {
if file.ID == fileID {
@@ -534,7 +448,7 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
return nil
}
func (s *UploadService) saveBox(box Box) error {
func (s *UploadService) SaveBox(box Box) error {
data, err := json.Marshal(box)
if err != nil {
return err
@@ -618,83 +532,6 @@ func previewKind(contentType string) string {
}
}
func needsThumbnail(file File) bool {
return file.PreviewKind == "image" || file.PreviewKind == "video"
}
func (s *UploadService) generateThumbnail(boxID, fileID, path, contentType string) string {
thumbnailName := "@thumb@" + fileID + ".jpg"
thumbnailPath := filepath.Join(s.filesDir, boxID, thumbnailName)
var err error
switch {
case strings.HasPrefix(contentType, "image/"):
err = createImageThumbnail(path, thumbnailPath)
case strings.HasPrefix(contentType, "video/"):
err = createVideoThumbnail(path, thumbnailPath)
default:
return ""
}
if err != nil {
s.logger.Warn("thumbnail generation failed", "source", "thumbnail", "severity", "warn", "code", 4101, "file_id", fileID, "error", err.Error())
return ""
}
return thumbnailName
}
func createImageThumbnail(sourcePath, targetPath string) error {
source, err := os.Open(sourcePath)
if err != nil {
return err
}
defer source.Close()
img, _, err := image.Decode(source)
if err != nil {
return err
}
thumb := resizeNearest(img, 360, 240)
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return err
}
defer target.Close()
return jpeg.Encode(target, thumb, &jpeg.Options{Quality: 82})
}
func createVideoThumbnail(sourcePath, targetPath string) error {
return exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourcePath, "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run()
}
func resizeNearest(src image.Image, maxWidth, maxHeight int) *image.RGBA {
bounds := src.Bounds()
width := bounds.Dx()
height := bounds.Dy()
if width <= 0 || height <= 0 {
return image.NewRGBA(image.Rect(0, 0, 1, 1))
}
scale := min(float64(maxWidth)/float64(width), float64(maxHeight)/float64(height))
if scale > 1 {
scale = 1
}
targetWidth := max(1, int(float64(width)*scale))
targetHeight := max(1, int(float64(height)*scale))
dst := image.NewRGBA(image.Rect(0, 0, targetWidth, targetHeight))
for y := 0; y < targetHeight; y++ {
for x := 0; x < targetWidth; x++ {
srcX := bounds.Min.X + int(float64(x)/scale)
srcY := bounds.Min.Y + int(float64(y)/scale)
dst.Set(x, y, src.At(srcX, srcY))
}
}
return dst
}
func (s *UploadService) writeBoxMetadata(box Box) error {
path := s.BoxMetadataPath(box)
data, err := json.MarshalIndent(box, "", " ")