feat: implement configurable background jobs and toggle flags
Introduce environment variables to globally and individually control background jobs: - `WARPBOX_JOBS_ENABLED` to toggle all background workers. - `WARPBOX_CLEANUP_ENABLED` to toggle the expired box cleanup job. - `WARPBOX_THUMBNAIL_ENABLED` to toggle the thumbnail generation job. Refactor background tasks into a dedicated `backend/libs/jobs` package, allowing jobs to be registered, scheduled, and conditionally run based on the new configuration flags. Additionally, update the default maximum upload size in `.env.example` to 16GB and document the new settings in the README.
This commit is contained in:
@@ -4,9 +4,12 @@ WARPBOX_ADDR=:8080
|
|||||||
WARPBOX_BASE_URL=http://localhost:8080
|
WARPBOX_BASE_URL=http://localhost:8080
|
||||||
WARPBOX_DATA_DIR=./data
|
WARPBOX_DATA_DIR=./data
|
||||||
WARPBOX_ADMIN_TOKEN=change-me
|
WARPBOX_ADMIN_TOKEN=change-me
|
||||||
|
WARPBOX_JOBS_ENABLED=true
|
||||||
|
WARPBOX_CLEANUP_ENABLED=true
|
||||||
WARPBOX_CLEANUP_EVERY=1h
|
WARPBOX_CLEANUP_EVERY=1h
|
||||||
|
WARPBOX_THUMBNAIL_ENABLED=true
|
||||||
WARPBOX_THUMBNAIL_EVERY=1m
|
WARPBOX_THUMBNAIL_EVERY=1m
|
||||||
WARPBOX_MAX_UPLOAD_SIZE_MB=2048
|
WARPBOX_MAX_UPLOAD_SIZE_MB=16384
|
||||||
WARPBOX_READ_TIMEOUT=15s
|
WARPBOX_READ_TIMEOUT=15s
|
||||||
WARPBOX_WRITE_TIMEOUT=60s
|
WARPBOX_WRITE_TIMEOUT=60s
|
||||||
WARPBOX_IDLE_TIMEOUT=120s
|
WARPBOX_IDLE_TIMEOUT=120s
|
||||||
|
|||||||
@@ -16,6 +16,10 @@ Fractions are supported, so `0.5Mb` is 512 KiB and `1.5Mb` is 1536 KiB.
|
|||||||
Runtime data is configured with `WARPBOX_DATA_DIR` and defaults to `./data` in the dev environment.
|
Runtime data is configured with `WARPBOX_DATA_DIR` and defaults to `./data` in the dev environment.
|
||||||
The dev script resolves that path from the repository root.
|
The dev script resolves that path from the repository root.
|
||||||
|
|
||||||
|
Background jobs are enabled with `WARPBOX_JOBS_ENABLED=true`. Individual jobs can be toggled with
|
||||||
|
`WARPBOX_CLEANUP_ENABLED` and `WARPBOX_THUMBNAIL_ENABLED`, and their schedules are configured with
|
||||||
|
`WARPBOX_CLEANUP_EVERY` and `WARPBOX_THUMBNAIL_EVERY`.
|
||||||
|
|
||||||
The basic admin console is available at `/admin`. Set `WARPBOX_ADMIN_TOKEN` and use that value to sign in.
|
The basic admin console is available at `/admin`. Set `WARPBOX_ADMIN_TOKEN` and use that value to sign in.
|
||||||
|
|
||||||
For one-off Go commands, run them from the backend module:
|
For one-off Go commands, run them from the backend module:
|
||||||
@@ -31,6 +35,7 @@ go run ./cmd/warpbox
|
|||||||
- `backend/libs/config` - environment-backed configuration.
|
- `backend/libs/config` - environment-backed configuration.
|
||||||
- `backend/libs/httpserver` - server construction and route composition.
|
- `backend/libs/httpserver` - server construction and route composition.
|
||||||
- `backend/libs/handlers` - HTTP handlers for pages, API, health, static files.
|
- `backend/libs/handlers` - HTTP handlers for pages, API, health, static files.
|
||||||
|
- `backend/libs/jobs` - background job registration and job loop definitions.
|
||||||
- `backend/libs/middleware` - request logging, recovery, security headers, gzip, request IDs.
|
- `backend/libs/middleware` - request logging, recovery, security headers, gzip, request IDs.
|
||||||
- `backend/libs/services` - business logic boundaries, starting with upload limits.
|
- `backend/libs/services` - business logic boundaries, starting with upload limits.
|
||||||
- `backend/libs/helpers` - small reusable helpers.
|
- `backend/libs/helpers` - small reusable helpers.
|
||||||
@@ -46,8 +51,8 @@ go run ./cmd/warpbox
|
|||||||
- `/admin/login` - token-based admin login.
|
- `/admin/login` - token-based admin login.
|
||||||
- `/admin` - overview metrics: boxes, files, storage, recent uploads, protected/expired boxes.
|
- `/admin` - overview metrics: boxes, files, storage, recent uploads, protected/expired boxes.
|
||||||
- `/admin/files` - recent upload table with view and delete actions.
|
- `/admin/files` - recent upload table with view and delete actions.
|
||||||
- Expired boxes are cleaned on startup and then every `WARPBOX_CLEANUP_EVERY`.
|
- Expired boxes and boxes that have reached their download limit are cleaned on startup and then every `WARPBOX_CLEANUP_EVERY` when `WARPBOX_CLEANUP_ENABLED=true`.
|
||||||
- Missing image/video thumbnails are generated in a background worker every `WARPBOX_THUMBNAIL_EVERY`.
|
- Missing image/video thumbnails are generated in a background worker every `WARPBOX_THUMBNAIL_EVERY` when `WARPBOX_THUMBNAIL_ENABLED=true`.
|
||||||
|
|
||||||
## Runtime Data
|
## Runtime Data
|
||||||
|
|
||||||
|
|||||||
@@ -11,38 +11,44 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
AppName string
|
AppName string
|
||||||
Environment string
|
Environment string
|
||||||
Addr string
|
Addr string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
DataDir string
|
DataDir string
|
||||||
AdminToken string
|
AdminToken string
|
||||||
StaticDir string
|
StaticDir string
|
||||||
TemplateDir string
|
TemplateDir string
|
||||||
ReadTimeout time.Duration
|
ReadTimeout time.Duration
|
||||||
WriteTimeout time.Duration
|
WriteTimeout time.Duration
|
||||||
IdleTimeout time.Duration
|
IdleTimeout time.Duration
|
||||||
CleanupEvery time.Duration
|
JobsEnabled bool
|
||||||
ThumbnailEvery time.Duration
|
CleanupEnabled bool
|
||||||
MaxUploadSize int64
|
CleanupEvery time.Duration
|
||||||
|
ThumbnailEnabled bool
|
||||||
|
ThumbnailEvery time.Duration
|
||||||
|
MaxUploadSize int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load() (Config, error) {
|
func Load() (Config, error) {
|
||||||
cfg := Config{
|
cfg := Config{
|
||||||
AppName: envString("WARPBOX_APP_NAME", "warpbox.dev"),
|
AppName: envString("WARPBOX_APP_NAME", "warpbox.dev"),
|
||||||
Environment: envString("WARPBOX_ENV", "development"),
|
Environment: envString("WARPBOX_ENV", "development"),
|
||||||
Addr: envString("WARPBOX_ADDR", ":8080"),
|
Addr: envString("WARPBOX_ADDR", ":8080"),
|
||||||
BaseURL: strings.TrimRight(envString("WARPBOX_BASE_URL", "http://localhost:8080"), "/"),
|
BaseURL: strings.TrimRight(envString("WARPBOX_BASE_URL", "http://localhost:8080"), "/"),
|
||||||
DataDir: envString("WARPBOX_DATA_DIR", defaultPath("data")),
|
DataDir: envString("WARPBOX_DATA_DIR", defaultPath("data")),
|
||||||
AdminToken: envString("WARPBOX_ADMIN_TOKEN", ""),
|
AdminToken: envString("WARPBOX_ADMIN_TOKEN", ""),
|
||||||
StaticDir: envString("WARPBOX_STATIC_DIR", defaultPath("static")),
|
StaticDir: envString("WARPBOX_STATIC_DIR", defaultPath("static")),
|
||||||
TemplateDir: envString("WARPBOX_TEMPLATE_DIR", defaultPath("templates")),
|
TemplateDir: envString("WARPBOX_TEMPLATE_DIR", defaultPath("templates")),
|
||||||
ReadTimeout: envDuration("WARPBOX_READ_TIMEOUT", 15*time.Second),
|
ReadTimeout: envDuration("WARPBOX_READ_TIMEOUT", 15*time.Second),
|
||||||
WriteTimeout: envDuration("WARPBOX_WRITE_TIMEOUT", 60*time.Second),
|
WriteTimeout: envDuration("WARPBOX_WRITE_TIMEOUT", 60*time.Second),
|
||||||
IdleTimeout: envDuration("WARPBOX_IDLE_TIMEOUT", 120*time.Second),
|
IdleTimeout: envDuration("WARPBOX_IDLE_TIMEOUT", 120*time.Second),
|
||||||
CleanupEvery: envDuration("WARPBOX_CLEANUP_EVERY", time.Hour),
|
JobsEnabled: envBool("WARPBOX_JOBS_ENABLED", true),
|
||||||
ThumbnailEvery: envDuration("WARPBOX_THUMBNAIL_EVERY", time.Minute),
|
CleanupEnabled: envBool("WARPBOX_CLEANUP_ENABLED", true),
|
||||||
MaxUploadSize: envMegabytes("WARPBOX_MAX_UPLOAD_SIZE_MB", 2048), // 2 GiB default.
|
CleanupEvery: envDuration("WARPBOX_CLEANUP_EVERY", time.Hour),
|
||||||
|
ThumbnailEnabled: envBool("WARPBOX_THUMBNAIL_ENABLED", true),
|
||||||
|
ThumbnailEvery: envDuration("WARPBOX_THUMBNAIL_EVERY", time.Minute),
|
||||||
|
MaxUploadSize: envMegabytes("WARPBOX_MAX_UPLOAD_SIZE_MB", 2048), // 2 GiB default.
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.BaseURL == "" {
|
if cfg.BaseURL == "" {
|
||||||
@@ -90,6 +96,19 @@ func envDuration(key string, fallback time.Duration) time.Duration {
|
|||||||
return parsed
|
return parsed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func envBool(key string, fallback bool) bool {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := strconv.ParseBool(value)
|
||||||
|
if err != nil {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
func envMegabytes(key string, fallback float64) int64 {
|
func envMegabytes(key string, fallback float64) int64 {
|
||||||
value := strings.TrimSpace(os.Getenv(key))
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
if value == "" {
|
if value == "" {
|
||||||
|
|||||||
@@ -32,3 +32,20 @@ func TestParseMegabytesRejectsInvalidValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEnvBool(t *testing.T) {
|
||||||
|
t.Setenv("WARPBOX_TEST_BOOL", "false")
|
||||||
|
if got := envBool("WARPBOX_TEST_BOOL", true); got {
|
||||||
|
t.Fatalf("envBool() = true, want false")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("WARPBOX_TEST_BOOL", "1")
|
||||||
|
if got := envBool("WARPBOX_TEST_BOOL", false); !got {
|
||||||
|
t.Fatalf("envBool() = false, want true")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("WARPBOX_TEST_BOOL", "not-a-bool")
|
||||||
|
if got := envBool("WARPBOX_TEST_BOOL", true); !got {
|
||||||
|
t.Fatalf("envBool() did not fall back to true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,10 +3,10 @@ package httpserver
|
|||||||
import (
|
import (
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
|
||||||
|
|
||||||
"warpbox.dev/backend/libs/config"
|
"warpbox.dev/backend/libs/config"
|
||||||
"warpbox.dev/backend/libs/handlers"
|
"warpbox.dev/backend/libs/handlers"
|
||||||
|
"warpbox.dev/backend/libs/jobs"
|
||||||
"warpbox.dev/backend/libs/middleware"
|
"warpbox.dev/backend/libs/middleware"
|
||||||
"warpbox.dev/backend/libs/services"
|
"warpbox.dev/backend/libs/services"
|
||||||
"warpbox.dev/backend/libs/web"
|
"warpbox.dev/backend/libs/web"
|
||||||
@@ -22,8 +22,7 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
stopCleanup := startCleanup(uploadService, cfg.CleanupEvery, logger)
|
stopJobs := jobs.StartAll(cfg, logger, uploadService)
|
||||||
stopThumbnails := startThumbnails(uploadService, cfg.ThumbnailEvery, logger)
|
|
||||||
app := handlers.NewApp(cfg, logger, renderer, uploadService)
|
app := handlers.NewApp(cfg, logger, renderer, uploadService)
|
||||||
|
|
||||||
router := http.NewServeMux()
|
router := http.NewServeMux()
|
||||||
@@ -46,8 +45,7 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
|
|||||||
IdleTimeout: cfg.IdleTimeout,
|
IdleTimeout: cfg.IdleTimeout,
|
||||||
}
|
}
|
||||||
server.RegisterOnShutdown(func() {
|
server.RegisterOnShutdown(func() {
|
||||||
stopCleanup()
|
stopJobs()
|
||||||
stopThumbnails()
|
|
||||||
if err := uploadService.Close(); err != nil {
|
if err := uploadService.Close(); err != nil {
|
||||||
logger.Error("failed to close upload service", "source", "shutdown", "severity", "error", "error", err.Error())
|
logger.Error("failed to close upload service", "source", "shutdown", "severity", "error", "error", err.Error())
|
||||||
}
|
}
|
||||||
@@ -55,72 +53,3 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
|
|||||||
|
|
||||||
return server, nil
|
return server, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func startCleanup(uploadService *services.UploadService, interval time.Duration, logger *slog.Logger) func() {
|
|
||||||
if interval <= 0 {
|
|
||||||
return func() {}
|
|
||||||
}
|
|
||||||
|
|
||||||
stop := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
if cleaned, err := uploadService.CleanupExpired(); err != nil {
|
|
||||||
logger.Warn("initial cleanup failed", "source", "housekeeping", "severity", "warn", "code", 4201, "error", err.Error())
|
|
||||||
} else if cleaned > 0 {
|
|
||||||
logger.Info("initial cleanup complete", "source", "housekeeping", "severity", "user_activity", "code", 2202, "cleaned", cleaned)
|
|
||||||
}
|
|
||||||
|
|
||||||
ticker := time.NewTicker(interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
if _, err := uploadService.CleanupExpired(); err != nil {
|
|
||||||
logger.Warn("scheduled cleanup failed", "source", "housekeeping", "severity", "warn", "code", 4202, "error", err.Error())
|
|
||||||
}
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return func() {
|
|
||||||
close(stop)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func startThumbnails(uploadService *services.UploadService, interval time.Duration, logger *slog.Logger) func() {
|
|
||||||
if interval <= 0 {
|
|
||||||
return func() {}
|
|
||||||
}
|
|
||||||
|
|
||||||
stop := make(chan struct{})
|
|
||||||
run := func(source string) {
|
|
||||||
result, err := uploadService.GenerateMissingThumbnails()
|
|
||||||
if err != nil {
|
|
||||||
logger.Warn("thumbnail job failed", "source", "thumbnail", "severity", "warn", "code", 4203, "error", err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if result.Generated > 0 || result.Failed > 0 {
|
|
||||||
logger.Info("thumbnail job run", "source", source, "severity", "user_activity", "code", 2204, "generated", result.Generated, "failed", result.Failed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
run("thumbnail")
|
|
||||||
|
|
||||||
ticker := time.NewTicker(interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
run("thumbnail")
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return func() {
|
|
||||||
close(stop)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
57
backend/libs/jobs/cleanup.go
Normal file
57
backend/libs/jobs/cleanup.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"warpbox.dev/backend/libs/config"
|
||||||
|
"warpbox.dev/backend/libs/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCleanupJob(cfg config.Config, logger *slog.Logger, uploadService *services.UploadService) job {
|
||||||
|
return job{
|
||||||
|
name: "cleanup",
|
||||||
|
enabled: cfg.CleanupEnabled,
|
||||||
|
interval: cfg.CleanupEvery,
|
||||||
|
run: func() {
|
||||||
|
cleaned, err := cleanupUnavailableBoxes(uploadService, logger)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("cleanup job failed", "source", "housekeeping", "severity", "warn", "code", 4202, "error", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cleaned > 0 {
|
||||||
|
logger.Info("cleanup job complete", "source", "housekeeping", "severity", "user_activity", "code", 2202, "cleaned", cleaned)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanupUnavailableBoxes(uploadService *services.UploadService, logger *slog.Logger) (int, error) {
|
||||||
|
boxes, err := uploadService.ListBoxes(0)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
cleaned := 0
|
||||||
|
for _, box := range boxes {
|
||||||
|
if !shouldDeleteBox(box, now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := uploadService.DeleteBoxWithSource(box.ID, "housekeeping"); err != nil {
|
||||||
|
return cleaned, err
|
||||||
|
}
|
||||||
|
cleaned++
|
||||||
|
}
|
||||||
|
if cleaned > 0 {
|
||||||
|
logger.Info("unavailable boxes cleaned", "source", "housekeeping", "severity", "user_activity", "code", 2201, "cleaned", cleaned)
|
||||||
|
}
|
||||||
|
return cleaned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func shouldDeleteBox(box services.Box, now time.Time) bool {
|
||||||
|
if !box.ExpiresAt.After(now) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return box.MaxDownloads > 0 && box.DownloadCount >= box.MaxDownloads
|
||||||
|
}
|
||||||
50
backend/libs/jobs/cleanup_test.go
Normal file
50
backend/libs/jobs/cleanup_test.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"warpbox.dev/backend/libs/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestShouldDeleteBox(t *testing.T) {
|
||||||
|
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
tests := map[string]struct {
|
||||||
|
box services.Box
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
"expired": {
|
||||||
|
box: services.Box{ExpiresAt: now.Add(-time.Second)},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
"expires now": {
|
||||||
|
box: services.Box{ExpiresAt: now},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
"download limit reached": {
|
||||||
|
box: services.Box{ExpiresAt: now.Add(time.Hour), MaxDownloads: 3, DownloadCount: 3},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
"download limit exceeded": {
|
||||||
|
box: services.Box{ExpiresAt: now.Add(time.Hour), MaxDownloads: 3, DownloadCount: 4},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
"active unlimited": {
|
||||||
|
box: services.Box{ExpiresAt: now.Add(time.Hour)},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
"active under limit": {
|
||||||
|
box: services.Box{ExpiresAt: now.Add(time.Hour), MaxDownloads: 3, DownloadCount: 2},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tt := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if got := shouldDeleteBox(tt.box, now); got != tt.want {
|
||||||
|
t.Fatalf("shouldDeleteBox() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
75
backend/libs/jobs/jobs.go
Normal file
75
backend/libs/jobs/jobs.go
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"warpbox.dev/backend/libs/config"
|
||||||
|
"warpbox.dev/backend/libs/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
type job struct {
|
||||||
|
name string
|
||||||
|
enabled bool
|
||||||
|
interval time.Duration
|
||||||
|
run func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func StartAll(cfg config.Config, logger *slog.Logger, uploadService *services.UploadService) func() {
|
||||||
|
if !cfg.JobsEnabled {
|
||||||
|
logger.Info("background jobs disabled", "source", "jobs", "severity", "dev")
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
stops := []func(){
|
||||||
|
start(newCleanupJob(cfg, logger, uploadService), logger),
|
||||||
|
start(newThumbnailsJob(cfg, logger, uploadService), logger),
|
||||||
|
}
|
||||||
|
|
||||||
|
var once sync.Once
|
||||||
|
return func() {
|
||||||
|
once.Do(func() {
|
||||||
|
for _, stop := range stops {
|
||||||
|
stop()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func start(j job, logger *slog.Logger) func() {
|
||||||
|
if !j.enabled {
|
||||||
|
logger.Info("background job disabled", "source", "jobs", "severity", "dev", "job", j.name)
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
if j.interval <= 0 {
|
||||||
|
logger.Info("background job disabled by interval", "source", "jobs", "severity", "dev", "job", j.name, "interval", j.interval.String())
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
stop := make(chan struct{})
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
j.run()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(j.interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
j.run()
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
var once sync.Once
|
||||||
|
return func() {
|
||||||
|
once.Do(func() {
|
||||||
|
close(stop)
|
||||||
|
<-done
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
161
backend/libs/jobs/thumbnails.go
Normal file
161
backend/libs/jobs/thumbnails.go
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"image"
|
||||||
|
_ "image/gif"
|
||||||
|
"image/jpeg"
|
||||||
|
_ "image/jpeg"
|
||||||
|
_ "image/png"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "golang.org/x/image/webp"
|
||||||
|
"warpbox.dev/backend/libs/config"
|
||||||
|
"warpbox.dev/backend/libs/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
type thumbnailJobResult struct {
|
||||||
|
Scanned int
|
||||||
|
Generated int
|
||||||
|
Failed int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newThumbnailsJob(cfg config.Config, logger *slog.Logger, uploadService *services.UploadService) job {
|
||||||
|
return job{
|
||||||
|
name: "thumbnail",
|
||||||
|
enabled: cfg.ThumbnailEnabled,
|
||||||
|
interval: cfg.ThumbnailEvery,
|
||||||
|
run: func() {
|
||||||
|
result, err := generateMissingThumbnails(uploadService, logger)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thumbnail job failed", "source", "thumbnail", "severity", "warn", "code", 4203, "error", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if result.Generated > 0 || result.Failed > 0 {
|
||||||
|
logger.Info("thumbnail job complete", "source", "thumbnail", "severity", "user_activity", "code", 2204, "generated", result.Generated, "failed", result.Failed)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateMissingThumbnails(uploadService *services.UploadService, logger *slog.Logger) (thumbnailJobResult, error) {
|
||||||
|
boxes, err := uploadService.ListBoxes(0)
|
||||||
|
if err != nil {
|
||||||
|
return thumbnailJobResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var result thumbnailJobResult
|
||||||
|
now := time.Now().UTC()
|
||||||
|
for _, box := range boxes {
|
||||||
|
if !box.ExpiresAt.After(now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
changed := false
|
||||||
|
for i := range box.Files {
|
||||||
|
file := &box.Files[i]
|
||||||
|
if file.Thumbnail != "" || !needsThumbnail(*file) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result.Scanned++
|
||||||
|
|
||||||
|
thumbnail, err := generateThumbnail(uploadService, box, *file)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thumbnail generation failed", "source", "thumbnail", "severity", "warn", "code", 4101, "file_id", file.ID, "error", err.Error())
|
||||||
|
result.Failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if thumbnail == "" {
|
||||||
|
result.Failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
file.Thumbnail = thumbnail
|
||||||
|
changed = true
|
||||||
|
result.Generated++
|
||||||
|
}
|
||||||
|
|
||||||
|
if changed {
|
||||||
|
if err := uploadService.SaveBox(box); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsThumbnail(file services.File) bool {
|
||||||
|
return file.PreviewKind == "image" || file.PreviewKind == "video"
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateThumbnail(uploadService *services.UploadService, box services.Box, file services.File) (string, error) {
|
||||||
|
thumbnailName := "@thumb@" + file.ID + ".jpg"
|
||||||
|
thumbnailPath := uploadService.ThumbnailPath(box, services.File{Thumbnail: thumbnailName})
|
||||||
|
sourcePath := uploadService.FilePath(box, file)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(file.ContentType, "image/"):
|
||||||
|
return thumbnailName, createImageThumbnail(sourcePath, thumbnailPath)
|
||||||
|
case strings.HasPrefix(file.ContentType, "video/"):
|
||||||
|
return thumbnailName, createVideoThumbnail(sourcePath, thumbnailPath)
|
||||||
|
default:
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func createImageThumbnail(sourcePath, targetPath string) error {
|
||||||
|
source, err := os.Open(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
|
||||||
|
img, _, err := image.Decode(source)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
thumb := resizeNearest(img, 360, 240)
|
||||||
|
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer target.Close()
|
||||||
|
|
||||||
|
return jpeg.Encode(target, thumb, &jpeg.Options{Quality: 82})
|
||||||
|
}
|
||||||
|
|
||||||
|
func createVideoThumbnail(sourcePath, targetPath string) error {
|
||||||
|
return exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourcePath, "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func resizeNearest(src image.Image, maxWidth, maxHeight int) *image.RGBA {
|
||||||
|
bounds := src.Bounds()
|
||||||
|
width := bounds.Dx()
|
||||||
|
height := bounds.Dy()
|
||||||
|
if width <= 0 || height <= 0 {
|
||||||
|
return image.NewRGBA(image.Rect(0, 0, 1, 1))
|
||||||
|
}
|
||||||
|
|
||||||
|
scale := min(float64(maxWidth)/float64(width), float64(maxHeight)/float64(height))
|
||||||
|
if scale > 1 {
|
||||||
|
scale = 1
|
||||||
|
}
|
||||||
|
targetWidth := max(1, int(float64(width)*scale))
|
||||||
|
targetHeight := max(1, int(float64(height)*scale))
|
||||||
|
dst := image.NewRGBA(image.Rect(0, 0, targetWidth, targetHeight))
|
||||||
|
|
||||||
|
for y := 0; y < targetHeight; y++ {
|
||||||
|
for x := 0; x < targetWidth; x++ {
|
||||||
|
srcX := bounds.Min.X + int(float64(x)/scale)
|
||||||
|
srcY := bounds.Min.Y + int(float64(y)/scale)
|
||||||
|
dst.Set(x, y, src.At(srcX, srcY))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dst
|
||||||
|
}
|
||||||
@@ -9,23 +9,15 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"image"
|
|
||||||
_ "image/gif"
|
|
||||||
"image/jpeg"
|
|
||||||
_ "image/jpeg"
|
|
||||||
_ "image/png"
|
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.etcd.io/bbolt"
|
"go.etcd.io/bbolt"
|
||||||
_ "golang.org/x/image/webp"
|
|
||||||
|
|
||||||
"warpbox.dev/backend/libs/helpers"
|
"warpbox.dev/backend/libs/helpers"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -96,12 +88,6 @@ type AdminStats struct {
|
|||||||
TotalSizeLabel string
|
TotalSizeLabel string
|
||||||
}
|
}
|
||||||
|
|
||||||
type ThumbnailJobResult struct {
|
|
||||||
Scanned int
|
|
||||||
Generated int
|
|
||||||
Failed int
|
|
||||||
}
|
|
||||||
|
|
||||||
type AdminBox struct {
|
type AdminBox struct {
|
||||||
ID string
|
ID string
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
@@ -229,7 +215,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.saveBox(box); err != nil {
|
if err := s.SaveBox(box); err != nil {
|
||||||
return UploadResult{}, err
|
return UploadResult{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -338,6 +324,10 @@ func (s *UploadService) AdminBoxes(limit int) ([]AdminBox, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *UploadService) DeleteBox(boxID string) error {
|
func (s *UploadService) DeleteBox(boxID string) error {
|
||||||
|
return s.DeleteBoxWithSource(boxID, "admin")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *UploadService) DeleteBoxWithSource(boxID, source string) error {
|
||||||
if err := s.db.Update(func(tx *bbolt.Tx) error {
|
if err := s.db.Update(func(tx *bbolt.Tx) error {
|
||||||
return tx.Bucket(boxesBucket).Delete([]byte(boxID))
|
return tx.Bucket(boxesBucket).Delete([]byte(boxID))
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -346,86 +336,10 @@ func (s *UploadService) DeleteBox(boxID string) error {
|
|||||||
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
|
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
s.logger.Info("box deleted", "source", "admin", "severity", "user_activity", "code", 2101, "box_id", boxID)
|
s.logger.Info("box deleted", "source", source, "severity", "user_activity", "code", 2101, "box_id", boxID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *UploadService) CleanupExpired() (int, error) {
|
|
||||||
boxes, err := s.ListBoxes(0)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
cleaned := 0
|
|
||||||
for _, box := range boxes {
|
|
||||||
if box.ExpiresAt.After(now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := s.DeleteBox(box.ID); err != nil {
|
|
||||||
return cleaned, err
|
|
||||||
}
|
|
||||||
cleaned++
|
|
||||||
}
|
|
||||||
if cleaned > 0 {
|
|
||||||
s.logger.Info("expired boxes cleaned", "source", "housekeeping", "severity", "user_activity", "code", 2201, "cleaned", cleaned)
|
|
||||||
}
|
|
||||||
return cleaned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *UploadService) GenerateMissingThumbnails() (ThumbnailJobResult, error) {
|
|
||||||
boxes, err := s.ListBoxes(0)
|
|
||||||
if err != nil {
|
|
||||||
return ThumbnailJobResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var result ThumbnailJobResult
|
|
||||||
for _, box := range boxes {
|
|
||||||
if time.Now().UTC().After(box.ExpiresAt) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
changed := false
|
|
||||||
for i := range box.Files {
|
|
||||||
file := &box.Files[i]
|
|
||||||
if file.Thumbnail != "" || !needsThumbnail(*file) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
result.Scanned++
|
|
||||||
|
|
||||||
path := s.FilePath(box, *file)
|
|
||||||
thumbnail := s.generateThumbnail(box.ID, file.ID, path, file.ContentType)
|
|
||||||
if thumbnail == "" {
|
|
||||||
result.Failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
file.Thumbnail = thumbnail
|
|
||||||
changed = true
|
|
||||||
result.Generated++
|
|
||||||
}
|
|
||||||
|
|
||||||
if changed {
|
|
||||||
if err := s.saveBox(box); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if result.Generated > 0 || result.Failed > 0 {
|
|
||||||
s.logger.Info("thumbnail job complete",
|
|
||||||
"source", "thumbnail",
|
|
||||||
"severity", "user_activity",
|
|
||||||
"code", 2203,
|
|
||||||
"scanned", result.Scanned,
|
|
||||||
"generated", result.Generated,
|
|
||||||
"failed", result.Failed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *UploadService) FindFile(box Box, fileID string) (File, error) {
|
func (s *UploadService) FindFile(box Box, fileID string) (File, error) {
|
||||||
for _, file := range box.Files {
|
for _, file := range box.Files {
|
||||||
if file.ID == fileID {
|
if file.ID == fileID {
|
||||||
@@ -534,7 +448,7 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *UploadService) saveBox(box Box) error {
|
func (s *UploadService) SaveBox(box Box) error {
|
||||||
data, err := json.Marshal(box)
|
data, err := json.Marshal(box)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -618,83 +532,6 @@ func previewKind(contentType string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func needsThumbnail(file File) bool {
|
|
||||||
return file.PreviewKind == "image" || file.PreviewKind == "video"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *UploadService) generateThumbnail(boxID, fileID, path, contentType string) string {
|
|
||||||
thumbnailName := "@thumb@" + fileID + ".jpg"
|
|
||||||
thumbnailPath := filepath.Join(s.filesDir, boxID, thumbnailName)
|
|
||||||
|
|
||||||
var err error
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(contentType, "image/"):
|
|
||||||
err = createImageThumbnail(path, thumbnailPath)
|
|
||||||
case strings.HasPrefix(contentType, "video/"):
|
|
||||||
err = createVideoThumbnail(path, thumbnailPath)
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Warn("thumbnail generation failed", "source", "thumbnail", "severity", "warn", "code", 4101, "file_id", fileID, "error", err.Error())
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return thumbnailName
|
|
||||||
}
|
|
||||||
|
|
||||||
func createImageThumbnail(sourcePath, targetPath string) error {
|
|
||||||
source, err := os.Open(sourcePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer source.Close()
|
|
||||||
|
|
||||||
img, _, err := image.Decode(source)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
thumb := resizeNearest(img, 360, 240)
|
|
||||||
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer target.Close()
|
|
||||||
|
|
||||||
return jpeg.Encode(target, thumb, &jpeg.Options{Quality: 82})
|
|
||||||
}
|
|
||||||
|
|
||||||
func createVideoThumbnail(sourcePath, targetPath string) error {
|
|
||||||
return exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourcePath, "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run()
|
|
||||||
}
|
|
||||||
|
|
||||||
func resizeNearest(src image.Image, maxWidth, maxHeight int) *image.RGBA {
|
|
||||||
bounds := src.Bounds()
|
|
||||||
width := bounds.Dx()
|
|
||||||
height := bounds.Dy()
|
|
||||||
if width <= 0 || height <= 0 {
|
|
||||||
return image.NewRGBA(image.Rect(0, 0, 1, 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
scale := min(float64(maxWidth)/float64(width), float64(maxHeight)/float64(height))
|
|
||||||
if scale > 1 {
|
|
||||||
scale = 1
|
|
||||||
}
|
|
||||||
targetWidth := max(1, int(float64(width)*scale))
|
|
||||||
targetHeight := max(1, int(float64(height)*scale))
|
|
||||||
dst := image.NewRGBA(image.Rect(0, 0, targetWidth, targetHeight))
|
|
||||||
|
|
||||||
for y := 0; y < targetHeight; y++ {
|
|
||||||
for x := 0; x < targetWidth; x++ {
|
|
||||||
srcX := bounds.Min.X + int(float64(x)/scale)
|
|
||||||
srcY := bounds.Min.Y + int(float64(y)/scale)
|
|
||||||
dst.Set(x, y, src.At(srcX, srcY))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return dst
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *UploadService) writeBoxMetadata(box Box) error {
|
func (s *UploadService) writeBoxMetadata(box Box) error {
|
||||||
path := s.BoxMetadataPath(box)
|
path := s.BoxMetadataPath(box)
|
||||||
data, err := json.MarshalIndent(box, "", " ")
|
data, err := json.MarshalIndent(box, "", " ")
|
||||||
|
|||||||
3
scripts/env/dev.env.example
vendored
3
scripts/env/dev.env.example
vendored
@@ -4,7 +4,10 @@ WARPBOX_ADDR=:8080
|
|||||||
WARPBOX_BASE_URL=http://localhost:8080
|
WARPBOX_BASE_URL=http://localhost:8080
|
||||||
WARPBOX_DATA_DIR=./data
|
WARPBOX_DATA_DIR=./data
|
||||||
WARPBOX_ADMIN_TOKEN=change-me
|
WARPBOX_ADMIN_TOKEN=change-me
|
||||||
|
WARPBOX_JOBS_ENABLED=true
|
||||||
|
WARPBOX_CLEANUP_ENABLED=true
|
||||||
WARPBOX_CLEANUP_EVERY=1h
|
WARPBOX_CLEANUP_EVERY=1h
|
||||||
|
WARPBOX_THUMBNAIL_ENABLED=true
|
||||||
WARPBOX_THUMBNAIL_EVERY=1m
|
WARPBOX_THUMBNAIL_EVERY=1m
|
||||||
WARPBOX_MAX_UPLOAD_SIZE_MB=2048
|
WARPBOX_MAX_UPLOAD_SIZE_MB=2048
|
||||||
WARPBOX_READ_TIMEOUT=15s
|
WARPBOX_READ_TIMEOUT=15s
|
||||||
|
|||||||
Reference in New Issue
Block a user