feat: implement configurable background jobs and toggle flags
Introduce environment variables to globally and individually control background jobs: - `WARPBOX_JOBS_ENABLED` to toggle all background workers. - `WARPBOX_CLEANUP_ENABLED` to toggle the expired box cleanup job. - `WARPBOX_THUMBNAIL_ENABLED` to toggle the thumbnail generation job. Refactor background tasks into a dedicated `backend/libs/jobs` package, allowing jobs to be registered, scheduled, and conditionally run based on the new configuration flags. Additionally, update the default maximum upload size in `.env.example` to 16GB and document the new settings in the README.
This commit is contained in:
@@ -9,23 +9,15 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
"image/jpeg"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/bbolt"
|
||||
_ "golang.org/x/image/webp"
|
||||
|
||||
"warpbox.dev/backend/libs/helpers"
|
||||
)
|
||||
|
||||
@@ -96,12 +88,6 @@ type AdminStats struct {
|
||||
TotalSizeLabel string
|
||||
}
|
||||
|
||||
type ThumbnailJobResult struct {
|
||||
Scanned int
|
||||
Generated int
|
||||
Failed int
|
||||
}
|
||||
|
||||
type AdminBox struct {
|
||||
ID string
|
||||
CreatedAt time.Time
|
||||
@@ -229,7 +215,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
|
||||
})
|
||||
}
|
||||
|
||||
if err := s.saveBox(box); err != nil {
|
||||
if err := s.SaveBox(box); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
|
||||
@@ -338,6 +324,10 @@ func (s *UploadService) AdminBoxes(limit int) ([]AdminBox, error) {
|
||||
}
|
||||
|
||||
func (s *UploadService) DeleteBox(boxID string) error {
|
||||
return s.DeleteBoxWithSource(boxID, "admin")
|
||||
}
|
||||
|
||||
func (s *UploadService) DeleteBoxWithSource(boxID, source string) error {
|
||||
if err := s.db.Update(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(boxesBucket).Delete([]byte(boxID))
|
||||
}); err != nil {
|
||||
@@ -346,86 +336,10 @@ func (s *UploadService) DeleteBox(boxID string) error {
|
||||
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
|
||||
return err
|
||||
}
|
||||
s.logger.Info("box deleted", "source", "admin", "severity", "user_activity", "code", 2101, "box_id", boxID)
|
||||
s.logger.Info("box deleted", "source", source, "severity", "user_activity", "code", 2101, "box_id", boxID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *UploadService) CleanupExpired() (int, error) {
|
||||
boxes, err := s.ListBoxes(0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
cleaned := 0
|
||||
for _, box := range boxes {
|
||||
if box.ExpiresAt.After(now) {
|
||||
continue
|
||||
}
|
||||
if err := s.DeleteBox(box.ID); err != nil {
|
||||
return cleaned, err
|
||||
}
|
||||
cleaned++
|
||||
}
|
||||
if cleaned > 0 {
|
||||
s.logger.Info("expired boxes cleaned", "source", "housekeeping", "severity", "user_activity", "code", 2201, "cleaned", cleaned)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func (s *UploadService) GenerateMissingThumbnails() (ThumbnailJobResult, error) {
|
||||
boxes, err := s.ListBoxes(0)
|
||||
if err != nil {
|
||||
return ThumbnailJobResult{}, err
|
||||
}
|
||||
|
||||
var result ThumbnailJobResult
|
||||
for _, box := range boxes {
|
||||
if time.Now().UTC().After(box.ExpiresAt) {
|
||||
continue
|
||||
}
|
||||
|
||||
changed := false
|
||||
for i := range box.Files {
|
||||
file := &box.Files[i]
|
||||
if file.Thumbnail != "" || !needsThumbnail(*file) {
|
||||
continue
|
||||
}
|
||||
result.Scanned++
|
||||
|
||||
path := s.FilePath(box, *file)
|
||||
thumbnail := s.generateThumbnail(box.ID, file.ID, path, file.ContentType)
|
||||
if thumbnail == "" {
|
||||
result.Failed++
|
||||
continue
|
||||
}
|
||||
|
||||
file.Thumbnail = thumbnail
|
||||
changed = true
|
||||
result.Generated++
|
||||
}
|
||||
|
||||
if changed {
|
||||
if err := s.saveBox(box); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if result.Generated > 0 || result.Failed > 0 {
|
||||
s.logger.Info("thumbnail job complete",
|
||||
"source", "thumbnail",
|
||||
"severity", "user_activity",
|
||||
"code", 2203,
|
||||
"scanned", result.Scanned,
|
||||
"generated", result.Generated,
|
||||
"failed", result.Failed,
|
||||
)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *UploadService) FindFile(box Box, fileID string) (File, error) {
|
||||
for _, file := range box.Files {
|
||||
if file.ID == fileID {
|
||||
@@ -534,7 +448,7 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *UploadService) saveBox(box Box) error {
|
||||
func (s *UploadService) SaveBox(box Box) error {
|
||||
data, err := json.Marshal(box)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -618,83 +532,6 @@ func previewKind(contentType string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func needsThumbnail(file File) bool {
|
||||
return file.PreviewKind == "image" || file.PreviewKind == "video"
|
||||
}
|
||||
|
||||
func (s *UploadService) generateThumbnail(boxID, fileID, path, contentType string) string {
|
||||
thumbnailName := "@thumb@" + fileID + ".jpg"
|
||||
thumbnailPath := filepath.Join(s.filesDir, boxID, thumbnailName)
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case strings.HasPrefix(contentType, "image/"):
|
||||
err = createImageThumbnail(path, thumbnailPath)
|
||||
case strings.HasPrefix(contentType, "video/"):
|
||||
err = createVideoThumbnail(path, thumbnailPath)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
s.logger.Warn("thumbnail generation failed", "source", "thumbnail", "severity", "warn", "code", 4101, "file_id", fileID, "error", err.Error())
|
||||
return ""
|
||||
}
|
||||
return thumbnailName
|
||||
}
|
||||
|
||||
func createImageThumbnail(sourcePath, targetPath string) error {
|
||||
source, err := os.Open(sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer source.Close()
|
||||
|
||||
img, _, err := image.Decode(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
thumb := resizeNearest(img, 360, 240)
|
||||
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
|
||||
return jpeg.Encode(target, thumb, &jpeg.Options{Quality: 82})
|
||||
}
|
||||
|
||||
func createVideoThumbnail(sourcePath, targetPath string) error {
|
||||
return exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourcePath, "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run()
|
||||
}
|
||||
|
||||
func resizeNearest(src image.Image, maxWidth, maxHeight int) *image.RGBA {
|
||||
bounds := src.Bounds()
|
||||
width := bounds.Dx()
|
||||
height := bounds.Dy()
|
||||
if width <= 0 || height <= 0 {
|
||||
return image.NewRGBA(image.Rect(0, 0, 1, 1))
|
||||
}
|
||||
|
||||
scale := min(float64(maxWidth)/float64(width), float64(maxHeight)/float64(height))
|
||||
if scale > 1 {
|
||||
scale = 1
|
||||
}
|
||||
targetWidth := max(1, int(float64(width)*scale))
|
||||
targetHeight := max(1, int(float64(height)*scale))
|
||||
dst := image.NewRGBA(image.Rect(0, 0, targetWidth, targetHeight))
|
||||
|
||||
for y := 0; y < targetHeight; y++ {
|
||||
for x := 0; x < targetWidth; x++ {
|
||||
srcX := bounds.Min.X + int(float64(x)/scale)
|
||||
srcY := bounds.Min.Y + int(float64(y)/scale)
|
||||
dst.Set(x, y, src.At(srcX, srcY))
|
||||
}
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
func (s *UploadService) writeBoxMetadata(box Box) error {
|
||||
path := s.BoxMetadataPath(box)
|
||||
data, err := json.MarshalIndent(box, "", " ")
|
||||
|
||||
Reference in New Issue
Block a user