feat(config): add expired box cleanup functionality
Adds dedicated config setting and cleanup logic for managing expired content boxes via admin tools and CLI.
This commit is contained in:
60
lib/boxstore/cleanup.go
Normal file
60
lib/boxstore/cleanup.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package boxstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
type CleanupExpiredResult struct {
|
||||
Scanned int
|
||||
Deleted int
|
||||
Skipped int
|
||||
DeletedIDs []string
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
func CleanupExpiredBoxes() (CleanupExpiredResult, error) {
|
||||
entries, err := os.ReadDir(uploadRoot)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return CleanupExpiredResult{}, nil
|
||||
}
|
||||
return CleanupExpiredResult{}, err
|
||||
}
|
||||
|
||||
result := CleanupExpiredResult{
|
||||
DeletedIDs: make([]string, 0),
|
||||
Warnings: make([]string, 0),
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
boxID := entry.Name()
|
||||
if !ValidBoxID(boxID) {
|
||||
continue
|
||||
}
|
||||
result.Scanned++
|
||||
|
||||
manifest, err := ReadManifest(boxID)
|
||||
if err != nil {
|
||||
result.Skipped++
|
||||
if !os.IsNotExist(err) {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("%s: %v", boxID, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !IsExpired(manifest) {
|
||||
continue
|
||||
}
|
||||
if err := DeleteBox(boxID); err != nil {
|
||||
result.Skipped++
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("%s: %v", boxID, err))
|
||||
continue
|
||||
}
|
||||
result.Deleted++
|
||||
result.DeletedIDs = append(result.DeletedIDs, boxID)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
58
lib/boxstore/cleanup_test.go
Normal file
58
lib/boxstore/cleanup_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package boxstore
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"warpbox/lib/models"
|
||||
)
|
||||
|
||||
func TestCleanupExpiredBoxesDeletesOnlyExpiredManifestBoxes(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "uploads")
|
||||
previousRoot := UploadRoot()
|
||||
t.Cleanup(func() { SetUploadRoot(previousRoot) })
|
||||
SetUploadRoot(root)
|
||||
|
||||
expiredID := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
activeID := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
legacyID := "cccccccccccccccccccccccccccccccc"
|
||||
|
||||
if err := os.MkdirAll(BoxPath(expiredID), 0755); err != nil {
|
||||
t.Fatalf("mkdir expired: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(BoxPath(activeID), 0755); err != nil {
|
||||
t.Fatalf("mkdir active: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(BoxPath(legacyID), 0755); err != nil {
|
||||
t.Fatalf("mkdir legacy: %v", err)
|
||||
}
|
||||
|
||||
if err := WriteManifest(expiredID, models.BoxManifest{CreatedAt: time.Now().UTC().Add(-2 * time.Hour), ExpiresAt: time.Now().UTC().Add(-time.Minute)}); err != nil {
|
||||
t.Fatalf("write expired manifest: %v", err)
|
||||
}
|
||||
if err := WriteManifest(activeID, models.BoxManifest{CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour)}); err != nil {
|
||||
t.Fatalf("write active manifest: %v", err)
|
||||
}
|
||||
|
||||
result, err := CleanupExpiredBoxes()
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup failed: %v", err)
|
||||
}
|
||||
if result.Deleted != 1 {
|
||||
t.Fatalf("expected 1 deleted box, got %d", result.Deleted)
|
||||
}
|
||||
if len(result.DeletedIDs) != 1 || result.DeletedIDs[0] != expiredID {
|
||||
t.Fatalf("expected deleted id %s, got %#v", expiredID, result.DeletedIDs)
|
||||
}
|
||||
if _, err := os.Stat(BoxPath(expiredID)); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected expired box dir removed, stat err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(BoxPath(activeID)); err != nil {
|
||||
t.Fatalf("expected active box to remain, stat err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(BoxPath(legacyID)); err != nil {
|
||||
t.Fatalf("expected legacy box to remain, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -199,6 +199,7 @@ func clearConfigEnv(t *testing.T) {
|
||||
"WARPBOX_THUMBNAIL_BATCH_SIZE",
|
||||
"WARPBOX_THUMBNAIL_INTERVAL_SECONDS",
|
||||
"WARPBOX_SECURITY_ENABLED",
|
||||
"WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS",
|
||||
} {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ var Definitions = []SettingDefinition{
|
||||
{Key: SettingSecurityUploadWindowSecs, EnvName: "WARPBOX_SECURITY_UPLOAD_WINDOW_SECONDS", Label: "Upload limit window seconds", Type: SettingTypeInt64, Editable: true, Minimum: 10},
|
||||
{Key: SettingSecurityUploadMaxRequests, EnvName: "WARPBOX_SECURITY_UPLOAD_MAX_REQUESTS", Label: "Upload max requests per window", Type: SettingTypeInt, Editable: true, Minimum: 1},
|
||||
{Key: SettingSecurityUploadMaxGB, EnvName: "WARPBOX_SECURITY_UPLOAD_MAX_GB", Label: "Upload max total GB per window", Type: SettingTypeSizeGB, Editable: true, Minimum: 0},
|
||||
{Key: SettingExpiredCleanupIntervalSecs, EnvName: "WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS", Label: "Expired boxes cleanup interval seconds", Type: SettingTypeInt64, Editable: true, Minimum: 0},
|
||||
}
|
||||
|
||||
func (cfg *Config) SettingRows() []SettingRow {
|
||||
|
||||
@@ -36,6 +36,7 @@ func Load() (*Config, error) {
|
||||
SecurityUploadWindowSeconds: 60,
|
||||
SecurityUploadMaxRequests: 20,
|
||||
SecurityUploadMaxBytes: 10 * 1024 * 1024 * 1024,
|
||||
ExpiredCleanupIntervalSeconds: 300,
|
||||
sources: make(map[string]Source),
|
||||
values: make(map[string]string),
|
||||
defaults: make(map[string]string),
|
||||
@@ -115,6 +116,7 @@ func Load() (*Config, error) {
|
||||
{SettingSecurityBanSeconds, "WARPBOX_SECURITY_BAN_SECONDS", 10, &cfg.SecurityBanSeconds},
|
||||
{SettingSecurityScanWindowSecs, "WARPBOX_SECURITY_SCAN_WINDOW_SECONDS", 10, &cfg.SecurityScanWindowSeconds},
|
||||
{SettingSecurityUploadWindowSecs, "WARPBOX_SECURITY_UPLOAD_WINDOW_SECONDS", 10, &cfg.SecurityUploadWindowSeconds},
|
||||
{SettingExpiredCleanupIntervalSecs, "WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS", 0, &cfg.ExpiredCleanupIntervalSeconds},
|
||||
}
|
||||
for _, item := range envInt64s {
|
||||
if err := cfg.applyInt64Env(item.key, item.name, item.min, item.target); err != nil {
|
||||
@@ -223,6 +225,7 @@ func (cfg *Config) captureDefaults() {
|
||||
cfg.captureDefaultValue(SettingSecurityUploadWindowSecs, strconv.FormatInt(cfg.SecurityUploadWindowSeconds, 10))
|
||||
cfg.captureDefaultValue(SettingSecurityUploadMaxRequests, strconv.Itoa(cfg.SecurityUploadMaxRequests))
|
||||
cfg.captureDefaultValue(SettingSecurityUploadMaxGB, formatGigabytesFromBytes(cfg.SecurityUploadMaxBytes))
|
||||
cfg.captureDefaultValue(SettingExpiredCleanupIntervalSecs, strconv.FormatInt(cfg.ExpiredCleanupIntervalSeconds, 10))
|
||||
}
|
||||
|
||||
func (cfg *Config) captureDefaultValue(key string, value string) {
|
||||
|
||||
@@ -49,6 +49,7 @@ const (
|
||||
SettingSecurityUploadWindowSecs = "security_upload_window_seconds"
|
||||
SettingSecurityUploadMaxRequests = "security_upload_max_requests"
|
||||
SettingSecurityUploadMaxGB = "security_upload_max_gb"
|
||||
SettingExpiredCleanupIntervalSecs = "expired_cleanup_interval_seconds"
|
||||
)
|
||||
|
||||
type SettingType string
|
||||
@@ -121,6 +122,7 @@ type Config struct {
|
||||
SecurityUploadWindowSeconds int64
|
||||
SecurityUploadMaxRequests int
|
||||
SecurityUploadMaxBytes int64
|
||||
ExpiredCleanupIntervalSeconds int64
|
||||
|
||||
sources map[string]Source
|
||||
values map[string]string
|
||||
|
||||
@@ -131,6 +131,8 @@ func (cfg *Config) assignInt64(key string, value int64, source Source) {
|
||||
cfg.SecurityUploadWindowSeconds = value
|
||||
case SettingSecurityUploadMaxGB:
|
||||
cfg.SecurityUploadMaxBytes = value
|
||||
case SettingExpiredCleanupIntervalSecs:
|
||||
cfg.ExpiredCleanupIntervalSeconds = value
|
||||
}
|
||||
if key == SettingGlobalMaxFileSizeBytes || key == SettingGlobalMaxBoxSizeBytes || key == SettingDefaultUserMaxFileBytes || key == SettingDefaultUserMaxBoxBytes || key == SettingSecurityUploadMaxGB {
|
||||
cfg.setValue(key, formatGigabytesFromBytes(value), source)
|
||||
|
||||
@@ -84,22 +84,41 @@ func (app *App) handleAdminBoxesAction(ctx *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if len(request.BoxIDs) == 0 {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "Select one or more boxes first"})
|
||||
return
|
||||
}
|
||||
|
||||
switch request.Action {
|
||||
case "delete", "expire", "bump":
|
||||
case "delete", "expire", "bump", "cleanup_expired":
|
||||
default:
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "Unknown action"})
|
||||
return
|
||||
}
|
||||
|
||||
if request.Action != "cleanup_expired" && len(request.BoxIDs) == 0 {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "Select one or more boxes first"})
|
||||
return
|
||||
}
|
||||
|
||||
if request.Action == "bump" && request.DeltaSeconds <= 0 {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "Missing bump duration"})
|
||||
return
|
||||
}
|
||||
if request.Action == "cleanup_expired" {
|
||||
result, err := app.runExpiredCleanup("admin")
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Expired cleanup job failed"})
|
||||
return
|
||||
}
|
||||
boxes, listErr := app.listAdminBoxes()
|
||||
if listErr != nil {
|
||||
ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Cleanup finished, but boxes could not be reloaded"})
|
||||
return
|
||||
}
|
||||
ctx.JSON(http.StatusOK, gin.H{
|
||||
"ok": len(result.Warnings) == 0,
|
||||
"message": fmt.Sprintf("Expired cleanup done: deleted %d box(es), skipped %d", result.Deleted, result.Skipped),
|
||||
"warnings": result.Warnings,
|
||||
"boxes": boxes,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
processed := 0
|
||||
warnings := make([]string, 0)
|
||||
@@ -299,6 +318,8 @@ func adminBoxesActionMessage(action string, processed int, deltaSeconds int64) s
|
||||
return fmt.Sprintf("Expired %d box(es)", processed)
|
||||
case "bump":
|
||||
return fmt.Sprintf("Extended %d box(es) by %s", processed, adminBoxesDeltaLabel(deltaSeconds))
|
||||
case "cleanup_expired":
|
||||
return fmt.Sprintf("Expired cleanup processed %d box(es)", processed)
|
||||
default:
|
||||
return "Action complete"
|
||||
}
|
||||
|
||||
@@ -451,6 +451,8 @@ func settingsCategoryForKey(key string) string {
|
||||
return "storage"
|
||||
case config.SettingBoxPollIntervalMS, config.SettingThumbnailBatchSize, config.SettingThumbnailIntervalSeconds:
|
||||
return "workers"
|
||||
case config.SettingExpiredCleanupIntervalSecs:
|
||||
return "workers"
|
||||
default:
|
||||
return "accounts"
|
||||
}
|
||||
@@ -489,6 +491,7 @@ func settingsDescription(key string) string {
|
||||
config.SettingSecurityUploadWindowSecs: "Window used for per-IP upload throttling.",
|
||||
config.SettingSecurityUploadMaxRequests: "Max upload requests per IP per upload window.",
|
||||
config.SettingSecurityUploadMaxGB: "Max upload volume in GB per IP per upload window.",
|
||||
config.SettingExpiredCleanupIntervalSecs: "Background interval for deleting expired boxes. Set 0 to disable periodic cleanup.",
|
||||
}
|
||||
return descriptions[key]
|
||||
}
|
||||
|
||||
@@ -279,6 +279,7 @@ func clearAdminSettingsEnv(t *testing.T) {
|
||||
"WARPBOX_SECURITY_UPLOAD_MAX_GB",
|
||||
"WARPBOX_SECURITY_UPLOAD_MAX_MB",
|
||||
"WARPBOX_SECURITY_UPLOAD_MAX_BYTES",
|
||||
"WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS",
|
||||
} {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
|
||||
62
lib/server/cleanup.go
Normal file
62
lib/server/cleanup.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"warpbox/lib/boxstore"
|
||||
)
|
||||
|
||||
func (app *App) runExpiredCleanup(trigger string) (boxstore.CleanupExpiredResult, error) {
|
||||
result, err := boxstore.CleanupExpiredBoxes()
|
||||
if err != nil {
|
||||
log.Printf("warpbox cleanup[%s] failed: %v", trigger, err)
|
||||
app.logActivity("boxes.cleanup.failed", "high", "Expired boxes cleanup failed", nil, map[string]string{
|
||||
"trigger": trigger,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
meta := map[string]string{
|
||||
"trigger": trigger,
|
||||
"scanned": intToString(result.Scanned),
|
||||
"deleted": intToString(result.Deleted),
|
||||
"skipped": intToString(result.Skipped),
|
||||
}
|
||||
if len(result.DeletedIDs) > 0 {
|
||||
limit := len(result.DeletedIDs)
|
||||
if limit > 20 {
|
||||
limit = 20
|
||||
}
|
||||
meta["deleted_ids"] = strings.Join(result.DeletedIDs[:limit], ",")
|
||||
}
|
||||
if len(result.Warnings) > 0 {
|
||||
limit := len(result.Warnings)
|
||||
if limit > 3 {
|
||||
limit = 3
|
||||
}
|
||||
meta["warnings"] = strings.Join(result.Warnings[:limit], " | ")
|
||||
}
|
||||
app.logActivity("boxes.cleanup", "medium", "Expired boxes cleanup run completed", nil, meta)
|
||||
log.Printf("warpbox cleanup[%s] scanned=%d deleted=%d skipped=%d", trigger, result.Scanned, result.Deleted, result.Skipped)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (app *App) startExpiredCleanupWorker() {
|
||||
if app == nil || app.config == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
interval := app.config.ExpiredCleanupIntervalSeconds
|
||||
if interval <= 0 {
|
||||
time.Sleep(30 * time.Second)
|
||||
continue
|
||||
}
|
||||
time.Sleep(time.Duration(interval) * time.Second)
|
||||
_, _ = app.runExpiredCleanup("worker")
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -104,6 +104,7 @@ func Run(addr string) error {
|
||||
compressed.Static("/static", "./static")
|
||||
|
||||
boxstore.StartThumbnailWorker(cfg.ThumbnailBatchSize, time.Duration(cfg.ThumbnailIntervalSeconds)*time.Second)
|
||||
app.startExpiredCleanupWorker()
|
||||
|
||||
return router.Run(addr)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user