diff --git a/lib/boxstore/cleanup.go b/lib/boxstore/cleanup.go new file mode 100644 index 0000000..acf8331 --- /dev/null +++ b/lib/boxstore/cleanup.go @@ -0,0 +1,60 @@ +package boxstore + +import ( + "fmt" + "os" +) + +type CleanupExpiredResult struct { + Scanned int + Deleted int + Skipped int + DeletedIDs []string + Warnings []string +} + +func CleanupExpiredBoxes() (CleanupExpiredResult, error) { + entries, err := os.ReadDir(uploadRoot) + if err != nil { + if os.IsNotExist(err) { + return CleanupExpiredResult{}, nil + } + return CleanupExpiredResult{}, err + } + + result := CleanupExpiredResult{ + DeletedIDs: make([]string, 0), + Warnings: make([]string, 0), + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + boxID := entry.Name() + if !ValidBoxID(boxID) { + continue + } + result.Scanned++ + + manifest, err := ReadManifest(boxID) + if err != nil { + result.Skipped++ + if !os.IsNotExist(err) { + result.Warnings = append(result.Warnings, fmt.Sprintf("%s: %v", boxID, err)) + } + continue + } + if !IsExpired(manifest) { + continue + } + if err := DeleteBox(boxID); err != nil { + result.Skipped++ + result.Warnings = append(result.Warnings, fmt.Sprintf("%s: %v", boxID, err)) + continue + } + result.Deleted++ + result.DeletedIDs = append(result.DeletedIDs, boxID) + } + + return result, nil +} diff --git a/lib/boxstore/cleanup_test.go b/lib/boxstore/cleanup_test.go new file mode 100644 index 0000000..d7f588a --- /dev/null +++ b/lib/boxstore/cleanup_test.go @@ -0,0 +1,58 @@ +package boxstore + +import ( + "os" + "path/filepath" + "testing" + "time" + + "warpbox/lib/models" +) + +func TestCleanupExpiredBoxesDeletesOnlyExpiredManifestBoxes(t *testing.T) { + root := filepath.Join(t.TempDir(), "uploads") + previousRoot := UploadRoot() + t.Cleanup(func() { SetUploadRoot(previousRoot) }) + SetUploadRoot(root) + + expiredID := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + activeID := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + legacyID := "cccccccccccccccccccccccccccccccc" + + if err := os.MkdirAll(BoxPath(expiredID), 0755); err != nil { + t.Fatalf("mkdir expired: %v", err) + } + if err := os.MkdirAll(BoxPath(activeID), 0755); err != nil { + t.Fatalf("mkdir active: %v", err) + } + if err := os.MkdirAll(BoxPath(legacyID), 0755); err != nil { + t.Fatalf("mkdir legacy: %v", err) + } + + if err := WriteManifest(expiredID, models.BoxManifest{CreatedAt: time.Now().UTC().Add(-2 * time.Hour), ExpiresAt: time.Now().UTC().Add(-time.Minute)}); err != nil { + t.Fatalf("write expired manifest: %v", err) + } + if err := WriteManifest(activeID, models.BoxManifest{CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour)}); err != nil { + t.Fatalf("write active manifest: %v", err) + } + + result, err := CleanupExpiredBoxes() + if err != nil { + t.Fatalf("cleanup failed: %v", err) + } + if result.Deleted != 1 { + t.Fatalf("expected 1 deleted box, got %d", result.Deleted) + } + if len(result.DeletedIDs) != 1 || result.DeletedIDs[0] != expiredID { + t.Fatalf("expected deleted id %s, got %#v", expiredID, result.DeletedIDs) + } + if _, err := os.Stat(BoxPath(expiredID)); !os.IsNotExist(err) { + t.Fatalf("expected expired box dir removed, stat err=%v", err) + } + if _, err := os.Stat(BoxPath(activeID)); err != nil { + t.Fatalf("expected active box to remain, stat err=%v", err) + } + if _, err := os.Stat(BoxPath(legacyID)); err != nil { + t.Fatalf("expected legacy box to remain, stat err=%v", err) + } +} diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 790f990..5667273 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -199,6 +199,7 @@ func clearConfigEnv(t *testing.T) { "WARPBOX_THUMBNAIL_BATCH_SIZE", "WARPBOX_THUMBNAIL_INTERVAL_SECONDS", "WARPBOX_SECURITY_ENABLED", + "WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS", } { t.Setenv(name, "") } diff --git a/lib/config/definitions.go b/lib/config/definitions.go index ff67ead..b9ea3c0 100644 --- a/lib/config/definitions.go +++ b/lib/config/definitions.go @@ -33,6 +33,7 @@ var Definitions = []SettingDefinition{ {Key: SettingSecurityUploadWindowSecs, EnvName: "WARPBOX_SECURITY_UPLOAD_WINDOW_SECONDS", Label: "Upload limit window seconds", Type: SettingTypeInt64, Editable: true, Minimum: 10}, {Key: SettingSecurityUploadMaxRequests, EnvName: "WARPBOX_SECURITY_UPLOAD_MAX_REQUESTS", Label: "Upload max requests per window", Type: SettingTypeInt, Editable: true, Minimum: 1}, {Key: SettingSecurityUploadMaxGB, EnvName: "WARPBOX_SECURITY_UPLOAD_MAX_GB", Label: "Upload max total GB per window", Type: SettingTypeSizeGB, Editable: true, Minimum: 0}, + {Key: SettingExpiredCleanupIntervalSecs, EnvName: "WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS", Label: "Expired boxes cleanup interval seconds", Type: SettingTypeInt64, Editable: true, Minimum: 0}, } func (cfg *Config) SettingRows() []SettingRow { diff --git a/lib/config/load.go b/lib/config/load.go index cdf6acf..7bfe7d0 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -36,6 +36,7 @@ func Load() (*Config, error) { SecurityUploadWindowSeconds: 60, SecurityUploadMaxRequests: 20, SecurityUploadMaxBytes: 10 * 1024 * 1024 * 1024, + ExpiredCleanupIntervalSeconds: 300, sources: make(map[string]Source), values: make(map[string]string), defaults: make(map[string]string), @@ -115,6 +116,7 @@ func Load() (*Config, error) { {SettingSecurityBanSeconds, "WARPBOX_SECURITY_BAN_SECONDS", 10, &cfg.SecurityBanSeconds}, {SettingSecurityScanWindowSecs, "WARPBOX_SECURITY_SCAN_WINDOW_SECONDS", 10, &cfg.SecurityScanWindowSeconds}, {SettingSecurityUploadWindowSecs, "WARPBOX_SECURITY_UPLOAD_WINDOW_SECONDS", 10, &cfg.SecurityUploadWindowSeconds}, + {SettingExpiredCleanupIntervalSecs, "WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS", 0, &cfg.ExpiredCleanupIntervalSeconds}, } for _, item := range envInt64s { if err := cfg.applyInt64Env(item.key, item.name, item.min, item.target); err != nil { @@ -223,6 +225,7 @@ func (cfg *Config) captureDefaults() { cfg.captureDefaultValue(SettingSecurityUploadWindowSecs, strconv.FormatInt(cfg.SecurityUploadWindowSeconds, 10)) cfg.captureDefaultValue(SettingSecurityUploadMaxRequests, strconv.Itoa(cfg.SecurityUploadMaxRequests)) cfg.captureDefaultValue(SettingSecurityUploadMaxGB, formatGigabytesFromBytes(cfg.SecurityUploadMaxBytes)) + cfg.captureDefaultValue(SettingExpiredCleanupIntervalSecs, strconv.FormatInt(cfg.ExpiredCleanupIntervalSeconds, 10)) } func (cfg *Config) captureDefaultValue(key string, value string) { diff --git a/lib/config/models.go b/lib/config/models.go index 367cb4f..6522564 100644 --- a/lib/config/models.go +++ b/lib/config/models.go @@ -49,6 +49,7 @@ const ( SettingSecurityUploadWindowSecs = "security_upload_window_seconds" SettingSecurityUploadMaxRequests = "security_upload_max_requests" SettingSecurityUploadMaxGB = "security_upload_max_gb" + SettingExpiredCleanupIntervalSecs = "expired_cleanup_interval_seconds" ) type SettingType string @@ -121,6 +122,7 @@ type Config struct { SecurityUploadWindowSeconds int64 SecurityUploadMaxRequests int SecurityUploadMaxBytes int64 + ExpiredCleanupIntervalSeconds int64 sources map[string]Source values map[string]string diff --git a/lib/config/overrides.go b/lib/config/overrides.go index 0bb2f55..b42a567 100644 --- a/lib/config/overrides.go +++ b/lib/config/overrides.go @@ -131,6 +131,8 @@ func (cfg *Config) assignInt64(key string, value int64, source Source) { cfg.SecurityUploadWindowSeconds = value case SettingSecurityUploadMaxGB: cfg.SecurityUploadMaxBytes = value + case SettingExpiredCleanupIntervalSecs: + cfg.ExpiredCleanupIntervalSeconds = value } if key == SettingGlobalMaxFileSizeBytes || key == SettingGlobalMaxBoxSizeBytes || key == SettingDefaultUserMaxFileBytes || key == SettingDefaultUserMaxBoxBytes || key == SettingSecurityUploadMaxGB { cfg.setValue(key, formatGigabytesFromBytes(value), source) diff --git a/lib/server/admin_boxes.go b/lib/server/admin_boxes.go index 8e79227..a490c9b 100644 --- a/lib/server/admin_boxes.go +++ b/lib/server/admin_boxes.go @@ -84,22 +84,41 @@ func (app *App) handleAdminBoxesAction(ctx *gin.Context) { return } - if len(request.BoxIDs) == 0 { - ctx.JSON(http.StatusBadRequest, gin.H{"error": "Select one or more boxes first"}) - return - } - switch request.Action { - case "delete", "expire", "bump": + case "delete", "expire", "bump", "cleanup_expired": default: ctx.JSON(http.StatusBadRequest, gin.H{"error": "Unknown action"}) return } + if request.Action != "cleanup_expired" && len(request.BoxIDs) == 0 { + ctx.JSON(http.StatusBadRequest, gin.H{"error": "Select one or more boxes first"}) + return + } + if request.Action == "bump" && request.DeltaSeconds <= 0 { ctx.JSON(http.StatusBadRequest, gin.H{"error": "Missing bump duration"}) return } + if request.Action == "cleanup_expired" { + result, err := app.runExpiredCleanup("admin") + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Expired cleanup job failed"}) + return + } + boxes, listErr := app.listAdminBoxes() + if listErr != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Cleanup finished, but boxes could not be reloaded"}) + return + } + ctx.JSON(http.StatusOK, gin.H{ + "ok": len(result.Warnings) == 0, + "message": fmt.Sprintf("Expired cleanup done: deleted %d box(es), skipped %d", result.Deleted, result.Skipped), + "warnings": result.Warnings, + "boxes": boxes, + }) + return + } processed := 0 warnings := make([]string, 0) @@ -299,6 +318,8 @@ func adminBoxesActionMessage(action string, processed int, deltaSeconds int64) s return fmt.Sprintf("Expired %d box(es)", processed) case "bump": return fmt.Sprintf("Extended %d box(es) by %s", processed, adminBoxesDeltaLabel(deltaSeconds)) + case "cleanup_expired": + return fmt.Sprintf("Expired cleanup processed %d box(es)", processed) default: return "Action complete" } diff --git a/lib/server/admin_settings.go b/lib/server/admin_settings.go index 10f8a78..3a10473 100644 --- a/lib/server/admin_settings.go +++ b/lib/server/admin_settings.go @@ -451,6 +451,8 @@ func settingsCategoryForKey(key string) string { return "storage" case config.SettingBoxPollIntervalMS, config.SettingThumbnailBatchSize, config.SettingThumbnailIntervalSeconds: return "workers" + case config.SettingExpiredCleanupIntervalSecs: + return "workers" default: return "accounts" } @@ -489,6 +491,7 @@ func settingsDescription(key string) string { config.SettingSecurityUploadWindowSecs: "Window used for per-IP upload throttling.", config.SettingSecurityUploadMaxRequests: "Max upload requests per IP per upload window.", config.SettingSecurityUploadMaxGB: "Max upload volume in GB per IP per upload window.", + config.SettingExpiredCleanupIntervalSecs: "Background interval for deleting expired boxes. Set 0 to disable periodic cleanup.", } return descriptions[key] } diff --git a/lib/server/admin_settings_test.go b/lib/server/admin_settings_test.go index 36d393b..89e92eb 100644 --- a/lib/server/admin_settings_test.go +++ b/lib/server/admin_settings_test.go @@ -279,6 +279,7 @@ func clearAdminSettingsEnv(t *testing.T) { "WARPBOX_SECURITY_UPLOAD_MAX_GB", "WARPBOX_SECURITY_UPLOAD_MAX_MB", "WARPBOX_SECURITY_UPLOAD_MAX_BYTES", + "WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS", } { t.Setenv(name, "") } diff --git a/lib/server/cleanup.go b/lib/server/cleanup.go new file mode 100644 index 0000000..57c1600 --- /dev/null +++ b/lib/server/cleanup.go @@ -0,0 +1,62 @@ +package server + +import ( + "log" + "strings" + "time" + + "warpbox/lib/boxstore" +) + +func (app *App) runExpiredCleanup(trigger string) (boxstore.CleanupExpiredResult, error) { + result, err := boxstore.CleanupExpiredBoxes() + if err != nil { + log.Printf("warpbox cleanup[%s] failed: %v", trigger, err) + app.logActivity("boxes.cleanup.failed", "high", "Expired boxes cleanup failed", nil, map[string]string{ + "trigger": trigger, + "error": err.Error(), + }) + return result, err + } + + meta := map[string]string{ + "trigger": trigger, + "scanned": intToString(result.Scanned), + "deleted": intToString(result.Deleted), + "skipped": intToString(result.Skipped), + } + if len(result.DeletedIDs) > 0 { + limit := len(result.DeletedIDs) + if limit > 20 { + limit = 20 + } + meta["deleted_ids"] = strings.Join(result.DeletedIDs[:limit], ",") + } + if len(result.Warnings) > 0 { + limit := len(result.Warnings) + if limit > 3 { + limit = 3 + } + meta["warnings"] = strings.Join(result.Warnings[:limit], " | ") + } + app.logActivity("boxes.cleanup", "medium", "Expired boxes cleanup run completed", nil, meta) + log.Printf("warpbox cleanup[%s] scanned=%d deleted=%d skipped=%d", trigger, result.Scanned, result.Deleted, result.Skipped) + return result, nil +} + +func (app *App) startExpiredCleanupWorker() { + if app == nil || app.config == nil { + return + } + go func() { + for { + interval := app.config.ExpiredCleanupIntervalSeconds + if interval <= 0 { + time.Sleep(30 * time.Second) + continue + } + time.Sleep(time.Duration(interval) * time.Second) + _, _ = app.runExpiredCleanup("worker") + } + }() +} diff --git a/lib/server/server.go b/lib/server/server.go index b780543..8e68c40 100644 --- a/lib/server/server.go +++ b/lib/server/server.go @@ -104,6 +104,7 @@ func Run(addr string) error { compressed.Static("/static", "./static") boxstore.StartThumbnailWorker(cfg.ThumbnailBatchSize, time.Duration(cfg.ThumbnailIntervalSeconds)*time.Second) + app.startExpiredCleanupWorker() return router.Run(addr) } diff --git a/run.sh b/run.sh index 1ef3674..c0376db 100755 --- a/run.sh +++ b/run.sh @@ -37,6 +37,7 @@ export WARPBOX_SECURITY_SCAN_MAX_ATTEMPTS="${WARPBOX_SECURITY_SCAN_MAX_ATTEMPTS: export WARPBOX_SECURITY_UPLOAD_WINDOW_SECONDS="${WARPBOX_SECURITY_UPLOAD_WINDOW_SECONDS:-60}" export WARPBOX_SECURITY_UPLOAD_MAX_REQUESTS="${WARPBOX_SECURITY_UPLOAD_MAX_REQUESTS:-20}" export WARPBOX_SECURITY_UPLOAD_MAX_GB="${WARPBOX_SECURITY_UPLOAD_MAX_GB:-10}" +export WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS="${WARPBOX_EXPIRED_CLEANUP_INTERVAL_SECONDS:-300}" # Data location. export WARPBOX_DATA_DIR="${WARPBOX_DATA_DIR:-./data}" diff --git a/static/js/admin/boxes.js b/static/js/admin/boxes.js index 0c88596..11e8f36 100644 --- a/static/js/admin/boxes.js +++ b/static/js/admin/boxes.js @@ -373,6 +373,35 @@ } } + async function runCleanupAction() { + try { + const response = await fetch("/admin/boxes/actions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action: "cleanup_expired", box_ids: [] }) + }); + const payload = await response.json(); + if (!response.ok) { + const message = payload.error || payload.message || "Cleanup failed"; + showToast(message, "error", 3200); + return; + } + state.boxes = Array.isArray(payload.boxes) ? payload.boxes : state.boxes; + state.selected.clear(); + if (state.activeId && !state.boxes.some((box) => box.id === state.activeId)) { + state.activeId = null; + } + renderTable(); + let message = payload.message || "Expired cleanup completed"; + if (Array.isArray(payload.warnings) && payload.warnings.length) { + message += ` (${payload.warnings.length} warning${payload.warnings.length === 1 ? "" : "s"})`; + } + showToast(message, Array.isArray(payload.warnings) && payload.warnings.length ? "warning" : "success", 3200); + } catch (_) { + showToast("Network error while running cleanup", "error", 3200); + } + } + function selectedIDsOrActive() { if (state.selected.size) return Array.from(state.selected); const active = currentActiveBox(); @@ -419,6 +448,10 @@ if (!window.confirm("Delete selected boxes? This removes stored files.")) return; await runBulkAction("delete", selectedIDsOrActive()); return; + case "cleanup-expired": + if (!window.confirm("Run cleanup for expired boxes now?")) return; + await runCleanupAction(); + return; case "help-scope": showToast("Ownership filter waits for account + box owner data in backend", "info", 3400); return; diff --git a/templates/admin/boxes.html b/templates/admin/boxes.html index 86e02d8..0b6ba7c 100644 --- a/templates/admin/boxes.html +++ b/templates/admin/boxes.html @@ -54,6 +54,7 @@
+ @@ -112,6 +113,7 @@ + @@ -211,6 +213,9 @@ +
+ +