kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(upload): add resumable chunk configuration and file validation
Some checks failed
Build and Publish Docker Image / deploy (push) Failing after 56s
Details

Browse Source 
- Add `WARPBOX_RESUMABLE_CHUNK_MODE` and `WARPBOX_RESUMABLE_CHUNK_PATH` environment variables to configure temporary chunk storage.
- Implement strict file validation for resuming uploads to ensure selected files match the pending session's metadata.
- Add `PLANS.md` to document development stages, roadmap, and API specifications (including batching and resumable flows).
This commit is contained in:
Daniel Legt
2026-06-02 22:13:54 +03:00
parent 5cd476e7f3
commit 313c89483c
22 changed files with 1809 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
backend/libs/handlers/resumable.go
View File

@@ -1,6 +1,7 @@
package handlers
import (
"context"
"encoding/json"
"fmt"
"net/http"
@@ -24,19 +25,16 @@ type resumableCreateRequest struct {
}
type resumableSessionResponse struct {
SessionID string `json:"sessionId"`
ChunkSize int64 `json:"chunkSize"`
Status string `json:"status"`
BoxID string `json:"boxId,omitempty"`
ExpiresAt string `json:"expiresAt"`
Files []services.ResumableFile `json:"files"`
SessionID string `json:"sessionId"`
ResumeToken string `json:"resumeToken,omitempty"`
ChunkSize int64 `json:"chunkSize"`
Status string `json:"status"`
BoxID string `json:"boxId,omitempty"`
ExpiresAt string `json:"expiresAt"`
Files []services.ResumableFile `json:"files"`
}
func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
if !a.cfg.ResumableUploadsEnabled {
helpers.WriteJSONError(w, http.StatusForbidden, "resumable uploads are disabled")
return
}
user, loggedIn, authErr := a.currentUserWithAuthError(r)
if authErr != nil {
a.logger.Warn("resumable upload rejected invalid bearer token", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4011)...)
@@ -48,6 +46,10 @@ func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
if !ok {
return
}
if !settings.ResumableUploadsEnabled {
helpers.WriteJSONError(w, http.StatusForbidden, "resumable uploads are disabled")
return
}
if !loggedIn && !settings.AnonymousUploadsEnabled {
a.logger.Warn("resumable anonymous upload rejected disabled", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4013)...)
helpers.WriteJSONError(w, http.StatusForbidden, "anonymous uploads are disabled")
@@ -89,7 +91,9 @@ func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, err.Error())
return
}
session, err := a.uploadService.CreateResumableSession(payload.Files, opts, a.cfg.ResumableChunkSize, a.cfg.ResumableRetention)
chunkSize := int64(settings.ResumableChunkSizeMB * 1024 * 1024)
retention := time.Duration(settings.ResumableRetentionHours) * time.Hour
session, err := a.uploadService.CreateResumableSession(payload.Files, opts, chunkSize, retention, resumableChunkRoot(settings))
if err != nil {
a.logger.Warn("resumable session create failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4002, "user_id", user.ID, "error", err.Error())...)
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
@@ -176,6 +180,17 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
if !ok {
return
}
if session.Status == services.ResumableStatusCompleted || session.Status == services.ResumableStatusProcessing {
result, completed, err := a.uploadService.CompleteResumableSession(r.Context(), session.ID)
if err != nil {
a.logger.Warn("resumable upload completion replay failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
return
}
a.logger.Info("resumable upload completion replayed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files))...)
helpers.WriteJSON(w, http.StatusOK, result)
return
}
user, loggedIn, _ := a.currentUserWithAuthError(r)
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
@@ -202,7 +217,7 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
return
}
}
result, completed, err := a.uploadService.CompleteResumableSession(r.Context(), session.ID)
result, completed, err := a.uploadService.CreateProcessingBoxFromResumable(session.ID)
if err != nil {
a.logger.Warn("resumable upload complete failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
@@ -216,11 +231,90 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
a.logger.Warn("failed to cleanup upload usage", "source", "quota", "severity", "warn", "code", 4405, "error", err.Error())
}
}
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
a.logger.Info("resumable upload completed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
a.finalizeResumableUploadAsync(completed.ID, result.BoxID)
a.logger.Info("resumable upload queued for processing", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
helpers.WriteJSON(w, http.StatusCreated, result)
}
func (a *App) CompleteUploadedResumableUpload(w http.ResponseWriter, r *http.Request) {
session, ok := a.authorizedResumableSession(w, r)
if !ok {
return
}
user, loggedIn, _ := a.currentUserWithAuthError(r)
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
if !ok {
return
}
fileSizes := make([]int64, 0, len(session.Files))
var totalBytes int64
var completeCount int
for _, file := range session.Files {
if len(file.UploadedChunks) != file.ChunkCount {
continue
}
fileSizes = append(fileSizes, file.Size)
totalBytes += file.Size
completeCount++
}
if completeCount == 0 {
helpers.WriteJSONError(w, http.StatusBadRequest, "no fully uploaded files to finish")
return
}
if !isAdminUpload {
if status, message := a.checkUploadPolicyForSizes(r, user, loggedIn, settings, policy, fileSizes, totalBytes); message != "" {
helpers.WriteJSONError(w, status, message)
return
}
if status, message := a.checkBoxCreationPolicy(r, user, loggedIn, policy); message != "" {
helpers.WriteJSONError(w, status, message)
return
}
if status, message := a.checkStorageBackendCapacity(session.Options.StorageBackendID, settings, totalBytes); message != "" {
helpers.WriteJSONError(w, status, message)
return
}
}
result, completed, err := a.uploadService.CompleteUploadedResumableSession(r.Context(), session.ID)
if err != nil {
a.logger.Warn("resumable partial complete failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4005, "session_id", session.ID, "error", err.Error())...)
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
return
}
if !isAdminUpload {
if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, 1); err != nil {
a.logger.Warn("failed to record partial resumable upload usage", "source", "quota", "severity", "warn", "code", 4406, "error", err.Error())
}
if err := a.settingsService.CleanupUsage(time.Now().UTC(), settings.UsageRetentionDays); err != nil {
a.logger.Warn("failed to cleanup upload usage", "source", "quota", "severity", "warn", "code", 4405, "error", err.Error())
}
}
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
a.logger.Info("resumable uploaded files completed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2007, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
helpers.WriteJSON(w, http.StatusCreated, result)
}
func (a *App) finalizeResumableUploadAsync(sessionID, boxID string) {
go func() {
a.logger.Info("resumable upload processing started", "source", "user-upload", "severity", "user_activity", "code", 2009, "session_id", sessionID, "box_id", boxID)
result, err := a.uploadService.FinalizeProcessingResumableSession(context.Background(), sessionID)
if err != nil {
a.logger.Warn("resumable upload processing failed", "source", "user-upload", "severity", "warn", "code", 4010, "session_id", sessionID, "box_id", boxID, "error", err.Error())
return
}
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
a.logger.Info("resumable upload processing completed", "source", "user-upload", "severity", "user_activity", "code", 2010, "session_id", sessionID, "box_id", result.BoxID, "files", len(result.Files))
}()
}
func resumableChunkRoot(settings services.UploadPolicySettings) string {
if settings.ResumableChunkMode != "custom" {
return ""
}
return strings.TrimSpace(settings.ResumableChunkPath)
}
func (a *App) CancelResumableUpload(w http.ResponseWriter, r *http.Request) {
session, ok := a.authorizedResumableSession(w, r)
if !ok {
@@ -245,6 +339,10 @@ func (a *App) authorizedResumableSession(w http.ResponseWriter, r *http.Request)
helpers.WriteJSONError(w, http.StatusNotFound, "upload session not found")
return services.ResumableSession{}, false
}
if !a.uploadService.VerifyResumableToken(session, r.Header.Get("X-Warpbox-Resume-Token")) {
helpers.WriteJSONError(w, http.StatusUnauthorized, "upload session not found")
return services.ResumableSession{}, false
}
if loggedIn {
if session.Options.OwnerID != user.ID {
helpers.WriteJSONError(w, http.StatusForbidden, "upload session not found")
@@ -318,11 +416,12 @@ func (a *App) resumableUploadOptions(r *http.Request, payload resumableCreateReq
func resumableResponse(session services.ResumableSession) resumableSessionResponse {
return resumableSessionResponse{
SessionID: session.ID,
ChunkSize: session.ChunkSize,
Status: session.Status,
BoxID: session.BoxID,
ExpiresAt: session.ExpiresAt.Format(time.RFC3339),
Files: session.Files,
SessionID: session.ID,
ResumeToken: session.ResumeToken,
ChunkSize: session.ChunkSize,
Status: session.Status,
BoxID: session.BoxID,
ExpiresAt: session.ExpiresAt.Format(time.RFC3339),
Files: session.Files,
}
}