feat(uploads): add native resumable upload support
Implement a native chunked resumable upload API and frontend integration to support reliable large file uploads. Changes include: - Added a 3-step resumable upload API flow (create session, upload chunks, complete session). - Introduced configuration options for chunk size, retention hours, and toggling the feature. - Updated the frontend to utilize resumable uploads with progress tracking. - Configured temporary chunk storage under `data/tmp/uploads` with automatic cleanup. - Documented the API flow and configuration in the README.
This commit is contained in:
328
backend/libs/handlers/resumable.go
Normal file
328
backend/libs/handlers/resumable.go
Normal file
@@ -0,0 +1,328 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"warpbox.dev/backend/libs/helpers"
|
||||
"warpbox.dev/backend/libs/jobs"
|
||||
"warpbox.dev/backend/libs/services"
|
||||
)
|
||||
|
||||
type resumableCreateRequest struct {
|
||||
Files []services.ResumableFileInput `json:"files"`
|
||||
MaxDays int `json:"maxDays"`
|
||||
ExpiresMinutes int `json:"expiresMinutes"`
|
||||
MaxDownloads int `json:"maxDownloads"`
|
||||
Password string `json:"password"`
|
||||
ObfuscateMetadata bool `json:"obfuscateMetadata"`
|
||||
CollectionID string `json:"collectionId"`
|
||||
}
|
||||
|
||||
type resumableSessionResponse struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
Status string `json:"status"`
|
||||
BoxID string `json:"boxId,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt"`
|
||||
Files []services.ResumableFile `json:"files"`
|
||||
}
|
||||
|
||||
func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.cfg.ResumableUploadsEnabled {
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "resumable uploads are disabled")
|
||||
return
|
||||
}
|
||||
user, loggedIn, authErr := a.currentUserWithAuthError(r)
|
||||
if authErr != nil {
|
||||
a.logger.Warn("resumable upload rejected invalid bearer token", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4011)...)
|
||||
helpers.WriteJSONError(w, http.StatusUnauthorized, "invalid bearer token")
|
||||
return
|
||||
}
|
||||
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
|
||||
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !loggedIn && !settings.AnonymousUploadsEnabled {
|
||||
a.logger.Warn("resumable anonymous upload rejected disabled", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4013)...)
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "anonymous uploads are disabled")
|
||||
return
|
||||
}
|
||||
rateKey := uploadRateKey(r, user, loggedIn)
|
||||
if !isAdminUpload && policy.ShortRequests > 0 && !a.rateLimiter.Allow("upload:"+rateKey, policy.ShortRequests, policy.ShortWindow, time.Now().UTC()) {
|
||||
a.logger.Warn("resumable upload rate limited", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4291, "user_id", user.ID)...)
|
||||
helpers.WriteJSONError(w, http.StatusTooManyRequests, "too many upload requests, please slow down")
|
||||
return
|
||||
}
|
||||
|
||||
var payload resumableCreateRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, "upload session request could not be read")
|
||||
return
|
||||
}
|
||||
fileSizes := make([]int64, 0, len(payload.Files))
|
||||
var totalBytes int64
|
||||
for _, file := range payload.Files {
|
||||
fileSizes = append(fileSizes, file.Size)
|
||||
totalBytes += file.Size
|
||||
}
|
||||
if !isAdminUpload {
|
||||
if status, message := a.checkUploadPolicyForSizes(r, user, loggedIn, settings, policy, fileSizes, totalBytes); message != "" {
|
||||
a.logger.Warn("resumable upload rejected by policy", withRequestLogAttrs(r, "source", "quota", "severity", "warn", "code", status, "user_id", user.ID, "message", message, "bytes", totalBytes, "files", len(payload.Files))...)
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
if status, message := a.checkBoxCreationPolicy(r, user, loggedIn, policy); message != "" {
|
||||
a.logger.Warn("resumable upload rejected by box policy", withRequestLogAttrs(r, "source", "quota", "severity", "warn", "code", status, "user_id", user.ID, "message", message, "bytes", totalBytes, "files", len(payload.Files))...)
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
opts, err := a.resumableUploadOptions(r, payload, user, loggedIn, isAdminUpload, policy)
|
||||
if err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, err.Error())
|
||||
return
|
||||
}
|
||||
session, err := a.uploadService.CreateResumableSession(payload.Files, opts, a.cfg.ResumableChunkSize, a.cfg.ResumableRetention)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable session create failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4002, "user_id", user.ID, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.logger.Info("resumable upload session created", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2002, "user_id", user.ID, "session_id", session.ID, "files", len(session.Files), "bytes", totalBytes, "anonymous", !loggedIn)...)
|
||||
helpers.WriteJSON(w, http.StatusCreated, resumableResponse(session))
|
||||
}
|
||||
|
||||
func (a *App) ResumableUploadStatus(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
helpers.WriteJSON(w, http.StatusOK, resumableResponse(session))
|
||||
}
|
||||
|
||||
func (a *App) AddResumableFiles(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, loggedIn, _ := a.currentUserWithAuthError(r)
|
||||
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
|
||||
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var payload struct {
|
||||
Files []services.ResumableFileInput `json:"files"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, "upload files request could not be read")
|
||||
return
|
||||
}
|
||||
fileSizes := make([]int64, 0, len(session.Files)+len(payload.Files))
|
||||
var totalBytes int64
|
||||
for _, file := range session.Files {
|
||||
fileSizes = append(fileSizes, file.Size)
|
||||
totalBytes += file.Size
|
||||
}
|
||||
for _, file := range payload.Files {
|
||||
fileSizes = append(fileSizes, file.Size)
|
||||
totalBytes += file.Size
|
||||
}
|
||||
if !isAdminUpload {
|
||||
if status, message := a.checkUploadPolicyForSizes(r, user, loggedIn, settings, policy, fileSizes, totalBytes); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
}
|
||||
updated, err := a.uploadService.AddResumableFiles(session.ID, payload.Files)
|
||||
if err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.logger.Info("resumable upload files added", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2006, "session_id", session.ID, "added", len(updated.Files)-len(session.Files), "files", len(updated.Files))...)
|
||||
helpers.WriteJSON(w, http.StatusOK, resumableResponse(updated))
|
||||
}
|
||||
|
||||
func (a *App) PutResumableChunk(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
fileID := r.PathValue("fileID")
|
||||
index, err := strconv.Atoi(r.PathValue("index"))
|
||||
if err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, "chunk index is invalid")
|
||||
return
|
||||
}
|
||||
updated, err := a.uploadService.PutResumableChunk(r.Context(), session.ID, fileID, index, r.Body)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable chunk failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4003, "session_id", session.ID, "file_id", fileID, "chunk", index, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.logger.Info("resumable chunk uploaded", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2003, "session_id", session.ID, "file_id", fileID, "chunk", index)...)
|
||||
helpers.WriteJSON(w, http.StatusOK, resumableResponse(updated))
|
||||
}
|
||||
|
||||
func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, loggedIn, _ := a.currentUserWithAuthError(r)
|
||||
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
|
||||
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
fileSizes := make([]int64, 0, len(session.Files))
|
||||
var totalBytes int64
|
||||
for _, file := range session.Files {
|
||||
fileSizes = append(fileSizes, file.Size)
|
||||
totalBytes += file.Size
|
||||
}
|
||||
if !isAdminUpload {
|
||||
if status, message := a.checkUploadPolicyForSizes(r, user, loggedIn, settings, policy, fileSizes, totalBytes); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
if status, message := a.checkBoxCreationPolicy(r, user, loggedIn, policy); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
if status, message := a.checkStorageBackendCapacity(session.Options.StorageBackendID, settings, totalBytes); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
}
|
||||
result, completed, err := a.uploadService.CompleteResumableSession(r.Context(), session.ID)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable upload complete failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !isAdminUpload {
|
||||
if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, 1); err != nil {
|
||||
a.logger.Warn("failed to record resumable upload usage", "source", "quota", "severity", "warn", "code", 4404, "error", err.Error())
|
||||
}
|
||||
if err := a.settingsService.CleanupUsage(time.Now().UTC(), settings.UsageRetentionDays); err != nil {
|
||||
a.logger.Warn("failed to cleanup upload usage", "source", "quota", "severity", "warn", "code", 4405, "error", err.Error())
|
||||
}
|
||||
}
|
||||
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
|
||||
a.logger.Info("resumable upload completed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
|
||||
helpers.WriteJSON(w, http.StatusCreated, result)
|
||||
}
|
||||
|
||||
func (a *App) CancelResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := a.uploadService.CancelResumableSession(session.ID); err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.logger.Info("resumable upload cancelled", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2005, "session_id", session.ID)...)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (a *App) authorizedResumableSession(w http.ResponseWriter, r *http.Request) (services.ResumableSession, bool) {
|
||||
user, loggedIn, authErr := a.currentUserWithAuthError(r)
|
||||
if authErr != nil {
|
||||
helpers.WriteJSONError(w, http.StatusUnauthorized, "invalid bearer token")
|
||||
return services.ResumableSession{}, false
|
||||
}
|
||||
session, err := a.uploadService.GetResumableSession(r.PathValue("sessionID"))
|
||||
if err != nil {
|
||||
helpers.WriteJSONError(w, http.StatusNotFound, "upload session not found")
|
||||
return services.ResumableSession{}, false
|
||||
}
|
||||
if loggedIn {
|
||||
if session.Options.OwnerID != user.ID {
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "upload session not found")
|
||||
return services.ResumableSession{}, false
|
||||
}
|
||||
return session, true
|
||||
}
|
||||
if session.Options.OwnerID != "" || session.Options.CreatorIP != uploadClientIP(r) {
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "upload session not found")
|
||||
return services.ResumableSession{}, false
|
||||
}
|
||||
return session, true
|
||||
}
|
||||
|
||||
func (a *App) loadUploadPolicyForAPI(w http.ResponseWriter, r *http.Request, user services.User, loggedIn bool) (services.UploadPolicySettings, services.EffectiveUploadPolicy, bool) {
|
||||
settings, err := a.settingsService.UploadPolicy()
|
||||
if err != nil {
|
||||
a.logger.Error("failed to load upload policy", "source", "settings", "severity", "error", "code", 5006, "error", err.Error())
|
||||
helpers.WriteJSONError(w, http.StatusInternalServerError, "upload policy could not be loaded")
|
||||
return services.UploadPolicySettings{}, services.EffectiveUploadPolicy{}, false
|
||||
}
|
||||
return settings, a.effectiveUploadPolicy(settings, user, loggedIn), true
|
||||
}
|
||||
|
||||
func (a *App) resumableUploadOptions(r *http.Request, payload resumableCreateRequest, user services.User, loggedIn, isAdminUpload bool, policy services.EffectiveUploadPolicy) (services.UploadOptions, error) {
|
||||
var ownerID string
|
||||
var collectionID string
|
||||
if loggedIn {
|
||||
ownerID = user.ID
|
||||
collectionID = strings.TrimSpace(payload.CollectionID)
|
||||
if !a.authService.CollectionOwnedBy(collectionID, user.ID) {
|
||||
return services.UploadOptions{}, fmt.Errorf("collection not found")
|
||||
}
|
||||
}
|
||||
unlimitedExpiry := isAdminUpload || policy.MaxDays < 0
|
||||
rawMaxDays := payload.MaxDays
|
||||
maxDays := rawMaxDays
|
||||
if maxDays <= 0 {
|
||||
maxDays = 7
|
||||
if policy.MaxDays > 0 && policy.MaxDays < maxDays {
|
||||
maxDays = policy.MaxDays
|
||||
}
|
||||
}
|
||||
expiresMinutes := payload.ExpiresMinutes
|
||||
if expiresMinutes < 0 || rawMaxDays < 0 {
|
||||
if !unlimitedExpiry {
|
||||
return services.UploadOptions{}, fmt.Errorf("expiration cannot exceed %d days", policy.MaxDays)
|
||||
}
|
||||
expiresMinutes = -1
|
||||
} else if !unlimitedExpiry {
|
||||
if maxDays > policy.MaxDays {
|
||||
return services.UploadOptions{}, fmt.Errorf("expiration cannot exceed %d days", policy.MaxDays)
|
||||
}
|
||||
if expiresMinutes > 0 && expiresMinutes > policy.MaxDays*24*60 {
|
||||
return services.UploadOptions{}, fmt.Errorf("expiration cannot exceed %d days", policy.MaxDays)
|
||||
}
|
||||
}
|
||||
return services.UploadOptions{
|
||||
MaxDays: maxDays,
|
||||
ExpiresInMinutes: expiresMinutes,
|
||||
MaxDownloads: payload.MaxDownloads,
|
||||
Password: payload.Password,
|
||||
ObfuscateMetadata: payload.ObfuscateMetadata,
|
||||
OwnerID: ownerID,
|
||||
CollectionID: collectionID,
|
||||
SkipSizeLimit: isAdminUpload || policy.MaxUploadMB < 0,
|
||||
CreatorIP: uploadClientIP(r),
|
||||
StorageBackendID: policy.StorageBackendID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resumableResponse(session services.ResumableSession) resumableSessionResponse {
|
||||
return resumableSessionResponse{
|
||||
SessionID: session.ID,
|
||||
ChunkSize: session.ChunkSize,
|
||||
Status: session.Status,
|
||||
BoxID: session.BoxID,
|
||||
ExpiresAt: session.ExpiresAt.Format(time.RFC3339),
|
||||
Files: session.Files,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user