feat(upload): add resumable chunk configuration and file validation
Some checks failed
Build and Publish Docker Image / deploy (push) Failing after 56s
Some checks failed
Build and Publish Docker Image / deploy (push) Failing after 56s
- Add `WARPBOX_RESUMABLE_CHUNK_MODE` and `WARPBOX_RESUMABLE_CHUNK_PATH` environment variables to configure temporary chunk storage. - Implement strict file validation for resuming uploads to ensure selected files match the pending session's metadata. - Add `PLANS.md` to document development stages, roadmap, and API specifications (including batching and resumable flows).
This commit is contained in:
@@ -574,6 +574,7 @@ func (a *App) AdminSettingsPost(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
settings.AnonymousUploadsEnabled = r.FormValue("anonymous_uploads_enabled") == "on"
|
||||
settings.ResumableUploadsEnabled = r.FormValue("resumable_uploads_enabled") == "on"
|
||||
if value := parsePositiveInt(r.FormValue("usage_retention_days")); value > 0 {
|
||||
settings.UsageRetentionDays = value
|
||||
}
|
||||
@@ -604,6 +605,16 @@ func (a *App) AdminSettingsPost(w http.ResponseWriter, r *http.Request) {
|
||||
if value := parsePositiveInt(r.FormValue("short_window_seconds")); value > 0 {
|
||||
settings.ShortWindowSeconds = value
|
||||
}
|
||||
if value := parsePositiveFloat(r.FormValue("resumable_chunk_size_mb")); value > 0 {
|
||||
settings.ResumableChunkSizeMB = value
|
||||
}
|
||||
if value := parsePositiveInt(r.FormValue("resumable_retention_hours")); value > 0 {
|
||||
settings.ResumableRetentionHours = value
|
||||
}
|
||||
if value := strings.TrimSpace(r.FormValue("resumable_chunk_mode")); value != "" {
|
||||
settings.ResumableChunkMode = value
|
||||
}
|
||||
settings.ResumableChunkPath = strings.TrimSpace(r.FormValue("resumable_chunk_path"))
|
||||
if value := r.FormValue("anonymous_storage_backend"); value != "" {
|
||||
settings.AnonymousStorageBackend = value
|
||||
}
|
||||
|
||||
@@ -146,6 +146,7 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/uploads/resumable/{sessionID}/files", a.AddResumableFiles)
|
||||
mux.HandleFunc("PUT /api/v1/uploads/resumable/{sessionID}/files/{fileID}/chunks/{index}", a.PutResumableChunk)
|
||||
mux.HandleFunc("POST /api/v1/uploads/resumable/{sessionID}/complete", a.CompleteResumableUpload)
|
||||
mux.HandleFunc("POST /api/v1/uploads/resumable/{sessionID}/complete-uploaded", a.CompleteUploadedResumableUpload)
|
||||
mux.HandleFunc("DELETE /api/v1/uploads/resumable/{sessionID}", a.CancelResumableUpload)
|
||||
mux.HandleFunc("GET /emoji/{pack}/{file}", a.EmojiAsset)
|
||||
mux.Handle("GET /static/", a.Static())
|
||||
|
||||
@@ -52,6 +52,7 @@ type fileView struct {
|
||||
Reactions []reactionView
|
||||
ReactionMore int
|
||||
Reacted bool
|
||||
Processing bool
|
||||
}
|
||||
|
||||
type reactionView struct {
|
||||
@@ -101,6 +102,15 @@ func (a *App) DownloadPage(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
locked := a.uploadService.IsProtected(box) && !a.isBoxUnlocked(r, box)
|
||||
if isSocialPreviewBot(r) && !locked && len(box.Files) == 1 {
|
||||
if box.Files[0].Processing {
|
||||
http.Error(w, "file is still processing", http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
a.serveFileContent(w, r, box, box.Files[0], false)
|
||||
a.logger.Info("single-file box served inline for social preview", withRequestLogAttrs(r, "source", "download", "severity", "user_activity", "code", 2008, "box_id", box.ID, "file_id", box.Files[0].ID)...)
|
||||
return
|
||||
}
|
||||
visitorID := a.reactionVisitorID(w, r)
|
||||
reactionsByFile, reactedByFile, err := a.reactionService.SummaryForBox(box.ID, visitorID)
|
||||
if err != nil {
|
||||
@@ -159,6 +169,15 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
locked := a.uploadService.IsProtected(box) && !a.isBoxUnlocked(r, box)
|
||||
if isSocialPreviewBot(r) && !locked {
|
||||
if file.Processing {
|
||||
http.Error(w, "file is still processing", http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
a.serveFileContent(w, r, box, file, false)
|
||||
a.logger.Info("file served inline for social preview", withRequestLogAttrs(r, "source", "download", "severity", "user_activity", "code", 2009, "box_id", box.ID, "file_id", file.ID)...)
|
||||
return
|
||||
}
|
||||
view := a.fileView(box, file)
|
||||
title := file.Name
|
||||
description := fmt.Sprintf("%s shared via Warpbox", helpers.FormatBytes(file.Size))
|
||||
@@ -193,6 +212,10 @@ func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "password required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if file.Processing {
|
||||
http.Error(w, "file is still processing", http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
|
||||
a.serveFileContent(w, r, box, file, r.URL.Query().Get("inline") != "1")
|
||||
a.logger.Info("file content served", withRequestLogAttrs(r, "source", "download", "severity", "user_activity", "code", 2005, "box_id", box.ID, "file_id", file.ID, "attachment", r.URL.Query().Get("inline") != "1")...)
|
||||
@@ -373,6 +396,7 @@ func (a *App) fileViewWithReactions(box services.Box, file services.File, reacti
|
||||
Reactions: reactionViews,
|
||||
ReactionMore: reactionOverflowCount(reactionViews),
|
||||
Reacted: reacted,
|
||||
Processing: file.Processing,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -583,3 +607,31 @@ func absoluteURL(r *http.Request, path string) string {
|
||||
}
|
||||
return fmt.Sprintf("%s://%s%s", scheme, r.Host, path)
|
||||
}
|
||||
|
||||
func isSocialPreviewBot(r *http.Request) bool {
|
||||
agent := strings.ToLower(r.UserAgent())
|
||||
if agent == "" {
|
||||
return false
|
||||
}
|
||||
bots := []string{
|
||||
"discordbot",
|
||||
"twitterbot",
|
||||
"facebookexternalhit",
|
||||
"telegrambot",
|
||||
"whatsapp",
|
||||
"slackbot",
|
||||
"linkedinbot",
|
||||
"skypeuripreview",
|
||||
"embedly",
|
||||
"pinterest",
|
||||
"vkshare",
|
||||
"mattermost",
|
||||
"mastodon",
|
||||
}
|
||||
for _, bot := range bots {
|
||||
if strings.Contains(agent, bot) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -24,19 +25,16 @@ type resumableCreateRequest struct {
|
||||
}
|
||||
|
||||
type resumableSessionResponse struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
Status string `json:"status"`
|
||||
BoxID string `json:"boxId,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt"`
|
||||
Files []services.ResumableFile `json:"files"`
|
||||
SessionID string `json:"sessionId"`
|
||||
ResumeToken string `json:"resumeToken,omitempty"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
Status string `json:"status"`
|
||||
BoxID string `json:"boxId,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt"`
|
||||
Files []services.ResumableFile `json:"files"`
|
||||
}
|
||||
|
||||
func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.cfg.ResumableUploadsEnabled {
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "resumable uploads are disabled")
|
||||
return
|
||||
}
|
||||
user, loggedIn, authErr := a.currentUserWithAuthError(r)
|
||||
if authErr != nil {
|
||||
a.logger.Warn("resumable upload rejected invalid bearer token", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4011)...)
|
||||
@@ -48,6 +46,10 @@ func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !settings.ResumableUploadsEnabled {
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "resumable uploads are disabled")
|
||||
return
|
||||
}
|
||||
if !loggedIn && !settings.AnonymousUploadsEnabled {
|
||||
a.logger.Warn("resumable anonymous upload rejected disabled", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4013)...)
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "anonymous uploads are disabled")
|
||||
@@ -89,7 +91,9 @@ func (a *App) CreateResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, err.Error())
|
||||
return
|
||||
}
|
||||
session, err := a.uploadService.CreateResumableSession(payload.Files, opts, a.cfg.ResumableChunkSize, a.cfg.ResumableRetention)
|
||||
chunkSize := int64(settings.ResumableChunkSizeMB * 1024 * 1024)
|
||||
retention := time.Duration(settings.ResumableRetentionHours) * time.Hour
|
||||
session, err := a.uploadService.CreateResumableSession(payload.Files, opts, chunkSize, retention, resumableChunkRoot(settings))
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable session create failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4002, "user_id", user.ID, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
@@ -176,6 +180,17 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if session.Status == services.ResumableStatusCompleted || session.Status == services.ResumableStatusProcessing {
|
||||
result, completed, err := a.uploadService.CompleteResumableSession(r.Context(), session.ID)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable upload completion replay failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.logger.Info("resumable upload completion replayed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files))...)
|
||||
helpers.WriteJSON(w, http.StatusOK, result)
|
||||
return
|
||||
}
|
||||
user, loggedIn, _ := a.currentUserWithAuthError(r)
|
||||
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
|
||||
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
|
||||
@@ -202,7 +217,7 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
result, completed, err := a.uploadService.CompleteResumableSession(r.Context(), session.ID)
|
||||
result, completed, err := a.uploadService.CreateProcessingBoxFromResumable(session.ID)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable upload complete failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
@@ -216,11 +231,90 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
a.logger.Warn("failed to cleanup upload usage", "source", "quota", "severity", "warn", "code", 4405, "error", err.Error())
|
||||
}
|
||||
}
|
||||
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
|
||||
a.logger.Info("resumable upload completed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
|
||||
a.finalizeResumableUploadAsync(completed.ID, result.BoxID)
|
||||
a.logger.Info("resumable upload queued for processing", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
|
||||
helpers.WriteJSON(w, http.StatusCreated, result)
|
||||
}
|
||||
|
||||
func (a *App) CompleteUploadedResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, loggedIn, _ := a.currentUserWithAuthError(r)
|
||||
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
|
||||
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
fileSizes := make([]int64, 0, len(session.Files))
|
||||
var totalBytes int64
|
||||
var completeCount int
|
||||
for _, file := range session.Files {
|
||||
if len(file.UploadedChunks) != file.ChunkCount {
|
||||
continue
|
||||
}
|
||||
fileSizes = append(fileSizes, file.Size)
|
||||
totalBytes += file.Size
|
||||
completeCount++
|
||||
}
|
||||
if completeCount == 0 {
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, "no fully uploaded files to finish")
|
||||
return
|
||||
}
|
||||
if !isAdminUpload {
|
||||
if status, message := a.checkUploadPolicyForSizes(r, user, loggedIn, settings, policy, fileSizes, totalBytes); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
if status, message := a.checkBoxCreationPolicy(r, user, loggedIn, policy); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
if status, message := a.checkStorageBackendCapacity(session.Options.StorageBackendID, settings, totalBytes); message != "" {
|
||||
helpers.WriteJSONError(w, status, message)
|
||||
return
|
||||
}
|
||||
}
|
||||
result, completed, err := a.uploadService.CompleteUploadedResumableSession(r.Context(), session.ID)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable partial complete failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4005, "session_id", session.ID, "error", err.Error())...)
|
||||
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !isAdminUpload {
|
||||
if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, 1); err != nil {
|
||||
a.logger.Warn("failed to record partial resumable upload usage", "source", "quota", "severity", "warn", "code", 4406, "error", err.Error())
|
||||
}
|
||||
if err := a.settingsService.CleanupUsage(time.Now().UTC(), settings.UsageRetentionDays); err != nil {
|
||||
a.logger.Warn("failed to cleanup upload usage", "source", "quota", "severity", "warn", "code", 4405, "error", err.Error())
|
||||
}
|
||||
}
|
||||
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
|
||||
a.logger.Info("resumable uploaded files completed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2007, "user_id", user.ID, "session_id", completed.ID, "box_id", result.BoxID, "files", len(result.Files), "bytes", totalBytes, "admin", isAdminUpload, "anonymous", !loggedIn)...)
|
||||
helpers.WriteJSON(w, http.StatusCreated, result)
|
||||
}
|
||||
|
||||
func (a *App) finalizeResumableUploadAsync(sessionID, boxID string) {
|
||||
go func() {
|
||||
a.logger.Info("resumable upload processing started", "source", "user-upload", "severity", "user_activity", "code", 2009, "session_id", sessionID, "box_id", boxID)
|
||||
result, err := a.uploadService.FinalizeProcessingResumableSession(context.Background(), sessionID)
|
||||
if err != nil {
|
||||
a.logger.Warn("resumable upload processing failed", "source", "user-upload", "severity", "warn", "code", 4010, "session_id", sessionID, "box_id", boxID, "error", err.Error())
|
||||
return
|
||||
}
|
||||
jobs.GenerateThumbnailsForBoxAsync(a.uploadService, a.logger, result.BoxID)
|
||||
a.logger.Info("resumable upload processing completed", "source", "user-upload", "severity", "user_activity", "code", 2010, "session_id", sessionID, "box_id", result.BoxID, "files", len(result.Files))
|
||||
}()
|
||||
}
|
||||
|
||||
func resumableChunkRoot(settings services.UploadPolicySettings) string {
|
||||
if settings.ResumableChunkMode != "custom" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(settings.ResumableChunkPath)
|
||||
}
|
||||
|
||||
func (a *App) CancelResumableUpload(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := a.authorizedResumableSession(w, r)
|
||||
if !ok {
|
||||
@@ -245,6 +339,10 @@ func (a *App) authorizedResumableSession(w http.ResponseWriter, r *http.Request)
|
||||
helpers.WriteJSONError(w, http.StatusNotFound, "upload session not found")
|
||||
return services.ResumableSession{}, false
|
||||
}
|
||||
if !a.uploadService.VerifyResumableToken(session, r.Header.Get("X-Warpbox-Resume-Token")) {
|
||||
helpers.WriteJSONError(w, http.StatusUnauthorized, "upload session not found")
|
||||
return services.ResumableSession{}, false
|
||||
}
|
||||
if loggedIn {
|
||||
if session.Options.OwnerID != user.ID {
|
||||
helpers.WriteJSONError(w, http.StatusForbidden, "upload session not found")
|
||||
@@ -318,11 +416,12 @@ func (a *App) resumableUploadOptions(r *http.Request, payload resumableCreateReq
|
||||
|
||||
func resumableResponse(session services.ResumableSession) resumableSessionResponse {
|
||||
return resumableSessionResponse{
|
||||
SessionID: session.ID,
|
||||
ChunkSize: session.ChunkSize,
|
||||
Status: session.Status,
|
||||
BoxID: session.BoxID,
|
||||
ExpiresAt: session.ExpiresAt.Format(time.RFC3339),
|
||||
Files: session.Files,
|
||||
SessionID: session.ID,
|
||||
ResumeToken: session.ResumeToken,
|
||||
ChunkSize: session.ChunkSize,
|
||||
Status: session.Status,
|
||||
BoxID: session.BoxID,
|
||||
ExpiresAt: session.ExpiresAt.Format(time.RFC3339),
|
||||
Files: session.Files,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,51 @@ func TestUploadTextResponseReturnsOnlyBoxURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSocialPreviewBotGetsRawSingleFileBox(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
payload := uploadThroughApp(t, app)
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/d/"+payload.BoxID, nil)
|
||||
request.Header.Set("User-Agent", "Discordbot/2.0")
|
||||
request.SetPathValue("boxID", payload.BoxID)
|
||||
response := httptest.NewRecorder()
|
||||
app.DownloadPage(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
if strings.Contains(response.Body.String(), "Shared files on Warpbox") {
|
||||
t.Fatalf("social preview bot received HTML download page")
|
||||
}
|
||||
if response.Body.String() != "hello" {
|
||||
t.Fatalf("social preview body = %q", response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSocialPreviewBotGetsRawFilePreview(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
payload := uploadThroughApp(t, app)
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/d/"+payload.BoxID+"/f/"+payload.Files[0].ID, nil)
|
||||
request.Header.Set("User-Agent", "TelegramBot")
|
||||
request.SetPathValue("boxID", payload.BoxID)
|
||||
request.SetPathValue("fileID", payload.Files[0].ID)
|
||||
response := httptest.NewRecorder()
|
||||
app.DownloadFile(response, request)
|
||||
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
if strings.Contains(response.Body.String(), "preview-title") {
|
||||
t.Fatalf("social preview bot received HTML preview page")
|
||||
}
|
||||
if response.Body.String() != "hello" {
|
||||
t.Fatalf("social preview body = %q", response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableUploadFlowCreatesNormalBox(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
@@ -119,9 +164,10 @@ func TestResumableUploadFlowCreatesNormalBox(t *testing.T) {
|
||||
t.Fatalf("create status = %d, body = %s", createResponse.Code, createResponse.Body.String())
|
||||
}
|
||||
var session struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
Files []struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ResumeToken string `json:"resumeToken"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
Files []struct {
|
||||
ID string `json:"id"`
|
||||
ChunkCount int `json:"chunkCount"`
|
||||
UploadedChunks []int `json:"uploadedChunks"`
|
||||
@@ -130,7 +176,7 @@ func TestResumableUploadFlowCreatesNormalBox(t *testing.T) {
|
||||
if err := json.Unmarshal(createResponse.Body.Bytes(), &session); err != nil {
|
||||
t.Fatalf("json.Unmarshal session returned error: %v", err)
|
||||
}
|
||||
if session.SessionID == "" || session.ChunkSize != 4 || len(session.Files) != 1 || session.Files[0].ChunkCount != 3 {
|
||||
if session.SessionID == "" || session.ResumeToken == "" || session.ChunkSize != 4 || len(session.Files) != 1 || session.Files[0].ChunkCount != 3 {
|
||||
t.Fatalf("unexpected session response: %+v", session)
|
||||
}
|
||||
|
||||
@@ -140,6 +186,7 @@ func TestResumableUploadFlowCreatesNormalBox(t *testing.T) {
|
||||
request.SetPathValue("sessionID", session.SessionID)
|
||||
request.SetPathValue("fileID", session.Files[0].ID)
|
||||
request.SetPathValue("index", strconv.Itoa(index))
|
||||
request.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
response := httptest.NewRecorder()
|
||||
app.PutResumableChunk(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
@@ -149,6 +196,7 @@ func TestResumableUploadFlowCreatesNormalBox(t *testing.T) {
|
||||
|
||||
completeRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/complete", nil)
|
||||
completeRequest.SetPathValue("sessionID", session.SessionID)
|
||||
completeRequest.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
completeResponse := httptest.NewRecorder()
|
||||
app.CompleteResumableUpload(completeResponse, completeRequest)
|
||||
if completeResponse.Code != http.StatusCreated {
|
||||
@@ -158,10 +206,22 @@ func TestResumableUploadFlowCreatesNormalBox(t *testing.T) {
|
||||
if err := json.Unmarshal(completeResponse.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal result returned error: %v", err)
|
||||
}
|
||||
box, err := app.uploadService.GetBox(payload.BoxID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBox returned error: %v", err)
|
||||
replayRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/complete", nil)
|
||||
replayRequest.SetPathValue("sessionID", session.SessionID)
|
||||
replayRequest.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
replayResponse := httptest.NewRecorder()
|
||||
app.CompleteResumableUpload(replayResponse, replayRequest)
|
||||
if replayResponse.Code != http.StatusOK {
|
||||
t.Fatalf("complete replay status = %d, body = %s", replayResponse.Code, replayResponse.Body.String())
|
||||
}
|
||||
var replayPayload services.UploadResult
|
||||
if err := json.Unmarshal(replayResponse.Body.Bytes(), &replayPayload); err != nil {
|
||||
t.Fatalf("json.Unmarshal replay result returned error: %v", err)
|
||||
}
|
||||
if replayPayload.BoxID != payload.BoxID || replayPayload.BoxURL == "" {
|
||||
t.Fatalf("unexpected replay result: %+v, original: %+v", replayPayload, payload)
|
||||
}
|
||||
box := waitForProcessedBox(t, app, payload.BoxID)
|
||||
if len(box.Files) != 1 || box.Files[0].Name != "note.txt" || box.Files[0].Size != 11 {
|
||||
t.Fatalf("unexpected box files: %+v", box.Files)
|
||||
}
|
||||
@@ -190,8 +250,9 @@ func TestResumableUploadRequiresAllChunks(t *testing.T) {
|
||||
t.Fatalf("create status = %d, body = %s", createResponse.Code, createResponse.Body.String())
|
||||
}
|
||||
var session struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
Files []struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ResumeToken string `json:"resumeToken"`
|
||||
Files []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"files"`
|
||||
}
|
||||
@@ -202,6 +263,7 @@ func TestResumableUploadRequiresAllChunks(t *testing.T) {
|
||||
chunkRequest.SetPathValue("sessionID", session.SessionID)
|
||||
chunkRequest.SetPathValue("fileID", session.Files[0].ID)
|
||||
chunkRequest.SetPathValue("index", "0")
|
||||
chunkRequest.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
chunkResponse := httptest.NewRecorder()
|
||||
app.PutResumableChunk(chunkResponse, chunkRequest)
|
||||
if chunkResponse.Code != http.StatusOK {
|
||||
@@ -210,6 +272,7 @@ func TestResumableUploadRequiresAllChunks(t *testing.T) {
|
||||
|
||||
completeRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/complete", nil)
|
||||
completeRequest.SetPathValue("sessionID", session.SessionID)
|
||||
completeRequest.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
completeResponse := httptest.NewRecorder()
|
||||
app.CompleteResumableUpload(completeResponse, completeRequest)
|
||||
if completeResponse.Code != http.StatusBadRequest {
|
||||
@@ -217,6 +280,54 @@ func TestResumableUploadRequiresAllChunks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableStatusRequiresResumeToken(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
|
||||
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable", strings.NewReader(`{"files":[{"name":"note.txt","size":4,"contentType":"text/plain"}]}`))
|
||||
createResponse := httptest.NewRecorder()
|
||||
app.CreateResumableUpload(createResponse, createRequest)
|
||||
if createResponse.Code != http.StatusCreated {
|
||||
t.Fatalf("create status = %d, body = %s", createResponse.Code, createResponse.Body.String())
|
||||
}
|
||||
var session struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ResumeToken string `json:"resumeToken"`
|
||||
}
|
||||
if err := json.Unmarshal(createResponse.Body.Bytes(), &session); err != nil {
|
||||
t.Fatalf("json.Unmarshal session returned error: %v", err)
|
||||
}
|
||||
|
||||
missing := httptest.NewRequest(http.MethodGet, "/api/v1/uploads/resumable/"+session.SessionID, nil)
|
||||
missing.SetPathValue("sessionID", session.SessionID)
|
||||
missingResponse := httptest.NewRecorder()
|
||||
app.ResumableUploadStatus(missingResponse, missing)
|
||||
if missingResponse.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("missing token status = %d, body = %s", missingResponse.Code, missingResponse.Body.String())
|
||||
}
|
||||
|
||||
wrong := httptest.NewRequest(http.MethodGet, "/api/v1/uploads/resumable/"+session.SessionID, nil)
|
||||
wrong.SetPathValue("sessionID", session.SessionID)
|
||||
wrong.Header.Set("X-Warpbox-Resume-Token", "wrong")
|
||||
wrongResponse := httptest.NewRecorder()
|
||||
app.ResumableUploadStatus(wrongResponse, wrong)
|
||||
if wrongResponse.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("wrong token status = %d, body = %s", wrongResponse.Code, wrongResponse.Body.String())
|
||||
}
|
||||
|
||||
valid := httptest.NewRequest(http.MethodGet, "/api/v1/uploads/resumable/"+session.SessionID, nil)
|
||||
valid.SetPathValue("sessionID", session.SessionID)
|
||||
valid.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
validResponse := httptest.NewRecorder()
|
||||
app.ResumableUploadStatus(validResponse, valid)
|
||||
if validResponse.Code != http.StatusOK {
|
||||
t.Fatalf("valid token status = %d, body = %s", validResponse.Code, validResponse.Body.String())
|
||||
}
|
||||
if strings.Contains(validResponse.Body.String(), "resumeTokenHash") {
|
||||
t.Fatalf("status response leaked token hash: %s", validResponse.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
@@ -229,8 +340,9 @@ func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
t.Fatalf("create status = %d, body = %s", createResponse.Code, createResponse.Body.String())
|
||||
}
|
||||
var session struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
Files []struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ResumeToken string `json:"resumeToken"`
|
||||
Files []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"files"`
|
||||
}
|
||||
@@ -241,6 +353,7 @@ func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
firstChunk.SetPathValue("sessionID", session.SessionID)
|
||||
firstChunk.SetPathValue("fileID", session.Files[0].ID)
|
||||
firstChunk.SetPathValue("index", "0")
|
||||
firstChunk.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
firstChunkResponse := httptest.NewRecorder()
|
||||
app.PutResumableChunk(firstChunkResponse, firstChunk)
|
||||
if firstChunkResponse.Code != http.StatusOK {
|
||||
@@ -249,6 +362,7 @@ func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
|
||||
addRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/files", strings.NewReader(`{"files":[{"name":"two.txt","size":4,"contentType":"text/plain","fingerprint":"two"}]}`))
|
||||
addRequest.SetPathValue("sessionID", session.SessionID)
|
||||
addRequest.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
addResponse := httptest.NewRecorder()
|
||||
app.AddResumableFiles(addResponse, addRequest)
|
||||
if addResponse.Code != http.StatusOK {
|
||||
@@ -271,6 +385,7 @@ func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
secondChunk.SetPathValue("sessionID", session.SessionID)
|
||||
secondChunk.SetPathValue("fileID", updated.Files[1].ID)
|
||||
secondChunk.SetPathValue("index", "0")
|
||||
secondChunk.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
secondChunkResponse := httptest.NewRecorder()
|
||||
app.PutResumableChunk(secondChunkResponse, secondChunk)
|
||||
if secondChunkResponse.Code != http.StatusOK {
|
||||
@@ -278,6 +393,7 @@ func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
}
|
||||
completeRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/complete", nil)
|
||||
completeRequest.SetPathValue("sessionID", session.SessionID)
|
||||
completeRequest.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
completeResponse := httptest.NewRecorder()
|
||||
app.CompleteResumableUpload(completeResponse, completeRequest)
|
||||
if completeResponse.Code != http.StatusCreated {
|
||||
@@ -296,6 +412,79 @@ func TestResumableUploadCanAddFilesToExistingSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableCompleteUploadedRequiresTokenAndKeepsFinishedFiles(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
|
||||
createBody := `{"files":[{"name":"done.txt","size":4,"contentType":"text/plain","fingerprint":"done"},{"name":"partial.txt","size":8,"contentType":"text/plain","fingerprint":"partial"}],"expiresMinutes":60}`
|
||||
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable", strings.NewReader(createBody))
|
||||
createResponse := httptest.NewRecorder()
|
||||
app.CreateResumableUpload(createResponse, createRequest)
|
||||
if createResponse.Code != http.StatusCreated {
|
||||
t.Fatalf("create status = %d, body = %s", createResponse.Code, createResponse.Body.String())
|
||||
}
|
||||
var session struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
ResumeToken string `json:"resumeToken"`
|
||||
Files []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"files"`
|
||||
}
|
||||
if err := json.Unmarshal(createResponse.Body.Bytes(), &session); err != nil {
|
||||
t.Fatalf("json.Unmarshal session returned error: %v", err)
|
||||
}
|
||||
for _, chunk := range []struct {
|
||||
fileIndex int
|
||||
index string
|
||||
body string
|
||||
}{
|
||||
{fileIndex: 0, index: "0", body: "done"},
|
||||
{fileIndex: 1, index: "0", body: "part"},
|
||||
} {
|
||||
request := httptest.NewRequest(http.MethodPut, "/api/v1/uploads/resumable/"+session.SessionID+"/files/"+session.Files[chunk.fileIndex].ID+"/chunks/"+chunk.index, strings.NewReader(chunk.body))
|
||||
request.SetPathValue("sessionID", session.SessionID)
|
||||
request.SetPathValue("fileID", session.Files[chunk.fileIndex].ID)
|
||||
request.SetPathValue("index", chunk.index)
|
||||
request.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
response := httptest.NewRecorder()
|
||||
app.PutResumableChunk(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("chunk status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
missing := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/complete-uploaded", nil)
|
||||
missing.SetPathValue("sessionID", session.SessionID)
|
||||
missingResponse := httptest.NewRecorder()
|
||||
app.CompleteUploadedResumableUpload(missingResponse, missing)
|
||||
if missingResponse.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("missing token status = %d, body = %s", missingResponse.Code, missingResponse.Body.String())
|
||||
}
|
||||
|
||||
complete := httptest.NewRequest(http.MethodPost, "/api/v1/uploads/resumable/"+session.SessionID+"/complete-uploaded", nil)
|
||||
complete.SetPathValue("sessionID", session.SessionID)
|
||||
complete.Header.Set("X-Warpbox-Resume-Token", session.ResumeToken)
|
||||
completeResponse := httptest.NewRecorder()
|
||||
app.CompleteUploadedResumableUpload(completeResponse, complete)
|
||||
if completeResponse.Code != http.StatusCreated {
|
||||
t.Fatalf("complete-uploaded status = %d, body = %s", completeResponse.Code, completeResponse.Body.String())
|
||||
}
|
||||
var payload services.UploadResult
|
||||
if err := json.Unmarshal(completeResponse.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal result returned error: %v", err)
|
||||
}
|
||||
box, err := app.uploadService.GetBox(payload.BoxID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBox returned error: %v", err)
|
||||
}
|
||||
if len(box.Files) != 1 || box.Files[0].Name != "done.txt" {
|
||||
t.Fatalf("complete-uploaded box files = %+v", box.Files)
|
||||
}
|
||||
if _, err := app.uploadService.GetResumableSession(session.SessionID); !os.IsNotExist(err) {
|
||||
t.Fatalf("GetResumableSession after complete-uploaded error = %v, want os.ErrNotExist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManageBoxAndDeleteFlow(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
@@ -424,6 +613,10 @@ func newTestApp(t *testing.T) (*App, func()) {
|
||||
UserDailyUploadMB: 8,
|
||||
DefaultUserStorageMB: 16,
|
||||
UsageRetentionDays: 30,
|
||||
ResumableUploadsEnabled: true,
|
||||
ResumableChunkSizeMB: 0.000003814697265625,
|
||||
ResumableRetentionHours: 1,
|
||||
ResumableChunkMode: "same",
|
||||
},
|
||||
}
|
||||
service, err := services.NewUploadService(cfg.MaxUploadSize, cfg.DataDir, cfg.BaseURL, logger)
|
||||
@@ -538,6 +731,31 @@ func tokenFromURL(t *testing.T, value string) string {
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
func waitForProcessedBox(t *testing.T, app *App, boxID string) services.Box {
|
||||
t.Helper()
|
||||
var box services.Box
|
||||
for i := 0; i < 50; i++ {
|
||||
next, err := app.uploadService.GetBox(boxID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBox returned error: %v", err)
|
||||
}
|
||||
box = next
|
||||
processing := false
|
||||
for _, file := range box.Files {
|
||||
if file.Processing {
|
||||
processing = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !processing {
|
||||
return box
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("box %s was still processing: %+v", boxID, box.Files)
|
||||
return box
|
||||
}
|
||||
|
||||
func copyDir(src, dst string) error {
|
||||
return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user