From adb1a12dfd9b6aeb49790b3ef8624ec96cdd819b Mon Sep 17 00:00:00 2001 From: Daniel Legt Date: Sun, 31 May 2026 22:27:43 +0300 Subject: [PATCH] feat(upload): support batching via header and update ShareX config Introduce support for grouping multiple sequential file uploads into a single box using the `X-Warpbox-Batch` header. This is particularly useful for ShareX multi-file selections, which are sent as separate back-to-back requests. Additionally, this change: - Updates the ShareX configuration template to opt-in to batching by default. - Switches ShareX configuration placeholders to the modern `{json:...}` format. - Adds `thumbnailUrl` to the upload response schema and documents its usage. --- README.md | 15 ++- backend/libs/handlers/api_docs.go | 33 ++++--- backend/libs/handlers/app.go | 5 + backend/libs/handlers/upload.go | 68 +++++++++++++- backend/libs/handlers/upload_group.go | 49 ++++++++++ backend/libs/services/upload.go | 125 +++++++++++++++++-------- backend/static/css/40-docs.css | 16 ++++ backend/templates/layouts/base.html | 2 +- backend/templates/pages/api.html | 44 +++++++-- examples/sharex/warpbox-anonymous.sxcu | 9 +- 10 files changed, 298 insertions(+), 68 deletions(-) create mode 100644 backend/libs/handlers/upload_group.go diff --git a/README.md b/README.md index a2a6f0b..a924376 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ Curl and custom uploaders can use the same endpoint: # Terminal-friendly output: one plain box URL. curl -F file=@./report.pdf http://localhost:8080/api/v1/upload -# JSON output with boxUrl, manageUrl, deleteUrl, zipUrl, and file entries. +# JSON output with boxUrl, thumbnailUrl, manageUrl, deleteUrl, zipUrl, and file entries. curl -F sharex=@./screenshot.png \ -H 'Accept: application/json' \ http://localhost:8080/api/v1/upload @@ -190,6 +190,19 @@ curl -F sharex=@./screenshot.png \ The upload endpoint accepts multipart fields named `file` and `sharex`. ShareX users can start from `examples/sharex/warpbox-anonymous.sxcu`; update `RequestURL` to match your instance URL. +Authenticated uploads (your account's limits) add an `Authorization: Bearer ` header — mint +a token under **Account → Access tokens**. The JSON response uses ShareX placeholders +`{json:boxUrl}` (URL), `{json:thumbnailUrl}` (thumbnail), `{json:deleteUrl}` (deletion), and +`{json:error}` (error message). + +### Grouping multiple files into one box (`X-Warpbox-Batch`) + +By default every uploaded file becomes its own box. To put several files in a **single** box, send +the opt-in `X-Warpbox-Batch` header: requests that share the same header value (scoped per account, +or per IP for anonymous uploads) within 20s are appended to the same box. This lets a multi-file +ShareX selection — which ShareX sends as separate back-to-back requests — land as one shareable +link. The shipped `.sxcu` sets `X-Warpbox-Batch: sharex`; remove that header for one box per file. +Requests without the header behave exactly as before. ## Stage 4 Accounts + Personal Boxes diff --git a/backend/libs/handlers/api_docs.go b/backend/libs/handlers/api_docs.go index 07c3af6..30e7188 100644 --- a/backend/libs/handlers/api_docs.go +++ b/backend/libs/handlers/api_docs.go @@ -17,6 +17,7 @@ type apiDocsData struct { ShareXExampleURL string ShareXDownloadURL string ShareXFileFieldName string + ShareXGroupWindow string } func (a *App) APIDocs(w http.ResponseWriter, r *http.Request) { @@ -33,6 +34,7 @@ func (a *App) APIDocs(w http.ResponseWriter, r *http.Request) { ShareXExampleURL: a.cfg.BaseURL + "/api/v1/upload", ShareXDownloadURL: a.cfg.BaseURL + "/api/v1/sharex/warpbox-anonymous.sxcu", ShareXFileFieldName: "sharex", + ShareXGroupWindow: uploadGroupWindow.String(), }, }) } @@ -47,11 +49,16 @@ func (a *App) ShareXAnonymousConfig(w http.ResponseWriter, r *http.Request) { "RequestURL": a.cfg.BaseURL + "/api/v1/upload", "Headers": map[string]string{ "Accept": "application/json", + // Group a multi-file selection (sent as back-to-back requests) into + // one box. Remove this header for one box per file. + uploadBatchHeader: "sharex", }, "Body": "MultipartFormData", "FileFormName": "sharex", - "URL": "$json:boxUrl$", - "DeletionURL": "$json:manageUrl$", + "URL": "{json:boxUrl}", + "ThumbnailURL": "{json:thumbnailUrl}", + "DeletionURL": "{json:deleteUrl}", + "ErrorMessage": "{json:error}", }) } @@ -109,22 +116,24 @@ func (a *App) UploadResponseSchema(w http.ResponseWriter, r *http.Request) { "type": "object", "required": []string{"boxId", "boxUrl", "zipUrl", "manageUrl", "deleteUrl", "expiresAt", "files"}, "properties": map[string]any{ - "boxId": map[string]any{"type": "string"}, - "boxUrl": map[string]any{"type": "string", "format": "uri"}, - "zipUrl": map[string]any{"type": "string", "format": "uri"}, - "manageUrl": map[string]any{"type": "string", "format": "uri", "description": "Private bearer URL for managing/deleting this upload. Returned only at upload time."}, - "deleteUrl": map[string]any{"type": "string", "format": "uri", "description": "Private bearer POST URL for deleting this upload. Returned only at upload time."}, - "expiresAt": map[string]any{"type": "string", "format": "date-time"}, + "boxId": map[string]any{"type": "string"}, + "boxUrl": map[string]any{"type": "string", "format": "uri"}, + "zipUrl": map[string]any{"type": "string", "format": "uri"}, + "thumbnailUrl": map[string]any{"type": "string", "format": "uri", "description": "Thumbnail of the most recently uploaded file (placeholder until generated)."}, + "manageUrl": map[string]any{"type": "string", "format": "uri", "description": "Private bearer URL for managing/deleting this upload. Returned only at upload time."}, + "deleteUrl": map[string]any{"type": "string", "format": "uri", "description": "Private bearer URL for deleting this upload (GET or POST). Returned only at upload time."}, + "expiresAt": map[string]any{"type": "string", "format": "date-time"}, "files": map[string]any{ "type": "array", "items": map[string]any{ "type": "object", "required": []string{"id", "name", "size", "url"}, "properties": map[string]any{ - "id": map[string]any{"type": "string"}, - "name": map[string]any{"type": "string"}, - "size": map[string]any{"type": "string"}, - "url": map[string]any{"type": "string", "format": "uri"}, + "id": map[string]any{"type": "string"}, + "name": map[string]any{"type": "string"}, + "size": map[string]any{"type": "string"}, + "url": map[string]any{"type": "string", "format": "uri"}, + "thumbnailUrl": map[string]any{"type": "string", "format": "uri"}, }, }, }, diff --git a/backend/libs/handlers/app.go b/backend/libs/handlers/app.go index 9046624..8f03fdb 100644 --- a/backend/libs/handlers/app.go +++ b/backend/libs/handlers/app.go @@ -18,6 +18,7 @@ type App struct { settingsService *services.SettingsService banService *services.BanService rateLimiter *rateLimiter + uploadGroups *uploadGrouper } func NewApp(cfg config.Config, logger *slog.Logger, renderer *web.Renderer, uploadService *services.UploadService, authService *services.AuthService, settingsService *services.SettingsService, banService *services.BanService) *App { @@ -30,6 +31,7 @@ func NewApp(cfg config.Config, logger *slog.Logger, renderer *web.Renderer, uplo settingsService: settingsService, banService: banService, rateLimiter: newRateLimiter(), + uploadGroups: newUploadGrouper(), } } @@ -112,6 +114,9 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("GET /d/{boxID}/deleted", a.ManageDeleted) mux.HandleFunc("GET /d/{boxID}/manage/{token}", a.ManageBox) mux.HandleFunc("POST /d/{boxID}/manage/{token}/delete", a.ManageDeleteBox) + // GET variant so ShareX (which issues a GET to the configured DeletionURL) + // can delete a box via its secret one-time delete token. + mux.HandleFunc("GET /d/{boxID}/manage/{token}/delete", a.ManageDeleteBox) mux.HandleFunc("POST /d/{boxID}/unlock", a.UnlockBox) mux.HandleFunc("GET /d/{boxID}/zip", a.DownloadZip) mux.HandleFunc("GET /d/{boxID}/f/{fileID}", a.DownloadFile) diff --git a/backend/libs/handlers/upload.go b/backend/libs/handlers/upload.go index d8e4a43..111cecb 100644 --- a/backend/libs/handlers/upload.go +++ b/backend/libs/handlers/upload.go @@ -92,7 +92,7 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("expiration cannot exceed %d days", effectivePolicy.MaxDays)) return } - result, err := a.uploadService.CreateBox(files, services.UploadOptions{ + opts := services.UploadOptions{ MaxDays: maxDays, ExpiresInMinutes: expiresMinutes, MaxDownloads: parseInt(r.FormValue("max_downloads")), @@ -103,14 +103,15 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { SkipSizeLimit: isAdminUpload || effectivePolicy.MaxUploadMB < 0, CreatorIP: uploadClientIP(r), StorageBackendID: effectivePolicy.StorageBackendID, - }) + } + result, boxesAdded, err := a.createOrAppendBox(r, user, loggedIn, files, opts) if err != nil { a.logger.Warn("upload failed", "source", "user-upload", "severity", "warn", "code", 4001, "ip", uploadClientIP(r), "user_id", user.ID, "error", err.Error()) helpers.WriteJSONError(w, http.StatusBadRequest, err.Error()) return } if !isAdminUpload { - if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, 1); err != nil { + if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, boxesAdded); err != nil { a.logger.Warn("failed to record upload usage", "source", "quota", "severity", "warn", "code", 4402, "error", err.Error()) } if err := a.settingsService.CleanupUsage(time.Now().UTC(), settings.UsageRetentionDays); err != nil { @@ -130,6 +131,67 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprintln(w, result.BoxURL) } +// createOrAppendBox creates a new box. It only ever appends to an existing box +// when the request opts in via the X-Warpbox-Batch header: requests sharing the +// same batch value (per account, or per IP for anonymous) within +// uploadGroupWindow are folded into one box. Without the header the behaviour is +// identical to creating a fresh box every time. Returns the result and how many +// boxes were created (1 for a new box, 0 for an append) for usage accounting. +func (a *App) createOrAppendBox(r *http.Request, user services.User, loggedIn bool, files []*multipart.FileHeader, opts services.UploadOptions) (services.UploadResult, int, error) { + batch := strings.TrimSpace(r.Header.Get(uploadBatchHeader)) + if batch == "" { + result, err := a.uploadService.CreateBox(files, opts) + if err != nil { + return services.UploadResult{}, 0, err + } + return result, 1, nil + } + + // Group key is scoped to the uploader so batches never cross accounts/IPs. + identity := "ip:" + uploadClientIP(r) + if loggedIn { + identity = "user:" + user.ID + } + entry := a.uploadGroups.entryFor(identity + "|" + batch) + + // Hold the per-key lock across the whole create/append so concurrent batched + // uploads serialise into the same box instead of racing. + entry.mu.Lock() + defer entry.mu.Unlock() + + if entry.boxID != "" && time.Since(entry.at) < uploadGroupWindow { + if box, err := a.uploadService.GetBox(entry.boxID); err == nil && a.batchBoxMatches(box, user, loggedIn, r) && a.uploadService.CanDownload(box) == nil { + if result, err := a.uploadService.AppendFiles(entry.boxID, files, opts); err == nil { + // Re-attach the manage/delete URLs from the box's creation so every + // upload in the batch returns a working deletion URL. + result.ManageURL = entry.manageURL + result.DeleteURL = entry.deleteURL + entry.at = time.Now() + return result, 0, nil + } + } + } + + result, err := a.uploadService.CreateBox(files, opts) + if err != nil { + return services.UploadResult{}, 0, err + } + entry.boxID = result.BoxID + entry.manageURL = result.ManageURL + entry.deleteURL = result.DeleteURL + entry.at = time.Now() + return result, 1, nil +} + +// batchBoxMatches guards that a batched append only ever touches a box owned by +// the same uploader (account for logged-in users, creator IP for anonymous). +func (a *App) batchBoxMatches(box services.Box, user services.User, loggedIn bool, r *http.Request) bool { + if loggedIn { + return box.OwnerID == user.ID + } + return box.OwnerID == "" && box.CreatorIP == uploadClientIP(r) +} + func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bool, settings services.UploadPolicySettings, policy services.EffectiveUploadPolicy, files []*multipart.FileHeader, totalBytes int64) (int, string) { if len(files) == 0 { return 0, "" diff --git a/backend/libs/handlers/upload_group.go b/backend/libs/handlers/upload_group.go new file mode 100644 index 0000000..7115150 --- /dev/null +++ b/backend/libs/handlers/upload_group.go @@ -0,0 +1,49 @@ +package handlers + +import ( + "sync" + "time" +) + +// uploadGroupWindow is how long after a batched upload a follow-up upload with +// the same X-Warpbox-Batch value (and same account/IP) is folded into the same +// box. ShareX sends a multi-file selection as separate back-to-back requests; +// the batch header lets it land them in one box. +const uploadGroupWindow = 20 * time.Second + +// uploadBatchHeader is the opt-in request header. Without it, uploads behave +// exactly as before (one box per request). With it, requests sharing the same +// value (per account/IP) within uploadGroupWindow are grouped into one box. +const uploadBatchHeader = "X-Warpbox-Batch" + +// uploadGrouper tracks the most recent box per batch key so opt-in batched +// uploads land in a single box. Each key has its own lock, which also serialises +// that key's concurrent uploads so they append to the same box instead of racing +// to create several. +type uploadGrouper struct { + mu sync.Mutex + entries map[string]*uploadGroupEntry +} + +type uploadGroupEntry struct { + mu sync.Mutex + boxID string + manageURL string + deleteURL string + at time.Time +} + +func newUploadGrouper() *uploadGrouper { + return &uploadGrouper{entries: make(map[string]*uploadGroupEntry)} +} + +func (g *uploadGrouper) entryFor(key string) *uploadGroupEntry { + g.mu.Lock() + defer g.mu.Unlock() + entry, ok := g.entries[key] + if !ok { + entry = &uploadGroupEntry{} + g.entries[key] = entry + } + return entry +} diff --git a/backend/libs/services/upload.go b/backend/libs/services/upload.go index df70c43..cfbe46f 100644 --- a/backend/libs/services/upload.go +++ b/backend/libs/services/upload.go @@ -82,20 +82,22 @@ type File struct { } type UploadResult struct { - BoxID string `json:"boxId"` - BoxURL string `json:"boxUrl"` - ZipURL string `json:"zipUrl"` - ManageURL string `json:"manageUrl"` - DeleteURL string `json:"deleteUrl"` - ExpiresAt string `json:"expiresAt"` - Files []ResultFile `json:"files"` + BoxID string `json:"boxId"` + BoxURL string `json:"boxUrl"` + ZipURL string `json:"zipUrl"` + ThumbnailURL string `json:"thumbnailUrl"` + ManageURL string `json:"manageUrl"` + DeleteURL string `json:"deleteUrl"` + ExpiresAt string `json:"expiresAt"` + Files []ResultFile `json:"files"` } type ResultFile struct { - ID string `json:"id"` - Name string `json:"name"` - Size string `json:"size"` - URL string `json:"url"` + ID string `json:"id"` + Name string `json:"name"` + Size string `json:"size"` + URL string `json:"url"` + ThumbnailURL string `json:"thumbnailUrl"` } type AdminStats struct { @@ -226,15 +228,66 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti box.PasswordHash = hash } - backend, err := s.storage.Backend(box.StorageBackendID) + if err := s.writeFilesToBox(&box, files, opts); err != nil { + return UploadResult{}, err + } + + if err := s.SaveBox(box); err != nil { + return UploadResult{}, err + } + + s.logger.Info("upload complete", + "source", "user-upload", + "severity", "user_activity", + "code", 2001, + "box_id", box.ID, + "file_count", len(box.Files), + ) + + return s.resultForBox(box, deleteToken), nil +} + +// AppendFiles adds files to an existing box (used to group a ShareX multi-file +// selection into a single box). The box keeps its original expiry, password and +// other settings; only the new files are written. +func (s *UploadService) AppendFiles(boxID string, files []*multipart.FileHeader, opts UploadOptions) (UploadResult, error) { + if len(files) == 0 { + return UploadResult{}, fmt.Errorf("no files were uploaded") + } + box, err := s.GetBox(boxID) if err != nil { return UploadResult{}, err } + if err := s.writeFilesToBox(&box, files, opts); err != nil { + return UploadResult{}, err + } + if err := s.SaveBox(box); err != nil { + return UploadResult{}, err + } + s.logger.Info("upload appended", + "source", "user-upload", + "severity", "user_activity", + "code", 2001, + "box_id", box.ID, + "added", len(files), + "file_count", len(box.Files), + ) + return s.resultForBox(box, ""), nil +} + +// writeFilesToBox streams each uploaded file into the box's storage backend and +// appends the file metadata to box.Files. The box's StorageBackendID determines +// where files land, so it works for both new and existing boxes. +func (s *UploadService) writeFilesToBox(box *Box, files []*multipart.FileHeader, opts UploadOptions) error { + backend, err := s.storage.Backend(box.StorageBackendID) + if err != nil { + return err + } for _, header := range files { if !opts.SkipSizeLimit { if err := s.ValidateSize(header.Size); err != nil { - return UploadResult{}, err + return err } } @@ -245,7 +298,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti file, err := header.Open() if err != nil { - return UploadResult{}, err + return err } fileID := randomID(8) @@ -263,7 +316,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti if err := s.writeUploadedObject(context.Background(), backend, objectKey, file, header.Size, maxSize, contentType); err != nil { file.Close() - return UploadResult{}, err + return err } file.Close() @@ -278,20 +331,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti UploadedAt: time.Now().UTC(), }) } - - if err := s.SaveBox(box); err != nil { - return UploadResult{}, err - } - - s.logger.Info("upload complete", - "source", "user-upload", - "severity", "user_activity", - "code", 2001, - "box_id", box.ID, - "file_count", len(box.Files), - ) - - return s.resultForBox(box, deleteToken), nil + return nil } func (s *UploadService) GetBox(id string) (Box, error) { @@ -726,19 +766,28 @@ func (s *UploadService) resultForBox(box Box, deleteToken string) UploadResult { files := make([]ResultFile, 0, len(box.Files)) for _, file := range box.Files { files = append(files, ResultFile{ - ID: file.ID, - Name: file.Name, - Size: helpers.FormatBytes(file.Size), - URL: fmt.Sprintf("%s/d/%s/f/%s", s.baseURL, box.ID, file.ID), + ID: file.ID, + Name: file.Name, + Size: helpers.FormatBytes(file.Size), + URL: fmt.Sprintf("%s/d/%s/f/%s", s.baseURL, box.ID, file.ID), + ThumbnailURL: fmt.Sprintf("%s/d/%s/thumb/%s", s.baseURL, box.ID, file.ID), }) } + // The box-level thumbnail points at the most recently added file, so a + // per-file ShareX upload previews the file it just sent. + thumbnailURL := fmt.Sprintf("%s/d/%s/og-image.jpg", s.baseURL, box.ID) + if len(files) > 0 { + thumbnailURL = files[len(files)-1].ThumbnailURL + } + result := UploadResult{ - BoxID: box.ID, - BoxURL: fmt.Sprintf("%s/d/%s", s.baseURL, box.ID), - ZipURL: fmt.Sprintf("%s/d/%s/zip", s.baseURL, box.ID), - ExpiresAt: box.ExpiresAt.Format(time.RFC3339), - Files: files, + BoxID: box.ID, + BoxURL: fmt.Sprintf("%s/d/%s", s.baseURL, box.ID), + ZipURL: fmt.Sprintf("%s/d/%s/zip", s.baseURL, box.ID), + ThumbnailURL: thumbnailURL, + ExpiresAt: box.ExpiresAt.Format(time.RFC3339), + Files: files, } if deleteToken != "" { result.ManageURL = fmt.Sprintf("%s/d/%s/manage/%s", s.baseURL, box.ID, deleteToken) diff --git a/backend/static/css/40-docs.css b/backend/static/css/40-docs.css index 0e6dbf4..5c2675d 100644 --- a/backend/static/css/40-docs.css +++ b/backend/static/css/40-docs.css @@ -36,6 +36,22 @@ font-size: 1rem; } +.docs-card h3 { + margin: 1.35rem 0 0; + font-size: 0.9rem; + font-weight: 650; + color: var(--foreground); +} + +/* Highlights where the API token goes in the ShareX config snippet. */ +.sxcu-highlight { + background: #fde047; + color: #1a1a1a; + font-weight: 700; + padding: 0 0.2rem; + border-radius: 3px; +} + .docs-card p { margin: 0.65rem 0 0; color: var(--muted-foreground); diff --git a/backend/templates/layouts/base.html b/backend/templates/layouts/base.html index a50b357..be830b4 100644 --- a/backend/templates/layouts/base.html +++ b/backend/templates/layouts/base.html @@ -60,7 +60,7 @@