kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): add S3 backend support and advanced upload limits

Browse Source 
- Introduce S3-compatible storage backend support using minio-go.
- Add configuration options for local storage limits, box limits, and rate limiting.
- Implement storage backend selection (local vs S3) for anonymous and registered users.
- Add an `/admin/storage` management interface.
- Update documentation and environment examples with the new configuration variables.
This commit is contained in:
Daniel Legt
2026-05-31 02:14:10 +03:00
parent 830d2a885c
commit c3558fd353
34 changed files with 2668 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
backend/libs/handlers/upload.go
View File

@@ -1,6 +1,7 @@
package handlers
import (
"context"
"errors"
"fmt"
"mime/multipart"
@@ -27,11 +28,17 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) {
helpers.WriteJSONError(w, http.StatusForbidden, "anonymous uploads are disabled")
return
}
effectivePolicy := a.effectiveUploadPolicy(settings, user, loggedIn)
rateKey := uploadRateKey(r, user, loggedIn)
if !isAdminUpload && !a.rateLimiter.Allow("upload:"+rateKey, effectivePolicy.ShortRequests, effectivePolicy.ShortWindow, time.Now().UTC()) {
helpers.WriteJSONError(w, http.StatusTooManyRequests, "too many upload requests, please slow down")
return
}
if !isAdminUpload {
r.Body = http.MaxBytesReader(w, r.Body, uploadParseLimit(settings, loggedIn, a.uploadService.MaxUploadSize()))
r.Body = http.MaxBytesReader(w, r.Body, uploadParseLimit(effectivePolicy, loggedIn, a.uploadService.MaxUploadSize()))
}
parseLimit := uploadParseLimit(settings, loggedIn, a.uploadService.MaxUploadSize())
parseLimit := uploadParseLimit(effectivePolicy, loggedIn, a.uploadService.MaxUploadSize())
if isAdminUpload {
parseLimit = 32 << 20
}
@@ -53,19 +60,29 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) {
}
}
if !isAdminUpload {
if status, message := a.checkUploadPolicy(r, user, loggedIn, settings, files, totalBytes); message != "" {
if status, message := a.checkUploadPolicy(r, user, loggedIn, settings, effectivePolicy, files, totalBytes); message != "" {
helpers.WriteJSONError(w, status, message)
return
}
}
maxDays := parseInt(r.FormValue("max_days"))
if maxDays <= 0 {
maxDays = min(7, effectivePolicy.MaxDays)
}
if !isAdminUpload && maxDays > effectivePolicy.MaxDays {
helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("expiration cannot exceed %d days", effectivePolicy.MaxDays))
return
}
result, err := a.uploadService.CreateBox(files, services.UploadOptions{
MaxDays: parseInt(r.FormValue("max_days")),
MaxDays: maxDays,
MaxDownloads: parseInt(r.FormValue("max_downloads")),
Password: r.FormValue("password"),
ObfuscateMetadata: r.FormValue("obfuscate_metadata") == "on",
OwnerID: ownerID,
CollectionID: collectionID,
SkipSizeLimit: isAdminUpload,
CreatorIP: uploadClientIP(r),
StorageBackendID: effectivePolicy.StorageBackendID,
})
if err != nil {
a.logger.Warn("upload failed", "source", "user-upload", "severity", "warn", "code", 4001, "error", err.Error())
@@ -73,7 +90,7 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) {
return
}
if !isAdminUpload {
if err := a.recordUploadUsage(r, user, loggedIn, totalBytes); err != nil {
if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, 1); err != nil {
a.logger.Warn("failed to record upload usage", "source", "quota", "severity", "warn", "code", 4402, "error", err.Error())
}
if err := a.settingsService.CleanupUsage(time.Now().UTC(), settings.UsageRetentionDays); err != nil {
@@ -92,25 +109,40 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintln(w, result.BoxURL)
}
func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bool, settings services.UploadPolicySettings, files []*multipart.FileHeader, totalBytes int64) (int, string) {
func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bool, settings services.UploadPolicySettings, policy services.EffectiveUploadPolicy, files []*multipart.FileHeader, totalBytes int64) (int, string) {
if len(files) == 0 {
return 0, ""
}
now := time.Now().UTC()
if !loggedIn {
anonymousMaxBytes := services.MegabytesToBytes(settings.AnonymousMaxUploadMB)
if policy.MaxUploadMB > 0 {
maxBytes := services.MegabytesToBytes(policy.MaxUploadMB)
for _, file := range files {
if file.Size > anonymousMaxBytes {
return http.StatusRequestEntityTooLarge, "file exceeds anonymous upload size limit"
if file.Size > maxBytes {
return http.StatusRequestEntityTooLarge, "file exceeds upload size limit"
}
}
}
if !loggedIn {
usage, err := a.settingsService.UsageForIP(uploadClientIP(r), now)
if err != nil {
return http.StatusInternalServerError, "upload usage could not be checked"
}
if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(settings.AnonymousDailyUploadMB) {
if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(policy.DailyUploadMB) {
return http.StatusTooManyRequests, "anonymous daily upload limit reached"
}
if usage.UploadedBoxes+1 > policy.DailyBoxes {
return http.StatusTooManyRequests, "anonymous daily box limit reached"
}
activeBoxes, err := a.uploadService.ActiveBoxCountForIP(uploadClientIP(r))
if err != nil {
return http.StatusInternalServerError, "active box limit could not be checked"
}
if activeBoxes+1 > policy.ActiveBoxes {
return http.StatusTooManyRequests, "anonymous active box limit reached"
}
if status, message := a.checkStorageBackendCapacity(policy.StorageBackendID, settings, totalBytes); message != "" {
return status, message
}
return 0, ""
}
@@ -118,42 +150,86 @@ func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bo
if err != nil {
return http.StatusInternalServerError, "upload usage could not be checked"
}
if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(settings.UserDailyUploadMB) {
if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(policy.DailyUploadMB) {
return http.StatusTooManyRequests, "daily upload limit reached"
}
if usage.UploadedBoxes+1 > policy.DailyBoxes {
return http.StatusTooManyRequests, "daily box limit reached"
}
activeBoxes, err := a.uploadService.ActiveBoxCountForUser(user.ID)
if err != nil {
return http.StatusInternalServerError, "active box limit could not be checked"
}
if activeBoxes+1 > policy.ActiveBoxes {
return http.StatusTooManyRequests, "active box limit reached"
}
activeStorage, err := a.uploadService.UserActiveStorageUsed(user.ID)
if err != nil {
return http.StatusInternalServerError, "storage quota could not be checked"
}
quotaMB := settings.DefaultUserStorageMB
if user.StorageQuotaMB != nil {
quotaMB = *user.StorageQuotaMB
}
if activeStorage+totalBytes > services.MegabytesToBytes(quotaMB) {
if policy.StorageQuotaSet && activeStorage+totalBytes > services.MegabytesToBytes(policy.StorageQuotaMB) {
return http.StatusRequestEntityTooLarge, "storage quota reached"
}
if status, message := a.checkStorageBackendCapacity(policy.StorageBackendID, settings, totalBytes); message != "" {
return status, message
}
return 0, ""
}
func (a *App) recordUploadUsage(r *http.Request, user services.User, loggedIn bool, totalBytes int64) error {
func (a *App) recordUploadUsage(r *http.Request, user services.User, loggedIn bool, totalBytes int64, boxes int) error {
now := time.Now().UTC()
if loggedIn {
return a.settingsService.AddUsage("user", user.ID, totalBytes, now)
return a.settingsService.AddUploadUsage("user", user.ID, totalBytes, boxes, now)
}
return a.settingsService.AddUsage("ip", uploadClientIP(r), totalBytes, now)
return a.settingsService.AddUploadUsage("ip", uploadClientIP(r), totalBytes, boxes, now)
}
func uploadParseLimit(settings services.UploadPolicySettings, loggedIn bool, fallback int64) int64 {
func (a *App) effectiveUploadPolicy(settings services.UploadPolicySettings, user services.User, loggedIn bool) services.EffectiveUploadPolicy {
if loggedIn {
return a.settingsService.EffectivePolicyForUser(settings, user)
}
return a.settingsService.EffectivePolicyForAnonymous(settings)
}
func (a *App) checkStorageBackendCapacity(backendID string, settings services.UploadPolicySettings, totalBytes int64) (int, string) {
if backendID != services.StorageBackendLocal {
return 0, ""
}
backend, err := a.uploadService.Storage().Backend(services.StorageBackendLocal)
if err != nil {
return http.StatusInternalServerError, "storage backend could not be checked"
}
used, err := backend.Usage(context.Background())
if err != nil {
return http.StatusInternalServerError, "storage backend usage could not be checked"
}
if used+totalBytes > services.GigabytesToBytes(settings.LocalStorageMaxGB) {
return http.StatusRequestEntityTooLarge, "local storage limit reached"
}
return 0, ""
}
func uploadParseLimit(policy services.EffectiveUploadPolicy, loggedIn bool, fallback int64) int64 {
if loggedIn && policy.MaxUploadMB <= 0 {
return fallback * 8
}
return services.MegabytesToBytes(settings.AnonymousMaxUploadMB) * 8
if policy.MaxUploadMB > 0 {
return services.MegabytesToBytes(policy.MaxUploadMB) * 8
}
return fallback * 8
}
func uploadClientIP(r *http.Request) string {
return services.ClientIP(r.RemoteAddr, r.Header.Get("X-Forwarded-For"))
}
func uploadRateKey(r *http.Request, user services.User, loggedIn bool) string {
if loggedIn {
return "user:" + user.ID
}
return "ip:" + uploadClientIP(r)
}
func totalUploadBytes(files []*multipart.FileHeader) int64 {
var total int64
for _, file := range files {