kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): add S3 backend support and advanced upload limits

Browse Source 
- Introduce S3-compatible storage backend support using minio-go.
- Add configuration options for local storage limits, box limits, and rate limiting.
- Implement storage backend selection (local vs S3) for anonymous and registered users.
- Add an `/admin/storage` management interface.
- Update documentation and environment examples with the new configuration variables.
This commit is contained in:
Daniel Legt
2026-05-31 02:14:10 +03:00
parent 830d2a885c
commit c3558fd353
34 changed files with 2668 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
backend/libs/handlers/accounts_test.go
View File

@@ -269,6 +269,104 @@ func TestSignedInDailyCap(t *testing.T) {
}
}
func TestLayeredUploadLimits(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
policy := testPolicy(t, app)
policy.AnonymousDailyBoxes = 1
policy.AnonymousActiveBoxes = 10
policy.AnonymousMaxDays = 3
policy.LocalStorageMaxGB = 0.001
if err := app.settingsService.UpdateUploadPolicy(policy); err != nil {
t.Fatalf("UpdateUploadPolicy returned error: %v", err)
}
first := uploadThroughApp(t, app)
if first.BoxID == "" {
t.Fatalf("first upload did not return a box id")
}
secondRequest := multipartUploadRequest(t, "/api/v1/upload", "file", "second.txt", "hello")
secondRequest.Header.Set("Accept", "application/json")
secondResponse := httptest.NewRecorder()
app.Upload(secondResponse, secondRequest)
if secondResponse.Code != http.StatusTooManyRequests {
t.Fatalf("daily box status = %d, body = %s", secondResponse.Code, secondResponse.Body.String())
}
policy.AnonymousDailyBoxes = 10
if err := app.settingsService.UpdateUploadPolicy(policy); err != nil {
t.Fatalf("UpdateUploadPolicy returned error: %v", err)
}
expiryRequest := multipartUploadRequestWithField(t, "/api/v1/upload", "file", "expiry.txt", "hello", "max_days", "30")
expiryRequest.Header.Set("Accept", "application/json")
expiryResponse := httptest.NewRecorder()
app.Upload(expiryResponse, expiryRequest)
if expiryResponse.Code != http.StatusRequestEntityTooLarge && expiryResponse.Code != http.StatusTooManyRequests {
t.Fatalf("expiry/box status = %d, body = %s", expiryResponse.Code, expiryResponse.Body.String())
}
}
func TestUserPolicyOverrideChangesUploadEnforcement(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
admin, err := app.authService.CreateBootstrapUser("admin", "admin@example.test", "password123")
if err != nil {
t.Fatalf("CreateBootstrapUser returned error: %v", err)
}
invite, err := app.authService.CreateInvite("user@example.test", services.UserRoleUser, admin.ID, 0)
if err != nil {
t.Fatalf("CreateInvite returned error: %v", err)
}
user, err := app.authService.AcceptInvite(invite.Token, "user", "password123")
if err != nil {
t.Fatalf("AcceptInvite returned error: %v", err)
}
dailyBoxes := 1
maxDays := 1
if err := app.authService.SetUserPolicy(user.ID, services.UserPolicy{DailyBoxes: &dailyBoxes, MaxDays: &maxDays}); err != nil {
t.Fatalf("SetUserPolicy returned error: %v", err)
}
_, token, err := app.authService.Login(user.Email, "password123")
if err != nil {
t.Fatalf("Login returned error: %v", err)
}
first := multipartUploadRequest(t, "/api/v1/upload", "file", "one.txt", "hello")
first.Header.Set("Accept", "application/json")
first.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: token})
firstResponse := httptest.NewRecorder()
app.Upload(firstResponse, first)
if firstResponse.Code != http.StatusCreated {
t.Fatalf("first status = %d, body = %s", firstResponse.Code, firstResponse.Body.String())
}
second := multipartUploadRequest(t, "/api/v1/upload", "file", "two.txt", "hello")
second.Header.Set("Accept", "application/json")
second.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: token})
secondResponse := httptest.NewRecorder()
app.Upload(secondResponse, second)
if secondResponse.Code != http.StatusTooManyRequests {
t.Fatalf("second status = %d, body = %s", secondResponse.Code, secondResponse.Body.String())
}
}
func TestLocalStorageCapRejectsUpload(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
policy := testPolicy(t, app)
policy.AnonymousMaxUploadMB = 4
policy.AnonymousDailyUploadMB = 8
policy.LocalStorageMaxGB = 0.001
if err := app.settingsService.UpdateUploadPolicy(policy); err != nil {
t.Fatalf("UpdateUploadPolicy returned error: %v", err)
}
request := multipartUploadRequest(t, "/api/v1/upload", "file", "large.txt", strings.Repeat("x", 2*1024*1024))
request.Header.Set("Accept", "application/json")
response := httptest.NewRecorder()
app.Upload(response, request)
if response.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("storage cap status = %d, body = %s", response.Code, response.Body.String())
}
}
func TestAdminSettingsPostChangesUploadEnforcement(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
@@ -281,10 +379,11 @@ func TestAdminSettingsPostChangesUploadEnforcement(t *testing.T) {
t.Fatalf("Login returned error: %v", err)
}
settingsForm := strings.NewReader("anonymous_max_upload_mb=512&anonymous_daily_upload_mb=2048&user_daily_upload_mb=8192&default_user_storage_mb=51200&usage_retention_days=30")
settingsForm := strings.NewReader("anonymous_max_upload_mb=512&anonymous_daily_upload_mb=2048&user_daily_upload_mb=8192&default_user_storage_mb=51200&usage_retention_days=30&csrf_token=test-csrf")
settingsRequest := httptest.NewRequest(http.MethodPost, "/admin/settings", settingsForm)
settingsRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded")
settingsRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: token})
settingsRequest.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"})
settingsResponse := httptest.NewRecorder()
app.AdminSettingsPost(settingsResponse, settingsRequest)
if settingsResponse.Code != http.StatusSeeOther {
@@ -320,9 +419,10 @@ func TestAdminUserQuotaPostChangesEnforcement(t *testing.T) {
t.Fatalf("admin Login returned error: %v", err)
}
quotaRequest := httptest.NewRequest(http.MethodPost, "/admin/users/"+user.ID+"/quota", strings.NewReader("storage_quota_mb=0.001"))
quotaRequest := httptest.NewRequest(http.MethodPost, "/admin/users/"+user.ID+"/quota", strings.NewReader("storage_quota_mb=0.001&csrf_token=test-csrf"))
quotaRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded")
quotaRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
quotaRequest.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"})
quotaRequest.SetPathValue("userID", user.ID)
quotaResponse := httptest.NewRecorder()
app.AdminUpdateUserQuota(quotaResponse, quotaRequest)