diff --git a/.env.example b/.env.example index 12a654c..2c91642 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,17 @@ WARPBOX_ANONYMOUS_DAILY_UPLOAD_MB=2048 WARPBOX_USER_DAILY_UPLOAD_MB=8192 WARPBOX_DEFAULT_USER_STORAGE_MB=51200 WARPBOX_USAGE_RETENTION_DAYS=30 +WARPBOX_LOCAL_STORAGE_MAX_GB=100 +WARPBOX_ANONYMOUS_MAX_DAYS=30 +WARPBOX_USER_MAX_DAYS=90 +WARPBOX_ANONYMOUS_DAILY_BOXES=100 +WARPBOX_USER_DAILY_BOXES=250 +WARPBOX_ANONYMOUS_ACTIVE_BOXES=500 +WARPBOX_USER_ACTIVE_BOXES=1000 +WARPBOX_SHORT_WINDOW_REQUESTS=60 +WARPBOX_SHORT_WINDOW_SECONDS=60 +WARPBOX_ANONYMOUS_STORAGE_BACKEND=local +WARPBOX_USER_STORAGE_BACKEND=local WARPBOX_READ_TIMEOUT=15s WARPBOX_WRITE_TIMEOUT=60s WARPBOX_IDLE_TIMEOUT=120s diff --git a/README.md b/README.md index 593f54b..b5a43a4 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,17 @@ Upload policy defaults are also configured in megabytes and can later be changed - `WARPBOX_USER_DAILY_UPLOAD_MB=8192` - `WARPBOX_DEFAULT_USER_STORAGE_MB=51200` - `WARPBOX_USAGE_RETENTION_DAYS=30` +- `WARPBOX_LOCAL_STORAGE_MAX_GB=100` +- `WARPBOX_ANONYMOUS_MAX_DAYS=30` +- `WARPBOX_USER_MAX_DAYS=90` +- `WARPBOX_ANONYMOUS_DAILY_BOXES=100` +- `WARPBOX_USER_DAILY_BOXES=250` +- `WARPBOX_ANONYMOUS_ACTIVE_BOXES=500` +- `WARPBOX_USER_ACTIVE_BOXES=1000` +- `WARPBOX_SHORT_WINDOW_REQUESTS=60` +- `WARPBOX_SHORT_WINDOW_SECONDS=60` +- `WARPBOX_ANONYMOUS_STORAGE_BACKEND=local` +- `WARPBOX_USER_STORAGE_BACKEND=local` Runtime data is configured with `WARPBOX_DATA_DIR` and defaults to `./data` in the dev environment. The dev script resolves that path from the repository root. @@ -126,6 +137,11 @@ from `examples/sharex/warpbox-anonymous.sxcu`; update `RequestURL` to match your - `/admin/settings` controls anonymous uploads, anonymous max upload size, daily upload caps, default user storage quota, and usage retention. - `/admin/users` shows storage/daily usage and lets admins set per-user storage quota overrides. +- `/admin/storage` manages the built-in local file backend and S3-compatible bucket backends. +- Upload limits now include daily bytes, daily box counts, active box counts, short-window request + limits, max expiration days, local storage capacity in GB, and per-user policy overrides. +- Uploaded file content, thumbnails, and private box metadata use the selected storage backend. + The bbolt database and JSON logs remain local under `./data/db` and `./data/logs`. - Anonymous uploads, ShareX uploads, unlisted public box links, password protection, expiry, delete tokens, thumbnails, and cleanup continue to work as before. @@ -136,8 +152,8 @@ support will power public forgot-password and optional email delivery. Warpbox keeps local runtime data under the configured data directory: -- `data/files/{box_id}/@each@{file_id}.ext` - uploaded file contents. -- `data/files/{box_id}/@thumb@{file_id}.jpg` - generated previews where available. +- `data/files/{box_id}/@each@{file_id}.ext` - uploaded file contents when the local backend is selected. +- `data/files/{box_id}/@thumb@{file_id}.jpg` - generated previews when the local backend is selected. - `data/db/warpbox.bbolt` - bbolt metadata database for boxes and file records. - `data/db/warpbox.bbolt` also stores users, sessions, invites, and collections. - `data/db/warpbox.bbolt` stores upload policy settings and daily usage records keyed by plain IP diff --git a/backend/go.mod b/backend/go.mod index 52b04b0..07ff494 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,9 +3,28 @@ module warpbox.dev/backend go 1.26 require ( + github.com/minio/minio-go/v7 v7.2.0 go.etcd.io/bbolt v1.4.3 - golang.org/x/crypto v0.33.0 + golang.org/x/crypto v0.51.0 golang.org/x/image v0.41.0 ) -require golang.org/x/sys v0.30.0 // indirect +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.44.0 // indirect + golang.org/x/text v0.37.0 // indirect + gopkg.in/ini.v1 v1.67.2 // indirect +) diff --git a/backend/go.sum b/backend/go.sum index a6acba3..2933360 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,18 +1,66 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs= +github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo= golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= +gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/backend/libs/config/config.go b/backend/libs/config/config.go index 1d68827..9358f5b 100644 --- a/backend/libs/config/config.go +++ b/backend/libs/config/config.go @@ -38,6 +38,17 @@ type SettingsDefaults struct { UserDailyUploadMB float64 DefaultUserStorageMB float64 UsageRetentionDays int + LocalStorageMaxGB float64 + AnonymousMaxDays int + UserMaxDays int + AnonymousDailyBoxes int + UserDailyBoxes int + AnonymousActiveBoxes int + UserActiveBoxes int + ShortWindowRequests int + ShortWindowSeconds int + AnonymousStorageBackend string + UserStorageBackend string } func Load() (Config, error) { @@ -66,6 +77,17 @@ func Load() (Config, error) { UserDailyUploadMB: envMegabytesFloat("WARPBOX_USER_DAILY_UPLOAD_MB", 8192), DefaultUserStorageMB: envMegabytesFloat("WARPBOX_DEFAULT_USER_STORAGE_MB", 51200), UsageRetentionDays: envInt("WARPBOX_USAGE_RETENTION_DAYS", 30), + LocalStorageMaxGB: envGigabytesFloat("WARPBOX_LOCAL_STORAGE_MAX_GB", 100), + AnonymousMaxDays: envInt("WARPBOX_ANONYMOUS_MAX_DAYS", 30), + UserMaxDays: envInt("WARPBOX_USER_MAX_DAYS", 90), + AnonymousDailyBoxes: envInt("WARPBOX_ANONYMOUS_DAILY_BOXES", 100), + UserDailyBoxes: envInt("WARPBOX_USER_DAILY_BOXES", 250), + AnonymousActiveBoxes: envInt("WARPBOX_ANONYMOUS_ACTIVE_BOXES", 500), + UserActiveBoxes: envInt("WARPBOX_USER_ACTIVE_BOXES", 1000), + ShortWindowRequests: envInt("WARPBOX_SHORT_WINDOW_REQUESTS", 60), + ShortWindowSeconds: envInt("WARPBOX_SHORT_WINDOW_SECONDS", 60), + AnonymousStorageBackend: envString("WARPBOX_ANONYMOUS_STORAGE_BACKEND", "local"), + UserStorageBackend: envString("WARPBOX_USER_STORAGE_BACKEND", "local"), }, } @@ -79,7 +101,16 @@ func Load() (Config, error) { cfg.DefaultSettings.AnonymousDailyUploadMB <= 0 || cfg.DefaultSettings.UserDailyUploadMB <= 0 || cfg.DefaultSettings.DefaultUserStorageMB <= 0 || - cfg.DefaultSettings.UsageRetentionDays <= 0 { + cfg.DefaultSettings.UsageRetentionDays <= 0 || + cfg.DefaultSettings.LocalStorageMaxGB <= 0 || + cfg.DefaultSettings.AnonymousMaxDays <= 0 || + cfg.DefaultSettings.UserMaxDays <= 0 || + cfg.DefaultSettings.AnonymousDailyBoxes <= 0 || + cfg.DefaultSettings.UserDailyBoxes <= 0 || + cfg.DefaultSettings.AnonymousActiveBoxes <= 0 || + cfg.DefaultSettings.UserActiveBoxes <= 0 || + cfg.DefaultSettings.ShortWindowRequests <= 0 || + cfg.DefaultSettings.ShortWindowSeconds <= 0 { return Config{}, fmt.Errorf("upload policy settings must be positive") } @@ -172,6 +203,23 @@ func envMegabytesFloat(key string, fallback float64) float64 { return parsed } +func envGigabytesFloat(key string, fallback float64) float64 { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + normalized := strings.TrimSpace(value) + normalized = strings.TrimSuffix(normalized, "GB") + normalized = strings.TrimSuffix(normalized, "Gb") + normalized = strings.TrimSuffix(normalized, "gb") + normalized = strings.TrimSpace(normalized) + parsed, err := strconv.ParseFloat(normalized, 64) + if err != nil || parsed <= 0 { + return fallback + } + return parsed +} + func parseMegabytes(value string) (int64, error) { sizeMB, err := parseMegabytesFloat(value) if err != nil { diff --git a/backend/libs/handlers/accounts_test.go b/backend/libs/handlers/accounts_test.go index 8ed0182..fceef0a 100644 --- a/backend/libs/handlers/accounts_test.go +++ b/backend/libs/handlers/accounts_test.go @@ -269,6 +269,104 @@ func TestSignedInDailyCap(t *testing.T) { } } +func TestLayeredUploadLimits(t *testing.T) { + app, cleanup := newTestApp(t) + defer cleanup() + policy := testPolicy(t, app) + policy.AnonymousDailyBoxes = 1 + policy.AnonymousActiveBoxes = 10 + policy.AnonymousMaxDays = 3 + policy.LocalStorageMaxGB = 0.001 + if err := app.settingsService.UpdateUploadPolicy(policy); err != nil { + t.Fatalf("UpdateUploadPolicy returned error: %v", err) + } + + first := uploadThroughApp(t, app) + if first.BoxID == "" { + t.Fatalf("first upload did not return a box id") + } + secondRequest := multipartUploadRequest(t, "/api/v1/upload", "file", "second.txt", "hello") + secondRequest.Header.Set("Accept", "application/json") + secondResponse := httptest.NewRecorder() + app.Upload(secondResponse, secondRequest) + if secondResponse.Code != http.StatusTooManyRequests { + t.Fatalf("daily box status = %d, body = %s", secondResponse.Code, secondResponse.Body.String()) + } + + policy.AnonymousDailyBoxes = 10 + if err := app.settingsService.UpdateUploadPolicy(policy); err != nil { + t.Fatalf("UpdateUploadPolicy returned error: %v", err) + } + expiryRequest := multipartUploadRequestWithField(t, "/api/v1/upload", "file", "expiry.txt", "hello", "max_days", "30") + expiryRequest.Header.Set("Accept", "application/json") + expiryResponse := httptest.NewRecorder() + app.Upload(expiryResponse, expiryRequest) + if expiryResponse.Code != http.StatusRequestEntityTooLarge && expiryResponse.Code != http.StatusTooManyRequests { + t.Fatalf("expiry/box status = %d, body = %s", expiryResponse.Code, expiryResponse.Body.String()) + } +} + +func TestUserPolicyOverrideChangesUploadEnforcement(t *testing.T) { + app, cleanup := newTestApp(t) + defer cleanup() + admin, err := app.authService.CreateBootstrapUser("admin", "admin@example.test", "password123") + if err != nil { + t.Fatalf("CreateBootstrapUser returned error: %v", err) + } + invite, err := app.authService.CreateInvite("user@example.test", services.UserRoleUser, admin.ID, 0) + if err != nil { + t.Fatalf("CreateInvite returned error: %v", err) + } + user, err := app.authService.AcceptInvite(invite.Token, "user", "password123") + if err != nil { + t.Fatalf("AcceptInvite returned error: %v", err) + } + dailyBoxes := 1 + maxDays := 1 + if err := app.authService.SetUserPolicy(user.ID, services.UserPolicy{DailyBoxes: &dailyBoxes, MaxDays: &maxDays}); err != nil { + t.Fatalf("SetUserPolicy returned error: %v", err) + } + _, token, err := app.authService.Login(user.Email, "password123") + if err != nil { + t.Fatalf("Login returned error: %v", err) + } + first := multipartUploadRequest(t, "/api/v1/upload", "file", "one.txt", "hello") + first.Header.Set("Accept", "application/json") + first.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: token}) + firstResponse := httptest.NewRecorder() + app.Upload(firstResponse, first) + if firstResponse.Code != http.StatusCreated { + t.Fatalf("first status = %d, body = %s", firstResponse.Code, firstResponse.Body.String()) + } + second := multipartUploadRequest(t, "/api/v1/upload", "file", "two.txt", "hello") + second.Header.Set("Accept", "application/json") + second.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: token}) + secondResponse := httptest.NewRecorder() + app.Upload(secondResponse, second) + if secondResponse.Code != http.StatusTooManyRequests { + t.Fatalf("second status = %d, body = %s", secondResponse.Code, secondResponse.Body.String()) + } +} + +func TestLocalStorageCapRejectsUpload(t *testing.T) { + app, cleanup := newTestApp(t) + defer cleanup() + policy := testPolicy(t, app) + policy.AnonymousMaxUploadMB = 4 + policy.AnonymousDailyUploadMB = 8 + policy.LocalStorageMaxGB = 0.001 + if err := app.settingsService.UpdateUploadPolicy(policy); err != nil { + t.Fatalf("UpdateUploadPolicy returned error: %v", err) + } + request := multipartUploadRequest(t, "/api/v1/upload", "file", "large.txt", strings.Repeat("x", 2*1024*1024)) + request.Header.Set("Accept", "application/json") + response := httptest.NewRecorder() + app.Upload(response, request) + if response.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("storage cap status = %d, body = %s", response.Code, response.Body.String()) + } +} + func TestAdminSettingsPostChangesUploadEnforcement(t *testing.T) { app, cleanup := newTestApp(t) defer cleanup() @@ -281,10 +379,11 @@ func TestAdminSettingsPostChangesUploadEnforcement(t *testing.T) { t.Fatalf("Login returned error: %v", err) } - settingsForm := strings.NewReader("anonymous_max_upload_mb=512&anonymous_daily_upload_mb=2048&user_daily_upload_mb=8192&default_user_storage_mb=51200&usage_retention_days=30") + settingsForm := strings.NewReader("anonymous_max_upload_mb=512&anonymous_daily_upload_mb=2048&user_daily_upload_mb=8192&default_user_storage_mb=51200&usage_retention_days=30&csrf_token=test-csrf") settingsRequest := httptest.NewRequest(http.MethodPost, "/admin/settings", settingsForm) settingsRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded") settingsRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: token}) + settingsRequest.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"}) settingsResponse := httptest.NewRecorder() app.AdminSettingsPost(settingsResponse, settingsRequest) if settingsResponse.Code != http.StatusSeeOther { @@ -320,9 +419,10 @@ func TestAdminUserQuotaPostChangesEnforcement(t *testing.T) { t.Fatalf("admin Login returned error: %v", err) } - quotaRequest := httptest.NewRequest(http.MethodPost, "/admin/users/"+user.ID+"/quota", strings.NewReader("storage_quota_mb=0.001")) + quotaRequest := httptest.NewRequest(http.MethodPost, "/admin/users/"+user.ID+"/quota", strings.NewReader("storage_quota_mb=0.001&csrf_token=test-csrf")) quotaRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded") quotaRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken}) + quotaRequest.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"}) quotaRequest.SetPathValue("userID", user.ID) quotaResponse := httptest.NewRecorder() app.AdminUpdateUserQuota(quotaResponse, quotaRequest) diff --git a/backend/libs/handlers/admin.go b/backend/libs/handlers/admin.go index c10024a..f1ca333 100644 --- a/backend/libs/handlers/admin.go +++ b/backend/libs/handlers/admin.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "crypto/sha256" "encoding/hex" "net/http" @@ -19,6 +20,8 @@ type adminPageData struct { Boxes []adminBoxView Users []adminUserView Settings services.UploadPolicySettings + Storage []services.StorageBackendView + UserEdit adminUserEditView Section string PageTitle string LastInviteURL string @@ -39,15 +42,40 @@ type adminBoxView struct { } type adminUserView struct { - ID string - Username string - Email string - Role string - Status string - StorageUsed string - StorageQuota string - DailyUsed string - CreatedAt string + ID string + Username string + Email string + Role string + Status string + StorageUsed string + StorageQuota string + DailyUsed string + StorageBackend string + CreatedAt string +} + +type adminUserEditView struct { + ID string + Username string + Email string + Role string + Status string + StorageUsed string + DailyUsed string + EffectiveStorage string + EffectiveDaily string + EffectiveMaxDays int + EffectiveDailyBoxes int + EffectiveActiveBoxes int + EffectiveBackend string + MaxUploadMB string + DailyUploadMB string + StorageQuotaMB string + MaxDays string + DailyBoxes string + ActiveBoxes string + ShortWindowRequests string + StorageBackendID string } func (a *App) AdminLogin(w http.ResponseWriter, r *http.Request) { @@ -59,6 +87,10 @@ func (a *App) AdminLogin(w http.ResponseWriter, r *http.Request) { } func (a *App) AdminLoginPost(w http.ResponseWriter, r *http.Request) { + if !a.rateLimiter.Allow("admin-login:"+uploadClientIP(r), 10, time.Minute, time.Now().UTC()) { + a.renderAdminLogin(w, r, http.StatusTooManyRequests, "Too many admin login attempts.") + return + } if err := r.ParseForm(); err != nil { a.renderAdminLogin(w, r, http.StatusBadRequest, "Unable to read login form.") return @@ -83,6 +115,9 @@ func (a *App) AdminLoginPost(w http.ResponseWriter, r *http.Request) { } func (a *App) AdminLogout(w http.ResponseWriter, r *http.Request) { + if !a.validateCSRF(w, r) { + return + } a.clearUserSessionCookie(w) http.SetCookie(w, &http.Cookie{ Name: adminCookieName, @@ -176,20 +211,22 @@ func (a *App) AdminUsers(w http.ResponseWriter, r *http.Request) { for _, user := range users { storageUsed, _ := a.uploadService.UserActiveStorageUsed(user.ID) usage, _ := a.settingsService.UsageForUser(user.ID, time.Now().UTC()) - quotaMB := settings.DefaultUserStorageMB - if user.StorageQuotaMB != nil { - quotaMB = *user.StorageQuotaMB + policy := a.settingsService.EffectivePolicyForUser(settings, user) + quota := "unlimited" + if policy.StorageQuotaSet { + quota = formatMB(policy.StorageQuotaMB) } rows = append(rows, adminUserView{ - ID: user.ID, - Username: user.Username, - Email: user.Email, - Role: user.Role, - Status: user.Status, - StorageUsed: services.FormatMegabytesFromBytes(storageUsed), - StorageQuota: formatMB(quotaMB), - DailyUsed: services.FormatMegabytesFromBytes(usage.UploadedBytes), - CreatedAt: user.CreatedAt.Format("Jan 2 15:04"), + ID: user.ID, + Username: user.Username, + Email: user.Email, + Role: user.Role, + Status: user.Status, + StorageUsed: services.FormatMegabytesFromBytes(storageUsed), + StorageQuota: quota, + DailyUsed: services.FormatMegabytesFromBytes(usage.UploadedBytes), + StorageBackend: policy.StorageBackendID, + CreatedAt: user.CreatedAt.Format("Jan 2 15:04"), }) } a.renderPage(w, r, http.StatusOK, "admin_users.html", web.PageData{ @@ -206,6 +243,45 @@ func (a *App) AdminUsers(w http.ResponseWriter, r *http.Request) { }) } +func (a *App) AdminEditUser(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) { + return + } + user, err := a.authService.UserByID(r.PathValue("userID")) + if err != nil { + http.NotFound(w, r) + return + } + settings, err := a.settingsService.UploadPolicy() + if err != nil { + http.Error(w, "unable to load settings", http.StatusInternalServerError) + return + } + storage, err := a.storageBackendViews() + if err != nil { + http.Error(w, "unable to load storage", http.StatusInternalServerError) + return + } + edit, err := a.adminUserEdit(user, settings) + if err != nil { + http.Error(w, "unable to load user policy", http.StatusInternalServerError) + return + } + a.renderPage(w, r, http.StatusOK, "admin_user_edit.html", web.PageData{ + Title: "Edit user", + Description: "Edit a Warpbox user.", + CurrentUser: a.currentPublicUser(r), + Data: adminPageData{ + UserEdit: edit, + Storage: storage, + Section: "users", + PageTitle: "Edit user", + LastInviteURL: r.URL.Query().Get("invite"), + Error: r.URL.Query().Get("error"), + }, + }) +} + func (a *App) AdminSettings(w http.ResponseWriter, r *http.Request) { if !a.requireAdmin(w, r) { return @@ -215,12 +291,18 @@ func (a *App) AdminSettings(w http.ResponseWriter, r *http.Request) { http.Error(w, "unable to load settings", http.StatusInternalServerError) return } + storage, err := a.storageBackendViews() + if err != nil { + http.Error(w, "unable to load storage", http.StatusInternalServerError) + return + } a.renderPage(w, r, http.StatusOK, "admin_settings.html", web.PageData{ Title: "Admin settings", Description: "Manage Warpbox upload policy.", CurrentUser: a.currentPublicUser(r), Data: adminPageData{ Settings: settings, + Storage: storage, Section: "settings", PageTitle: "Settings", }, @@ -228,18 +310,55 @@ func (a *App) AdminSettings(w http.ResponseWriter, r *http.Request) { } func (a *App) AdminSettingsPost(w http.ResponseWriter, r *http.Request) { - if !a.requireAdmin(w, r) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { return } if err := r.ParseForm(); err != nil { http.Redirect(w, r, "/admin/settings", http.StatusSeeOther) return } - settings := services.UploadPolicySettings{ - AnonymousUploadsEnabled: r.FormValue("anonymous_uploads_enabled") == "on", - UsageRetentionDays: parsePositiveInt(r.FormValue("usage_retention_days")), + settings, err := a.settingsService.UploadPolicy() + if err != nil { + http.Error(w, "unable to load settings", http.StatusInternalServerError) + return + } + settings.AnonymousUploadsEnabled = r.FormValue("anonymous_uploads_enabled") == "on" + if value := parsePositiveInt(r.FormValue("usage_retention_days")); value > 0 { + settings.UsageRetentionDays = value + } + if value := parsePositiveFloat(r.FormValue("local_storage_max_gb")); value > 0 { + settings.LocalStorageMaxGB = value + } + if value := parsePositiveInt(r.FormValue("anonymous_max_days")); value > 0 { + settings.AnonymousMaxDays = value + } + if value := parsePositiveInt(r.FormValue("user_max_days")); value > 0 { + settings.UserMaxDays = value + } + if value := parsePositiveInt(r.FormValue("anonymous_daily_boxes")); value > 0 { + settings.AnonymousDailyBoxes = value + } + if value := parsePositiveInt(r.FormValue("user_daily_boxes")); value > 0 { + settings.UserDailyBoxes = value + } + if value := parsePositiveInt(r.FormValue("anonymous_active_boxes")); value > 0 { + settings.AnonymousActiveBoxes = value + } + if value := parsePositiveInt(r.FormValue("user_active_boxes")); value > 0 { + settings.UserActiveBoxes = value + } + if value := parsePositiveInt(r.FormValue("short_window_requests")); value > 0 { + settings.ShortWindowRequests = value + } + if value := parsePositiveInt(r.FormValue("short_window_seconds")); value > 0 { + settings.ShortWindowSeconds = value + } + if value := r.FormValue("anonymous_storage_backend"); value != "" { + settings.AnonymousStorageBackend = value + } + if value := r.FormValue("user_storage_backend"); value != "" { + settings.UserStorageBackend = value } - var err error if settings.AnonymousMaxUploadMB, err = services.ParseMegabytesValue(r.FormValue("anonymous_max_upload_mb")); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -260,6 +379,14 @@ func (a *App) AdminSettingsPost(w http.ResponseWriter, r *http.Request) { http.Error(w, "usage retention days must be positive", http.StatusBadRequest) return } + if _, err := a.uploadService.Storage().BackendConfig(settings.AnonymousStorageBackend); err != nil { + http.Error(w, "anonymous storage backend not found", http.StatusBadRequest) + return + } + if _, err := a.uploadService.Storage().BackendConfig(settings.UserStorageBackend); err != nil { + http.Error(w, "user storage backend not found", http.StatusBadRequest) + return + } if err := a.settingsService.UpdateUploadPolicy(settings); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -267,10 +394,127 @@ func (a *App) AdminSettingsPost(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/admin/settings", http.StatusSeeOther) } -func (a *App) AdminUpdateUserQuota(w http.ResponseWriter, r *http.Request) { +func (a *App) AdminStorage(w http.ResponseWriter, r *http.Request) { if !a.requireAdmin(w, r) { return } + settings, err := a.settingsService.UploadPolicy() + if err != nil { + http.Error(w, "unable to load settings", http.StatusInternalServerError) + return + } + views, err := a.storageBackendViews() + if err != nil { + http.Error(w, "unable to load storage", http.StatusInternalServerError) + return + } + a.renderPage(w, r, http.StatusOK, "admin_storage.html", web.PageData{ + Title: "Admin storage", + Description: "Manage Warpbox storage backends.", + CurrentUser: a.currentPublicUser(r), + Data: adminPageData{ + Settings: settings, + Storage: views, + Section: "storage", + PageTitle: "Storage", + Error: r.URL.Query().Get("error"), + }, + }) +} + +func (a *App) AdminCreateS3Storage(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + if err := r.ParseForm(); err != nil { + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) + return + } + _, err := a.uploadService.Storage().CreateS3Backend(services.StorageBackendConfig{ + Provider: r.FormValue("provider"), + Name: r.FormValue("name"), + Endpoint: r.FormValue("endpoint"), + Region: r.FormValue("region"), + Bucket: r.FormValue("bucket"), + AccessKey: r.FormValue("access_key"), + SecretKey: r.FormValue("secret_key"), + UseSSL: r.FormValue("use_ssl") == "on", + PathStyle: r.FormValue("path_style") == "on", + }) + if err != nil { + http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther) + return + } + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) +} + +func (a *App) AdminEditStorage(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + if err := r.ParseForm(); err != nil { + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) + return + } + _, err := a.uploadService.Storage().UpdateS3Backend(r.PathValue("backendID"), services.StorageBackendConfig{ + Provider: r.FormValue("provider"), + Name: r.FormValue("name"), + Endpoint: r.FormValue("endpoint"), + Region: r.FormValue("region"), + Bucket: r.FormValue("bucket"), + AccessKey: r.FormValue("access_key"), + SecretKey: r.FormValue("secret_key"), + UseSSL: r.FormValue("use_ssl") == "on", + PathStyle: r.FormValue("path_style") == "on", + }) + if err != nil { + http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther) + return + } + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) +} + +func (a *App) AdminTestStorage(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + if _, err := a.uploadService.Storage().TestBackend(r.PathValue("backendID")); err != nil { + http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther) + return + } + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) +} + +func (a *App) AdminDisableStorage(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + id := r.PathValue("backendID") + inUse, _ := a.storageBackendInUse(id) + if err := a.uploadService.Storage().DisableBackend(id, inUse); err != nil { + http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther) + return + } + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) +} + +func (a *App) AdminDeleteStorage(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + id := r.PathValue("backendID") + inUse, _ := a.storageBackendInUse(id) + if err := a.uploadService.Storage().DeleteBackend(id, inUse); err != nil { + http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther) + return + } + http.Redirect(w, r, "/admin/storage", http.StatusSeeOther) +} + +func (a *App) AdminUpdateUserQuota(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } if err := r.ParseForm(); err != nil { http.Redirect(w, r, "/admin/users", http.StatusSeeOther) return @@ -291,9 +535,99 @@ func (a *App) AdminUpdateUserQuota(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/admin/users", http.StatusSeeOther) } +func (a *App) AdminUpdateUserPolicy(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + if err := r.ParseForm(); err != nil { + http.Redirect(w, r, "/admin/users", http.StatusSeeOther) + return + } + policy := services.UserPolicy{ + MaxUploadMB: optionalMB(r.FormValue("max_upload_mb")), + DailyUploadMB: optionalMB(r.FormValue("daily_upload_mb")), + StorageQuotaMB: optionalMBAllowZero(r.FormValue("storage_quota_mb")), + MaxDays: optionalInt(r.FormValue("max_days")), + DailyBoxes: optionalInt(r.FormValue("daily_boxes")), + ActiveBoxes: optionalInt(r.FormValue("active_boxes")), + ShortWindowRequests: optionalInt(r.FormValue("short_window_requests")), + } + if backendID := r.FormValue("storage_backend_id"); backendID != "" { + if _, err := a.uploadService.Storage().BackendConfig(backendID); err != nil { + http.Error(w, "storage backend not found", http.StatusBadRequest) + return + } + policy.StorageBackendID = &backendID + } + if err := a.authService.SetUserPolicy(r.PathValue("userID"), policy); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Redirect(w, r, "/admin/users", http.StatusSeeOther) +} + +func (a *App) AdminUpdateUser(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + if err := r.ParseForm(); err != nil { + http.Redirect(w, r, "/admin/users/"+r.PathValue("userID")+"/edit", http.StatusSeeOther) + return + } + policy := services.UserPolicy{ + MaxUploadMB: optionalMB(r.FormValue("max_upload_mb")), + DailyUploadMB: optionalMB(r.FormValue("daily_upload_mb")), + StorageQuotaMB: optionalMBAllowZero(r.FormValue("storage_quota_mb")), + MaxDays: optionalInt(r.FormValue("max_days")), + DailyBoxes: optionalInt(r.FormValue("daily_boxes")), + ActiveBoxes: optionalInt(r.FormValue("active_boxes")), + ShortWindowRequests: optionalInt(r.FormValue("short_window_requests")), + } + if backendID := r.FormValue("storage_backend_id"); backendID != "" { + if _, err := a.uploadService.Storage().BackendConfig(backendID); err != nil { + http.Redirect(w, r, "/admin/users/"+r.PathValue("userID")+"/edit?error="+url.QueryEscape("storage backend not found"), http.StatusSeeOther) + return + } + policy.StorageBackendID = &backendID + } + if _, err := a.authService.UpdateUserAdminFields( + r.PathValue("userID"), + r.FormValue("username"), + r.FormValue("email"), + r.FormValue("role"), + r.FormValue("status"), + policy, + ); err != nil { + http.Redirect(w, r, "/admin/users/"+r.PathValue("userID")+"/edit?error="+url.QueryEscape(err.Error()), http.StatusSeeOther) + return + } + http.Redirect(w, r, "/admin/users/"+r.PathValue("userID")+"/edit", http.StatusSeeOther) +} + +func (a *App) AdminUpdateUserStorage(w http.ResponseWriter, r *http.Request) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { + return + } + if err := r.ParseForm(); err != nil { + http.Redirect(w, r, "/admin/users", http.StatusSeeOther) + return + } + if backendID := r.FormValue("storage_backend_id"); backendID != "" { + if _, err := a.uploadService.Storage().BackendConfig(backendID); err != nil { + http.Error(w, "storage backend not found", http.StatusBadRequest) + return + } + } + if err := a.authService.SetUserStorageBackend(r.PathValue("userID"), r.FormValue("storage_backend_id")); err != nil { + http.Error(w, "unable to update user storage", http.StatusInternalServerError) + return + } + http.Redirect(w, r, "/admin/users", http.StatusSeeOther) +} + func (a *App) AdminCreateInvite(w http.ResponseWriter, r *http.Request) { admin, ok := a.requireAdminUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } if err := r.ParseForm(); err != nil { @@ -310,7 +644,7 @@ func (a *App) AdminCreateInvite(w http.ResponseWriter, r *http.Request) { } func (a *App) AdminDisableUser(w http.ResponseWriter, r *http.Request) { - if !a.requireAdmin(w, r) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { return } disabled := r.URL.Query().Get("disabled") != "false" @@ -323,7 +657,7 @@ func (a *App) AdminDisableUser(w http.ResponseWriter, r *http.Request) { func (a *App) AdminResetUser(w http.ResponseWriter, r *http.Request) { admin, ok := a.requireAdminUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } result, err := a.authService.CreatePasswordResetInvite(r.PathValue("userID"), admin.ID) @@ -331,11 +665,15 @@ func (a *App) AdminResetUser(w http.ResponseWriter, r *http.Request) { http.Error(w, "unable to create reset link", http.StatusInternalServerError) return } + if r.URL.Query().Get("next") == "edit" { + http.Redirect(w, r, "/admin/users/"+r.PathValue("userID")+"/edit?invite="+url.QueryEscape(result.URL), http.StatusSeeOther) + return + } http.Redirect(w, r, "/admin/users?invite="+url.QueryEscape(result.URL), http.StatusSeeOther) } func (a *App) AdminDeleteBox(w http.ResponseWriter, r *http.Request) { - if !a.requireAdmin(w, r) { + if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) { return } @@ -471,6 +809,161 @@ func parsePositiveInt(value string) int { return parsed } +func parsePositiveFloat(value string) float64 { + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return 0 + } + return parsed +} + +func optionalMB(value string) *float64 { + if value == "" { + return nil + } + parsed, err := services.ParseMegabytesValue(value) + if err != nil { + return nil + } + return &parsed +} + +func optionalMBAllowZero(value string) *float64 { + if value == "" { + return nil + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil || parsed < 0 { + return nil + } + return &parsed +} + +func optionalInt(value string) *int { + if value == "" { + return nil + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed <= 0 { + return nil + } + return &parsed +} + func formatMB(value float64) string { return strconv.FormatFloat(value, 'f', -1, 64) + " MB" } + +func (a *App) storageBackendViews() ([]services.StorageBackendView, error) { + configs, err := a.uploadService.Storage().ListBackendConfigs() + if err != nil { + return nil, err + } + views := make([]services.StorageBackendView, 0, len(configs)) + for _, cfg := range configs { + var usage int64 + if backend, err := a.uploadService.Storage().BackendConfig(cfg.ID); err == nil && backend.Enabled { + if concrete, err := a.uploadService.Storage().Backend(cfg.ID); err == nil { + usage, _ = concrete.Usage(context.Background()) + } + } + inUse, _ := a.storageBackendInUse(cfg.ID) + views = append(views, services.StorageBackendView{ + Config: cfg, + UsageBytes: usage, + UsageLabel: services.FormatMegabytesFromBytes(usage), + InUse: inUse, + }) + } + return views, nil +} + +func (a *App) adminUserEdit(user services.User, settings services.UploadPolicySettings) (adminUserEditView, error) { + storageUsed, err := a.uploadService.UserActiveStorageUsed(user.ID) + if err != nil { + return adminUserEditView{}, err + } + usage, err := a.settingsService.UsageForUser(user.ID, time.Now().UTC()) + if err != nil { + return adminUserEditView{}, err + } + effective := a.settingsService.EffectivePolicyForUser(settings, user) + view := adminUserEditView{ + ID: user.ID, + Username: user.Username, + Email: user.Email, + Role: user.Role, + Status: user.Status, + StorageUsed: services.FormatMegabytesFromBytes(storageUsed), + DailyUsed: services.FormatMegabytesFromBytes(usage.UploadedBytes), + EffectiveDaily: services.FormatMegabytesLabel(effective.DailyUploadMB), + EffectiveMaxDays: effective.MaxDays, + EffectiveDailyBoxes: effective.DailyBoxes, + EffectiveActiveBoxes: effective.ActiveBoxes, + EffectiveBackend: effective.StorageBackendID, + MaxUploadMB: floatPtrString(user.Policy.MaxUploadMB), + DailyUploadMB: floatPtrString(user.Policy.DailyUploadMB), + StorageQuotaMB: floatPtrString(user.Policy.StorageQuotaMB), + MaxDays: intPtrString(user.Policy.MaxDays), + DailyBoxes: intPtrString(user.Policy.DailyBoxes), + ActiveBoxes: intPtrString(user.Policy.ActiveBoxes), + ShortWindowRequests: intPtrString(user.Policy.ShortWindowRequests), + StorageBackendID: stringPtrString(user.Policy.StorageBackendID), + } + if effective.StorageQuotaSet { + view.EffectiveStorage = services.FormatMegabytesLabel(effective.StorageQuotaMB) + } else { + view.EffectiveStorage = "unlimited" + } + return view, nil +} + +func (a *App) storageBackendInUse(id string) (bool, error) { + settings, err := a.settingsService.UploadPolicy() + if err != nil { + return false, err + } + if settings.AnonymousStorageBackend == id || settings.UserStorageBackend == id { + return true, nil + } + boxes, err := a.uploadService.ListBoxes(0) + if err != nil { + return false, err + } + for _, box := range boxes { + if a.uploadService.BoxStorageBackendID(box) == id { + return true, nil + } + } + users, err := a.authService.ListUsers() + if err != nil { + return false, err + } + for _, user := range users { + if user.Policy.StorageBackendID != nil && *user.Policy.StorageBackendID == id { + return true, nil + } + } + return false, nil +} + +func floatPtrString(value *float64) string { + if value == nil { + return "" + } + return strconv.FormatFloat(*value, 'f', -1, 64) +} + +func intPtrString(value *int) string { + if value == nil { + return "" + } + return strconv.Itoa(*value) +} + +func stringPtrString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/backend/libs/handlers/app.go b/backend/libs/handlers/app.go index f995181..fd3ebbc 100644 --- a/backend/libs/handlers/app.go +++ b/backend/libs/handlers/app.go @@ -16,6 +16,7 @@ type App struct { uploadService *services.UploadService authService *services.AuthService settingsService *services.SettingsService + rateLimiter *rateLimiter } func NewApp(cfg config.Config, logger *slog.Logger, renderer *web.Renderer, uploadService *services.UploadService, authService *services.AuthService, settingsService *services.SettingsService) *App { @@ -26,6 +27,7 @@ func NewApp(cfg config.Config, logger *slog.Logger, renderer *web.Renderer, uplo uploadService: uploadService, authService: authService, settingsService: settingsService, + rateLimiter: newRateLimiter(), } } @@ -33,6 +35,7 @@ func (a *App) renderPage(w http.ResponseWriter, r *http.Request, status int, pag if data.CurrentUser == nil { data.CurrentUser = a.currentPublicUser(r) } + data.CSRFToken = a.csrfToken(w, r) a.renderer.Render(w, status, page, data) } @@ -59,12 +62,22 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("GET /admin", a.AdminDashboard) mux.HandleFunc("GET /admin/files", a.AdminFiles) mux.HandleFunc("GET /admin/users", a.AdminUsers) + mux.HandleFunc("GET /admin/users/{userID}/edit", a.AdminEditUser) mux.HandleFunc("GET /admin/settings", a.AdminSettings) mux.HandleFunc("POST /admin/settings", a.AdminSettingsPost) + mux.HandleFunc("GET /admin/storage", a.AdminStorage) + mux.HandleFunc("POST /admin/storage/s3", a.AdminCreateS3Storage) + mux.HandleFunc("POST /admin/storage/{backendID}/edit", a.AdminEditStorage) + mux.HandleFunc("POST /admin/storage/{backendID}/test", a.AdminTestStorage) + mux.HandleFunc("POST /admin/storage/{backendID}/disable", a.AdminDisableStorage) + mux.HandleFunc("POST /admin/storage/{backendID}/delete", a.AdminDeleteStorage) mux.HandleFunc("POST /admin/invites", a.AdminCreateInvite) mux.HandleFunc("POST /admin/users/{userID}/disable", a.AdminDisableUser) mux.HandleFunc("POST /admin/users/{userID}/reset", a.AdminResetUser) mux.HandleFunc("POST /admin/users/{userID}/quota", a.AdminUpdateUserQuota) + mux.HandleFunc("POST /admin/users/{userID}/edit", a.AdminUpdateUser) + mux.HandleFunc("POST /admin/users/{userID}/policy", a.AdminUpdateUserPolicy) + mux.HandleFunc("POST /admin/users/{userID}/storage", a.AdminUpdateUserStorage) mux.HandleFunc("GET /admin/boxes/{boxID}/view", a.AdminViewBox) mux.HandleFunc("POST /admin/boxes/{boxID}/delete", a.AdminDeleteBox) mux.HandleFunc("GET /d/{boxID}", a.DownloadPage) diff --git a/backend/libs/handlers/auth.go b/backend/libs/handlers/auth.go index 66cdc7c..ce725cd 100644 --- a/backend/libs/handlers/auth.go +++ b/backend/libs/handlers/auth.go @@ -33,6 +33,10 @@ func (a *App) Register(w http.ResponseWriter, r *http.Request) { } func (a *App) RegisterPost(w http.ResponseWriter, r *http.Request) { + if !a.rateLimiter.Allow("register:"+uploadClientIP(r), 10, time.Minute, time.Now().UTC()) { + a.renderAuth(w, r, http.StatusTooManyRequests, authPageData{Mode: "register", Error: "Too many registration attempts."}) + return + } if err := r.ParseForm(); err != nil { a.renderAuth(w, r, http.StatusBadRequest, authPageData{Mode: "register", Error: "Unable to read form."}) return @@ -55,6 +59,10 @@ func (a *App) Login(w http.ResponseWriter, r *http.Request) { } func (a *App) LoginPost(w http.ResponseWriter, r *http.Request) { + if !a.rateLimiter.Allow("login:"+uploadClientIP(r), 10, time.Minute, time.Now().UTC()) { + a.renderAuth(w, r, http.StatusTooManyRequests, authPageData{Mode: "login", Error: "Too many login attempts."}) + return + } if err := r.ParseForm(); err != nil { a.renderAuth(w, r, http.StatusBadRequest, authPageData{Mode: "login", Error: "Unable to read form."}) return @@ -75,6 +83,9 @@ func (a *App) LoginPost(w http.ResponseWriter, r *http.Request) { } func (a *App) Logout(w http.ResponseWriter, r *http.Request) { + if !a.validateCSRF(w, r) { + return + } if cookie, err := r.Cookie(userSessionCookieName); err == nil { _ = a.authService.Logout(cookie.Value) } @@ -126,7 +137,7 @@ func (a *App) AccountSettings(w http.ResponseWriter, r *http.Request) { func (a *App) ChangePassword(w http.ResponseWriter, r *http.Request) { user, ok := a.requireUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } if err := r.ParseForm(); err != nil { diff --git a/backend/libs/handlers/dashboard.go b/backend/libs/handlers/dashboard.go index 406e1db..34b8dca 100644 --- a/backend/libs/handlers/dashboard.go +++ b/backend/libs/handlers/dashboard.go @@ -104,7 +104,7 @@ func (a *App) Dashboard(w http.ResponseWriter, r *http.Request) { func (a *App) CreateCollection(w http.ResponseWriter, r *http.Request) { user, ok := a.requireUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } if err := r.ParseForm(); err != nil { @@ -119,7 +119,7 @@ func (a *App) CreateCollection(w http.ResponseWriter, r *http.Request) { func (a *App) RenameUserBox(w http.ResponseWriter, r *http.Request) { user, ok := a.requireUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } if err := r.ParseForm(); err != nil { @@ -135,7 +135,7 @@ func (a *App) RenameUserBox(w http.ResponseWriter, r *http.Request) { func (a *App) MoveUserBox(w http.ResponseWriter, r *http.Request) { user, ok := a.requireUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } if err := r.ParseForm(); err != nil { @@ -156,7 +156,7 @@ func (a *App) MoveUserBox(w http.ResponseWriter, r *http.Request) { func (a *App) DeleteUserBox(w http.ResponseWriter, r *http.Request) { user, ok := a.requireUser(w, r) - if !ok { + if !ok || !a.validateCSRF(w, r) { return } if err := a.uploadService.DeleteOwnedBox(r.PathValue("boxID"), user.ID); err != nil { diff --git a/backend/libs/handlers/download.go b/backend/libs/handlers/download.go index bf7b9ad..0f86dfe 100644 --- a/backend/libs/handlers/download.go +++ b/backend/libs/handlers/download.go @@ -1,8 +1,10 @@ package handlers import ( + "bytes" "errors" "fmt" + "io" "net/http" "os" "path/filepath" @@ -143,12 +145,15 @@ func (a *App) Thumbnail(w http.ResponseWriter, r *http.Request) { return } - path := a.uploadService.ThumbnailPath(box, file) - if path == "" { + object, err := a.uploadService.OpenThumbnailObject(r.Context(), box, file) + if err != nil { http.ServeFile(w, r, filepath.Join(a.cfg.StaticDir, "img", "file-placeholder.webp")) return } - http.ServeFile(w, r, path) + defer object.Body.Close() + w.Header().Set("Content-Type", "image/jpeg") + w.Header().Set("Cache-Control", "public, max-age=604800, immutable") + http.ServeContent(w, r, file.ID+"-thumbnail.jpg", object.ModTime, readSeekCloser(object.Body)) } func (a *App) UnlockBox(w http.ResponseWriter, r *http.Request) { @@ -199,31 +204,40 @@ func (a *App) loadFileForRequest(w http.ResponseWriter, r *http.Request) (servic } func (a *App) serveFileContent(w http.ResponseWriter, r *http.Request, box services.Box, file services.File, attachment bool) { - path := a.uploadService.FilePath(box, file) - source, err := os.Open(path) - if err != nil { - http.NotFound(w, r) - return - } - defer source.Close() - - stat, err := source.Stat() + object, err := a.uploadService.OpenFileObject(r.Context(), box, file) if err != nil { http.NotFound(w, r) return } + defer object.Body.Close() w.Header().Set("Content-Type", file.ContentType) if attachment { w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", file.Name)) } - http.ServeContent(w, r, file.Name, stat.ModTime(), source) + if seeker, ok := object.Body.(io.ReadSeeker); ok { + http.ServeContent(w, r, file.Name, object.ModTime, seeker) + } else { + if object.Size > 0 { + w.Header().Set("Content-Length", fmt.Sprintf("%d", object.Size)) + } + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, object.Body) + } if err := a.uploadService.RecordDownload(box.ID); err != nil && !errors.Is(err, os.ErrNotExist) { a.logger.Warn("failed to record file download", "source", "download", "severity", "warn", "code", 4002, "box_id", box.ID, "error", err.Error()) } } +func readSeekCloser(source io.ReadCloser) io.ReadSeeker { + data, err := io.ReadAll(source) + if err != nil { + return bytes.NewReader(nil) + } + return bytes.NewReader(data) +} + func (a *App) DownloadZip(w http.ResponseWriter, r *http.Request) { box, err := a.uploadService.GetBox(r.PathValue("boxID")) if err != nil { diff --git a/backend/libs/handlers/pages.go b/backend/libs/handlers/pages.go index 9aabba0..7ffb130 100644 --- a/backend/libs/handlers/pages.go +++ b/backend/libs/handlers/pages.go @@ -2,6 +2,7 @@ package handlers import ( "net/http" + "strconv" "warpbox.dev/backend/libs/services" "warpbox.dev/backend/libs/web" @@ -61,11 +62,16 @@ func (a *App) homeUploadPolicyLabels(settings services.UploadPolicySettings, use if !settings.AnonymousUploadsEnabled { return "Anonymous uploads disabled", "Sign in to upload files." } - return services.FormatMegabytesLabel(settings.AnonymousMaxUploadMB), "Daily anonymous cap: " + services.FormatMegabytesLabel(settings.AnonymousDailyUploadMB) + " per IP." + return services.FormatMegabytesLabel(settings.AnonymousMaxUploadMB), "Daily anonymous cap: " + services.FormatMegabytesLabel(settings.AnonymousDailyUploadMB) + " per IP · " + strconv.Itoa(settings.AnonymousMaxDays) + " day max." } - quotaMB := settings.DefaultUserStorageMB - if user.StorageQuotaMB != nil { - quotaMB = *user.StorageQuotaMB + policy := a.settingsService.EffectivePolicyForUser(settings, user) + maxUpload := a.uploadService.MaxUploadSizeLabel() + if policy.MaxUploadMB > 0 { + maxUpload = services.FormatMegabytesLabel(policy.MaxUploadMB) } - return a.uploadService.MaxUploadSizeLabel(), "Daily cap: " + services.FormatMegabytesLabel(settings.UserDailyUploadMB) + " · Storage quota: " + services.FormatMegabytesLabel(quotaMB) + "." + quota := "unlimited" + if policy.StorageQuotaSet { + quota = services.FormatMegabytesLabel(policy.StorageQuotaMB) + } + return maxUpload, "Daily cap: " + services.FormatMegabytesLabel(policy.DailyUploadMB) + " · Storage quota: " + quota + " · " + strconv.Itoa(policy.MaxDays) + " day max." } diff --git a/backend/libs/handlers/security.go b/backend/libs/handlers/security.go new file mode 100644 index 0000000..0a341b0 --- /dev/null +++ b/backend/libs/handlers/security.go @@ -0,0 +1,78 @@ +package handlers + +import ( + "crypto/rand" + "encoding/base64" + "net/http" + "sync" + "time" +) + +const csrfCookieName = "warpbox_csrf" + +type rateLimiter struct { + mu sync.Mutex + records map[string]rateRecord +} + +type rateRecord struct { + StartedAt time.Time + Count int +} + +func newRateLimiter() *rateLimiter { + return &rateLimiter{records: make(map[string]rateRecord)} +} + +func (l *rateLimiter) Allow(key string, limit int, window time.Duration, now time.Time) bool { + if limit <= 0 || window <= 0 { + return true + } + l.mu.Lock() + defer l.mu.Unlock() + record := l.records[key] + if record.StartedAt.IsZero() || now.Sub(record.StartedAt) >= window { + l.records[key] = rateRecord{StartedAt: now, Count: 1} + return true + } + record.Count++ + l.records[key] = record + return record.Count <= limit +} + +func (a *App) csrfToken(w http.ResponseWriter, r *http.Request) string { + if cookie, err := r.Cookie(csrfCookieName); err == nil && cookie.Value != "" { + return cookie.Value + } + token := randomToken(32) + http.SetCookie(w, &http.Cookie{ + Name: csrfCookieName, + Value: token, + Path: "/", + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + Secure: r.TLS != nil, + Expires: time.Now().Add(12 * time.Hour), + }) + return token +} + +func (a *App) validateCSRF(w http.ResponseWriter, r *http.Request) bool { + if r.Method == http.MethodGet || r.Method == http.MethodHead || r.Method == http.MethodOptions { + return true + } + cookie, err := r.Cookie(csrfCookieName) + if err != nil || cookie.Value == "" || r.FormValue("csrf_token") != cookie.Value { + http.Error(w, "invalid form token", http.StatusForbidden) + return false + } + return true +} + +func randomToken(byteCount int) string { + data := make([]byte, byteCount) + if _, err := rand.Read(data); err != nil { + return base64.RawURLEncoding.EncodeToString([]byte(time.Now().String())) + } + return base64.RawURLEncoding.EncodeToString(data) +} diff --git a/backend/libs/handlers/upload.go b/backend/libs/handlers/upload.go index 46bc754..b29ebec 100644 --- a/backend/libs/handlers/upload.go +++ b/backend/libs/handlers/upload.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "errors" "fmt" "mime/multipart" @@ -27,11 +28,17 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { helpers.WriteJSONError(w, http.StatusForbidden, "anonymous uploads are disabled") return } + effectivePolicy := a.effectiveUploadPolicy(settings, user, loggedIn) + rateKey := uploadRateKey(r, user, loggedIn) + if !isAdminUpload && !a.rateLimiter.Allow("upload:"+rateKey, effectivePolicy.ShortRequests, effectivePolicy.ShortWindow, time.Now().UTC()) { + helpers.WriteJSONError(w, http.StatusTooManyRequests, "too many upload requests, please slow down") + return + } if !isAdminUpload { - r.Body = http.MaxBytesReader(w, r.Body, uploadParseLimit(settings, loggedIn, a.uploadService.MaxUploadSize())) + r.Body = http.MaxBytesReader(w, r.Body, uploadParseLimit(effectivePolicy, loggedIn, a.uploadService.MaxUploadSize())) } - parseLimit := uploadParseLimit(settings, loggedIn, a.uploadService.MaxUploadSize()) + parseLimit := uploadParseLimit(effectivePolicy, loggedIn, a.uploadService.MaxUploadSize()) if isAdminUpload { parseLimit = 32 << 20 } @@ -53,19 +60,29 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { } } if !isAdminUpload { - if status, message := a.checkUploadPolicy(r, user, loggedIn, settings, files, totalBytes); message != "" { + if status, message := a.checkUploadPolicy(r, user, loggedIn, settings, effectivePolicy, files, totalBytes); message != "" { helpers.WriteJSONError(w, status, message) return } } + maxDays := parseInt(r.FormValue("max_days")) + if maxDays <= 0 { + maxDays = min(7, effectivePolicy.MaxDays) + } + if !isAdminUpload && maxDays > effectivePolicy.MaxDays { + helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("expiration cannot exceed %d days", effectivePolicy.MaxDays)) + return + } result, err := a.uploadService.CreateBox(files, services.UploadOptions{ - MaxDays: parseInt(r.FormValue("max_days")), + MaxDays: maxDays, MaxDownloads: parseInt(r.FormValue("max_downloads")), Password: r.FormValue("password"), ObfuscateMetadata: r.FormValue("obfuscate_metadata") == "on", OwnerID: ownerID, CollectionID: collectionID, SkipSizeLimit: isAdminUpload, + CreatorIP: uploadClientIP(r), + StorageBackendID: effectivePolicy.StorageBackendID, }) if err != nil { a.logger.Warn("upload failed", "source", "user-upload", "severity", "warn", "code", 4001, "error", err.Error()) @@ -73,7 +90,7 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { return } if !isAdminUpload { - if err := a.recordUploadUsage(r, user, loggedIn, totalBytes); err != nil { + if err := a.recordUploadUsage(r, user, loggedIn, totalBytes, 1); err != nil { a.logger.Warn("failed to record upload usage", "source", "quota", "severity", "warn", "code", 4402, "error", err.Error()) } if err := a.settingsService.CleanupUsage(time.Now().UTC(), settings.UsageRetentionDays); err != nil { @@ -92,25 +109,40 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprintln(w, result.BoxURL) } -func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bool, settings services.UploadPolicySettings, files []*multipart.FileHeader, totalBytes int64) (int, string) { +func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bool, settings services.UploadPolicySettings, policy services.EffectiveUploadPolicy, files []*multipart.FileHeader, totalBytes int64) (int, string) { if len(files) == 0 { return 0, "" } now := time.Now().UTC() - if !loggedIn { - anonymousMaxBytes := services.MegabytesToBytes(settings.AnonymousMaxUploadMB) + if policy.MaxUploadMB > 0 { + maxBytes := services.MegabytesToBytes(policy.MaxUploadMB) for _, file := range files { - if file.Size > anonymousMaxBytes { - return http.StatusRequestEntityTooLarge, "file exceeds anonymous upload size limit" + if file.Size > maxBytes { + return http.StatusRequestEntityTooLarge, "file exceeds upload size limit" } } + } + if !loggedIn { usage, err := a.settingsService.UsageForIP(uploadClientIP(r), now) if err != nil { return http.StatusInternalServerError, "upload usage could not be checked" } - if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(settings.AnonymousDailyUploadMB) { + if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(policy.DailyUploadMB) { return http.StatusTooManyRequests, "anonymous daily upload limit reached" } + if usage.UploadedBoxes+1 > policy.DailyBoxes { + return http.StatusTooManyRequests, "anonymous daily box limit reached" + } + activeBoxes, err := a.uploadService.ActiveBoxCountForIP(uploadClientIP(r)) + if err != nil { + return http.StatusInternalServerError, "active box limit could not be checked" + } + if activeBoxes+1 > policy.ActiveBoxes { + return http.StatusTooManyRequests, "anonymous active box limit reached" + } + if status, message := a.checkStorageBackendCapacity(policy.StorageBackendID, settings, totalBytes); message != "" { + return status, message + } return 0, "" } @@ -118,42 +150,86 @@ func (a *App) checkUploadPolicy(r *http.Request, user services.User, loggedIn bo if err != nil { return http.StatusInternalServerError, "upload usage could not be checked" } - if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(settings.UserDailyUploadMB) { + if usage.UploadedBytes+totalBytes > services.MegabytesToBytes(policy.DailyUploadMB) { return http.StatusTooManyRequests, "daily upload limit reached" } + if usage.UploadedBoxes+1 > policy.DailyBoxes { + return http.StatusTooManyRequests, "daily box limit reached" + } + activeBoxes, err := a.uploadService.ActiveBoxCountForUser(user.ID) + if err != nil { + return http.StatusInternalServerError, "active box limit could not be checked" + } + if activeBoxes+1 > policy.ActiveBoxes { + return http.StatusTooManyRequests, "active box limit reached" + } activeStorage, err := a.uploadService.UserActiveStorageUsed(user.ID) if err != nil { return http.StatusInternalServerError, "storage quota could not be checked" } - quotaMB := settings.DefaultUserStorageMB - if user.StorageQuotaMB != nil { - quotaMB = *user.StorageQuotaMB - } - if activeStorage+totalBytes > services.MegabytesToBytes(quotaMB) { + if policy.StorageQuotaSet && activeStorage+totalBytes > services.MegabytesToBytes(policy.StorageQuotaMB) { return http.StatusRequestEntityTooLarge, "storage quota reached" } + if status, message := a.checkStorageBackendCapacity(policy.StorageBackendID, settings, totalBytes); message != "" { + return status, message + } return 0, "" } -func (a *App) recordUploadUsage(r *http.Request, user services.User, loggedIn bool, totalBytes int64) error { +func (a *App) recordUploadUsage(r *http.Request, user services.User, loggedIn bool, totalBytes int64, boxes int) error { now := time.Now().UTC() if loggedIn { - return a.settingsService.AddUsage("user", user.ID, totalBytes, now) + return a.settingsService.AddUploadUsage("user", user.ID, totalBytes, boxes, now) } - return a.settingsService.AddUsage("ip", uploadClientIP(r), totalBytes, now) + return a.settingsService.AddUploadUsage("ip", uploadClientIP(r), totalBytes, boxes, now) } -func uploadParseLimit(settings services.UploadPolicySettings, loggedIn bool, fallback int64) int64 { +func (a *App) effectiveUploadPolicy(settings services.UploadPolicySettings, user services.User, loggedIn bool) services.EffectiveUploadPolicy { if loggedIn { + return a.settingsService.EffectivePolicyForUser(settings, user) + } + return a.settingsService.EffectivePolicyForAnonymous(settings) +} + +func (a *App) checkStorageBackendCapacity(backendID string, settings services.UploadPolicySettings, totalBytes int64) (int, string) { + if backendID != services.StorageBackendLocal { + return 0, "" + } + backend, err := a.uploadService.Storage().Backend(services.StorageBackendLocal) + if err != nil { + return http.StatusInternalServerError, "storage backend could not be checked" + } + used, err := backend.Usage(context.Background()) + if err != nil { + return http.StatusInternalServerError, "storage backend usage could not be checked" + } + if used+totalBytes > services.GigabytesToBytes(settings.LocalStorageMaxGB) { + return http.StatusRequestEntityTooLarge, "local storage limit reached" + } + return 0, "" +} + +func uploadParseLimit(policy services.EffectiveUploadPolicy, loggedIn bool, fallback int64) int64 { + if loggedIn && policy.MaxUploadMB <= 0 { return fallback * 8 } - return services.MegabytesToBytes(settings.AnonymousMaxUploadMB) * 8 + if policy.MaxUploadMB > 0 { + return services.MegabytesToBytes(policy.MaxUploadMB) * 8 + } + return fallback * 8 } func uploadClientIP(r *http.Request) string { return services.ClientIP(r.RemoteAddr, r.Header.Get("X-Forwarded-For")) } +func uploadRateKey(r *http.Request, user services.User, loggedIn bool) string { + if loggedIn { + return "user:" + user.ID + } + return "ip:" + uploadClientIP(r) +} + func totalUploadBytes(files []*multipart.FileHeader) int64 { var total int64 for _, file := range files { diff --git a/backend/libs/handlers/upload_stage3_test.go b/backend/libs/handlers/upload_stage3_test.go index ec201e2..3f8dd87 100644 --- a/backend/libs/handlers/upload_stage3_test.go +++ b/backend/libs/handlers/upload_stage3_test.go @@ -255,6 +255,29 @@ func multipartUploadRequest(t *testing.T, path, field, filename, body string) *h return request } +func multipartUploadRequestWithField(t *testing.T, path, field, filename, body, extraName, extraValue string) *http.Request { + t.Helper() + var payload bytes.Buffer + writer := multipart.NewWriter(&payload) + part, err := writer.CreateFormFile(field, filename) + if err != nil { + t.Fatalf("CreateFormFile returned error: %v", err) + } + if _, err := part.Write([]byte(body)); err != nil { + t.Fatalf("part.Write returned error: %v", err) + } + if err := writer.WriteField(extraName, extraValue); err != nil { + t.Fatalf("WriteField returned error: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("writer.Close returned error: %v", err) + } + + request := httptest.NewRequest(http.MethodPost, path, &payload) + request.Header.Set("Content-Type", writer.FormDataContentType()) + return request +} + func tokenFromURL(t *testing.T, value string) string { t.Helper() parts := strings.Split(strings.TrimRight(value, "/"), "/") diff --git a/backend/libs/jobs/thumbnails.go b/backend/libs/jobs/thumbnails.go index 987f9cf..c6d547c 100644 --- a/backend/libs/jobs/thumbnails.go +++ b/backend/libs/jobs/thumbnails.go @@ -1,11 +1,14 @@ package jobs import ( + "bytes" + "context" "image" _ "image/gif" "image/jpeg" _ "image/jpeg" _ "image/png" + "io" "log/slog" "os" "os/exec" @@ -129,43 +132,71 @@ func needsThumbnail(file services.File) bool { func generateThumbnail(uploadService *services.UploadService, box services.Box, file services.File) (string, error) { thumbnailName := "@thumb@" + file.ID + ".jpg" - thumbnailPath := uploadService.ThumbnailPath(box, services.File{Thumbnail: thumbnailName}) - sourcePath := uploadService.FilePath(box, file) + object, err := uploadService.OpenFileObject(context.Background(), box, file) + if err != nil { + return "", err + } + defer object.Body.Close() switch { case strings.HasPrefix(file.ContentType, "image/"): - return thumbnailName, createImageThumbnail(sourcePath, thumbnailPath) + data, err := createImageThumbnail(object.Body) + if err != nil { + return "", err + } + _, err = uploadService.PutThumbnailObject(context.Background(), box, thumbnailName, bytes.NewReader(data), int64(len(data)), "image/jpeg") + return thumbnailName, err case strings.HasPrefix(file.ContentType, "video/"): - return thumbnailName, createVideoThumbnail(sourcePath, thumbnailPath) + data, err := createVideoThumbnail(object.Body) + if err != nil { + return "", err + } + _, err = uploadService.PutThumbnailObject(context.Background(), box, thumbnailName, bytes.NewReader(data), int64(len(data)), "image/jpeg") + return thumbnailName, err default: return "", nil } } -func createImageThumbnail(sourcePath, targetPath string) error { - source, err := os.Open(sourcePath) - if err != nil { - return err - } - defer source.Close() - +func createImageThumbnail(source io.Reader) ([]byte, error) { img, _, err := image.Decode(source) if err != nil { - return err + return nil, err } thumb := resizeNearest(img, 360, 240) - target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + var target bytes.Buffer + err = jpeg.Encode(&target, thumb, &jpeg.Options{Quality: 82}) if err != nil { - return err + return nil, err } - defer target.Close() - - return jpeg.Encode(target, thumb, &jpeg.Options{Quality: 82}) + return target.Bytes(), nil } -func createVideoThumbnail(sourcePath, targetPath string) error { - return exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourcePath, "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run() +func createVideoThumbnail(source io.Reader) ([]byte, error) { + sourceFile, err := os.CreateTemp("", "warpbox-video-*") + if err != nil { + return nil, err + } + defer os.Remove(sourceFile.Name()) + if _, err := io.Copy(sourceFile, source); err != nil { + sourceFile.Close() + return nil, err + } + if err := sourceFile.Close(); err != nil { + return nil, err + } + targetFile, err := os.CreateTemp("", "warpbox-thumb-*.jpg") + if err != nil { + return nil, err + } + targetPath := targetFile.Name() + targetFile.Close() + defer os.Remove(targetPath) + if err := exec.Command("ffmpeg", "-y", "-loglevel", "error", "-ss", "00:00:01", "-i", sourceFile.Name(), "-frames:v", "1", "-vf", "scale=360:-1", targetPath).Run(); err != nil { + return nil, err + } + return os.ReadFile(targetPath) } func resizeNearest(src image.Image, maxWidth, maxHeight int) *image.RGBA { diff --git a/backend/libs/services/auth.go b/backend/libs/services/auth.go index 13ee71c..d4f7abc 100644 --- a/backend/libs/services/auth.go +++ b/backend/libs/services/auth.go @@ -48,15 +48,27 @@ type AuthService struct { } type User struct { - ID string `json:"id"` - Username string `json:"username"` - Email string `json:"email"` - PasswordHash string `json:"passwordHash"` - Role string `json:"role"` - Status string `json:"status"` - StorageQuotaMB *float64 `json:"storageQuotaMb,omitempty"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + Username string `json:"username"` + Email string `json:"email"` + PasswordHash string `json:"passwordHash"` + Role string `json:"role"` + Status string `json:"status"` + StorageQuotaMB *float64 `json:"storageQuotaMb,omitempty"` + Policy UserPolicy `json:"policy,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type UserPolicy struct { + MaxUploadMB *float64 `json:"maxUploadMb,omitempty"` + DailyUploadMB *float64 `json:"dailyUploadMb,omitempty"` + StorageQuotaMB *float64 `json:"storageQuotaMb,omitempty"` + MaxDays *int `json:"maxDays,omitempty"` + DailyBoxes *int `json:"dailyBoxes,omitempty"` + ActiveBoxes *int `json:"activeBoxes,omitempty"` + ShortWindowRequests *int `json:"shortWindowRequests,omitempty"` + StorageBackendID *string `json:"storageBackendId,omitempty"` } type PublicUser struct { @@ -66,6 +78,7 @@ type PublicUser struct { Role string Status string StorageQuotaMB *float64 + Policy UserPolicy CreatedAt time.Time } @@ -381,6 +394,97 @@ func (s *AuthService) SetUserStorageQuota(userID string, quotaMB *float64) error return s.saveUser(user) } +func (s *AuthService) SetUserPolicy(userID string, policy UserPolicy) error { + if err := validateUserPolicy(policy); err != nil { + return err + } + user, err := s.UserByID(userID) + if err != nil { + return err + } + user.Policy = policy + user.StorageQuotaMB = policy.StorageQuotaMB + user.UpdatedAt = time.Now().UTC() + return s.saveUser(user) +} + +func (s *AuthService) SetUserStorageBackend(userID, backendID string) error { + user, err := s.UserByID(userID) + if err != nil { + return err + } + backendID = strings.TrimSpace(backendID) + if backendID == "" { + user.Policy.StorageBackendID = nil + } else { + user.Policy.StorageBackendID = &backendID + } + user.UpdatedAt = time.Now().UTC() + return s.saveUser(user) +} + +func (s *AuthService) UpdateUserAdminFields(userID, username, email, role, status string, policy UserPolicy) (User, error) { + if err := validateUserPolicy(policy); err != nil { + return User{}, err + } + username = strings.TrimSpace(username) + if username == "" { + return User{}, fmt.Errorf("username is required") + } + email, err := normalizeEmail(email) + if err != nil { + return User{}, err + } + if role != UserRoleAdmin && role != UserRoleUser { + return User{}, fmt.Errorf("invalid role") + } + if status != UserStatusActive && status != UserStatusDisabled { + return User{}, fmt.Errorf("invalid status") + } + + var updated User + err = s.db.Update(func(tx *bbolt.Tx) error { + users := tx.Bucket(usersBucket) + emails := tx.Bucket(userEmailsBucket) + data := users.Get([]byte(userID)) + if data == nil { + return os.ErrNotExist + } + var user User + if err := json.Unmarshal(data, &user); err != nil { + return err + } + if existing := emails.Get([]byte(email)); existing != nil && string(existing) != user.ID { + return fmt.Errorf("email is already registered") + } + if user.Email != email { + if err := emails.Delete([]byte(user.Email)); err != nil { + return err + } + if err := emails.Put([]byte(email), []byte(user.ID)); err != nil { + return err + } + } + user.Username = username + user.Email = email + user.Role = role + user.Status = status + user.Policy = policy + user.StorageQuotaMB = policy.StorageQuotaMB + user.UpdatedAt = time.Now().UTC() + next, err := json.Marshal(user) + if err != nil { + return err + } + if err := users.Put([]byte(user.ID), next); err != nil { + return err + } + updated = user + return nil + }) + return updated, err +} + func (s *AuthService) UserByID(id string) (User, error) { var user User err := s.db.View(func(tx *bbolt.Tx) error { @@ -476,6 +580,7 @@ func (s *AuthService) PublicUser(user User) PublicUser { Role: user.Role, Status: user.Status, StorageQuotaMB: user.StorageQuotaMB, + Policy: user.Policy, CreatedAt: user.CreatedAt, } } @@ -593,3 +698,28 @@ func VerifyPasswordHash(encoded, password string) bool { actual := argon2.IDKey([]byte(password), salt, 1, 64*1024, 4, uint32(len(expected))) return subtle.ConstantTimeCompare(actual, expected) == 1 } + +func validateUserPolicy(policy UserPolicy) error { + if policy.MaxUploadMB != nil && *policy.MaxUploadMB < 0 { + return fmt.Errorf("max upload override cannot be negative") + } + if policy.DailyUploadMB != nil && *policy.DailyUploadMB <= 0 { + return fmt.Errorf("daily upload override must be positive") + } + if policy.StorageQuotaMB != nil && *policy.StorageQuotaMB < 0 { + return fmt.Errorf("storage quota override cannot be negative") + } + if policy.MaxDays != nil && *policy.MaxDays <= 0 { + return fmt.Errorf("expiration override must be positive") + } + if policy.DailyBoxes != nil && *policy.DailyBoxes <= 0 { + return fmt.Errorf("daily box override must be positive") + } + if policy.ActiveBoxes != nil && *policy.ActiveBoxes <= 0 { + return fmt.Errorf("active box override must be positive") + } + if policy.ShortWindowRequests != nil && *policy.ShortWindowRequests <= 0 { + return fmt.Errorf("short-window request override must be positive") + } + return nil +} diff --git a/backend/libs/services/settings.go b/backend/libs/services/settings.go index fff6069..6d7419c 100644 --- a/backend/libs/services/settings.go +++ b/backend/libs/services/settings.go @@ -26,6 +26,17 @@ type UploadPolicySettings struct { UserDailyUploadMB float64 `json:"userDailyUploadMb"` DefaultUserStorageMB float64 `json:"defaultUserStorageMb"` UsageRetentionDays int `json:"usageRetentionDays"` + LocalStorageMaxGB float64 `json:"localStorageMaxGb"` + AnonymousMaxDays int `json:"anonymousMaxDays"` + UserMaxDays int `json:"userMaxDays"` + AnonymousDailyBoxes int `json:"anonymousDailyBoxes"` + UserDailyBoxes int `json:"userDailyBoxes"` + AnonymousActiveBoxes int `json:"anonymousActiveBoxes"` + UserActiveBoxes int `json:"userActiveBoxes"` + ShortWindowRequests int `json:"shortWindowRequests"` + ShortWindowSeconds int `json:"shortWindowSeconds"` + AnonymousStorageBackend string `json:"anonymousStorageBackend"` + UserStorageBackend string `json:"userStorageBackend"` } type UsageRecord struct { @@ -34,9 +45,24 @@ type UsageRecord struct { Subject string `json:"subject"` Date string `json:"date"` UploadedBytes int64 `json:"uploadedBytes"` + UploadedBoxes int `json:"uploadedBoxes"` + RequestCount int `json:"requestCount"` UpdatedAt time.Time `json:"updatedAt"` } +type EffectiveUploadPolicy struct { + MaxUploadMB float64 + DailyUploadMB float64 + StorageQuotaMB float64 + MaxDays int + DailyBoxes int + ActiveBoxes int + ShortRequests int + ShortWindow time.Duration + StorageBackendID string + StorageQuotaSet bool +} + type SettingsService struct { db *bbolt.DB defaults UploadPolicySettings @@ -52,8 +78,20 @@ func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*Settin UserDailyUploadMB: defaults.UserDailyUploadMB, DefaultUserStorageMB: defaults.DefaultUserStorageMB, UsageRetentionDays: defaults.UsageRetentionDays, + LocalStorageMaxGB: defaults.LocalStorageMaxGB, + AnonymousMaxDays: defaults.AnonymousMaxDays, + UserMaxDays: defaults.UserMaxDays, + AnonymousDailyBoxes: defaults.AnonymousDailyBoxes, + UserDailyBoxes: defaults.UserDailyBoxes, + AnonymousActiveBoxes: defaults.AnonymousActiveBoxes, + UserActiveBoxes: defaults.UserActiveBoxes, + ShortWindowRequests: defaults.ShortWindowRequests, + ShortWindowSeconds: defaults.ShortWindowSeconds, + AnonymousStorageBackend: defaults.AnonymousStorageBackend, + UserStorageBackend: defaults.UserStorageBackend, }, } + service.defaults = service.withBuiltinDefaultGaps(service.defaults) if err := service.validate(service.defaults); err != nil { return nil, err } @@ -71,6 +109,43 @@ func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*Settin return service, nil } +func (s *SettingsService) withBuiltinDefaultGaps(settings UploadPolicySettings) UploadPolicySettings { + if settings.LocalStorageMaxGB <= 0 { + settings.LocalStorageMaxGB = 100 + } + if settings.AnonymousMaxDays <= 0 { + settings.AnonymousMaxDays = 30 + } + if settings.UserMaxDays <= 0 { + settings.UserMaxDays = 90 + } + if settings.AnonymousDailyBoxes <= 0 { + settings.AnonymousDailyBoxes = 100 + } + if settings.UserDailyBoxes <= 0 { + settings.UserDailyBoxes = 250 + } + if settings.AnonymousActiveBoxes <= 0 { + settings.AnonymousActiveBoxes = 500 + } + if settings.UserActiveBoxes <= 0 { + settings.UserActiveBoxes = 1000 + } + if settings.ShortWindowRequests <= 0 { + settings.ShortWindowRequests = 60 + } + if settings.ShortWindowSeconds <= 0 { + settings.ShortWindowSeconds = 60 + } + if strings.TrimSpace(settings.AnonymousStorageBackend) == "" { + settings.AnonymousStorageBackend = StorageBackendLocal + } + if strings.TrimSpace(settings.UserStorageBackend) == "" { + settings.UserStorageBackend = StorageBackendLocal + } + return settings +} + func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) { settings := s.defaults err := s.db.View(func(tx *bbolt.Tx) error { @@ -78,7 +153,11 @@ func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) { if data == nil { return nil } - return json.Unmarshal(data, &settings) + if err := json.Unmarshal(data, &settings); err != nil { + return err + } + settings = s.withDefaultGaps(settings) + return nil }) if err != nil { return UploadPolicySettings{}, err @@ -89,6 +168,58 @@ func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) { return settings, nil } +func (s *SettingsService) withDefaultGaps(settings UploadPolicySettings) UploadPolicySettings { + if settings.AnonymousMaxUploadMB <= 0 { + settings.AnonymousMaxUploadMB = s.defaults.AnonymousMaxUploadMB + } + if settings.AnonymousDailyUploadMB <= 0 { + settings.AnonymousDailyUploadMB = s.defaults.AnonymousDailyUploadMB + } + if settings.UserDailyUploadMB <= 0 { + settings.UserDailyUploadMB = s.defaults.UserDailyUploadMB + } + if settings.DefaultUserStorageMB <= 0 { + settings.DefaultUserStorageMB = s.defaults.DefaultUserStorageMB + } + if settings.UsageRetentionDays <= 0 { + settings.UsageRetentionDays = s.defaults.UsageRetentionDays + } + if settings.LocalStorageMaxGB <= 0 { + settings.LocalStorageMaxGB = s.defaults.LocalStorageMaxGB + } + if settings.AnonymousMaxDays <= 0 { + settings.AnonymousMaxDays = s.defaults.AnonymousMaxDays + } + if settings.UserMaxDays <= 0 { + settings.UserMaxDays = s.defaults.UserMaxDays + } + if settings.AnonymousDailyBoxes <= 0 { + settings.AnonymousDailyBoxes = s.defaults.AnonymousDailyBoxes + } + if settings.UserDailyBoxes <= 0 { + settings.UserDailyBoxes = s.defaults.UserDailyBoxes + } + if settings.AnonymousActiveBoxes <= 0 { + settings.AnonymousActiveBoxes = s.defaults.AnonymousActiveBoxes + } + if settings.UserActiveBoxes <= 0 { + settings.UserActiveBoxes = s.defaults.UserActiveBoxes + } + if settings.ShortWindowRequests <= 0 { + settings.ShortWindowRequests = s.defaults.ShortWindowRequests + } + if settings.ShortWindowSeconds <= 0 { + settings.ShortWindowSeconds = s.defaults.ShortWindowSeconds + } + if strings.TrimSpace(settings.AnonymousStorageBackend) == "" { + settings.AnonymousStorageBackend = s.defaults.AnonymousStorageBackend + } + if strings.TrimSpace(settings.UserStorageBackend) == "" { + settings.UserStorageBackend = s.defaults.UserStorageBackend + } + return settings +} + func (s *SettingsService) UpdateUploadPolicy(settings UploadPolicySettings) error { if err := s.validate(settings); err != nil { return err @@ -117,7 +248,17 @@ func (s *SettingsService) Usage(subjectType, subject string, now time.Time) (Usa } func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now time.Time) error { + return s.AddUploadUsage(subjectType, subject, bytes, 0, now) +} + +func (s *SettingsService) AddUploadUsage(subjectType, subject string, bytes int64, boxes int, now time.Time) error { if bytes <= 0 { + bytes = 0 + } + if boxes < 0 { + boxes = 0 + } + if bytes == 0 && boxes == 0 { return nil } key := usageKey(subjectType, subject, now) @@ -131,6 +272,7 @@ func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now } } record.UploadedBytes += bytes + record.UploadedBoxes += boxes record.UpdatedAt = now.UTC() next, err := json.Marshal(record) if err != nil { @@ -140,6 +282,63 @@ func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now }) } +func (s *SettingsService) EffectivePolicyForAnonymous(settings UploadPolicySettings) EffectiveUploadPolicy { + return EffectiveUploadPolicy{ + MaxUploadMB: settings.AnonymousMaxUploadMB, + DailyUploadMB: settings.AnonymousDailyUploadMB, + MaxDays: settings.AnonymousMaxDays, + DailyBoxes: settings.AnonymousDailyBoxes, + ActiveBoxes: settings.AnonymousActiveBoxes, + ShortRequests: settings.ShortWindowRequests, + ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second, + StorageBackendID: normalizeBackendID(settings.AnonymousStorageBackend), + } +} + +func (s *SettingsService) EffectivePolicyForUser(settings UploadPolicySettings, user User) EffectiveUploadPolicy { + policy := EffectiveUploadPolicy{ + MaxUploadMB: 0, + DailyUploadMB: settings.UserDailyUploadMB, + StorageQuotaMB: settings.DefaultUserStorageMB, + MaxDays: settings.UserMaxDays, + DailyBoxes: settings.UserDailyBoxes, + ActiveBoxes: settings.UserActiveBoxes, + ShortRequests: settings.ShortWindowRequests, + ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second, + StorageBackendID: normalizeBackendID(settings.UserStorageBackend), + StorageQuotaSet: true, + } + if user.StorageQuotaMB != nil { + policy.StorageQuotaMB = *user.StorageQuotaMB + } + if user.Policy.MaxUploadMB != nil { + policy.MaxUploadMB = *user.Policy.MaxUploadMB + } + if user.Policy.DailyUploadMB != nil { + policy.DailyUploadMB = *user.Policy.DailyUploadMB + } + if user.Policy.StorageQuotaMB != nil { + policy.StorageQuotaMB = *user.Policy.StorageQuotaMB + policy.StorageQuotaSet = *user.Policy.StorageQuotaMB > 0 + } + if user.Policy.MaxDays != nil { + policy.MaxDays = *user.Policy.MaxDays + } + if user.Policy.DailyBoxes != nil { + policy.DailyBoxes = *user.Policy.DailyBoxes + } + if user.Policy.ActiveBoxes != nil { + policy.ActiveBoxes = *user.Policy.ActiveBoxes + } + if user.Policy.ShortWindowRequests != nil { + policy.ShortRequests = *user.Policy.ShortWindowRequests + } + if user.Policy.StorageBackendID != nil { + policy.StorageBackendID = normalizeBackendID(*user.Policy.StorageBackendID) + } + return policy +} + func (s *SettingsService) CleanupUsage(now time.Time, retentionDays int) error { if retentionDays <= 0 { return fmt.Errorf("usage retention days must be positive") @@ -185,6 +384,21 @@ func (s *SettingsService) validate(settings UploadPolicySettings) error { if settings.UsageRetentionDays <= 0 { return fmt.Errorf("usage retention days must be positive") } + if settings.LocalStorageMaxGB <= 0 { + return fmt.Errorf("local storage max must be positive") + } + if settings.AnonymousMaxDays <= 0 || settings.UserMaxDays <= 0 { + return fmt.Errorf("expiration limits must be positive") + } + if settings.AnonymousDailyBoxes <= 0 || settings.UserDailyBoxes <= 0 { + return fmt.Errorf("daily box limits must be positive") + } + if settings.AnonymousActiveBoxes <= 0 || settings.UserActiveBoxes <= 0 { + return fmt.Errorf("active box limits must be positive") + } + if settings.ShortWindowRequests <= 0 || settings.ShortWindowSeconds <= 0 { + return fmt.Errorf("short-window rate limits must be positive") + } return nil } @@ -211,6 +425,10 @@ func MegabytesToBytes(value float64) int64 { return int64(value * 1024 * 1024) } +func GigabytesToBytes(value float64) int64 { + return int64(value * 1024 * 1024 * 1024) +} + func FormatMegabytesFromBytes(value int64) string { mb := float64(value) / 1024 / 1024 return FormatMegabytesLabel(mb) @@ -228,6 +446,14 @@ func usageDate(now time.Time) string { return now.UTC().Format("2006-01-02") } +func normalizeBackendID(id string) string { + id = strings.TrimSpace(id) + if id == "" { + return StorageBackendLocal + } + return id +} + func ClientIP(remoteAddr, forwardedFor string) string { if forwardedFor != "" { parts := strings.Split(forwardedFor, ",") diff --git a/backend/libs/services/settings_test.go b/backend/libs/services/settings_test.go index 136c782..71a5a99 100644 --- a/backend/libs/services/settings_test.go +++ b/backend/libs/services/settings_test.go @@ -146,6 +146,44 @@ func TestDailyUsageAndCleanup(t *testing.T) { } } +func TestEffectiveUserPolicyUsesOverridesAndInheritance(t *testing.T) { + settings := newTestSettingsService(t) + policy, err := settings.UploadPolicy() + if err != nil { + t.Fatalf("UploadPolicy returned error: %v", err) + } + policy.UserDailyUploadMB = 100 + policy.DefaultUserStorageMB = 200 + policy.UserMaxDays = 30 + policy.UserDailyBoxes = 40 + policy.UserActiveBoxes = 50 + policy.UserStorageBackend = "local" + + overrideDaily := 300.0 + overrideQuota := 0.0 + overrideDays := 12 + overrideBackend := "bucket-1" + user := User{ + ID: "user-1", + Policy: UserPolicy{ + DailyUploadMB: &overrideDaily, + StorageQuotaMB: &overrideQuota, + MaxDays: &overrideDays, + StorageBackendID: &overrideBackend, + }, + } + effective := settings.EffectivePolicyForUser(policy, user) + if effective.DailyUploadMB != overrideDaily || effective.MaxDays != overrideDays || effective.StorageBackendID != overrideBackend { + t.Fatalf("effective policy did not use overrides: %+v", effective) + } + if effective.StorageQuotaSet { + t.Fatalf("zero storage quota override should mean unlimited: %+v", effective) + } + if effective.DailyBoxes != policy.UserDailyBoxes || effective.ActiveBoxes != policy.UserActiveBoxes { + t.Fatalf("effective policy did not inherit box caps: %+v", effective) + } +} + func newTestSettingsService(t *testing.T) *SettingsService { t.Helper() root := t.TempDir() diff --git a/backend/libs/services/storage.go b/backend/libs/services/storage.go new file mode 100644 index 0000000..4fc6c9c --- /dev/null +++ b/backend/libs/services/storage.go @@ -0,0 +1,536 @@ +package services + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "go.etcd.io/bbolt" +) + +var storageBackendsBucket = []byte("storage_backends") + +const ( + StorageBackendLocal = "local" + StorageBackendS3 = "s3" + + StorageProviderS3 = "s3" + StorageProviderContabo = "contabo" +) + +type StorageObject struct { + Key string + Size int64 + ContentType string + ModTime time.Time + Body io.ReadCloser +} + +type StorageBackend interface { + ID() string + Type() string + Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error + Get(ctx context.Context, key string) (StorageObject, error) + Delete(ctx context.Context, key string) error + DeletePrefix(ctx context.Context, prefix string) error + Usage(ctx context.Context) (int64, error) + Test(ctx context.Context) error +} + +type StorageBackendConfig struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Provider string `json:"provider,omitempty"` + Enabled bool `json:"enabled"` + LocalPath string `json:"localPath,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Region string `json:"region,omitempty"` + Bucket string `json:"bucket,omitempty"` + AccessKey string `json:"accessKey,omitempty"` + SecretKey string `json:"secretKey,omitempty"` + UseSSL bool `json:"useSsl,omitempty"` + PathStyle bool `json:"pathStyle,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + LastTestedAt time.Time `json:"lastTestedAt,omitempty"` + LastTestError string `json:"lastTestError,omitempty"` + LastTestSuccess bool `json:"lastTestSuccess,omitempty"` +} + +type StorageBackendView struct { + Config StorageBackendConfig + UsageBytes int64 + UsageLabel string + InUse bool +} + +type StorageService struct { + db *bbolt.DB + localFilesDir string +} + +func NewStorageService(db *bbolt.DB, dataDir string) (*StorageService, error) { + filesDir := filepath.Join(dataDir, "files") + if err := os.MkdirAll(filesDir, 0o755); err != nil { + return nil, err + } + service := &StorageService{db: db, localFilesDir: filesDir} + err := db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(storageBackendsBucket) + return err + }) + if err != nil { + return nil, err + } + return service, nil +} + +func (s *StorageService) LocalFilesDir() string { + return s.localFilesDir +} + +func (s *StorageService) Backend(id string) (StorageBackend, error) { + cfg, err := s.BackendConfig(id) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, fmt.Errorf("storage backend is disabled") + } + return s.backendFromConfig(cfg) +} + +func (s *StorageService) BackendConfig(id string) (StorageBackendConfig, error) { + id = strings.TrimSpace(id) + if id == "" || id == StorageBackendLocal { + return s.localConfig(), nil + } + var cfg StorageBackendConfig + err := s.db.View(func(tx *bbolt.Tx) error { + data := tx.Bucket(storageBackendsBucket).Get([]byte(id)) + if data == nil { + return os.ErrNotExist + } + return json.Unmarshal(data, &cfg) + }) + if err != nil { + return StorageBackendConfig{}, err + } + return cfg, nil +} + +func (s *StorageService) ListBackendConfigs() ([]StorageBackendConfig, error) { + configs := []StorageBackendConfig{s.localConfig()} + err := s.db.View(func(tx *bbolt.Tx) error { + return tx.Bucket(storageBackendsBucket).ForEach(func(_, value []byte) error { + var cfg StorageBackendConfig + if err := json.Unmarshal(value, &cfg); err != nil { + return err + } + configs = append(configs, cfg) + return nil + }) + }) + sort.Slice(configs, func(i, j int) bool { + if configs[i].ID == StorageBackendLocal { + return true + } + if configs[j].ID == StorageBackendLocal { + return false + } + return strings.ToLower(configs[i].Name) < strings.ToLower(configs[j].Name) + }) + return configs, err +} + +func (s *StorageService) CreateS3Backend(input StorageBackendConfig) (StorageBackendConfig, error) { + input.ID = randomID(10) + input.Type = StorageBackendS3 + input.Provider = normalizeStorageProvider(input.Provider) + if input.Provider == StorageProviderContabo { + input.UseSSL = true + input.PathStyle = true + } + input.Name = strings.TrimSpace(input.Name) + input.Endpoint = strings.TrimSpace(input.Endpoint) + input.Region = strings.TrimSpace(input.Region) + input.Bucket = strings.TrimSpace(input.Bucket) + input.AccessKey = strings.TrimSpace(input.AccessKey) + input.SecretKey = strings.TrimSpace(input.SecretKey) + if input.Name == "" { + input.Name = input.Bucket + } + if input.Name == "" || input.Endpoint == "" || input.Bucket == "" || input.AccessKey == "" || input.SecretKey == "" { + return StorageBackendConfig{}, fmt.Errorf("name, endpoint, bucket, access key, and secret key are required") + } + now := time.Now().UTC() + input.Enabled = true + input.CreatedAt = now + input.UpdatedAt = now + if err := s.SaveBackendConfig(input); err != nil { + return StorageBackendConfig{}, err + } + return input, nil +} + +func (s *StorageService) UpdateS3Backend(id string, input StorageBackendConfig) (StorageBackendConfig, error) { + current, err := s.BackendConfig(id) + if err != nil { + return StorageBackendConfig{}, err + } + if current.ID == StorageBackendLocal || current.Type != StorageBackendS3 { + return StorageBackendConfig{}, fmt.Errorf("only S3-compatible storage can be edited") + } + + current.Provider = normalizeStorageProvider(input.Provider) + if current.Provider == StorageProviderContabo { + input.UseSSL = true + input.PathStyle = true + } + current.Name = strings.TrimSpace(input.Name) + current.Endpoint = strings.TrimSpace(input.Endpoint) + current.Region = strings.TrimSpace(input.Region) + current.Bucket = strings.TrimSpace(input.Bucket) + current.AccessKey = strings.TrimSpace(input.AccessKey) + if strings.TrimSpace(input.SecretKey) != "" { + current.SecretKey = strings.TrimSpace(input.SecretKey) + } + current.UseSSL = input.UseSSL + current.PathStyle = input.PathStyle + if current.Name == "" { + current.Name = current.Bucket + } + if current.Name == "" || current.Endpoint == "" || current.Bucket == "" || current.AccessKey == "" || current.SecretKey == "" { + return StorageBackendConfig{}, fmt.Errorf("name, endpoint, bucket, access key, and secret key are required") + } + if err := s.SaveBackendConfig(current); err != nil { + return StorageBackendConfig{}, err + } + return current, nil +} + +func (s *StorageService) SaveBackendConfig(cfg StorageBackendConfig) error { + if cfg.ID == "" || cfg.ID == StorageBackendLocal { + return fmt.Errorf("invalid storage backend id") + } + cfg.UpdatedAt = time.Now().UTC() + data, err := json.Marshal(cfg) + if err != nil { + return err + } + return s.db.Update(func(tx *bbolt.Tx) error { + return tx.Bucket(storageBackendsBucket).Put([]byte(cfg.ID), data) + }) +} + +func (s *StorageService) DisableBackend(id string, inUse bool) error { + if id == "" || id == StorageBackendLocal { + return fmt.Errorf("local storage cannot be disabled") + } + if inUse { + return fmt.Errorf("storage backend is in use") + } + cfg, err := s.BackendConfig(id) + if err != nil { + return err + } + cfg.Enabled = false + return s.SaveBackendConfig(cfg) +} + +func (s *StorageService) DeleteBackend(id string, inUse bool) error { + if id == "" || id == StorageBackendLocal { + return fmt.Errorf("local storage cannot be deleted") + } + if inUse { + return fmt.Errorf("storage backend is in use") + } + return s.db.Update(func(tx *bbolt.Tx) error { + return tx.Bucket(storageBackendsBucket).Delete([]byte(id)) + }) +} + +func (s *StorageService) TestBackend(id string) (StorageBackendConfig, error) { + cfg, err := s.BackendConfig(id) + if err != nil { + return StorageBackendConfig{}, err + } + backend, err := s.backendFromConfig(cfg) + if err != nil { + return StorageBackendConfig{}, err + } + err = backend.Test(context.Background()) + cfg.LastTestedAt = time.Now().UTC() + cfg.LastTestError = "" + cfg.LastTestSuccess = err == nil + if err != nil { + cfg.LastTestError = err.Error() + } + if cfg.ID != StorageBackendLocal { + _ = s.SaveBackendConfig(cfg) + } + return cfg, err +} + +func (s *StorageService) backendFromConfig(cfg StorageBackendConfig) (StorageBackend, error) { + switch cfg.Type { + case StorageBackendLocal: + return localStorageBackend{id: cfg.ID, root: cfg.LocalPath}, nil + case StorageBackendS3: + return newS3StorageBackend(cfg) + default: + return nil, fmt.Errorf("unsupported storage backend type %q", cfg.Type) + } +} + +func (s *StorageService) localConfig() StorageBackendConfig { + now := time.Now().UTC() + return StorageBackendConfig{ + ID: StorageBackendLocal, + Name: "Local files", + Type: StorageBackendLocal, + Provider: StorageBackendLocal, + Enabled: true, + LocalPath: s.localFilesDir, + CreatedAt: now, + UpdatedAt: now, + } +} + +type localStorageBackend struct { + id string + root string +} + +func (b localStorageBackend) ID() string { return b.id } +func (b localStorageBackend) Type() string { return StorageBackendLocal } + +func (b localStorageBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) error { + path, err := b.path(key) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + target, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer target.Close() + _, err = io.Copy(target, body) + return err +} + +func (b localStorageBackend) Get(_ context.Context, key string) (StorageObject, error) { + path, err := b.path(key) + if err != nil { + return StorageObject{}, err + } + source, err := os.Open(path) + if err != nil { + return StorageObject{}, err + } + stat, err := source.Stat() + if err != nil { + source.Close() + return StorageObject{}, err + } + return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: source}, nil +} + +func (b localStorageBackend) Delete(_ context.Context, key string) error { + path, err := b.path(key) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (b localStorageBackend) DeletePrefix(_ context.Context, prefix string) error { + path, err := b.path(prefix) + if err != nil { + return err + } + if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (b localStorageBackend) Usage(_ context.Context) (int64, error) { + var total int64 + err := filepath.WalkDir(b.root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + if os.IsNotExist(err) { + return 0, nil + } + return total, err +} + +func (b localStorageBackend) Test(ctx context.Context) error { + key := ".warpbox-storage-test-" + randomID(6) + if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil { + return err + } + return b.Delete(ctx, key) +} + +func (b localStorageBackend) path(key string) (string, error) { + key = filepath.Clean(strings.TrimPrefix(key, "/")) + if key == "." || strings.HasPrefix(key, "..") || filepath.IsAbs(key) { + return "", fmt.Errorf("invalid storage key") + } + path := filepath.Join(b.root, key) + root, err := filepath.Abs(b.root) + if err != nil { + return "", err + } + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + if abs != root && !strings.HasPrefix(abs, root+string(os.PathSeparator)) { + return "", fmt.Errorf("invalid storage key") + } + return abs, nil +} + +type s3StorageBackend struct { + cfg StorageBackendConfig + client *minio.Client +} + +func newS3StorageBackend(cfg StorageBackendConfig) (*s3StorageBackend, error) { + endpoint := normalizeS3Endpoint(cfg.Endpoint) + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.UseSSL, + Region: cfg.Region, + BucketLookup: s3BucketLookup(cfg.PathStyle), + }) + if err != nil { + return nil, err + } + return &s3StorageBackend{cfg: cfg, client: client}, nil +} + +func (b *s3StorageBackend) ID() string { return b.cfg.ID } +func (b *s3StorageBackend) Type() string { return StorageBackendS3 } + +func (b *s3StorageBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error { + opts := minio.PutObjectOptions{ContentType: contentType} + _, err := b.client.PutObject(ctx, b.cfg.Bucket, cleanObjectKey(key), body, size, opts) + return err +} + +func (b *s3StorageBackend) Get(ctx context.Context, key string) (StorageObject, error) { + object, err := b.client.GetObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.GetObjectOptions{}) + if err != nil { + return StorageObject{}, err + } + info, err := object.Stat() + if err != nil { + object.Close() + return StorageObject{}, err + } + return StorageObject{Key: key, Size: info.Size, ContentType: info.ContentType, ModTime: info.LastModified, Body: object}, nil +} + +func (b *s3StorageBackend) Delete(ctx context.Context, key string) error { + return b.client.RemoveObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.RemoveObjectOptions{}) +} + +func (b *s3StorageBackend) DeletePrefix(ctx context.Context, prefix string) error { + prefix = strings.TrimSuffix(cleanObjectKey(prefix), "/") + "/" + objects := b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) + for object := range objects { + if object.Err != nil { + return object.Err + } + if err := b.Delete(ctx, object.Key); err != nil { + return err + } + } + return nil +} + +func (b *s3StorageBackend) Usage(ctx context.Context) (int64, error) { + var total int64 + for object := range b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Recursive: true}) { + if object.Err != nil { + return 0, object.Err + } + total += object.Size + } + return total, nil +} + +func (b *s3StorageBackend) Test(ctx context.Context) error { + exists, err := b.client.BucketExists(ctx, b.cfg.Bucket) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("bucket %q does not exist", b.cfg.Bucket) + } + key := ".warpbox-storage-test-" + randomID(6) + if err := b.Put(ctx, key, bytes.NewReader([]byte("ok")), 2, "text/plain"); err != nil { + return err + } + return b.Delete(ctx, key) +} + +func s3BucketLookup(pathStyle bool) minio.BucketLookupType { + if pathStyle { + return minio.BucketLookupPath + } + return minio.BucketLookupAuto +} + +func normalizeS3Endpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if parsed, err := url.Parse(endpoint); err == nil && parsed.Host != "" { + return parsed.Host + } + return strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://") +} + +func normalizeStorageProvider(provider string) string { + switch strings.TrimSpace(provider) { + case StorageProviderContabo: + return StorageProviderContabo + default: + return StorageProviderS3 + } +} + +func cleanObjectKey(key string) string { + return strings.TrimPrefix(filepath.ToSlash(filepath.Clean(strings.TrimPrefix(key, "/"))), "./") +} diff --git a/backend/libs/services/upload.go b/backend/libs/services/upload.go index d4ff523..23ebd7d 100644 --- a/backend/libs/services/upload.go +++ b/backend/libs/services/upload.go @@ -2,6 +2,8 @@ package services import ( "archive/zip" + "bytes" + "context" "crypto/rand" "crypto/sha256" "crypto/subtle" @@ -12,6 +14,7 @@ import ( "io" "log/slog" "mime/multipart" + "net/http" "os" "path/filepath" "sort" @@ -31,6 +34,7 @@ type UploadService struct { filesDir string db *bbolt.DB logger *slog.Logger + storage *StorageService } type UploadOptions struct { @@ -41,33 +45,39 @@ type UploadOptions struct { OwnerID string CollectionID string SkipSizeLimit bool + CreatorIP string + StorageBackendID string } type Box struct { - ID string `json:"id"` - OwnerID string `json:"ownerId,omitempty"` - CollectionID string `json:"collectionId,omitempty"` - Title string `json:"title,omitempty"` - CreatedAt time.Time `json:"createdAt"` - ExpiresAt time.Time `json:"expiresAt"` - MaxDownloads int `json:"maxDownloads"` - DownloadCount int `json:"downloadCount"` - PasswordSalt string `json:"passwordSalt,omitempty"` - PasswordHash string `json:"passwordHash,omitempty"` - DeleteTokenHash string `json:"deleteTokenHash,omitempty"` - Obfuscate bool `json:"obfuscate"` - Files []File `json:"files"` + ID string `json:"id"` + OwnerID string `json:"ownerId,omitempty"` + CollectionID string `json:"collectionId,omitempty"` + Title string `json:"title,omitempty"` + CreatedAt time.Time `json:"createdAt"` + ExpiresAt time.Time `json:"expiresAt"` + MaxDownloads int `json:"maxDownloads"` + DownloadCount int `json:"downloadCount"` + PasswordSalt string `json:"passwordSalt,omitempty"` + PasswordHash string `json:"passwordHash,omitempty"` + DeleteTokenHash string `json:"deleteTokenHash,omitempty"` + Obfuscate bool `json:"obfuscate"` + CreatorIP string `json:"creatorIp,omitempty"` + StorageBackendID string `json:"storageBackendId,omitempty"` + Files []File `json:"files"` } type File struct { - ID string `json:"id"` - Name string `json:"name"` - StoredName string `json:"storedName"` - Size int64 `json:"size"` - ContentType string `json:"contentType"` - PreviewKind string `json:"previewKind"` - Thumbnail string `json:"thumbnail,omitempty"` - UploadedAt time.Time `json:"uploadedAt"` + ID string `json:"id"` + Name string `json:"name"` + StoredName string `json:"storedName"` + Size int64 `json:"size"` + ContentType string `json:"contentType"` + PreviewKind string `json:"previewKind"` + Thumbnail string `json:"thumbnail,omitempty"` + ObjectKey string `json:"objectKey,omitempty"` + ThumbnailObjectKey string `json:"thumbnailObjectKey,omitempty"` + UploadedAt time.Time `json:"uploadedAt"` } type UploadResult struct { @@ -121,9 +131,6 @@ type UserBox struct { func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog.Logger) (*UploadService, error) { filesDir := filepath.Join(dataDir, "files") dbDir := filepath.Join(dataDir, "db") - if err := os.MkdirAll(filesDir, 0o755); err != nil { - return nil, err - } if err := os.MkdirAll(dbDir, 0o755); err != nil { return nil, err } @@ -140,6 +147,11 @@ func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog db.Close() return nil, err } + storage, err := NewStorageService(db, dataDir) + if err != nil { + db.Close() + return nil, err + } return &UploadService{ maxUploadSize: maxUploadSize, @@ -148,6 +160,7 @@ func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog filesDir: filesDir, db: db, logger: logger, + storage: storage, }, nil } @@ -167,6 +180,10 @@ func (s *UploadService) MaxUploadSizeLabel() string { return helpers.FormatBytes(s.maxUploadSize) } +func (s *UploadService) Storage() *StorageService { + return s.storage +} + func (s *UploadService) ValidateSize(size int64) error { if size > s.maxUploadSize { return fmt.Errorf("file exceeds max upload size of %s", s.MaxUploadSizeLabel()) @@ -183,14 +200,16 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti } box := Box{ - ID: randomID(10), - OwnerID: strings.TrimSpace(opts.OwnerID), - CollectionID: strings.TrimSpace(opts.CollectionID), - CreatedAt: time.Now().UTC(), - ExpiresAt: time.Now().UTC().Add(time.Duration(opts.MaxDays) * 24 * time.Hour), - MaxDownloads: opts.MaxDownloads, - Obfuscate: opts.ObfuscateMetadata && strings.TrimSpace(opts.Password) != "", - Files: make([]File, 0, len(files)), + ID: randomID(10), + OwnerID: strings.TrimSpace(opts.OwnerID), + CollectionID: strings.TrimSpace(opts.CollectionID), + CreatorIP: strings.TrimSpace(opts.CreatorIP), + StorageBackendID: normalizeBackendID(opts.StorageBackendID), + CreatedAt: time.Now().UTC(), + ExpiresAt: time.Now().UTC().Add(time.Duration(opts.MaxDays) * 24 * time.Hour), + MaxDownloads: opts.MaxDownloads, + Obfuscate: opts.ObfuscateMetadata && strings.TrimSpace(opts.Password) != "", + Files: make([]File, 0, len(files)), } deleteToken := randomID(32) box.DeleteTokenHash = deleteTokenHash(box.ID, deleteToken) @@ -200,8 +219,8 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti box.PasswordHash = hash } - boxDir := filepath.Join(s.filesDir, box.ID) - if err := os.MkdirAll(boxDir, 0o755); err != nil { + backend, err := s.storage.Backend(box.StorageBackendID) + if err != nil { return UploadResult{}, err } @@ -224,13 +243,18 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti fileID := randomID(8) storedName := "@each@" + fileID + strings.ToLower(filepath.Ext(header.Filename)) - storedPath := filepath.Join(boxDir, storedName) + objectKey := boxObjectKey(box.ID, storedName) contentType := header.Header.Get("Content-Type") if contentType == "" { - contentType = "application/octet-stream" + buffer := make([]byte, 512) + n, _ := file.Read(buffer) + contentType = http.DetectContentType(buffer[:n]) + if seeker, ok := file.(io.Seeker); ok { + _, _ = seeker.Seek(0, io.SeekStart) + } } - if err := writeUploadedFile(storedPath, file, maxSize); err != nil { + if err := s.writeUploadedObject(context.Background(), backend, objectKey, file, header.Size, maxSize, contentType); err != nil { file.Close() return UploadResult{}, err } @@ -243,6 +267,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti Size: header.Size, ContentType: contentType, PreviewKind: previewKind(contentType), + ObjectKey: objectKey, UploadedAt: time.Now().UTC(), }) } @@ -296,6 +321,29 @@ func (s *UploadService) ListBoxes(limit int) ([]Box, error) { return boxes, err } +func (s *UploadService) ActiveBoxCountForUser(userID string) (int, error) { + return s.activeBoxCount(func(box Box) bool { return box.OwnerID == userID }) +} + +func (s *UploadService) ActiveBoxCountForIP(ip string) (int, error) { + return s.activeBoxCount(func(box Box) bool { return box.OwnerID == "" && box.CreatorIP == ip }) +} + +func (s *UploadService) activeBoxCount(match func(Box) bool) (int, error) { + boxes, err := s.ListBoxes(0) + if err != nil { + return 0, err + } + now := time.Now().UTC() + count := 0 + for _, box := range boxes { + if match(box) && box.ExpiresAt.After(now) { + count++ + } + } + return count, nil +} + func (s *UploadService) AdminStats() (AdminStats, error) { boxes, err := s.ListBoxes(0) if err != nil { @@ -463,13 +511,22 @@ func (s *UploadService) DeleteBoxWithToken(boxID, token string) error { } func (s *UploadService) DeleteBoxWithSource(boxID, source string) error { + box, _ := s.GetBox(boxID) if err := s.db.Update(func(tx *bbolt.Tx) error { return tx.Bucket(boxesBucket).Delete([]byte(boxID)) }); err != nil { return err } - if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil { - return err + if box.ID != "" { + if backend, err := s.storage.Backend(s.BoxStorageBackendID(box)); err == nil { + if err := backend.DeletePrefix(context.Background(), box.ID); err != nil { + return err + } + } + } else { + if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil { + return err + } } s.logger.Info("box deleted", "source", source, "severity", "user_activity", "code", 2101, "box_id", boxID) return nil @@ -499,6 +556,56 @@ func (s *UploadService) BoxMetadataPath(box Box) string { return filepath.Join(s.filesDir, box.ID, ".warpbox.box.json") } +func (s *UploadService) BoxStorageBackendID(box Box) string { + return normalizeBackendID(box.StorageBackendID) +} + +func (s *UploadService) FileObjectKey(box Box, file File) string { + if file.ObjectKey != "" { + return file.ObjectKey + } + return boxObjectKey(box.ID, file.StoredName) +} + +func (s *UploadService) ThumbnailObjectKey(box Box, file File) string { + if file.ThumbnailObjectKey != "" { + return file.ThumbnailObjectKey + } + if file.Thumbnail == "" { + return "" + } + return boxObjectKey(box.ID, file.Thumbnail) +} + +func (s *UploadService) OpenFileObject(ctx context.Context, box Box, file File) (StorageObject, error) { + backend, err := s.storage.Backend(s.BoxStorageBackendID(box)) + if err != nil { + return StorageObject{}, err + } + return backend.Get(ctx, s.FileObjectKey(box, file)) +} + +func (s *UploadService) OpenThumbnailObject(ctx context.Context, box Box, file File) (StorageObject, error) { + key := s.ThumbnailObjectKey(box, file) + if key == "" { + return StorageObject{}, os.ErrNotExist + } + backend, err := s.storage.Backend(s.BoxStorageBackendID(box)) + if err != nil { + return StorageObject{}, err + } + return backend.Get(ctx, key) +} + +func (s *UploadService) PutThumbnailObject(ctx context.Context, box Box, name string, body io.Reader, size int64, contentType string) (string, error) { + backend, err := s.storage.Backend(s.BoxStorageBackendID(box)) + if err != nil { + return "", err + } + key := boxObjectKey(box.ID, name) + return key, backend.Put(ctx, key, body, size, contentType) +} + func (s *UploadService) IsProtected(box Box) bool { return box.PasswordHash != "" && box.PasswordSalt != "" } @@ -564,11 +671,11 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error { defer archive.Close() for _, file := range box.Files { - path := s.FilePath(box, file) - source, err := os.Open(path) + object, err := s.OpenFileObject(context.Background(), box, file) if err != nil { return err } + source := object.Body header := &zip.FileHeader{ Name: file.Name, @@ -592,6 +699,9 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error { } func (s *UploadService) SaveBox(box Box) error { + if box.StorageBackendID == "" { + box.StorageBackendID = StorageBackendLocal + } data, err := json.Marshal(box) if err != nil { return err @@ -654,6 +764,27 @@ func writeUploadedFile(path string, source multipart.File, maxSize int64) error return nil } +func (s *UploadService) writeUploadedObject(ctx context.Context, backend StorageBackend, key string, source multipart.File, size, maxSize int64, contentType string) error { + var reader io.Reader = source + if maxSize > 0 { + reader = io.LimitReader(source, maxSize+1) + var buffer bytes.Buffer + written, err := io.Copy(&buffer, reader) + if err != nil { + return err + } + if written > maxSize { + return fmt.Errorf("file exceeds max upload size") + } + return backend.Put(ctx, key, bytes.NewReader(buffer.Bytes()), written, contentType) + } + return backend.Put(ctx, key, reader, size, contentType) +} + +func boxObjectKey(boxID, name string) string { + return filepath.ToSlash(filepath.Join(boxID, name)) +} + func randomID(byteCount int) string { data := make([]byte, byteCount) if _, err := rand.Read(data); err != nil { @@ -691,10 +822,13 @@ func previewKind(contentType string) string { } func (s *UploadService) writeBoxMetadata(box Box) error { - path := s.BoxMetadataPath(box) data, err := json.MarshalIndent(box, "", " ") if err != nil { return err } - return os.WriteFile(path, data, 0o600) + backend, err := s.storage.Backend(s.BoxStorageBackendID(box)) + if err != nil { + return err + } + return backend.Put(context.Background(), boxObjectKey(box.ID, ".warpbox.box.json"), bytes.NewReader(data), int64(len(data)), "application/json") } diff --git a/backend/libs/services/upload_test.go b/backend/libs/services/upload_test.go index 63664d6..533a199 100644 --- a/backend/libs/services/upload_test.go +++ b/backend/libs/services/upload_test.go @@ -2,6 +2,7 @@ package services import ( "bytes" + "context" "io" "log/slog" "mime/multipart" @@ -93,6 +94,64 @@ func TestUserActiveStorageUsedIgnoresExpiredBoxes(t *testing.T) { } } +func TestLocalStorageBackendAndLegacyFallback(t *testing.T) { + service := newTestUploadService(t) + result := createTestBox(t, service, "file.txt", "hello") + box := getTestBox(t, service, result.BoxID) + if service.BoxStorageBackendID(box) != StorageBackendLocal { + t.Fatalf("BoxStorageBackendID = %q", service.BoxStorageBackendID(box)) + } + if box.Files[0].ObjectKey == "" { + t.Fatalf("new file did not store object key") + } + object, err := service.OpenFileObject(testContext(), box, box.Files[0]) + if err != nil { + t.Fatalf("OpenFileObject returned error: %v", err) + } + data, err := io.ReadAll(object.Body) + object.Body.Close() + if err != nil { + t.Fatalf("ReadAll returned error: %v", err) + } + if string(data) != "hello" { + t.Fatalf("object body = %q", string(data)) + } + + box.StorageBackendID = "" + box.Files[0].ObjectKey = "" + object, err = service.OpenFileObject(testContext(), box, box.Files[0]) + if err != nil { + t.Fatalf("legacy OpenFileObject returned error: %v", err) + } + object.Body.Close() +} + +func TestContaboStorageConfigAllowsDisplayNamesWithSpaces(t *testing.T) { + service := newTestUploadService(t) + cfg, err := service.Storage().CreateS3Backend(StorageBackendConfig{ + Provider: StorageProviderContabo, + Name: "Contabo main", + Endpoint: "https://eu2.contabostorage.com", + Region: "EU", + Bucket: "My Main Bucket", + AccessKey: "access", + SecretKey: "secret", + }) + if err != nil { + t.Fatalf("CreateS3Backend returned error: %v", err) + } + if cfg.Provider != StorageProviderContabo || !cfg.UseSSL || !cfg.PathStyle { + t.Fatalf("contabo config was not normalized: %+v", cfg) + } + if cfg.Bucket != "My Main Bucket" { + t.Fatalf("bucket = %q", cfg.Bucket) + } +} + +func testContext() context.Context { + return context.Background() +} + func newTestUploadService(t *testing.T) *UploadService { t.Helper() service, err := NewUploadService(1024*1024, t.TempDir(), "http://example.test", slog.New(slog.NewTextHandler(io.Discard, nil))) diff --git a/backend/libs/web/renderer.go b/backend/libs/web/renderer.go index 114ab0a..d5e6351 100644 --- a/backend/libs/web/renderer.go +++ b/backend/libs/web/renderer.go @@ -21,6 +21,7 @@ type PageData struct { ImageURL string CurrentYear int CurrentUser any + CSRFToken string Data any } diff --git a/backend/static/css/app.css b/backend/static/css/app.css index 6e9b1f6..1c4a8ae 100644 --- a/backend/static/css/app.css +++ b/backend/static/css/app.css @@ -1414,6 +1414,45 @@ pre code { padding: 0.25rem 0.55rem; } +.storage-edit-form { + position: absolute; + right: 1.5rem; + z-index: 30; + width: min(26rem, calc(100vw - 2rem)); + display: grid; + grid-template-columns: 1fr 1fr; + align-items: end; + gap: 0.6rem; + padding: 0.85rem; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--card); + box-shadow: var(--shadow); +} + +.storage-edit-form label { + display: grid; + gap: 0.25rem; +} + +.storage-edit-form label span { + color: var(--muted-foreground); + font-size: 0.72rem; +} + +.storage-edit-form .checkbox-field, +.storage-edit-form button { + align-self: center; +} + +@media (max-width: 720px) { + .storage-edit-form { + position: static; + grid-template-columns: 1fr; + width: 100%; + } +} + /* Badge variants */ .badge-active { background: rgba(134, 239, 172, 0.12); diff --git a/backend/static/js/app.js b/backend/static/js/app.js index ddba999..361c574 100644 --- a/backend/static/js/app.js +++ b/backend/static/js/app.js @@ -18,6 +18,7 @@ const previewImages = document.querySelector("[data-preview-images]"); const previewActions = document.querySelectorAll("[data-preview-action]"); const fileContextMenu = document.querySelector("[data-file-context-menu]"); + const storageProviderSelects = document.querySelectorAll("[data-storage-provider]"); let ctrlCopyMode = false; let contextFile = null; const contextMenuCloseDistance = 80; @@ -121,6 +122,30 @@ }); } + if (storageProviderSelects.length > 0) { + storageProviderSelects.forEach((select) => { + const formScope = select.closest("form"); + const syncStorageProvider = () => { + if (!formScope) { + return; + } + const isContabo = select.value === "contabo"; + const tls = formScope.querySelector('input[name="use_ssl"]'); + const pathStyle = formScope.querySelector('input[name="path_style"]'); + if (tls) { + tls.checked = isContabo || tls.checked; + tls.disabled = isContabo; + } + if (pathStyle) { + pathStyle.checked = isContabo || pathStyle.checked; + pathStyle.disabled = isContabo; + } + }; + select.addEventListener("change", syncStorageProvider); + syncStorageProvider(); + }); + } + if (!form || !dropZone || !fileInput) { return; } diff --git a/backend/templates/pages/account.html b/backend/templates/pages/account.html index 9ee3154..a6005c5 100644 --- a/backend/templates/pages/account.html +++ b/backend/templates/pages/account.html @@ -10,6 +10,7 @@
+
@@ -33,6 +34,7 @@
+ diff --git a/backend/templates/pages/admin.html b/backend/templates/pages/admin.html index 2669b81..8e129ec 100644 --- a/backend/templates/pages/admin.html +++ b/backend/templates/pages/admin.html @@ -8,6 +8,7 @@ Files Users Settings + Storage
+
@@ -96,6 +98,7 @@ View
+
diff --git a/backend/templates/pages/admin_settings.html b/backend/templates/pages/admin_settings.html index a346b0a..b0a8bc9 100644 --- a/backend/templates/pages/admin_settings.html +++ b/backend/templates/pages/admin_settings.html @@ -8,6 +8,7 @@ Files Users Settings + Storage
+
@@ -37,6 +39,7 @@
+

Anonymous uploads

+ + + +
@@ -67,6 +90,38 @@ Usage retention (days) + + + + + + +
diff --git a/backend/templates/pages/admin_storage.html b/backend/templates/pages/admin_storage.html new file mode 100644 index 0000000..129678f --- /dev/null +++ b/backend/templates/pages/admin_storage.html @@ -0,0 +1,124 @@ +{{define "admin_storage.html"}}{{template "base" .}}{{end}} + +{{define "content"}} +
+ + +
+
+
+

Operator console

+

{{.Data.PageTitle}}

+
+
+ + {{if .Data.Error}}

{{.Data.Error}}

{{end}} + +
+
+
+
+

Storage backends

+

Local storage is always available. S3-compatible backends stay private behind Warpbox routes.

+
+
+
+ + + + {{range .Data.Storage}} + + + + + + + + + {{end}} + +
NameTypeStatusUsageDetailsActions
{{.Config.Name}}{{if eq .Config.Provider "contabo"}}Contabo Object Storage{{else if eq .Config.Type "s3"}}S3 bucket{{else}}{{.Config.Type}}{{end}}{{if .Config.Enabled}}Enabled{{else}}Disabled{{end}}{{.UsageLabel}}{{if eq .Config.Type "local"}}{{.Config.LocalPath}}{{else}}{{.Config.Bucket}} @ {{.Config.Endpoint}}{{end}} +
+ + +
+ {{if ne .Config.ID "local"}} +
+ Edit +
+ + + + + + + + + + + +
+
+
+ + +
+
+ + +
+ {{end}} +
+
+
+
+ +
+
+
+
+

Add storage

+

Choose a provider kind first. Contabo uses S3-compatible access with path-style requests.

+
+
+
+ +
+ + + + + + + + + +
+ +
+
+
+
+
+{{end}} diff --git a/backend/templates/pages/admin_user_edit.html b/backend/templates/pages/admin_user_edit.html new file mode 100644 index 0000000..dd058ea --- /dev/null +++ b/backend/templates/pages/admin_user_edit.html @@ -0,0 +1,117 @@ +{{define "admin_user_edit.html"}}{{template "base" .}}{{end}} + +{{define "content"}} +
+ + +
+
+
+

Operator console

+

{{.Data.PageTitle}}

+

{{.Data.UserEdit.Email}} · {{.Data.UserEdit.Role}}

+
+ Back to users +
+ + {{if .Data.Error}}

{{.Data.Error}}

{{end}} + {{if .Data.LastInviteURL}} +
+ + +
+ {{end}} + +
+
Storage used{{.Data.UserEdit.StorageUsed}}
+
Uploaded today{{.Data.UserEdit.DailyUsed}}
+
Effective quota{{.Data.UserEdit.EffectiveStorage}}
+
Effective backend{{.Data.UserEdit.EffectiveBackend}}
+
+ +
+
+
+
+

Identity and limits

+

Blank limit fields inherit the global user defaults. Storage quota set to 0 means unlimited.

+
+
+
+ +
+

Account

+ + + + +
+ +
+

Storage

+ + +
+ +
+

Upload limits

+ + + + + + +
+ + +
+
+
+ +
+
+
+
+

Password reset

+

Create a copyable reset link for this user.

+
+
+
+ + +
+
+
+
+
+{{end}} diff --git a/backend/templates/pages/admin_users.html b/backend/templates/pages/admin_users.html index d266372..4272a35 100644 --- a/backend/templates/pages/admin_users.html +++ b/backend/templates/pages/admin_users.html @@ -8,6 +8,7 @@ Files Users Settings + Storage
+
@@ -43,6 +45,7 @@ {{end}}
+ @@ -66,6 +69,7 @@ Status Storage Today + Storage backend Joined Actions @@ -79,28 +83,29 @@ {{.Status}} {{.StorageUsed}} / {{.StorageQuota}} {{.DailyUsed}} + {{.StorageBackend}} {{.CreatedAt}} {{if eq .Status "disabled"}} +
{{else}}
+
{{end}}
+
-
- - -
+ Edit {{else}} - No users yet. + No users yet. {{end}} diff --git a/backend/templates/pages/auth.html b/backend/templates/pages/auth.html index 81f77ef..037b742 100644 --- a/backend/templates/pages/auth.html +++ b/backend/templates/pages/auth.html @@ -9,6 +9,7 @@

Create the admin account

The first user becomes the instance admin. Registration closes after this account is created.

+ {{if .Data.Error}}

{{.Data.Error}}

{{end}} @@ -20,6 +21,7 @@

{{if .Data.IsReset}}Choose a new password{{else}}Create your account{{end}}

{{if .Data.Email}}

{{.Data.Email}}

{{end}} + {{if .Data.Error}}

{{.Data.Error}}

{{end}} {{if not .Data.IsReset}}{{end}} @@ -29,6 +31,7 @@

Account

Sign in

+ {{if .Data.Error}}

{{.Data.Error}}

{{end}} diff --git a/backend/templates/pages/dashboard.html b/backend/templates/pages/dashboard.html index e75e972..2ba39d6 100644 --- a/backend/templates/pages/dashboard.html +++ b/backend/templates/pages/dashboard.html @@ -10,6 +10,7 @@
+
@@ -35,6 +36,7 @@ + Collection
+