kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): add S3 backend support and advanced upload limits

Browse Source 
- Introduce S3-compatible storage backend support using minio-go.
- Add configuration options for local storage limits, box limits, and rate limiting.
- Implement storage backend selection (local vs S3) for anonymous and registered users.
- Add an `/admin/storage` management interface.
- Update documentation and environment examples with the new configuration variables.
This commit is contained in:
Daniel Legt
2026-05-31 02:14:10 +03:00
parent 830d2a885c
commit c3558fd353
34 changed files with 2668 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

228
backend/libs/services/settings.go
View File

@@ -26,6 +26,17 @@ type UploadPolicySettings struct {
UserDailyUploadMB float64 `json:"userDailyUploadMb"`
DefaultUserStorageMB float64 `json:"defaultUserStorageMb"`
UsageRetentionDays int `json:"usageRetentionDays"`
LocalStorageMaxGB float64 `json:"localStorageMaxGb"`
AnonymousMaxDays int `json:"anonymousMaxDays"`
UserMaxDays int `json:"userMaxDays"`
AnonymousDailyBoxes int `json:"anonymousDailyBoxes"`
UserDailyBoxes int `json:"userDailyBoxes"`
AnonymousActiveBoxes int `json:"anonymousActiveBoxes"`
UserActiveBoxes int `json:"userActiveBoxes"`
ShortWindowRequests int `json:"shortWindowRequests"`
ShortWindowSeconds int `json:"shortWindowSeconds"`
AnonymousStorageBackend string `json:"anonymousStorageBackend"`
UserStorageBackend string `json:"userStorageBackend"`
}
type UsageRecord struct {
@@ -34,9 +45,24 @@ type UsageRecord struct {
Subject string `json:"subject"`
Date string `json:"date"`
UploadedBytes int64 `json:"uploadedBytes"`
UploadedBoxes int `json:"uploadedBoxes"`
RequestCount int `json:"requestCount"`
UpdatedAt time.Time `json:"updatedAt"`
}
type EffectiveUploadPolicy struct {
MaxUploadMB float64
DailyUploadMB float64
StorageQuotaMB float64
MaxDays int
DailyBoxes int
ActiveBoxes int
ShortRequests int
ShortWindow time.Duration
StorageBackendID string
StorageQuotaSet bool
}
type SettingsService struct {
db *bbolt.DB
defaults UploadPolicySettings
@@ -52,8 +78,20 @@ func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*Settin
UserDailyUploadMB: defaults.UserDailyUploadMB,
DefaultUserStorageMB: defaults.DefaultUserStorageMB,
UsageRetentionDays: defaults.UsageRetentionDays,
LocalStorageMaxGB: defaults.LocalStorageMaxGB,
AnonymousMaxDays: defaults.AnonymousMaxDays,
UserMaxDays: defaults.UserMaxDays,
AnonymousDailyBoxes: defaults.AnonymousDailyBoxes,
UserDailyBoxes: defaults.UserDailyBoxes,
AnonymousActiveBoxes: defaults.AnonymousActiveBoxes,
UserActiveBoxes: defaults.UserActiveBoxes,
ShortWindowRequests: defaults.ShortWindowRequests,
ShortWindowSeconds: defaults.ShortWindowSeconds,
AnonymousStorageBackend: defaults.AnonymousStorageBackend,
UserStorageBackend: defaults.UserStorageBackend,
},
}
service.defaults = service.withBuiltinDefaultGaps(service.defaults)
if err := service.validate(service.defaults); err != nil {
return nil, err
}
@@ -71,6 +109,43 @@ func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*Settin
return service, nil
}
func (s *SettingsService) withBuiltinDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
if settings.LocalStorageMaxGB <= 0 {
settings.LocalStorageMaxGB = 100
}
if settings.AnonymousMaxDays <= 0 {
settings.AnonymousMaxDays = 30
}
if settings.UserMaxDays <= 0 {
settings.UserMaxDays = 90
}
if settings.AnonymousDailyBoxes <= 0 {
settings.AnonymousDailyBoxes = 100
}
if settings.UserDailyBoxes <= 0 {
settings.UserDailyBoxes = 250
}
if settings.AnonymousActiveBoxes <= 0 {
settings.AnonymousActiveBoxes = 500
}
if settings.UserActiveBoxes <= 0 {
settings.UserActiveBoxes = 1000
}
if settings.ShortWindowRequests <= 0 {
settings.ShortWindowRequests = 60
}
if settings.ShortWindowSeconds <= 0 {
settings.ShortWindowSeconds = 60
}
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
settings.AnonymousStorageBackend = StorageBackendLocal
}
if strings.TrimSpace(settings.UserStorageBackend) == "" {
settings.UserStorageBackend = StorageBackendLocal
}
return settings
}
func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
settings := s.defaults
err := s.db.View(func(tx *bbolt.Tx) error {
@@ -78,7 +153,11 @@ func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
if data == nil {
return nil
}
return json.Unmarshal(data, &settings)
if err := json.Unmarshal(data, &settings); err != nil {
return err
}
settings = s.withDefaultGaps(settings)
return nil
})
if err != nil {
return UploadPolicySettings{}, err
@@ -89,6 +168,58 @@ func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
return settings, nil
}
func (s *SettingsService) withDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
if settings.AnonymousMaxUploadMB <= 0 {
settings.AnonymousMaxUploadMB = s.defaults.AnonymousMaxUploadMB
}
if settings.AnonymousDailyUploadMB <= 0 {
settings.AnonymousDailyUploadMB = s.defaults.AnonymousDailyUploadMB
}
if settings.UserDailyUploadMB <= 0 {
settings.UserDailyUploadMB = s.defaults.UserDailyUploadMB
}
if settings.DefaultUserStorageMB <= 0 {
settings.DefaultUserStorageMB = s.defaults.DefaultUserStorageMB
}
if settings.UsageRetentionDays <= 0 {
settings.UsageRetentionDays = s.defaults.UsageRetentionDays
}
if settings.LocalStorageMaxGB <= 0 {
settings.LocalStorageMaxGB = s.defaults.LocalStorageMaxGB
}
if settings.AnonymousMaxDays <= 0 {
settings.AnonymousMaxDays = s.defaults.AnonymousMaxDays
}
if settings.UserMaxDays <= 0 {
settings.UserMaxDays = s.defaults.UserMaxDays
}
if settings.AnonymousDailyBoxes <= 0 {
settings.AnonymousDailyBoxes = s.defaults.AnonymousDailyBoxes
}
if settings.UserDailyBoxes <= 0 {
settings.UserDailyBoxes = s.defaults.UserDailyBoxes
}
if settings.AnonymousActiveBoxes <= 0 {
settings.AnonymousActiveBoxes = s.defaults.AnonymousActiveBoxes
}
if settings.UserActiveBoxes <= 0 {
settings.UserActiveBoxes = s.defaults.UserActiveBoxes
}
if settings.ShortWindowRequests <= 0 {
settings.ShortWindowRequests = s.defaults.ShortWindowRequests
}
if settings.ShortWindowSeconds <= 0 {
settings.ShortWindowSeconds = s.defaults.ShortWindowSeconds
}
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
settings.AnonymousStorageBackend = s.defaults.AnonymousStorageBackend
}
if strings.TrimSpace(settings.UserStorageBackend) == "" {
settings.UserStorageBackend = s.defaults.UserStorageBackend
}
return settings
}
func (s *SettingsService) UpdateUploadPolicy(settings UploadPolicySettings) error {
if err := s.validate(settings); err != nil {
return err
@@ -117,7 +248,17 @@ func (s *SettingsService) Usage(subjectType, subject string, now time.Time) (Usa
}
func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now time.Time) error {
return s.AddUploadUsage(subjectType, subject, bytes, 0, now)
}
func (s *SettingsService) AddUploadUsage(subjectType, subject string, bytes int64, boxes int, now time.Time) error {
if bytes <= 0 {
bytes = 0
}
if boxes < 0 {
boxes = 0
}
if bytes == 0 && boxes == 0 {
return nil
}
key := usageKey(subjectType, subject, now)
@@ -131,6 +272,7 @@ func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now
}
}
record.UploadedBytes += bytes
record.UploadedBoxes += boxes
record.UpdatedAt = now.UTC()
next, err := json.Marshal(record)
if err != nil {
@@ -140,6 +282,63 @@ func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now
})
}
func (s *SettingsService) EffectivePolicyForAnonymous(settings UploadPolicySettings) EffectiveUploadPolicy {
return EffectiveUploadPolicy{
MaxUploadMB: settings.AnonymousMaxUploadMB,
DailyUploadMB: settings.AnonymousDailyUploadMB,
MaxDays: settings.AnonymousMaxDays,
DailyBoxes: settings.AnonymousDailyBoxes,
ActiveBoxes: settings.AnonymousActiveBoxes,
ShortRequests: settings.ShortWindowRequests,
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
StorageBackendID: normalizeBackendID(settings.AnonymousStorageBackend),
}
}
func (s *SettingsService) EffectivePolicyForUser(settings UploadPolicySettings, user User) EffectiveUploadPolicy {
policy := EffectiveUploadPolicy{
MaxUploadMB: 0,
DailyUploadMB: settings.UserDailyUploadMB,
StorageQuotaMB: settings.DefaultUserStorageMB,
MaxDays: settings.UserMaxDays,
DailyBoxes: settings.UserDailyBoxes,
ActiveBoxes: settings.UserActiveBoxes,
ShortRequests: settings.ShortWindowRequests,
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
StorageBackendID: normalizeBackendID(settings.UserStorageBackend),
StorageQuotaSet: true,
}
if user.StorageQuotaMB != nil {
policy.StorageQuotaMB = *user.StorageQuotaMB
}
if user.Policy.MaxUploadMB != nil {
policy.MaxUploadMB = *user.Policy.MaxUploadMB
}
if user.Policy.DailyUploadMB != nil {
policy.DailyUploadMB = *user.Policy.DailyUploadMB
}
if user.Policy.StorageQuotaMB != nil {
policy.StorageQuotaMB = *user.Policy.StorageQuotaMB
policy.StorageQuotaSet = *user.Policy.StorageQuotaMB > 0
}
if user.Policy.MaxDays != nil {
policy.MaxDays = *user.Policy.MaxDays
}
if user.Policy.DailyBoxes != nil {
policy.DailyBoxes = *user.Policy.DailyBoxes
}
if user.Policy.ActiveBoxes != nil {
policy.ActiveBoxes = *user.Policy.ActiveBoxes
}
if user.Policy.ShortWindowRequests != nil {
policy.ShortRequests = *user.Policy.ShortWindowRequests
}
if user.Policy.StorageBackendID != nil {
policy.StorageBackendID = normalizeBackendID(*user.Policy.StorageBackendID)
}
return policy
}
func (s *SettingsService) CleanupUsage(now time.Time, retentionDays int) error {
if retentionDays <= 0 {
return fmt.Errorf("usage retention days must be positive")
@@ -185,6 +384,21 @@ func (s *SettingsService) validate(settings UploadPolicySettings) error {
if settings.UsageRetentionDays <= 0 {
return fmt.Errorf("usage retention days must be positive")
}
if settings.LocalStorageMaxGB <= 0 {
return fmt.Errorf("local storage max must be positive")
}
if settings.AnonymousMaxDays <= 0 || settings.UserMaxDays <= 0 {
return fmt.Errorf("expiration limits must be positive")
}
if settings.AnonymousDailyBoxes <= 0 || settings.UserDailyBoxes <= 0 {
return fmt.Errorf("daily box limits must be positive")
}
if settings.AnonymousActiveBoxes <= 0 || settings.UserActiveBoxes <= 0 {
return fmt.Errorf("active box limits must be positive")
}
if settings.ShortWindowRequests <= 0 || settings.ShortWindowSeconds <= 0 {
return fmt.Errorf("short-window rate limits must be positive")
}
return nil
}
@@ -211,6 +425,10 @@ func MegabytesToBytes(value float64) int64 {
return int64(value * 1024 * 1024)
}
func GigabytesToBytes(value float64) int64 {
return int64(value * 1024 * 1024 * 1024)
}
func FormatMegabytesFromBytes(value int64) string {
mb := float64(value) / 1024 / 1024
return FormatMegabytesLabel(mb)
@@ -228,6 +446,14 @@ func usageDate(now time.Time) string {
return now.UTC().Format("2006-01-02")
}
func normalizeBackendID(id string) string {
id = strings.TrimSpace(id)
if id == "" {
return StorageBackendLocal
}
return id
}
func ClientIP(remoteAddr, forwardedFor string) string {
if forwardedFor != "" {
parts := strings.Split(forwardedFor, ",")