All checks were successful
Build and Publish Docker Image / deploy (push) Successful in 1m38s
- Add `WARPBOX_TRUSTED_PROXIES` configuration to restrict accepted forwarded client IP headers to specific proxy IPs/CIDRs, securing client IP resolution. - Integrate `BanService` into the background cleanup job to automatically purge expired abuse and ban evidence events. - Update documentation with reverse proxy security guidelines and a production systemd deployment guide.
486 lines
15 KiB
Go
486 lines
15 KiB
Go
package services
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.etcd.io/bbolt"
|
|
"warpbox.dev/backend/libs/config"
|
|
)
|
|
|
|
var (
|
|
settingsBucket = []byte("settings")
|
|
usageBucket = []byte("usage")
|
|
)
|
|
|
|
var settingsKey = []byte("upload_policy")
|
|
|
|
type UploadPolicySettings struct {
|
|
AnonymousUploadsEnabled bool `json:"anonymousUploadsEnabled"`
|
|
AnonymousMaxUploadMB float64 `json:"anonymousMaxUploadMb"`
|
|
AnonymousDailyUploadMB float64 `json:"anonymousDailyUploadMb"`
|
|
UserDailyUploadMB float64 `json:"userDailyUploadMb"`
|
|
DefaultUserStorageMB float64 `json:"defaultUserStorageMb"`
|
|
UsageRetentionDays int `json:"usageRetentionDays"`
|
|
LocalStorageMaxGB float64 `json:"localStorageMaxGb"`
|
|
AnonymousMaxDays int `json:"anonymousMaxDays"`
|
|
UserMaxDays int `json:"userMaxDays"`
|
|
AnonymousDailyBoxes int `json:"anonymousDailyBoxes"`
|
|
UserDailyBoxes int `json:"userDailyBoxes"`
|
|
AnonymousActiveBoxes int `json:"anonymousActiveBoxes"`
|
|
UserActiveBoxes int `json:"userActiveBoxes"`
|
|
ShortWindowRequests int `json:"shortWindowRequests"`
|
|
ShortWindowSeconds int `json:"shortWindowSeconds"`
|
|
AnonymousStorageBackend string `json:"anonymousStorageBackend"`
|
|
UserStorageBackend string `json:"userStorageBackend"`
|
|
}
|
|
|
|
type UsageRecord struct {
|
|
Key string `json:"key"`
|
|
SubjectType string `json:"subjectType"`
|
|
Subject string `json:"subject"`
|
|
Date string `json:"date"`
|
|
UploadedBytes int64 `json:"uploadedBytes"`
|
|
UploadedBoxes int `json:"uploadedBoxes"`
|
|
RequestCount int `json:"requestCount"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type EffectiveUploadPolicy struct {
|
|
MaxUploadMB float64
|
|
DailyUploadMB float64
|
|
StorageQuotaMB float64
|
|
MaxDays int
|
|
DailyBoxes int
|
|
ActiveBoxes int
|
|
ShortRequests int
|
|
ShortWindow time.Duration
|
|
StorageBackendID string
|
|
StorageQuotaSet bool
|
|
}
|
|
|
|
type SettingsService struct {
|
|
db *bbolt.DB
|
|
defaults UploadPolicySettings
|
|
}
|
|
|
|
func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*SettingsService, error) {
|
|
service := &SettingsService{
|
|
db: db,
|
|
defaults: UploadPolicySettings{
|
|
AnonymousUploadsEnabled: defaults.AnonymousUploadsEnabled,
|
|
AnonymousMaxUploadMB: defaults.AnonymousMaxUploadMB,
|
|
AnonymousDailyUploadMB: defaults.AnonymousDailyUploadMB,
|
|
UserDailyUploadMB: defaults.UserDailyUploadMB,
|
|
DefaultUserStorageMB: defaults.DefaultUserStorageMB,
|
|
UsageRetentionDays: defaults.UsageRetentionDays,
|
|
LocalStorageMaxGB: defaults.LocalStorageMaxGB,
|
|
AnonymousMaxDays: defaults.AnonymousMaxDays,
|
|
UserMaxDays: defaults.UserMaxDays,
|
|
AnonymousDailyBoxes: defaults.AnonymousDailyBoxes,
|
|
UserDailyBoxes: defaults.UserDailyBoxes,
|
|
AnonymousActiveBoxes: defaults.AnonymousActiveBoxes,
|
|
UserActiveBoxes: defaults.UserActiveBoxes,
|
|
ShortWindowRequests: defaults.ShortWindowRequests,
|
|
ShortWindowSeconds: defaults.ShortWindowSeconds,
|
|
AnonymousStorageBackend: defaults.AnonymousStorageBackend,
|
|
UserStorageBackend: defaults.UserStorageBackend,
|
|
},
|
|
}
|
|
service.defaults = service.withBuiltinDefaultGaps(service.defaults)
|
|
if err := service.validate(service.defaults); err != nil {
|
|
return nil, err
|
|
}
|
|
err := db.Update(func(tx *bbolt.Tx) error {
|
|
for _, bucket := range [][]byte{settingsBucket, usageBucket} {
|
|
if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return service, nil
|
|
}
|
|
|
|
func (s *SettingsService) withBuiltinDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
|
|
if settings.LocalStorageMaxGB <= 0 {
|
|
settings.LocalStorageMaxGB = 100
|
|
}
|
|
if settings.AnonymousMaxDays <= 0 {
|
|
settings.AnonymousMaxDays = 30
|
|
}
|
|
if settings.UserMaxDays <= 0 {
|
|
settings.UserMaxDays = 90
|
|
}
|
|
if settings.AnonymousDailyBoxes <= 0 {
|
|
settings.AnonymousDailyBoxes = 100
|
|
}
|
|
if settings.UserDailyBoxes <= 0 {
|
|
settings.UserDailyBoxes = 250
|
|
}
|
|
if settings.AnonymousActiveBoxes <= 0 {
|
|
settings.AnonymousActiveBoxes = 500
|
|
}
|
|
if settings.UserActiveBoxes <= 0 {
|
|
settings.UserActiveBoxes = 1000
|
|
}
|
|
if settings.ShortWindowRequests <= 0 {
|
|
settings.ShortWindowRequests = 60
|
|
}
|
|
if settings.ShortWindowSeconds <= 0 {
|
|
settings.ShortWindowSeconds = 60
|
|
}
|
|
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
|
|
settings.AnonymousStorageBackend = StorageBackendLocal
|
|
}
|
|
if strings.TrimSpace(settings.UserStorageBackend) == "" {
|
|
settings.UserStorageBackend = StorageBackendLocal
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
|
|
settings := s.defaults
|
|
err := s.db.View(func(tx *bbolt.Tx) error {
|
|
data := tx.Bucket(settingsBucket).Get(settingsKey)
|
|
if data == nil {
|
|
return nil
|
|
}
|
|
if err := json.Unmarshal(data, &settings); err != nil {
|
|
return err
|
|
}
|
|
settings = s.withDefaultGaps(settings)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return UploadPolicySettings{}, err
|
|
}
|
|
if err := s.validate(settings); err != nil {
|
|
return UploadPolicySettings{}, err
|
|
}
|
|
return settings, nil
|
|
}
|
|
|
|
func (s *SettingsService) withDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
|
|
if settings.AnonymousMaxUploadMB == 0 {
|
|
settings.AnonymousMaxUploadMB = s.defaults.AnonymousMaxUploadMB
|
|
}
|
|
if settings.AnonymousDailyUploadMB == 0 {
|
|
settings.AnonymousDailyUploadMB = s.defaults.AnonymousDailyUploadMB
|
|
}
|
|
if settings.UserDailyUploadMB == 0 {
|
|
settings.UserDailyUploadMB = s.defaults.UserDailyUploadMB
|
|
}
|
|
if settings.DefaultUserStorageMB <= 0 {
|
|
settings.DefaultUserStorageMB = s.defaults.DefaultUserStorageMB
|
|
}
|
|
if settings.UsageRetentionDays <= 0 {
|
|
settings.UsageRetentionDays = s.defaults.UsageRetentionDays
|
|
}
|
|
if settings.LocalStorageMaxGB <= 0 {
|
|
settings.LocalStorageMaxGB = s.defaults.LocalStorageMaxGB
|
|
}
|
|
if settings.AnonymousMaxDays <= 0 {
|
|
settings.AnonymousMaxDays = s.defaults.AnonymousMaxDays
|
|
}
|
|
if settings.UserMaxDays <= 0 {
|
|
settings.UserMaxDays = s.defaults.UserMaxDays
|
|
}
|
|
if settings.AnonymousDailyBoxes <= 0 {
|
|
settings.AnonymousDailyBoxes = s.defaults.AnonymousDailyBoxes
|
|
}
|
|
if settings.UserDailyBoxes <= 0 {
|
|
settings.UserDailyBoxes = s.defaults.UserDailyBoxes
|
|
}
|
|
if settings.AnonymousActiveBoxes <= 0 {
|
|
settings.AnonymousActiveBoxes = s.defaults.AnonymousActiveBoxes
|
|
}
|
|
if settings.UserActiveBoxes <= 0 {
|
|
settings.UserActiveBoxes = s.defaults.UserActiveBoxes
|
|
}
|
|
if settings.ShortWindowRequests <= 0 {
|
|
settings.ShortWindowRequests = s.defaults.ShortWindowRequests
|
|
}
|
|
if settings.ShortWindowSeconds <= 0 {
|
|
settings.ShortWindowSeconds = s.defaults.ShortWindowSeconds
|
|
}
|
|
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
|
|
settings.AnonymousStorageBackend = s.defaults.AnonymousStorageBackend
|
|
}
|
|
if strings.TrimSpace(settings.UserStorageBackend) == "" {
|
|
settings.UserStorageBackend = s.defaults.UserStorageBackend
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func (s *SettingsService) UpdateUploadPolicy(settings UploadPolicySettings) error {
|
|
if err := s.validate(settings); err != nil {
|
|
return err
|
|
}
|
|
data, err := json.Marshal(settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket(settingsBucket).Put(settingsKey, data)
|
|
})
|
|
}
|
|
|
|
func (s *SettingsService) Usage(subjectType, subject string, now time.Time) (UsageRecord, error) {
|
|
key := usageKey(subjectType, subject, now)
|
|
var record UsageRecord
|
|
err := s.db.View(func(tx *bbolt.Tx) error {
|
|
data := tx.Bucket(usageBucket).Get([]byte(key))
|
|
if data == nil {
|
|
record = UsageRecord{Key: key, SubjectType: subjectType, Subject: subject, Date: usageDate(now)}
|
|
return nil
|
|
}
|
|
return json.Unmarshal(data, &record)
|
|
})
|
|
return record, err
|
|
}
|
|
|
|
func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now time.Time) error {
|
|
return s.AddUploadUsage(subjectType, subject, bytes, 0, now)
|
|
}
|
|
|
|
func (s *SettingsService) AddUploadUsage(subjectType, subject string, bytes int64, boxes int, now time.Time) error {
|
|
if bytes <= 0 {
|
|
bytes = 0
|
|
}
|
|
if boxes < 0 {
|
|
boxes = 0
|
|
}
|
|
if bytes == 0 && boxes == 0 {
|
|
return nil
|
|
}
|
|
key := usageKey(subjectType, subject, now)
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
bucket := tx.Bucket(usageBucket)
|
|
record := UsageRecord{Key: key, SubjectType: subjectType, Subject: subject, Date: usageDate(now)}
|
|
data := bucket.Get([]byte(key))
|
|
if data != nil {
|
|
if err := json.Unmarshal(data, &record); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
record.UploadedBytes += bytes
|
|
record.UploadedBoxes += boxes
|
|
record.UpdatedAt = now.UTC()
|
|
next, err := json.Marshal(record)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bucket.Put([]byte(key), next)
|
|
})
|
|
}
|
|
|
|
func (s *SettingsService) EffectivePolicyForAnonymous(settings UploadPolicySettings) EffectiveUploadPolicy {
|
|
return EffectiveUploadPolicy{
|
|
MaxUploadMB: settings.AnonymousMaxUploadMB,
|
|
DailyUploadMB: settings.AnonymousDailyUploadMB,
|
|
MaxDays: settings.AnonymousMaxDays,
|
|
DailyBoxes: settings.AnonymousDailyBoxes,
|
|
ActiveBoxes: settings.AnonymousActiveBoxes,
|
|
ShortRequests: settings.ShortWindowRequests,
|
|
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
|
|
StorageBackendID: normalizeBackendID(settings.AnonymousStorageBackend),
|
|
}
|
|
}
|
|
|
|
func (s *SettingsService) EffectivePolicyForUser(settings UploadPolicySettings, user User) EffectiveUploadPolicy {
|
|
policy := EffectiveUploadPolicy{
|
|
MaxUploadMB: 0,
|
|
DailyUploadMB: settings.UserDailyUploadMB,
|
|
StorageQuotaMB: settings.DefaultUserStorageMB,
|
|
MaxDays: settings.UserMaxDays,
|
|
DailyBoxes: settings.UserDailyBoxes,
|
|
ActiveBoxes: settings.UserActiveBoxes,
|
|
ShortRequests: settings.ShortWindowRequests,
|
|
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
|
|
StorageBackendID: normalizeBackendID(settings.UserStorageBackend),
|
|
StorageQuotaSet: true,
|
|
}
|
|
if user.StorageQuotaMB != nil {
|
|
policy.StorageQuotaMB = *user.StorageQuotaMB
|
|
}
|
|
if user.Policy.MaxUploadMB != nil {
|
|
policy.MaxUploadMB = *user.Policy.MaxUploadMB
|
|
}
|
|
if user.Policy.DailyUploadMB != nil {
|
|
policy.DailyUploadMB = *user.Policy.DailyUploadMB
|
|
}
|
|
if user.Policy.StorageQuotaMB != nil {
|
|
policy.StorageQuotaMB = *user.Policy.StorageQuotaMB
|
|
policy.StorageQuotaSet = *user.Policy.StorageQuotaMB > 0
|
|
}
|
|
if user.Policy.MaxDays != nil {
|
|
policy.MaxDays = *user.Policy.MaxDays
|
|
}
|
|
if user.Policy.DailyBoxes != nil {
|
|
policy.DailyBoxes = *user.Policy.DailyBoxes
|
|
}
|
|
if user.Policy.ActiveBoxes != nil {
|
|
policy.ActiveBoxes = *user.Policy.ActiveBoxes
|
|
}
|
|
if user.Policy.ShortWindowRequests != nil {
|
|
policy.ShortRequests = *user.Policy.ShortWindowRequests
|
|
}
|
|
if user.Policy.StorageBackendID != nil {
|
|
policy.StorageBackendID = normalizeBackendID(*user.Policy.StorageBackendID)
|
|
}
|
|
return policy
|
|
}
|
|
|
|
func (s *SettingsService) CleanupUsage(now time.Time, retentionDays int) error {
|
|
if retentionDays <= 0 {
|
|
return fmt.Errorf("usage retention days must be positive")
|
|
}
|
|
cutoff := now.UTC().AddDate(0, 0, -retentionDays)
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
bucket := tx.Bucket(usageBucket)
|
|
return bucket.ForEach(func(key, value []byte) error {
|
|
var record UsageRecord
|
|
if err := json.Unmarshal(value, &record); err != nil {
|
|
return err
|
|
}
|
|
date, err := time.Parse("2006-01-02", record.Date)
|
|
if err != nil || date.Before(cutoff) {
|
|
return bucket.Delete(key)
|
|
}
|
|
return nil
|
|
})
|
|
})
|
|
}
|
|
|
|
func (s *SettingsService) UsageForUser(userID string, now time.Time) (UsageRecord, error) {
|
|
return s.Usage("user", userID, now)
|
|
}
|
|
|
|
func (s *SettingsService) UsageForIP(ip string, now time.Time) (UsageRecord, error) {
|
|
return s.Usage("ip", ip, now)
|
|
}
|
|
|
|
func (s *SettingsService) validate(settings UploadPolicySettings) error {
|
|
if settings.AnonymousMaxUploadMB < 0 && settings.AnonymousMaxUploadMB != -1 || settings.AnonymousMaxUploadMB == 0 {
|
|
return fmt.Errorf("anonymous max upload must be positive or -1 for unlimited")
|
|
}
|
|
if settings.AnonymousDailyUploadMB < 0 && settings.AnonymousDailyUploadMB != -1 || settings.AnonymousDailyUploadMB == 0 {
|
|
return fmt.Errorf("anonymous daily upload must be positive or -1 for unlimited")
|
|
}
|
|
if settings.UserDailyUploadMB < 0 && settings.UserDailyUploadMB != -1 || settings.UserDailyUploadMB == 0 {
|
|
return fmt.Errorf("user daily upload must be positive or -1 for unlimited")
|
|
}
|
|
if settings.DefaultUserStorageMB <= 0 {
|
|
return fmt.Errorf("default user storage must be positive")
|
|
}
|
|
if settings.UsageRetentionDays <= 0 {
|
|
return fmt.Errorf("usage retention days must be positive")
|
|
}
|
|
if settings.LocalStorageMaxGB <= 0 {
|
|
return fmt.Errorf("local storage max must be positive")
|
|
}
|
|
if settings.AnonymousMaxDays <= 0 || settings.UserMaxDays <= 0 {
|
|
return fmt.Errorf("expiration limits must be positive")
|
|
}
|
|
if settings.AnonymousDailyBoxes <= 0 || settings.UserDailyBoxes <= 0 {
|
|
return fmt.Errorf("daily box limits must be positive")
|
|
}
|
|
if settings.AnonymousActiveBoxes <= 0 || settings.UserActiveBoxes <= 0 {
|
|
return fmt.Errorf("active box limits must be positive")
|
|
}
|
|
if settings.ShortWindowRequests <= 0 || settings.ShortWindowSeconds <= 0 {
|
|
return fmt.Errorf("short-window rate limits must be positive")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ParseMegabytesValue(value string) (float64, error) {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return 0, fmt.Errorf("megabyte value is required")
|
|
}
|
|
value = strings.TrimSuffix(value, "MB")
|
|
value = strings.TrimSuffix(value, "Mb")
|
|
value = strings.TrimSuffix(value, "mb")
|
|
value = strings.TrimSpace(value)
|
|
parsed, err := strconv.ParseFloat(value, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if parsed <= 0 {
|
|
return 0, fmt.Errorf("megabyte value must be positive")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func ParseMegabytesLimitValue(value string) (float64, error) {
|
|
parsed, err := parseMegabytesNumber(value)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if parsed == -1 {
|
|
return -1, nil
|
|
}
|
|
if parsed <= 0 {
|
|
return 0, fmt.Errorf("megabyte value must be positive or -1 for unlimited")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func parseMegabytesNumber(value string) (float64, error) {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return 0, fmt.Errorf("megabyte value is required")
|
|
}
|
|
value = strings.TrimSuffix(value, "MB")
|
|
value = strings.TrimSuffix(value, "Mb")
|
|
value = strings.TrimSuffix(value, "mb")
|
|
value = strings.TrimSpace(value)
|
|
return strconv.ParseFloat(value, 64)
|
|
}
|
|
|
|
func MegabytesToBytes(value float64) int64 {
|
|
return int64(value * 1024 * 1024)
|
|
}
|
|
|
|
func GigabytesToBytes(value float64) int64 {
|
|
return int64(value * 1024 * 1024 * 1024)
|
|
}
|
|
|
|
func FormatMegabytesFromBytes(value int64) string {
|
|
mb := float64(value) / 1024 / 1024
|
|
mb = math.Round(mb*100) / 100
|
|
return FormatMegabytesLabel(mb)
|
|
}
|
|
|
|
func FormatMegabytesLabel(value float64) string {
|
|
if value < 0 {
|
|
return "unlimited"
|
|
}
|
|
return strconv.FormatFloat(value, 'f', -1, 64) + " MB"
|
|
}
|
|
|
|
func usageKey(subjectType, subject string, now time.Time) string {
|
|
return subjectType + ":" + subject + ":" + usageDate(now)
|
|
}
|
|
|
|
func usageDate(now time.Time) string {
|
|
return now.UTC().Format("2006-01-02")
|
|
}
|
|
|
|
func normalizeBackendID(id string) string {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
return StorageBackendLocal
|
|
}
|
|
return id
|
|
}
|