- Add backend services to create, list, and delete API tokens. - Implement Bearer token authentication to resolve tokens to users. - Register HTTP routes for managing user tokens under `/account/tokens`. - Add tests to verify that uploads with valid Bearer tokens associate the upload with the correct user, while invalid tokens fall back to anonymous uploads.
474 lines
14 KiB
Go
474 lines
14 KiB
Go
package services
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"net"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.etcd.io/bbolt"
|
|
"warpbox.dev/backend/libs/config"
|
|
)
|
|
|
|
var (
|
|
settingsBucket = []byte("settings")
|
|
usageBucket = []byte("usage")
|
|
)
|
|
|
|
var settingsKey = []byte("upload_policy")
|
|
|
|
type UploadPolicySettings struct {
|
|
AnonymousUploadsEnabled bool `json:"anonymousUploadsEnabled"`
|
|
AnonymousMaxUploadMB float64 `json:"anonymousMaxUploadMb"`
|
|
AnonymousDailyUploadMB float64 `json:"anonymousDailyUploadMb"`
|
|
UserDailyUploadMB float64 `json:"userDailyUploadMb"`
|
|
DefaultUserStorageMB float64 `json:"defaultUserStorageMb"`
|
|
UsageRetentionDays int `json:"usageRetentionDays"`
|
|
LocalStorageMaxGB float64 `json:"localStorageMaxGb"`
|
|
AnonymousMaxDays int `json:"anonymousMaxDays"`
|
|
UserMaxDays int `json:"userMaxDays"`
|
|
AnonymousDailyBoxes int `json:"anonymousDailyBoxes"`
|
|
UserDailyBoxes int `json:"userDailyBoxes"`
|
|
AnonymousActiveBoxes int `json:"anonymousActiveBoxes"`
|
|
UserActiveBoxes int `json:"userActiveBoxes"`
|
|
ShortWindowRequests int `json:"shortWindowRequests"`
|
|
ShortWindowSeconds int `json:"shortWindowSeconds"`
|
|
AnonymousStorageBackend string `json:"anonymousStorageBackend"`
|
|
UserStorageBackend string `json:"userStorageBackend"`
|
|
}
|
|
|
|
type UsageRecord struct {
|
|
Key string `json:"key"`
|
|
SubjectType string `json:"subjectType"`
|
|
Subject string `json:"subject"`
|
|
Date string `json:"date"`
|
|
UploadedBytes int64 `json:"uploadedBytes"`
|
|
UploadedBoxes int `json:"uploadedBoxes"`
|
|
RequestCount int `json:"requestCount"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type EffectiveUploadPolicy struct {
|
|
MaxUploadMB float64
|
|
DailyUploadMB float64
|
|
StorageQuotaMB float64
|
|
MaxDays int
|
|
DailyBoxes int
|
|
ActiveBoxes int
|
|
ShortRequests int
|
|
ShortWindow time.Duration
|
|
StorageBackendID string
|
|
StorageQuotaSet bool
|
|
}
|
|
|
|
type SettingsService struct {
|
|
db *bbolt.DB
|
|
defaults UploadPolicySettings
|
|
}
|
|
|
|
func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*SettingsService, error) {
|
|
service := &SettingsService{
|
|
db: db,
|
|
defaults: UploadPolicySettings{
|
|
AnonymousUploadsEnabled: defaults.AnonymousUploadsEnabled,
|
|
AnonymousMaxUploadMB: defaults.AnonymousMaxUploadMB,
|
|
AnonymousDailyUploadMB: defaults.AnonymousDailyUploadMB,
|
|
UserDailyUploadMB: defaults.UserDailyUploadMB,
|
|
DefaultUserStorageMB: defaults.DefaultUserStorageMB,
|
|
UsageRetentionDays: defaults.UsageRetentionDays,
|
|
LocalStorageMaxGB: defaults.LocalStorageMaxGB,
|
|
AnonymousMaxDays: defaults.AnonymousMaxDays,
|
|
UserMaxDays: defaults.UserMaxDays,
|
|
AnonymousDailyBoxes: defaults.AnonymousDailyBoxes,
|
|
UserDailyBoxes: defaults.UserDailyBoxes,
|
|
AnonymousActiveBoxes: defaults.AnonymousActiveBoxes,
|
|
UserActiveBoxes: defaults.UserActiveBoxes,
|
|
ShortWindowRequests: defaults.ShortWindowRequests,
|
|
ShortWindowSeconds: defaults.ShortWindowSeconds,
|
|
AnonymousStorageBackend: defaults.AnonymousStorageBackend,
|
|
UserStorageBackend: defaults.UserStorageBackend,
|
|
},
|
|
}
|
|
service.defaults = service.withBuiltinDefaultGaps(service.defaults)
|
|
if err := service.validate(service.defaults); err != nil {
|
|
return nil, err
|
|
}
|
|
err := db.Update(func(tx *bbolt.Tx) error {
|
|
for _, bucket := range [][]byte{settingsBucket, usageBucket} {
|
|
if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return service, nil
|
|
}
|
|
|
|
func (s *SettingsService) withBuiltinDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
|
|
if settings.LocalStorageMaxGB <= 0 {
|
|
settings.LocalStorageMaxGB = 100
|
|
}
|
|
if settings.AnonymousMaxDays <= 0 {
|
|
settings.AnonymousMaxDays = 30
|
|
}
|
|
if settings.UserMaxDays <= 0 {
|
|
settings.UserMaxDays = 90
|
|
}
|
|
if settings.AnonymousDailyBoxes <= 0 {
|
|
settings.AnonymousDailyBoxes = 100
|
|
}
|
|
if settings.UserDailyBoxes <= 0 {
|
|
settings.UserDailyBoxes = 250
|
|
}
|
|
if settings.AnonymousActiveBoxes <= 0 {
|
|
settings.AnonymousActiveBoxes = 500
|
|
}
|
|
if settings.UserActiveBoxes <= 0 {
|
|
settings.UserActiveBoxes = 1000
|
|
}
|
|
if settings.ShortWindowRequests <= 0 {
|
|
settings.ShortWindowRequests = 60
|
|
}
|
|
if settings.ShortWindowSeconds <= 0 {
|
|
settings.ShortWindowSeconds = 60
|
|
}
|
|
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
|
|
settings.AnonymousStorageBackend = StorageBackendLocal
|
|
}
|
|
if strings.TrimSpace(settings.UserStorageBackend) == "" {
|
|
settings.UserStorageBackend = StorageBackendLocal
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
|
|
settings := s.defaults
|
|
err := s.db.View(func(tx *bbolt.Tx) error {
|
|
data := tx.Bucket(settingsBucket).Get(settingsKey)
|
|
if data == nil {
|
|
return nil
|
|
}
|
|
if err := json.Unmarshal(data, &settings); err != nil {
|
|
return err
|
|
}
|
|
settings = s.withDefaultGaps(settings)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return UploadPolicySettings{}, err
|
|
}
|
|
if err := s.validate(settings); err != nil {
|
|
return UploadPolicySettings{}, err
|
|
}
|
|
return settings, nil
|
|
}
|
|
|
|
func (s *SettingsService) withDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
|
|
if settings.AnonymousMaxUploadMB <= 0 {
|
|
settings.AnonymousMaxUploadMB = s.defaults.AnonymousMaxUploadMB
|
|
}
|
|
if settings.AnonymousDailyUploadMB <= 0 {
|
|
settings.AnonymousDailyUploadMB = s.defaults.AnonymousDailyUploadMB
|
|
}
|
|
if settings.UserDailyUploadMB <= 0 {
|
|
settings.UserDailyUploadMB = s.defaults.UserDailyUploadMB
|
|
}
|
|
if settings.DefaultUserStorageMB <= 0 {
|
|
settings.DefaultUserStorageMB = s.defaults.DefaultUserStorageMB
|
|
}
|
|
if settings.UsageRetentionDays <= 0 {
|
|
settings.UsageRetentionDays = s.defaults.UsageRetentionDays
|
|
}
|
|
if settings.LocalStorageMaxGB <= 0 {
|
|
settings.LocalStorageMaxGB = s.defaults.LocalStorageMaxGB
|
|
}
|
|
if settings.AnonymousMaxDays <= 0 {
|
|
settings.AnonymousMaxDays = s.defaults.AnonymousMaxDays
|
|
}
|
|
if settings.UserMaxDays <= 0 {
|
|
settings.UserMaxDays = s.defaults.UserMaxDays
|
|
}
|
|
if settings.AnonymousDailyBoxes <= 0 {
|
|
settings.AnonymousDailyBoxes = s.defaults.AnonymousDailyBoxes
|
|
}
|
|
if settings.UserDailyBoxes <= 0 {
|
|
settings.UserDailyBoxes = s.defaults.UserDailyBoxes
|
|
}
|
|
if settings.AnonymousActiveBoxes <= 0 {
|
|
settings.AnonymousActiveBoxes = s.defaults.AnonymousActiveBoxes
|
|
}
|
|
if settings.UserActiveBoxes <= 0 {
|
|
settings.UserActiveBoxes = s.defaults.UserActiveBoxes
|
|
}
|
|
if settings.ShortWindowRequests <= 0 {
|
|
settings.ShortWindowRequests = s.defaults.ShortWindowRequests
|
|
}
|
|
if settings.ShortWindowSeconds <= 0 {
|
|
settings.ShortWindowSeconds = s.defaults.ShortWindowSeconds
|
|
}
|
|
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
|
|
settings.AnonymousStorageBackend = s.defaults.AnonymousStorageBackend
|
|
}
|
|
if strings.TrimSpace(settings.UserStorageBackend) == "" {
|
|
settings.UserStorageBackend = s.defaults.UserStorageBackend
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func (s *SettingsService) UpdateUploadPolicy(settings UploadPolicySettings) error {
|
|
if err := s.validate(settings); err != nil {
|
|
return err
|
|
}
|
|
data, err := json.Marshal(settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket(settingsBucket).Put(settingsKey, data)
|
|
})
|
|
}
|
|
|
|
func (s *SettingsService) Usage(subjectType, subject string, now time.Time) (UsageRecord, error) {
|
|
key := usageKey(subjectType, subject, now)
|
|
var record UsageRecord
|
|
err := s.db.View(func(tx *bbolt.Tx) error {
|
|
data := tx.Bucket(usageBucket).Get([]byte(key))
|
|
if data == nil {
|
|
record = UsageRecord{Key: key, SubjectType: subjectType, Subject: subject, Date: usageDate(now)}
|
|
return nil
|
|
}
|
|
return json.Unmarshal(data, &record)
|
|
})
|
|
return record, err
|
|
}
|
|
|
|
func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now time.Time) error {
|
|
return s.AddUploadUsage(subjectType, subject, bytes, 0, now)
|
|
}
|
|
|
|
func (s *SettingsService) AddUploadUsage(subjectType, subject string, bytes int64, boxes int, now time.Time) error {
|
|
if bytes <= 0 {
|
|
bytes = 0
|
|
}
|
|
if boxes < 0 {
|
|
boxes = 0
|
|
}
|
|
if bytes == 0 && boxes == 0 {
|
|
return nil
|
|
}
|
|
key := usageKey(subjectType, subject, now)
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
bucket := tx.Bucket(usageBucket)
|
|
record := UsageRecord{Key: key, SubjectType: subjectType, Subject: subject, Date: usageDate(now)}
|
|
data := bucket.Get([]byte(key))
|
|
if data != nil {
|
|
if err := json.Unmarshal(data, &record); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
record.UploadedBytes += bytes
|
|
record.UploadedBoxes += boxes
|
|
record.UpdatedAt = now.UTC()
|
|
next, err := json.Marshal(record)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bucket.Put([]byte(key), next)
|
|
})
|
|
}
|
|
|
|
func (s *SettingsService) EffectivePolicyForAnonymous(settings UploadPolicySettings) EffectiveUploadPolicy {
|
|
return EffectiveUploadPolicy{
|
|
MaxUploadMB: settings.AnonymousMaxUploadMB,
|
|
DailyUploadMB: settings.AnonymousDailyUploadMB,
|
|
MaxDays: settings.AnonymousMaxDays,
|
|
DailyBoxes: settings.AnonymousDailyBoxes,
|
|
ActiveBoxes: settings.AnonymousActiveBoxes,
|
|
ShortRequests: settings.ShortWindowRequests,
|
|
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
|
|
StorageBackendID: normalizeBackendID(settings.AnonymousStorageBackend),
|
|
}
|
|
}
|
|
|
|
func (s *SettingsService) EffectivePolicyForUser(settings UploadPolicySettings, user User) EffectiveUploadPolicy {
|
|
policy := EffectiveUploadPolicy{
|
|
MaxUploadMB: 0,
|
|
DailyUploadMB: settings.UserDailyUploadMB,
|
|
StorageQuotaMB: settings.DefaultUserStorageMB,
|
|
MaxDays: settings.UserMaxDays,
|
|
DailyBoxes: settings.UserDailyBoxes,
|
|
ActiveBoxes: settings.UserActiveBoxes,
|
|
ShortRequests: settings.ShortWindowRequests,
|
|
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
|
|
StorageBackendID: normalizeBackendID(settings.UserStorageBackend),
|
|
StorageQuotaSet: true,
|
|
}
|
|
if user.StorageQuotaMB != nil {
|
|
policy.StorageQuotaMB = *user.StorageQuotaMB
|
|
}
|
|
if user.Policy.MaxUploadMB != nil {
|
|
policy.MaxUploadMB = *user.Policy.MaxUploadMB
|
|
}
|
|
if user.Policy.DailyUploadMB != nil {
|
|
policy.DailyUploadMB = *user.Policy.DailyUploadMB
|
|
}
|
|
if user.Policy.StorageQuotaMB != nil {
|
|
policy.StorageQuotaMB = *user.Policy.StorageQuotaMB
|
|
policy.StorageQuotaSet = *user.Policy.StorageQuotaMB > 0
|
|
}
|
|
if user.Policy.MaxDays != nil {
|
|
policy.MaxDays = *user.Policy.MaxDays
|
|
}
|
|
if user.Policy.DailyBoxes != nil {
|
|
policy.DailyBoxes = *user.Policy.DailyBoxes
|
|
}
|
|
if user.Policy.ActiveBoxes != nil {
|
|
policy.ActiveBoxes = *user.Policy.ActiveBoxes
|
|
}
|
|
if user.Policy.ShortWindowRequests != nil {
|
|
policy.ShortRequests = *user.Policy.ShortWindowRequests
|
|
}
|
|
if user.Policy.StorageBackendID != nil {
|
|
policy.StorageBackendID = normalizeBackendID(*user.Policy.StorageBackendID)
|
|
}
|
|
return policy
|
|
}
|
|
|
|
func (s *SettingsService) CleanupUsage(now time.Time, retentionDays int) error {
|
|
if retentionDays <= 0 {
|
|
return fmt.Errorf("usage retention days must be positive")
|
|
}
|
|
cutoff := now.UTC().AddDate(0, 0, -retentionDays)
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
bucket := tx.Bucket(usageBucket)
|
|
return bucket.ForEach(func(key, value []byte) error {
|
|
var record UsageRecord
|
|
if err := json.Unmarshal(value, &record); err != nil {
|
|
return err
|
|
}
|
|
date, err := time.Parse("2006-01-02", record.Date)
|
|
if err != nil || date.Before(cutoff) {
|
|
return bucket.Delete(key)
|
|
}
|
|
return nil
|
|
})
|
|
})
|
|
}
|
|
|
|
func (s *SettingsService) UsageForUser(userID string, now time.Time) (UsageRecord, error) {
|
|
return s.Usage("user", userID, now)
|
|
}
|
|
|
|
func (s *SettingsService) UsageForIP(ip string, now time.Time) (UsageRecord, error) {
|
|
return s.Usage("ip", ip, now)
|
|
}
|
|
|
|
func (s *SettingsService) validate(settings UploadPolicySettings) error {
|
|
if settings.AnonymousMaxUploadMB <= 0 {
|
|
return fmt.Errorf("anonymous max upload must be positive")
|
|
}
|
|
if settings.AnonymousDailyUploadMB <= 0 {
|
|
return fmt.Errorf("anonymous daily upload must be positive")
|
|
}
|
|
if settings.UserDailyUploadMB <= 0 {
|
|
return fmt.Errorf("user daily upload must be positive")
|
|
}
|
|
if settings.DefaultUserStorageMB <= 0 {
|
|
return fmt.Errorf("default user storage must be positive")
|
|
}
|
|
if settings.UsageRetentionDays <= 0 {
|
|
return fmt.Errorf("usage retention days must be positive")
|
|
}
|
|
if settings.LocalStorageMaxGB <= 0 {
|
|
return fmt.Errorf("local storage max must be positive")
|
|
}
|
|
if settings.AnonymousMaxDays <= 0 || settings.UserMaxDays <= 0 {
|
|
return fmt.Errorf("expiration limits must be positive")
|
|
}
|
|
if settings.AnonymousDailyBoxes <= 0 || settings.UserDailyBoxes <= 0 {
|
|
return fmt.Errorf("daily box limits must be positive")
|
|
}
|
|
if settings.AnonymousActiveBoxes <= 0 || settings.UserActiveBoxes <= 0 {
|
|
return fmt.Errorf("active box limits must be positive")
|
|
}
|
|
if settings.ShortWindowRequests <= 0 || settings.ShortWindowSeconds <= 0 {
|
|
return fmt.Errorf("short-window rate limits must be positive")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ParseMegabytesValue(value string) (float64, error) {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return 0, fmt.Errorf("megabyte value is required")
|
|
}
|
|
value = strings.TrimSuffix(value, "MB")
|
|
value = strings.TrimSuffix(value, "Mb")
|
|
value = strings.TrimSuffix(value, "mb")
|
|
value = strings.TrimSpace(value)
|
|
parsed, err := strconv.ParseFloat(value, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if parsed <= 0 {
|
|
return 0, fmt.Errorf("megabyte value must be positive")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func MegabytesToBytes(value float64) int64 {
|
|
return int64(value * 1024 * 1024)
|
|
}
|
|
|
|
func GigabytesToBytes(value float64) int64 {
|
|
return int64(value * 1024 * 1024 * 1024)
|
|
}
|
|
|
|
func FormatMegabytesFromBytes(value int64) string {
|
|
mb := float64(value) / 1024 / 1024
|
|
mb = math.Round(mb*100) / 100
|
|
return FormatMegabytesLabel(mb)
|
|
}
|
|
|
|
func FormatMegabytesLabel(value float64) string {
|
|
return strconv.FormatFloat(value, 'f', -1, 64) + " MB"
|
|
}
|
|
|
|
func usageKey(subjectType, subject string, now time.Time) string {
|
|
return subjectType + ":" + subject + ":" + usageDate(now)
|
|
}
|
|
|
|
func usageDate(now time.Time) string {
|
|
return now.UTC().Format("2006-01-02")
|
|
}
|
|
|
|
func normalizeBackendID(id string) string {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
return StorageBackendLocal
|
|
}
|
|
return id
|
|
}
|
|
|
|
func ClientIP(remoteAddr, forwardedFor string) string {
|
|
if forwardedFor != "" {
|
|
parts := strings.Split(forwardedFor, ",")
|
|
if ip := strings.TrimSpace(parts[0]); ip != "" {
|
|
return ip
|
|
}
|
|
}
|
|
host := remoteAddr
|
|
if strings.Contains(remoteAddr, ":") {
|
|
if splitHost, _, err := net.SplitHostPort(remoteAddr); err == nil {
|
|
host = splitHost
|
|
}
|
|
}
|
|
return host
|
|
}
|