feat(storage): add S3 backend support and advanced upload limits
- Introduce S3-compatible storage backend support using minio-go. - Add configuration options for local storage limits, box limits, and rate limiting. - Implement storage backend selection (local vs S3) for anonymous and registered users. - Add an `/admin/storage` management interface. - Update documentation and environment examples with the new configuration variables.
This commit is contained in:
@@ -48,15 +48,27 @@ type AuthService struct {
|
||||
}
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
PasswordHash string `json:"passwordHash"`
|
||||
Role string `json:"role"`
|
||||
Status string `json:"status"`
|
||||
StorageQuotaMB *float64 `json:"storageQuotaMb,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
PasswordHash string `json:"passwordHash"`
|
||||
Role string `json:"role"`
|
||||
Status string `json:"status"`
|
||||
StorageQuotaMB *float64 `json:"storageQuotaMb,omitempty"`
|
||||
Policy UserPolicy `json:"policy,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type UserPolicy struct {
|
||||
MaxUploadMB *float64 `json:"maxUploadMb,omitempty"`
|
||||
DailyUploadMB *float64 `json:"dailyUploadMb,omitempty"`
|
||||
StorageQuotaMB *float64 `json:"storageQuotaMb,omitempty"`
|
||||
MaxDays *int `json:"maxDays,omitempty"`
|
||||
DailyBoxes *int `json:"dailyBoxes,omitempty"`
|
||||
ActiveBoxes *int `json:"activeBoxes,omitempty"`
|
||||
ShortWindowRequests *int `json:"shortWindowRequests,omitempty"`
|
||||
StorageBackendID *string `json:"storageBackendId,omitempty"`
|
||||
}
|
||||
|
||||
type PublicUser struct {
|
||||
@@ -66,6 +78,7 @@ type PublicUser struct {
|
||||
Role string
|
||||
Status string
|
||||
StorageQuotaMB *float64
|
||||
Policy UserPolicy
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
@@ -381,6 +394,97 @@ func (s *AuthService) SetUserStorageQuota(userID string, quotaMB *float64) error
|
||||
return s.saveUser(user)
|
||||
}
|
||||
|
||||
func (s *AuthService) SetUserPolicy(userID string, policy UserPolicy) error {
|
||||
if err := validateUserPolicy(policy); err != nil {
|
||||
return err
|
||||
}
|
||||
user, err := s.UserByID(userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
user.Policy = policy
|
||||
user.StorageQuotaMB = policy.StorageQuotaMB
|
||||
user.UpdatedAt = time.Now().UTC()
|
||||
return s.saveUser(user)
|
||||
}
|
||||
|
||||
func (s *AuthService) SetUserStorageBackend(userID, backendID string) error {
|
||||
user, err := s.UserByID(userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
backendID = strings.TrimSpace(backendID)
|
||||
if backendID == "" {
|
||||
user.Policy.StorageBackendID = nil
|
||||
} else {
|
||||
user.Policy.StorageBackendID = &backendID
|
||||
}
|
||||
user.UpdatedAt = time.Now().UTC()
|
||||
return s.saveUser(user)
|
||||
}
|
||||
|
||||
func (s *AuthService) UpdateUserAdminFields(userID, username, email, role, status string, policy UserPolicy) (User, error) {
|
||||
if err := validateUserPolicy(policy); err != nil {
|
||||
return User{}, err
|
||||
}
|
||||
username = strings.TrimSpace(username)
|
||||
if username == "" {
|
||||
return User{}, fmt.Errorf("username is required")
|
||||
}
|
||||
email, err := normalizeEmail(email)
|
||||
if err != nil {
|
||||
return User{}, err
|
||||
}
|
||||
if role != UserRoleAdmin && role != UserRoleUser {
|
||||
return User{}, fmt.Errorf("invalid role")
|
||||
}
|
||||
if status != UserStatusActive && status != UserStatusDisabled {
|
||||
return User{}, fmt.Errorf("invalid status")
|
||||
}
|
||||
|
||||
var updated User
|
||||
err = s.db.Update(func(tx *bbolt.Tx) error {
|
||||
users := tx.Bucket(usersBucket)
|
||||
emails := tx.Bucket(userEmailsBucket)
|
||||
data := users.Get([]byte(userID))
|
||||
if data == nil {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
var user User
|
||||
if err := json.Unmarshal(data, &user); err != nil {
|
||||
return err
|
||||
}
|
||||
if existing := emails.Get([]byte(email)); existing != nil && string(existing) != user.ID {
|
||||
return fmt.Errorf("email is already registered")
|
||||
}
|
||||
if user.Email != email {
|
||||
if err := emails.Delete([]byte(user.Email)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := emails.Put([]byte(email), []byte(user.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
user.Username = username
|
||||
user.Email = email
|
||||
user.Role = role
|
||||
user.Status = status
|
||||
user.Policy = policy
|
||||
user.StorageQuotaMB = policy.StorageQuotaMB
|
||||
user.UpdatedAt = time.Now().UTC()
|
||||
next, err := json.Marshal(user)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := users.Put([]byte(user.ID), next); err != nil {
|
||||
return err
|
||||
}
|
||||
updated = user
|
||||
return nil
|
||||
})
|
||||
return updated, err
|
||||
}
|
||||
|
||||
func (s *AuthService) UserByID(id string) (User, error) {
|
||||
var user User
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
@@ -476,6 +580,7 @@ func (s *AuthService) PublicUser(user User) PublicUser {
|
||||
Role: user.Role,
|
||||
Status: user.Status,
|
||||
StorageQuotaMB: user.StorageQuotaMB,
|
||||
Policy: user.Policy,
|
||||
CreatedAt: user.CreatedAt,
|
||||
}
|
||||
}
|
||||
@@ -593,3 +698,28 @@ func VerifyPasswordHash(encoded, password string) bool {
|
||||
actual := argon2.IDKey([]byte(password), salt, 1, 64*1024, 4, uint32(len(expected)))
|
||||
return subtle.ConstantTimeCompare(actual, expected) == 1
|
||||
}
|
||||
|
||||
func validateUserPolicy(policy UserPolicy) error {
|
||||
if policy.MaxUploadMB != nil && *policy.MaxUploadMB < 0 {
|
||||
return fmt.Errorf("max upload override cannot be negative")
|
||||
}
|
||||
if policy.DailyUploadMB != nil && *policy.DailyUploadMB <= 0 {
|
||||
return fmt.Errorf("daily upload override must be positive")
|
||||
}
|
||||
if policy.StorageQuotaMB != nil && *policy.StorageQuotaMB < 0 {
|
||||
return fmt.Errorf("storage quota override cannot be negative")
|
||||
}
|
||||
if policy.MaxDays != nil && *policy.MaxDays <= 0 {
|
||||
return fmt.Errorf("expiration override must be positive")
|
||||
}
|
||||
if policy.DailyBoxes != nil && *policy.DailyBoxes <= 0 {
|
||||
return fmt.Errorf("daily box override must be positive")
|
||||
}
|
||||
if policy.ActiveBoxes != nil && *policy.ActiveBoxes <= 0 {
|
||||
return fmt.Errorf("active box override must be positive")
|
||||
}
|
||||
if policy.ShortWindowRequests != nil && *policy.ShortWindowRequests <= 0 {
|
||||
return fmt.Errorf("short-window request override must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -26,6 +26,17 @@ type UploadPolicySettings struct {
|
||||
UserDailyUploadMB float64 `json:"userDailyUploadMb"`
|
||||
DefaultUserStorageMB float64 `json:"defaultUserStorageMb"`
|
||||
UsageRetentionDays int `json:"usageRetentionDays"`
|
||||
LocalStorageMaxGB float64 `json:"localStorageMaxGb"`
|
||||
AnonymousMaxDays int `json:"anonymousMaxDays"`
|
||||
UserMaxDays int `json:"userMaxDays"`
|
||||
AnonymousDailyBoxes int `json:"anonymousDailyBoxes"`
|
||||
UserDailyBoxes int `json:"userDailyBoxes"`
|
||||
AnonymousActiveBoxes int `json:"anonymousActiveBoxes"`
|
||||
UserActiveBoxes int `json:"userActiveBoxes"`
|
||||
ShortWindowRequests int `json:"shortWindowRequests"`
|
||||
ShortWindowSeconds int `json:"shortWindowSeconds"`
|
||||
AnonymousStorageBackend string `json:"anonymousStorageBackend"`
|
||||
UserStorageBackend string `json:"userStorageBackend"`
|
||||
}
|
||||
|
||||
type UsageRecord struct {
|
||||
@@ -34,9 +45,24 @@ type UsageRecord struct {
|
||||
Subject string `json:"subject"`
|
||||
Date string `json:"date"`
|
||||
UploadedBytes int64 `json:"uploadedBytes"`
|
||||
UploadedBoxes int `json:"uploadedBoxes"`
|
||||
RequestCount int `json:"requestCount"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
type EffectiveUploadPolicy struct {
|
||||
MaxUploadMB float64
|
||||
DailyUploadMB float64
|
||||
StorageQuotaMB float64
|
||||
MaxDays int
|
||||
DailyBoxes int
|
||||
ActiveBoxes int
|
||||
ShortRequests int
|
||||
ShortWindow time.Duration
|
||||
StorageBackendID string
|
||||
StorageQuotaSet bool
|
||||
}
|
||||
|
||||
type SettingsService struct {
|
||||
db *bbolt.DB
|
||||
defaults UploadPolicySettings
|
||||
@@ -52,8 +78,20 @@ func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*Settin
|
||||
UserDailyUploadMB: defaults.UserDailyUploadMB,
|
||||
DefaultUserStorageMB: defaults.DefaultUserStorageMB,
|
||||
UsageRetentionDays: defaults.UsageRetentionDays,
|
||||
LocalStorageMaxGB: defaults.LocalStorageMaxGB,
|
||||
AnonymousMaxDays: defaults.AnonymousMaxDays,
|
||||
UserMaxDays: defaults.UserMaxDays,
|
||||
AnonymousDailyBoxes: defaults.AnonymousDailyBoxes,
|
||||
UserDailyBoxes: defaults.UserDailyBoxes,
|
||||
AnonymousActiveBoxes: defaults.AnonymousActiveBoxes,
|
||||
UserActiveBoxes: defaults.UserActiveBoxes,
|
||||
ShortWindowRequests: defaults.ShortWindowRequests,
|
||||
ShortWindowSeconds: defaults.ShortWindowSeconds,
|
||||
AnonymousStorageBackend: defaults.AnonymousStorageBackend,
|
||||
UserStorageBackend: defaults.UserStorageBackend,
|
||||
},
|
||||
}
|
||||
service.defaults = service.withBuiltinDefaultGaps(service.defaults)
|
||||
if err := service.validate(service.defaults); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -71,6 +109,43 @@ func NewSettingsService(db *bbolt.DB, defaults config.SettingsDefaults) (*Settin
|
||||
return service, nil
|
||||
}
|
||||
|
||||
func (s *SettingsService) withBuiltinDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
|
||||
if settings.LocalStorageMaxGB <= 0 {
|
||||
settings.LocalStorageMaxGB = 100
|
||||
}
|
||||
if settings.AnonymousMaxDays <= 0 {
|
||||
settings.AnonymousMaxDays = 30
|
||||
}
|
||||
if settings.UserMaxDays <= 0 {
|
||||
settings.UserMaxDays = 90
|
||||
}
|
||||
if settings.AnonymousDailyBoxes <= 0 {
|
||||
settings.AnonymousDailyBoxes = 100
|
||||
}
|
||||
if settings.UserDailyBoxes <= 0 {
|
||||
settings.UserDailyBoxes = 250
|
||||
}
|
||||
if settings.AnonymousActiveBoxes <= 0 {
|
||||
settings.AnonymousActiveBoxes = 500
|
||||
}
|
||||
if settings.UserActiveBoxes <= 0 {
|
||||
settings.UserActiveBoxes = 1000
|
||||
}
|
||||
if settings.ShortWindowRequests <= 0 {
|
||||
settings.ShortWindowRequests = 60
|
||||
}
|
||||
if settings.ShortWindowSeconds <= 0 {
|
||||
settings.ShortWindowSeconds = 60
|
||||
}
|
||||
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
|
||||
settings.AnonymousStorageBackend = StorageBackendLocal
|
||||
}
|
||||
if strings.TrimSpace(settings.UserStorageBackend) == "" {
|
||||
settings.UserStorageBackend = StorageBackendLocal
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
|
||||
settings := s.defaults
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
@@ -78,7 +153,11 @@ func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
|
||||
if data == nil {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(data, &settings)
|
||||
if err := json.Unmarshal(data, &settings); err != nil {
|
||||
return err
|
||||
}
|
||||
settings = s.withDefaultGaps(settings)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return UploadPolicySettings{}, err
|
||||
@@ -89,6 +168,58 @@ func (s *SettingsService) UploadPolicy() (UploadPolicySettings, error) {
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (s *SettingsService) withDefaultGaps(settings UploadPolicySettings) UploadPolicySettings {
|
||||
if settings.AnonymousMaxUploadMB <= 0 {
|
||||
settings.AnonymousMaxUploadMB = s.defaults.AnonymousMaxUploadMB
|
||||
}
|
||||
if settings.AnonymousDailyUploadMB <= 0 {
|
||||
settings.AnonymousDailyUploadMB = s.defaults.AnonymousDailyUploadMB
|
||||
}
|
||||
if settings.UserDailyUploadMB <= 0 {
|
||||
settings.UserDailyUploadMB = s.defaults.UserDailyUploadMB
|
||||
}
|
||||
if settings.DefaultUserStorageMB <= 0 {
|
||||
settings.DefaultUserStorageMB = s.defaults.DefaultUserStorageMB
|
||||
}
|
||||
if settings.UsageRetentionDays <= 0 {
|
||||
settings.UsageRetentionDays = s.defaults.UsageRetentionDays
|
||||
}
|
||||
if settings.LocalStorageMaxGB <= 0 {
|
||||
settings.LocalStorageMaxGB = s.defaults.LocalStorageMaxGB
|
||||
}
|
||||
if settings.AnonymousMaxDays <= 0 {
|
||||
settings.AnonymousMaxDays = s.defaults.AnonymousMaxDays
|
||||
}
|
||||
if settings.UserMaxDays <= 0 {
|
||||
settings.UserMaxDays = s.defaults.UserMaxDays
|
||||
}
|
||||
if settings.AnonymousDailyBoxes <= 0 {
|
||||
settings.AnonymousDailyBoxes = s.defaults.AnonymousDailyBoxes
|
||||
}
|
||||
if settings.UserDailyBoxes <= 0 {
|
||||
settings.UserDailyBoxes = s.defaults.UserDailyBoxes
|
||||
}
|
||||
if settings.AnonymousActiveBoxes <= 0 {
|
||||
settings.AnonymousActiveBoxes = s.defaults.AnonymousActiveBoxes
|
||||
}
|
||||
if settings.UserActiveBoxes <= 0 {
|
||||
settings.UserActiveBoxes = s.defaults.UserActiveBoxes
|
||||
}
|
||||
if settings.ShortWindowRequests <= 0 {
|
||||
settings.ShortWindowRequests = s.defaults.ShortWindowRequests
|
||||
}
|
||||
if settings.ShortWindowSeconds <= 0 {
|
||||
settings.ShortWindowSeconds = s.defaults.ShortWindowSeconds
|
||||
}
|
||||
if strings.TrimSpace(settings.AnonymousStorageBackend) == "" {
|
||||
settings.AnonymousStorageBackend = s.defaults.AnonymousStorageBackend
|
||||
}
|
||||
if strings.TrimSpace(settings.UserStorageBackend) == "" {
|
||||
settings.UserStorageBackend = s.defaults.UserStorageBackend
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
func (s *SettingsService) UpdateUploadPolicy(settings UploadPolicySettings) error {
|
||||
if err := s.validate(settings); err != nil {
|
||||
return err
|
||||
@@ -117,7 +248,17 @@ func (s *SettingsService) Usage(subjectType, subject string, now time.Time) (Usa
|
||||
}
|
||||
|
||||
func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now time.Time) error {
|
||||
return s.AddUploadUsage(subjectType, subject, bytes, 0, now)
|
||||
}
|
||||
|
||||
func (s *SettingsService) AddUploadUsage(subjectType, subject string, bytes int64, boxes int, now time.Time) error {
|
||||
if bytes <= 0 {
|
||||
bytes = 0
|
||||
}
|
||||
if boxes < 0 {
|
||||
boxes = 0
|
||||
}
|
||||
if bytes == 0 && boxes == 0 {
|
||||
return nil
|
||||
}
|
||||
key := usageKey(subjectType, subject, now)
|
||||
@@ -131,6 +272,7 @@ func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now
|
||||
}
|
||||
}
|
||||
record.UploadedBytes += bytes
|
||||
record.UploadedBoxes += boxes
|
||||
record.UpdatedAt = now.UTC()
|
||||
next, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
@@ -140,6 +282,63 @@ func (s *SettingsService) AddUsage(subjectType, subject string, bytes int64, now
|
||||
})
|
||||
}
|
||||
|
||||
func (s *SettingsService) EffectivePolicyForAnonymous(settings UploadPolicySettings) EffectiveUploadPolicy {
|
||||
return EffectiveUploadPolicy{
|
||||
MaxUploadMB: settings.AnonymousMaxUploadMB,
|
||||
DailyUploadMB: settings.AnonymousDailyUploadMB,
|
||||
MaxDays: settings.AnonymousMaxDays,
|
||||
DailyBoxes: settings.AnonymousDailyBoxes,
|
||||
ActiveBoxes: settings.AnonymousActiveBoxes,
|
||||
ShortRequests: settings.ShortWindowRequests,
|
||||
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
|
||||
StorageBackendID: normalizeBackendID(settings.AnonymousStorageBackend),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SettingsService) EffectivePolicyForUser(settings UploadPolicySettings, user User) EffectiveUploadPolicy {
|
||||
policy := EffectiveUploadPolicy{
|
||||
MaxUploadMB: 0,
|
||||
DailyUploadMB: settings.UserDailyUploadMB,
|
||||
StorageQuotaMB: settings.DefaultUserStorageMB,
|
||||
MaxDays: settings.UserMaxDays,
|
||||
DailyBoxes: settings.UserDailyBoxes,
|
||||
ActiveBoxes: settings.UserActiveBoxes,
|
||||
ShortRequests: settings.ShortWindowRequests,
|
||||
ShortWindow: time.Duration(settings.ShortWindowSeconds) * time.Second,
|
||||
StorageBackendID: normalizeBackendID(settings.UserStorageBackend),
|
||||
StorageQuotaSet: true,
|
||||
}
|
||||
if user.StorageQuotaMB != nil {
|
||||
policy.StorageQuotaMB = *user.StorageQuotaMB
|
||||
}
|
||||
if user.Policy.MaxUploadMB != nil {
|
||||
policy.MaxUploadMB = *user.Policy.MaxUploadMB
|
||||
}
|
||||
if user.Policy.DailyUploadMB != nil {
|
||||
policy.DailyUploadMB = *user.Policy.DailyUploadMB
|
||||
}
|
||||
if user.Policy.StorageQuotaMB != nil {
|
||||
policy.StorageQuotaMB = *user.Policy.StorageQuotaMB
|
||||
policy.StorageQuotaSet = *user.Policy.StorageQuotaMB > 0
|
||||
}
|
||||
if user.Policy.MaxDays != nil {
|
||||
policy.MaxDays = *user.Policy.MaxDays
|
||||
}
|
||||
if user.Policy.DailyBoxes != nil {
|
||||
policy.DailyBoxes = *user.Policy.DailyBoxes
|
||||
}
|
||||
if user.Policy.ActiveBoxes != nil {
|
||||
policy.ActiveBoxes = *user.Policy.ActiveBoxes
|
||||
}
|
||||
if user.Policy.ShortWindowRequests != nil {
|
||||
policy.ShortRequests = *user.Policy.ShortWindowRequests
|
||||
}
|
||||
if user.Policy.StorageBackendID != nil {
|
||||
policy.StorageBackendID = normalizeBackendID(*user.Policy.StorageBackendID)
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func (s *SettingsService) CleanupUsage(now time.Time, retentionDays int) error {
|
||||
if retentionDays <= 0 {
|
||||
return fmt.Errorf("usage retention days must be positive")
|
||||
@@ -185,6 +384,21 @@ func (s *SettingsService) validate(settings UploadPolicySettings) error {
|
||||
if settings.UsageRetentionDays <= 0 {
|
||||
return fmt.Errorf("usage retention days must be positive")
|
||||
}
|
||||
if settings.LocalStorageMaxGB <= 0 {
|
||||
return fmt.Errorf("local storage max must be positive")
|
||||
}
|
||||
if settings.AnonymousMaxDays <= 0 || settings.UserMaxDays <= 0 {
|
||||
return fmt.Errorf("expiration limits must be positive")
|
||||
}
|
||||
if settings.AnonymousDailyBoxes <= 0 || settings.UserDailyBoxes <= 0 {
|
||||
return fmt.Errorf("daily box limits must be positive")
|
||||
}
|
||||
if settings.AnonymousActiveBoxes <= 0 || settings.UserActiveBoxes <= 0 {
|
||||
return fmt.Errorf("active box limits must be positive")
|
||||
}
|
||||
if settings.ShortWindowRequests <= 0 || settings.ShortWindowSeconds <= 0 {
|
||||
return fmt.Errorf("short-window rate limits must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -211,6 +425,10 @@ func MegabytesToBytes(value float64) int64 {
|
||||
return int64(value * 1024 * 1024)
|
||||
}
|
||||
|
||||
func GigabytesToBytes(value float64) int64 {
|
||||
return int64(value * 1024 * 1024 * 1024)
|
||||
}
|
||||
|
||||
func FormatMegabytesFromBytes(value int64) string {
|
||||
mb := float64(value) / 1024 / 1024
|
||||
return FormatMegabytesLabel(mb)
|
||||
@@ -228,6 +446,14 @@ func usageDate(now time.Time) string {
|
||||
return now.UTC().Format("2006-01-02")
|
||||
}
|
||||
|
||||
func normalizeBackendID(id string) string {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return StorageBackendLocal
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func ClientIP(remoteAddr, forwardedFor string) string {
|
||||
if forwardedFor != "" {
|
||||
parts := strings.Split(forwardedFor, ",")
|
||||
|
||||
@@ -146,6 +146,44 @@ func TestDailyUsageAndCleanup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveUserPolicyUsesOverridesAndInheritance(t *testing.T) {
|
||||
settings := newTestSettingsService(t)
|
||||
policy, err := settings.UploadPolicy()
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPolicy returned error: %v", err)
|
||||
}
|
||||
policy.UserDailyUploadMB = 100
|
||||
policy.DefaultUserStorageMB = 200
|
||||
policy.UserMaxDays = 30
|
||||
policy.UserDailyBoxes = 40
|
||||
policy.UserActiveBoxes = 50
|
||||
policy.UserStorageBackend = "local"
|
||||
|
||||
overrideDaily := 300.0
|
||||
overrideQuota := 0.0
|
||||
overrideDays := 12
|
||||
overrideBackend := "bucket-1"
|
||||
user := User{
|
||||
ID: "user-1",
|
||||
Policy: UserPolicy{
|
||||
DailyUploadMB: &overrideDaily,
|
||||
StorageQuotaMB: &overrideQuota,
|
||||
MaxDays: &overrideDays,
|
||||
StorageBackendID: &overrideBackend,
|
||||
},
|
||||
}
|
||||
effective := settings.EffectivePolicyForUser(policy, user)
|
||||
if effective.DailyUploadMB != overrideDaily || effective.MaxDays != overrideDays || effective.StorageBackendID != overrideBackend {
|
||||
t.Fatalf("effective policy did not use overrides: %+v", effective)
|
||||
}
|
||||
if effective.StorageQuotaSet {
|
||||
t.Fatalf("zero storage quota override should mean unlimited: %+v", effective)
|
||||
}
|
||||
if effective.DailyBoxes != policy.UserDailyBoxes || effective.ActiveBoxes != policy.UserActiveBoxes {
|
||||
t.Fatalf("effective policy did not inherit box caps: %+v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestSettingsService(t *testing.T) *SettingsService {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
|
||||
536
backend/libs/services/storage.go
Normal file
536
backend/libs/services/storage.go
Normal file
@@ -0,0 +1,536 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
var storageBackendsBucket = []byte("storage_backends")
|
||||
|
||||
const (
|
||||
StorageBackendLocal = "local"
|
||||
StorageBackendS3 = "s3"
|
||||
|
||||
StorageProviderS3 = "s3"
|
||||
StorageProviderContabo = "contabo"
|
||||
)
|
||||
|
||||
type StorageObject struct {
|
||||
Key string
|
||||
Size int64
|
||||
ContentType string
|
||||
ModTime time.Time
|
||||
Body io.ReadCloser
|
||||
}
|
||||
|
||||
type StorageBackend interface {
|
||||
ID() string
|
||||
Type() string
|
||||
Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error
|
||||
Get(ctx context.Context, key string) (StorageObject, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
DeletePrefix(ctx context.Context, prefix string) error
|
||||
Usage(ctx context.Context) (int64, error)
|
||||
Test(ctx context.Context) error
|
||||
}
|
||||
|
||||
type StorageBackendConfig struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LocalPath string `json:"localPath,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
AccessKey string `json:"accessKey,omitempty"`
|
||||
SecretKey string `json:"secretKey,omitempty"`
|
||||
UseSSL bool `json:"useSsl,omitempty"`
|
||||
PathStyle bool `json:"pathStyle,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
||||
LastTestError string `json:"lastTestError,omitempty"`
|
||||
LastTestSuccess bool `json:"lastTestSuccess,omitempty"`
|
||||
}
|
||||
|
||||
type StorageBackendView struct {
|
||||
Config StorageBackendConfig
|
||||
UsageBytes int64
|
||||
UsageLabel string
|
||||
InUse bool
|
||||
}
|
||||
|
||||
type StorageService struct {
|
||||
db *bbolt.DB
|
||||
localFilesDir string
|
||||
}
|
||||
|
||||
func NewStorageService(db *bbolt.DB, dataDir string) (*StorageService, error) {
|
||||
filesDir := filepath.Join(dataDir, "files")
|
||||
if err := os.MkdirAll(filesDir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
service := &StorageService{db: db, localFilesDir: filesDir}
|
||||
err := db.Update(func(tx *bbolt.Tx) error {
|
||||
_, err := tx.CreateBucketIfNotExists(storageBackendsBucket)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return service, nil
|
||||
}
|
||||
|
||||
func (s *StorageService) LocalFilesDir() string {
|
||||
return s.localFilesDir
|
||||
}
|
||||
|
||||
func (s *StorageService) Backend(id string) (StorageBackend, error) {
|
||||
cfg, err := s.BackendConfig(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !cfg.Enabled {
|
||||
return nil, fmt.Errorf("storage backend is disabled")
|
||||
}
|
||||
return s.backendFromConfig(cfg)
|
||||
}
|
||||
|
||||
func (s *StorageService) BackendConfig(id string) (StorageBackendConfig, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == StorageBackendLocal {
|
||||
return s.localConfig(), nil
|
||||
}
|
||||
var cfg StorageBackendConfig
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
data := tx.Bucket(storageBackendsBucket).Get([]byte(id))
|
||||
if data == nil {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return json.Unmarshal(data, &cfg)
|
||||
})
|
||||
if err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (s *StorageService) ListBackendConfigs() ([]StorageBackendConfig, error) {
|
||||
configs := []StorageBackendConfig{s.localConfig()}
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(storageBackendsBucket).ForEach(func(_, value []byte) error {
|
||||
var cfg StorageBackendConfig
|
||||
if err := json.Unmarshal(value, &cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
configs = append(configs, cfg)
|
||||
return nil
|
||||
})
|
||||
})
|
||||
sort.Slice(configs, func(i, j int) bool {
|
||||
if configs[i].ID == StorageBackendLocal {
|
||||
return true
|
||||
}
|
||||
if configs[j].ID == StorageBackendLocal {
|
||||
return false
|
||||
}
|
||||
return strings.ToLower(configs[i].Name) < strings.ToLower(configs[j].Name)
|
||||
})
|
||||
return configs, err
|
||||
}
|
||||
|
||||
func (s *StorageService) CreateS3Backend(input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
input.ID = randomID(10)
|
||||
input.Type = StorageBackendS3
|
||||
input.Provider = normalizeStorageProvider(input.Provider)
|
||||
if input.Provider == StorageProviderContabo {
|
||||
input.UseSSL = true
|
||||
input.PathStyle = true
|
||||
}
|
||||
input.Name = strings.TrimSpace(input.Name)
|
||||
input.Endpoint = strings.TrimSpace(input.Endpoint)
|
||||
input.Region = strings.TrimSpace(input.Region)
|
||||
input.Bucket = strings.TrimSpace(input.Bucket)
|
||||
input.AccessKey = strings.TrimSpace(input.AccessKey)
|
||||
input.SecretKey = strings.TrimSpace(input.SecretKey)
|
||||
if input.Name == "" {
|
||||
input.Name = input.Bucket
|
||||
}
|
||||
if input.Name == "" || input.Endpoint == "" || input.Bucket == "" || input.AccessKey == "" || input.SecretKey == "" {
|
||||
return StorageBackendConfig{}, fmt.Errorf("name, endpoint, bucket, access key, and secret key are required")
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
input.Enabled = true
|
||||
input.CreatedAt = now
|
||||
input.UpdatedAt = now
|
||||
if err := s.SaveBackendConfig(input); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func (s *StorageService) UpdateS3Backend(id string, input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
current, err := s.BackendConfig(id)
|
||||
if err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
if current.ID == StorageBackendLocal || current.Type != StorageBackendS3 {
|
||||
return StorageBackendConfig{}, fmt.Errorf("only S3-compatible storage can be edited")
|
||||
}
|
||||
|
||||
current.Provider = normalizeStorageProvider(input.Provider)
|
||||
if current.Provider == StorageProviderContabo {
|
||||
input.UseSSL = true
|
||||
input.PathStyle = true
|
||||
}
|
||||
current.Name = strings.TrimSpace(input.Name)
|
||||
current.Endpoint = strings.TrimSpace(input.Endpoint)
|
||||
current.Region = strings.TrimSpace(input.Region)
|
||||
current.Bucket = strings.TrimSpace(input.Bucket)
|
||||
current.AccessKey = strings.TrimSpace(input.AccessKey)
|
||||
if strings.TrimSpace(input.SecretKey) != "" {
|
||||
current.SecretKey = strings.TrimSpace(input.SecretKey)
|
||||
}
|
||||
current.UseSSL = input.UseSSL
|
||||
current.PathStyle = input.PathStyle
|
||||
if current.Name == "" {
|
||||
current.Name = current.Bucket
|
||||
}
|
||||
if current.Name == "" || current.Endpoint == "" || current.Bucket == "" || current.AccessKey == "" || current.SecretKey == "" {
|
||||
return StorageBackendConfig{}, fmt.Errorf("name, endpoint, bucket, access key, and secret key are required")
|
||||
}
|
||||
if err := s.SaveBackendConfig(current); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
|
||||
func (s *StorageService) SaveBackendConfig(cfg StorageBackendConfig) error {
|
||||
if cfg.ID == "" || cfg.ID == StorageBackendLocal {
|
||||
return fmt.Errorf("invalid storage backend id")
|
||||
}
|
||||
cfg.UpdatedAt = time.Now().UTC()
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.db.Update(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(storageBackendsBucket).Put([]byte(cfg.ID), data)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageService) DisableBackend(id string, inUse bool) error {
|
||||
if id == "" || id == StorageBackendLocal {
|
||||
return fmt.Errorf("local storage cannot be disabled")
|
||||
}
|
||||
if inUse {
|
||||
return fmt.Errorf("storage backend is in use")
|
||||
}
|
||||
cfg, err := s.BackendConfig(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Enabled = false
|
||||
return s.SaveBackendConfig(cfg)
|
||||
}
|
||||
|
||||
func (s *StorageService) DeleteBackend(id string, inUse bool) error {
|
||||
if id == "" || id == StorageBackendLocal {
|
||||
return fmt.Errorf("local storage cannot be deleted")
|
||||
}
|
||||
if inUse {
|
||||
return fmt.Errorf("storage backend is in use")
|
||||
}
|
||||
return s.db.Update(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(storageBackendsBucket).Delete([]byte(id))
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageService) TestBackend(id string) (StorageBackendConfig, error) {
|
||||
cfg, err := s.BackendConfig(id)
|
||||
if err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
backend, err := s.backendFromConfig(cfg)
|
||||
if err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
err = backend.Test(context.Background())
|
||||
cfg.LastTestedAt = time.Now().UTC()
|
||||
cfg.LastTestError = ""
|
||||
cfg.LastTestSuccess = err == nil
|
||||
if err != nil {
|
||||
cfg.LastTestError = err.Error()
|
||||
}
|
||||
if cfg.ID != StorageBackendLocal {
|
||||
_ = s.SaveBackendConfig(cfg)
|
||||
}
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
func (s *StorageService) backendFromConfig(cfg StorageBackendConfig) (StorageBackend, error) {
|
||||
switch cfg.Type {
|
||||
case StorageBackendLocal:
|
||||
return localStorageBackend{id: cfg.ID, root: cfg.LocalPath}, nil
|
||||
case StorageBackendS3:
|
||||
return newS3StorageBackend(cfg)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage backend type %q", cfg.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StorageService) localConfig() StorageBackendConfig {
|
||||
now := time.Now().UTC()
|
||||
return StorageBackendConfig{
|
||||
ID: StorageBackendLocal,
|
||||
Name: "Local files",
|
||||
Type: StorageBackendLocal,
|
||||
Provider: StorageBackendLocal,
|
||||
Enabled: true,
|
||||
LocalPath: s.localFilesDir,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
type localStorageBackend struct {
|
||||
id string
|
||||
root string
|
||||
}
|
||||
|
||||
func (b localStorageBackend) ID() string { return b.id }
|
||||
func (b localStorageBackend) Type() string { return StorageBackendLocal }
|
||||
|
||||
func (b localStorageBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Get(_ context.Context, key string) (StorageObject, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
source, err := os.Open(path)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: source}, nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Delete(_ context.Context, key string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) DeletePrefix(_ context.Context, prefix string) error {
|
||||
path, err := b.path(prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Usage(_ context.Context) (int64, error) {
|
||||
var total int64
|
||||
err := filepath.WalkDir(b.root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
total += info.Size()
|
||||
return nil
|
||||
})
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return total, err
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b localStorageBackend) path(key string) (string, error) {
|
||||
key = filepath.Clean(strings.TrimPrefix(key, "/"))
|
||||
if key == "." || strings.HasPrefix(key, "..") || filepath.IsAbs(key) {
|
||||
return "", fmt.Errorf("invalid storage key")
|
||||
}
|
||||
path := filepath.Join(b.root, key)
|
||||
root, err := filepath.Abs(b.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if abs != root && !strings.HasPrefix(abs, root+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("invalid storage key")
|
||||
}
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
type s3StorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func newS3StorageBackend(cfg StorageBackendConfig) (*s3StorageBackend, error) {
|
||||
endpoint := normalizeS3Endpoint(cfg.Endpoint)
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
BucketLookup: s3BucketLookup(cfg.PathStyle),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3StorageBackend{cfg: cfg, client: client}, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b *s3StorageBackend) Type() string { return StorageBackendS3 }
|
||||
|
||||
func (b *s3StorageBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
opts := minio.PutObjectOptions{ContentType: contentType}
|
||||
_, err := b.client.PutObject(ctx, b.cfg.Bucket, cleanObjectKey(key), body, size, opts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
object, err := b.client.GetObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
info, err := object.Stat()
|
||||
if err != nil {
|
||||
object.Close()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: info.Size, ContentType: info.ContentType, ModTime: info.LastModified, Body: object}, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.client.RemoveObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
prefix = strings.TrimSuffix(cleanObjectKey(prefix), "/") + "/"
|
||||
objects := b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
|
||||
for object := range objects {
|
||||
if object.Err != nil {
|
||||
return object.Err
|
||||
}
|
||||
if err := b.Delete(ctx, object.Key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
var total int64
|
||||
for object := range b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Recursive: true}) {
|
||||
if object.Err != nil {
|
||||
return 0, object.Err
|
||||
}
|
||||
total += object.Size
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Test(ctx context.Context) error {
|
||||
exists, err := b.client.BucketExists(ctx, b.cfg.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", b.cfg.Bucket)
|
||||
}
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, bytes.NewReader([]byte("ok")), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func s3BucketLookup(pathStyle bool) minio.BucketLookupType {
|
||||
if pathStyle {
|
||||
return minio.BucketLookupPath
|
||||
}
|
||||
return minio.BucketLookupAuto
|
||||
}
|
||||
|
||||
func normalizeS3Endpoint(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if parsed, err := url.Parse(endpoint); err == nil && parsed.Host != "" {
|
||||
return parsed.Host
|
||||
}
|
||||
return strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
|
||||
}
|
||||
|
||||
func normalizeStorageProvider(provider string) string {
|
||||
switch strings.TrimSpace(provider) {
|
||||
case StorageProviderContabo:
|
||||
return StorageProviderContabo
|
||||
default:
|
||||
return StorageProviderS3
|
||||
}
|
||||
}
|
||||
|
||||
func cleanObjectKey(key string) string {
|
||||
return strings.TrimPrefix(filepath.ToSlash(filepath.Clean(strings.TrimPrefix(key, "/"))), "./")
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package services
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -31,6 +34,7 @@ type UploadService struct {
|
||||
filesDir string
|
||||
db *bbolt.DB
|
||||
logger *slog.Logger
|
||||
storage *StorageService
|
||||
}
|
||||
|
||||
type UploadOptions struct {
|
||||
@@ -41,33 +45,39 @@ type UploadOptions struct {
|
||||
OwnerID string
|
||||
CollectionID string
|
||||
SkipSizeLimit bool
|
||||
CreatorIP string
|
||||
StorageBackendID string
|
||||
}
|
||||
|
||||
type Box struct {
|
||||
ID string `json:"id"`
|
||||
OwnerID string `json:"ownerId,omitempty"`
|
||||
CollectionID string `json:"collectionId,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
MaxDownloads int `json:"maxDownloads"`
|
||||
DownloadCount int `json:"downloadCount"`
|
||||
PasswordSalt string `json:"passwordSalt,omitempty"`
|
||||
PasswordHash string `json:"passwordHash,omitempty"`
|
||||
DeleteTokenHash string `json:"deleteTokenHash,omitempty"`
|
||||
Obfuscate bool `json:"obfuscate"`
|
||||
Files []File `json:"files"`
|
||||
ID string `json:"id"`
|
||||
OwnerID string `json:"ownerId,omitempty"`
|
||||
CollectionID string `json:"collectionId,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
MaxDownloads int `json:"maxDownloads"`
|
||||
DownloadCount int `json:"downloadCount"`
|
||||
PasswordSalt string `json:"passwordSalt,omitempty"`
|
||||
PasswordHash string `json:"passwordHash,omitempty"`
|
||||
DeleteTokenHash string `json:"deleteTokenHash,omitempty"`
|
||||
Obfuscate bool `json:"obfuscate"`
|
||||
CreatorIP string `json:"creatorIp,omitempty"`
|
||||
StorageBackendID string `json:"storageBackendId,omitempty"`
|
||||
Files []File `json:"files"`
|
||||
}
|
||||
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
StoredName string `json:"storedName"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"contentType"`
|
||||
PreviewKind string `json:"previewKind"`
|
||||
Thumbnail string `json:"thumbnail,omitempty"`
|
||||
UploadedAt time.Time `json:"uploadedAt"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
StoredName string `json:"storedName"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"contentType"`
|
||||
PreviewKind string `json:"previewKind"`
|
||||
Thumbnail string `json:"thumbnail,omitempty"`
|
||||
ObjectKey string `json:"objectKey,omitempty"`
|
||||
ThumbnailObjectKey string `json:"thumbnailObjectKey,omitempty"`
|
||||
UploadedAt time.Time `json:"uploadedAt"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
@@ -121,9 +131,6 @@ type UserBox struct {
|
||||
func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog.Logger) (*UploadService, error) {
|
||||
filesDir := filepath.Join(dataDir, "files")
|
||||
dbDir := filepath.Join(dataDir, "db")
|
||||
if err := os.MkdirAll(filesDir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(dbDir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -140,6 +147,11 @@ func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
storage, err := NewStorageService(db, dataDir)
|
||||
if err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &UploadService{
|
||||
maxUploadSize: maxUploadSize,
|
||||
@@ -148,6 +160,7 @@ func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog
|
||||
filesDir: filesDir,
|
||||
db: db,
|
||||
logger: logger,
|
||||
storage: storage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -167,6 +180,10 @@ func (s *UploadService) MaxUploadSizeLabel() string {
|
||||
return helpers.FormatBytes(s.maxUploadSize)
|
||||
}
|
||||
|
||||
func (s *UploadService) Storage() *StorageService {
|
||||
return s.storage
|
||||
}
|
||||
|
||||
func (s *UploadService) ValidateSize(size int64) error {
|
||||
if size > s.maxUploadSize {
|
||||
return fmt.Errorf("file exceeds max upload size of %s", s.MaxUploadSizeLabel())
|
||||
@@ -183,14 +200,16 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
|
||||
}
|
||||
|
||||
box := Box{
|
||||
ID: randomID(10),
|
||||
OwnerID: strings.TrimSpace(opts.OwnerID),
|
||||
CollectionID: strings.TrimSpace(opts.CollectionID),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ExpiresAt: time.Now().UTC().Add(time.Duration(opts.MaxDays) * 24 * time.Hour),
|
||||
MaxDownloads: opts.MaxDownloads,
|
||||
Obfuscate: opts.ObfuscateMetadata && strings.TrimSpace(opts.Password) != "",
|
||||
Files: make([]File, 0, len(files)),
|
||||
ID: randomID(10),
|
||||
OwnerID: strings.TrimSpace(opts.OwnerID),
|
||||
CollectionID: strings.TrimSpace(opts.CollectionID),
|
||||
CreatorIP: strings.TrimSpace(opts.CreatorIP),
|
||||
StorageBackendID: normalizeBackendID(opts.StorageBackendID),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ExpiresAt: time.Now().UTC().Add(time.Duration(opts.MaxDays) * 24 * time.Hour),
|
||||
MaxDownloads: opts.MaxDownloads,
|
||||
Obfuscate: opts.ObfuscateMetadata && strings.TrimSpace(opts.Password) != "",
|
||||
Files: make([]File, 0, len(files)),
|
||||
}
|
||||
deleteToken := randomID(32)
|
||||
box.DeleteTokenHash = deleteTokenHash(box.ID, deleteToken)
|
||||
@@ -200,8 +219,8 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
|
||||
box.PasswordHash = hash
|
||||
}
|
||||
|
||||
boxDir := filepath.Join(s.filesDir, box.ID)
|
||||
if err := os.MkdirAll(boxDir, 0o755); err != nil {
|
||||
backend, err := s.storage.Backend(box.StorageBackendID)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
|
||||
@@ -224,13 +243,18 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
|
||||
|
||||
fileID := randomID(8)
|
||||
storedName := "@each@" + fileID + strings.ToLower(filepath.Ext(header.Filename))
|
||||
storedPath := filepath.Join(boxDir, storedName)
|
||||
objectKey := boxObjectKey(box.ID, storedName)
|
||||
contentType := header.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
buffer := make([]byte, 512)
|
||||
n, _ := file.Read(buffer)
|
||||
contentType = http.DetectContentType(buffer[:n])
|
||||
if seeker, ok := file.(io.Seeker); ok {
|
||||
_, _ = seeker.Seek(0, io.SeekStart)
|
||||
}
|
||||
}
|
||||
|
||||
if err := writeUploadedFile(storedPath, file, maxSize); err != nil {
|
||||
if err := s.writeUploadedObject(context.Background(), backend, objectKey, file, header.Size, maxSize, contentType); err != nil {
|
||||
file.Close()
|
||||
return UploadResult{}, err
|
||||
}
|
||||
@@ -243,6 +267,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
|
||||
Size: header.Size,
|
||||
ContentType: contentType,
|
||||
PreviewKind: previewKind(contentType),
|
||||
ObjectKey: objectKey,
|
||||
UploadedAt: time.Now().UTC(),
|
||||
})
|
||||
}
|
||||
@@ -296,6 +321,29 @@ func (s *UploadService) ListBoxes(limit int) ([]Box, error) {
|
||||
return boxes, err
|
||||
}
|
||||
|
||||
func (s *UploadService) ActiveBoxCountForUser(userID string) (int, error) {
|
||||
return s.activeBoxCount(func(box Box) bool { return box.OwnerID == userID })
|
||||
}
|
||||
|
||||
func (s *UploadService) ActiveBoxCountForIP(ip string) (int, error) {
|
||||
return s.activeBoxCount(func(box Box) bool { return box.OwnerID == "" && box.CreatorIP == ip })
|
||||
}
|
||||
|
||||
func (s *UploadService) activeBoxCount(match func(Box) bool) (int, error) {
|
||||
boxes, err := s.ListBoxes(0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
count := 0
|
||||
for _, box := range boxes {
|
||||
if match(box) && box.ExpiresAt.After(now) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *UploadService) AdminStats() (AdminStats, error) {
|
||||
boxes, err := s.ListBoxes(0)
|
||||
if err != nil {
|
||||
@@ -463,13 +511,22 @@ func (s *UploadService) DeleteBoxWithToken(boxID, token string) error {
|
||||
}
|
||||
|
||||
func (s *UploadService) DeleteBoxWithSource(boxID, source string) error {
|
||||
box, _ := s.GetBox(boxID)
|
||||
if err := s.db.Update(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(boxesBucket).Delete([]byte(boxID))
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
|
||||
return err
|
||||
if box.ID != "" {
|
||||
if backend, err := s.storage.Backend(s.BoxStorageBackendID(box)); err == nil {
|
||||
if err := backend.DeletePrefix(context.Background(), box.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.logger.Info("box deleted", "source", source, "severity", "user_activity", "code", 2101, "box_id", boxID)
|
||||
return nil
|
||||
@@ -499,6 +556,56 @@ func (s *UploadService) BoxMetadataPath(box Box) string {
|
||||
return filepath.Join(s.filesDir, box.ID, ".warpbox.box.json")
|
||||
}
|
||||
|
||||
func (s *UploadService) BoxStorageBackendID(box Box) string {
|
||||
return normalizeBackendID(box.StorageBackendID)
|
||||
}
|
||||
|
||||
func (s *UploadService) FileObjectKey(box Box, file File) string {
|
||||
if file.ObjectKey != "" {
|
||||
return file.ObjectKey
|
||||
}
|
||||
return boxObjectKey(box.ID, file.StoredName)
|
||||
}
|
||||
|
||||
func (s *UploadService) ThumbnailObjectKey(box Box, file File) string {
|
||||
if file.ThumbnailObjectKey != "" {
|
||||
return file.ThumbnailObjectKey
|
||||
}
|
||||
if file.Thumbnail == "" {
|
||||
return ""
|
||||
}
|
||||
return boxObjectKey(box.ID, file.Thumbnail)
|
||||
}
|
||||
|
||||
func (s *UploadService) OpenFileObject(ctx context.Context, box Box, file File) (StorageObject, error) {
|
||||
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return backend.Get(ctx, s.FileObjectKey(box, file))
|
||||
}
|
||||
|
||||
func (s *UploadService) OpenThumbnailObject(ctx context.Context, box Box, file File) (StorageObject, error) {
|
||||
key := s.ThumbnailObjectKey(box, file)
|
||||
if key == "" {
|
||||
return StorageObject{}, os.ErrNotExist
|
||||
}
|
||||
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return backend.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (s *UploadService) PutThumbnailObject(ctx context.Context, box Box, name string, body io.Reader, size int64, contentType string) (string, error) {
|
||||
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := boxObjectKey(box.ID, name)
|
||||
return key, backend.Put(ctx, key, body, size, contentType)
|
||||
}
|
||||
|
||||
func (s *UploadService) IsProtected(box Box) bool {
|
||||
return box.PasswordHash != "" && box.PasswordSalt != ""
|
||||
}
|
||||
@@ -564,11 +671,11 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
|
||||
defer archive.Close()
|
||||
|
||||
for _, file := range box.Files {
|
||||
path := s.FilePath(box, file)
|
||||
source, err := os.Open(path)
|
||||
object, err := s.OpenFileObject(context.Background(), box, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source := object.Body
|
||||
|
||||
header := &zip.FileHeader{
|
||||
Name: file.Name,
|
||||
@@ -592,6 +699,9 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
|
||||
}
|
||||
|
||||
func (s *UploadService) SaveBox(box Box) error {
|
||||
if box.StorageBackendID == "" {
|
||||
box.StorageBackendID = StorageBackendLocal
|
||||
}
|
||||
data, err := json.Marshal(box)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -654,6 +764,27 @@ func writeUploadedFile(path string, source multipart.File, maxSize int64) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *UploadService) writeUploadedObject(ctx context.Context, backend StorageBackend, key string, source multipart.File, size, maxSize int64, contentType string) error {
|
||||
var reader io.Reader = source
|
||||
if maxSize > 0 {
|
||||
reader = io.LimitReader(source, maxSize+1)
|
||||
var buffer bytes.Buffer
|
||||
written, err := io.Copy(&buffer, reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if written > maxSize {
|
||||
return fmt.Errorf("file exceeds max upload size")
|
||||
}
|
||||
return backend.Put(ctx, key, bytes.NewReader(buffer.Bytes()), written, contentType)
|
||||
}
|
||||
return backend.Put(ctx, key, reader, size, contentType)
|
||||
}
|
||||
|
||||
func boxObjectKey(boxID, name string) string {
|
||||
return filepath.ToSlash(filepath.Join(boxID, name))
|
||||
}
|
||||
|
||||
func randomID(byteCount int) string {
|
||||
data := make([]byte, byteCount)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
@@ -691,10 +822,13 @@ func previewKind(contentType string) string {
|
||||
}
|
||||
|
||||
func (s *UploadService) writeBoxMetadata(box Box) error {
|
||||
path := s.BoxMetadataPath(box)
|
||||
data, err := json.MarshalIndent(box, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o600)
|
||||
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return backend.Put(context.Background(), boxObjectKey(box.ID, ".warpbox.box.json"), bytes.NewReader(data), int64(len(data)), "application/json")
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
@@ -93,6 +94,64 @@ func TestUserActiveStorageUsedIgnoresExpiredBoxes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalStorageBackendAndLegacyFallback(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
result := createTestBox(t, service, "file.txt", "hello")
|
||||
box := getTestBox(t, service, result.BoxID)
|
||||
if service.BoxStorageBackendID(box) != StorageBackendLocal {
|
||||
t.Fatalf("BoxStorageBackendID = %q", service.BoxStorageBackendID(box))
|
||||
}
|
||||
if box.Files[0].ObjectKey == "" {
|
||||
t.Fatalf("new file did not store object key")
|
||||
}
|
||||
object, err := service.OpenFileObject(testContext(), box, box.Files[0])
|
||||
if err != nil {
|
||||
t.Fatalf("OpenFileObject returned error: %v", err)
|
||||
}
|
||||
data, err := io.ReadAll(object.Body)
|
||||
object.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll returned error: %v", err)
|
||||
}
|
||||
if string(data) != "hello" {
|
||||
t.Fatalf("object body = %q", string(data))
|
||||
}
|
||||
|
||||
box.StorageBackendID = ""
|
||||
box.Files[0].ObjectKey = ""
|
||||
object, err = service.OpenFileObject(testContext(), box, box.Files[0])
|
||||
if err != nil {
|
||||
t.Fatalf("legacy OpenFileObject returned error: %v", err)
|
||||
}
|
||||
object.Body.Close()
|
||||
}
|
||||
|
||||
func TestContaboStorageConfigAllowsDisplayNamesWithSpaces(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
cfg, err := service.Storage().CreateS3Backend(StorageBackendConfig{
|
||||
Provider: StorageProviderContabo,
|
||||
Name: "Contabo main",
|
||||
Endpoint: "https://eu2.contabostorage.com",
|
||||
Region: "EU",
|
||||
Bucket: "My Main Bucket",
|
||||
AccessKey: "access",
|
||||
SecretKey: "secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateS3Backend returned error: %v", err)
|
||||
}
|
||||
if cfg.Provider != StorageProviderContabo || !cfg.UseSSL || !cfg.PathStyle {
|
||||
t.Fatalf("contabo config was not normalized: %+v", cfg)
|
||||
}
|
||||
if cfg.Bucket != "My Main Bucket" {
|
||||
t.Fatalf("bucket = %q", cfg.Bucket)
|
||||
}
|
||||
}
|
||||
|
||||
func testContext() context.Context {
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
func newTestUploadService(t *testing.T) *UploadService {
|
||||
t.Helper()
|
||||
service, err := NewUploadService(1024*1024, t.TempDir(), "http://example.test", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
|
||||
Reference in New Issue
Block a user