kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): add S3 backend support and advanced upload limits

Browse Source 
- Introduce S3-compatible storage backend support using minio-go.
- Add configuration options for local storage limits, box limits, and rate limiting.
- Implement storage backend selection (local vs S3) for anonymous and registered users.
- Add an `/admin/storage` management interface.
- Update documentation and environment examples with the new configuration variables.
This commit is contained in:
Daniel Legt
2026-05-31 02:14:10 +03:00
parent 830d2a885c
commit c3558fd353
34 changed files with 2668 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
backend/libs/services/upload.go
View File

@@ -2,6 +2,8 @@ package services
import (
"archive/zip"
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
@@ -12,6 +14,7 @@ import (
"io"
"log/slog"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"sort"
@@ -31,6 +34,7 @@ type UploadService struct {
filesDir string
db *bbolt.DB
logger *slog.Logger
storage *StorageService
}
type UploadOptions struct {
@@ -41,33 +45,39 @@ type UploadOptions struct {
OwnerID string
CollectionID string
SkipSizeLimit bool
CreatorIP string
StorageBackendID string
}
type Box struct {
ID string `json:"id"`
OwnerID string `json:"ownerId,omitempty"`
CollectionID string `json:"collectionId,omitempty"`
Title string `json:"title,omitempty"`
CreatedAt time.Time `json:"createdAt"`
ExpiresAt time.Time `json:"expiresAt"`
MaxDownloads int `json:"maxDownloads"`
DownloadCount int `json:"downloadCount"`
PasswordSalt string `json:"passwordSalt,omitempty"`
PasswordHash string `json:"passwordHash,omitempty"`
DeleteTokenHash string `json:"deleteTokenHash,omitempty"`
Obfuscate bool `json:"obfuscate"`
Files []File `json:"files"`
ID string `json:"id"`
OwnerID string `json:"ownerId,omitempty"`
CollectionID string `json:"collectionId,omitempty"`
Title string `json:"title,omitempty"`
CreatedAt time.Time `json:"createdAt"`
ExpiresAt time.Time `json:"expiresAt"`
MaxDownloads int `json:"maxDownloads"`
DownloadCount int `json:"downloadCount"`
PasswordSalt string `json:"passwordSalt,omitempty"`
PasswordHash string `json:"passwordHash,omitempty"`
DeleteTokenHash string `json:"deleteTokenHash,omitempty"`
Obfuscate bool `json:"obfuscate"`
CreatorIP string `json:"creatorIp,omitempty"`
StorageBackendID string `json:"storageBackendId,omitempty"`
Files []File `json:"files"`
}
type File struct {
ID string `json:"id"`
Name string `json:"name"`
StoredName string `json:"storedName"`
Size int64 `json:"size"`
ContentType string `json:"contentType"`
PreviewKind string `json:"previewKind"`
Thumbnail string `json:"thumbnail,omitempty"`
UploadedAt time.Time `json:"uploadedAt"`
ID string `json:"id"`
Name string `json:"name"`
StoredName string `json:"storedName"`
Size int64 `json:"size"`
ContentType string `json:"contentType"`
PreviewKind string `json:"previewKind"`
Thumbnail string `json:"thumbnail,omitempty"`
ObjectKey string `json:"objectKey,omitempty"`
ThumbnailObjectKey string `json:"thumbnailObjectKey,omitempty"`
UploadedAt time.Time `json:"uploadedAt"`
}
type UploadResult struct {
@@ -121,9 +131,6 @@ type UserBox struct {
func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog.Logger) (*UploadService, error) {
filesDir := filepath.Join(dataDir, "files")
dbDir := filepath.Join(dataDir, "db")
if err := os.MkdirAll(filesDir, 0o755); err != nil {
return nil, err
}
if err := os.MkdirAll(dbDir, 0o755); err != nil {
return nil, err
}
@@ -140,6 +147,11 @@ func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog
db.Close()
return nil, err
}
storage, err := NewStorageService(db, dataDir)
if err != nil {
db.Close()
return nil, err
}
return &UploadService{
maxUploadSize: maxUploadSize,
@@ -148,6 +160,7 @@ func NewUploadService(maxUploadSize int64, dataDir, baseURL string, logger *slog
filesDir: filesDir,
db: db,
logger: logger,
storage: storage,
}, nil
}
@@ -167,6 +180,10 @@ func (s *UploadService) MaxUploadSizeLabel() string {
return helpers.FormatBytes(s.maxUploadSize)
}
func (s *UploadService) Storage() *StorageService {
return s.storage
}
func (s *UploadService) ValidateSize(size int64) error {
if size > s.maxUploadSize {
return fmt.Errorf("file exceeds max upload size of %s", s.MaxUploadSizeLabel())
@@ -183,14 +200,16 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
}
box := Box{
ID: randomID(10),
OwnerID: strings.TrimSpace(opts.OwnerID),
CollectionID: strings.TrimSpace(opts.CollectionID),
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Duration(opts.MaxDays) * 24 * time.Hour),
MaxDownloads: opts.MaxDownloads,
Obfuscate: opts.ObfuscateMetadata && strings.TrimSpace(opts.Password) != "",
Files: make([]File, 0, len(files)),
ID: randomID(10),
OwnerID: strings.TrimSpace(opts.OwnerID),
CollectionID: strings.TrimSpace(opts.CollectionID),
CreatorIP: strings.TrimSpace(opts.CreatorIP),
StorageBackendID: normalizeBackendID(opts.StorageBackendID),
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Duration(opts.MaxDays) * 24 * time.Hour),
MaxDownloads: opts.MaxDownloads,
Obfuscate: opts.ObfuscateMetadata && strings.TrimSpace(opts.Password) != "",
Files: make([]File, 0, len(files)),
}
deleteToken := randomID(32)
box.DeleteTokenHash = deleteTokenHash(box.ID, deleteToken)
@@ -200,8 +219,8 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
box.PasswordHash = hash
}
boxDir := filepath.Join(s.filesDir, box.ID)
if err := os.MkdirAll(boxDir, 0o755); err != nil {
backend, err := s.storage.Backend(box.StorageBackendID)
if err != nil {
return UploadResult{}, err
}
@@ -224,13 +243,18 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
fileID := randomID(8)
storedName := "@each@" + fileID + strings.ToLower(filepath.Ext(header.Filename))
storedPath := filepath.Join(boxDir, storedName)
objectKey := boxObjectKey(box.ID, storedName)
contentType := header.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream"
buffer := make([]byte, 512)
n, _ := file.Read(buffer)
contentType = http.DetectContentType(buffer[:n])
if seeker, ok := file.(io.Seeker); ok {
_, _ = seeker.Seek(0, io.SeekStart)
}
}
if err := writeUploadedFile(storedPath, file, maxSize); err != nil {
if err := s.writeUploadedObject(context.Background(), backend, objectKey, file, header.Size, maxSize, contentType); err != nil {
file.Close()
return UploadResult{}, err
}
@@ -243,6 +267,7 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
Size: header.Size,
ContentType: contentType,
PreviewKind: previewKind(contentType),
ObjectKey: objectKey,
UploadedAt: time.Now().UTC(),
})
}
@@ -296,6 +321,29 @@ func (s *UploadService) ListBoxes(limit int) ([]Box, error) {
return boxes, err
}
func (s *UploadService) ActiveBoxCountForUser(userID string) (int, error) {
return s.activeBoxCount(func(box Box) bool { return box.OwnerID == userID })
}
func (s *UploadService) ActiveBoxCountForIP(ip string) (int, error) {
return s.activeBoxCount(func(box Box) bool { return box.OwnerID == "" && box.CreatorIP == ip })
}
func (s *UploadService) activeBoxCount(match func(Box) bool) (int, error) {
boxes, err := s.ListBoxes(0)
if err != nil {
return 0, err
}
now := time.Now().UTC()
count := 0
for _, box := range boxes {
if match(box) && box.ExpiresAt.After(now) {
count++
}
}
return count, nil
}
func (s *UploadService) AdminStats() (AdminStats, error) {
boxes, err := s.ListBoxes(0)
if err != nil {
@@ -463,13 +511,22 @@ func (s *UploadService) DeleteBoxWithToken(boxID, token string) error {
}
func (s *UploadService) DeleteBoxWithSource(boxID, source string) error {
box, _ := s.GetBox(boxID)
if err := s.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket(boxesBucket).Delete([]byte(boxID))
}); err != nil {
return err
}
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
return err
if box.ID != "" {
if backend, err := s.storage.Backend(s.BoxStorageBackendID(box)); err == nil {
if err := backend.DeletePrefix(context.Background(), box.ID); err != nil {
return err
}
}
} else {
if err := os.RemoveAll(filepath.Join(s.filesDir, boxID)); err != nil {
return err
}
}
s.logger.Info("box deleted", "source", source, "severity", "user_activity", "code", 2101, "box_id", boxID)
return nil
@@ -499,6 +556,56 @@ func (s *UploadService) BoxMetadataPath(box Box) string {
return filepath.Join(s.filesDir, box.ID, ".warpbox.box.json")
}
func (s *UploadService) BoxStorageBackendID(box Box) string {
return normalizeBackendID(box.StorageBackendID)
}
func (s *UploadService) FileObjectKey(box Box, file File) string {
if file.ObjectKey != "" {
return file.ObjectKey
}
return boxObjectKey(box.ID, file.StoredName)
}
func (s *UploadService) ThumbnailObjectKey(box Box, file File) string {
if file.ThumbnailObjectKey != "" {
return file.ThumbnailObjectKey
}
if file.Thumbnail == "" {
return ""
}
return boxObjectKey(box.ID, file.Thumbnail)
}
func (s *UploadService) OpenFileObject(ctx context.Context, box Box, file File) (StorageObject, error) {
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
if err != nil {
return StorageObject{}, err
}
return backend.Get(ctx, s.FileObjectKey(box, file))
}
func (s *UploadService) OpenThumbnailObject(ctx context.Context, box Box, file File) (StorageObject, error) {
key := s.ThumbnailObjectKey(box, file)
if key == "" {
return StorageObject{}, os.ErrNotExist
}
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
if err != nil {
return StorageObject{}, err
}
return backend.Get(ctx, key)
}
func (s *UploadService) PutThumbnailObject(ctx context.Context, box Box, name string, body io.Reader, size int64, contentType string) (string, error) {
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
if err != nil {
return "", err
}
key := boxObjectKey(box.ID, name)
return key, backend.Put(ctx, key, body, size, contentType)
}
func (s *UploadService) IsProtected(box Box) bool {
return box.PasswordHash != "" && box.PasswordSalt != ""
}
@@ -564,11 +671,11 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
defer archive.Close()
for _, file := range box.Files {
path := s.FilePath(box, file)
source, err := os.Open(path)
object, err := s.OpenFileObject(context.Background(), box, file)
if err != nil {
return err
}
source := object.Body
header := &zip.FileHeader{
Name: file.Name,
@@ -592,6 +699,9 @@ func (s *UploadService) WriteZip(w io.Writer, box Box) error {
}
func (s *UploadService) SaveBox(box Box) error {
if box.StorageBackendID == "" {
box.StorageBackendID = StorageBackendLocal
}
data, err := json.Marshal(box)
if err != nil {
return err
@@ -654,6 +764,27 @@ func writeUploadedFile(path string, source multipart.File, maxSize int64) error
return nil
}
func (s *UploadService) writeUploadedObject(ctx context.Context, backend StorageBackend, key string, source multipart.File, size, maxSize int64, contentType string) error {
var reader io.Reader = source
if maxSize > 0 {
reader = io.LimitReader(source, maxSize+1)
var buffer bytes.Buffer
written, err := io.Copy(&buffer, reader)
if err != nil {
return err
}
if written > maxSize {
return fmt.Errorf("file exceeds max upload size")
}
return backend.Put(ctx, key, bytes.NewReader(buffer.Bytes()), written, contentType)
}
return backend.Put(ctx, key, reader, size, contentType)
}
func boxObjectKey(boxID, name string) string {
return filepath.ToSlash(filepath.Join(boxID, name))
}
func randomID(byteCount int) string {
data := make([]byte, byteCount)
if _, err := rand.Read(data); err != nil {
@@ -691,10 +822,13 @@ func previewKind(contentType string) string {
}
func (s *UploadService) writeBoxMetadata(box Box) error {
path := s.BoxMetadataPath(box)
data, err := json.MarshalIndent(box, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, data, 0o600)
backend, err := s.storage.Backend(s.BoxStorageBackendID(box))
if err != nil {
return err
}
return backend.Put(context.Background(), boxObjectKey(box.ID, ".warpbox.box.json"), bytes.NewReader(data), int64(len(data)), "application/json")
}