Daniel Legt
73bd14572d
All checks were successful
Build and Publish Docker Image / deploy (push) Successful in 1m41s
- Implement storage backend deletion, which automatically resets default storage settings and user-specific overrides when a backend is removed. - Add unit tests covering the delete action and its cleanup side effects. - Improve admin UI responsiveness, fixing table scrolling, flex wrapping, and text truncation for long storage backend names. - Update security documentation to clarify trusted proxy configurations and explain how trusted proxies are protected from automatic bans.
522 lines
15 KiB
Go
522 lines
15 KiB
Go
package services
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.etcd.io/bbolt"
|
|
)
|
|
|
|
var storageBackendsBucket = []byte("storage_backends")
|
|
var storageBackendTestStatusBucket = []byte("storage_backend_test_status")
|
|
|
|
const (
|
|
StorageBackendLocal = "local"
|
|
StorageBackendS3 = "s3"
|
|
StorageBackendSFTP = "sftp"
|
|
StorageBackendSMB = "smb"
|
|
StorageBackendWebDAV = "webdav"
|
|
|
|
StorageProviderS3 = "s3"
|
|
StorageProviderContabo = "contabo"
|
|
StorageProviderSFTP = "sftp"
|
|
StorageProviderSMB = "smb"
|
|
StorageProviderWebDAV = "webdav"
|
|
)
|
|
|
|
type StorageObject struct {
|
|
Key string
|
|
Size int64
|
|
ContentType string
|
|
ModTime time.Time
|
|
Body io.ReadCloser
|
|
}
|
|
|
|
type StorageBackend interface {
|
|
ID() string
|
|
Type() string
|
|
Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error
|
|
Get(ctx context.Context, key string) (StorageObject, error)
|
|
Delete(ctx context.Context, key string) error
|
|
DeletePrefix(ctx context.Context, prefix string) error
|
|
Usage(ctx context.Context) (int64, error)
|
|
Test(ctx context.Context) error
|
|
}
|
|
|
|
type StorageBackendConfig struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
Provider string `json:"provider,omitempty"`
|
|
Enabled bool `json:"enabled"`
|
|
LocalPath string `json:"localPath,omitempty"`
|
|
Endpoint string `json:"endpoint,omitempty"`
|
|
Region string `json:"region,omitempty"`
|
|
Bucket string `json:"bucket,omitempty"`
|
|
AccessKey string `json:"accessKey,omitempty"`
|
|
SecretKey string `json:"secretKey,omitempty"`
|
|
UseSSL bool `json:"useSsl,omitempty"`
|
|
PathStyle bool `json:"pathStyle,omitempty"`
|
|
Host string `json:"host,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
Username string `json:"username,omitempty"`
|
|
Password string `json:"password,omitempty"`
|
|
PrivateKey string `json:"privateKey,omitempty"`
|
|
HostKey string `json:"hostKey,omitempty"`
|
|
RemotePath string `json:"remotePath,omitempty"`
|
|
Share string `json:"share,omitempty"`
|
|
Domain string `json:"domain,omitempty"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
|
LastTestError string `json:"lastTestError,omitempty"`
|
|
LastTestSuccess bool `json:"lastTestSuccess,omitempty"`
|
|
}
|
|
|
|
type StorageBackendView struct {
|
|
Config StorageBackendConfig
|
|
UsageBytes int64
|
|
UsageLabel string
|
|
InUse bool
|
|
InUseReason string
|
|
SpeedTests []StorageSpeedTest
|
|
CanSpeedTest bool
|
|
}
|
|
|
|
type StorageService struct {
|
|
db *bbolt.DB
|
|
localFilesDir string
|
|
}
|
|
|
|
func NewStorageService(db *bbolt.DB, dataDir string) (*StorageService, error) {
|
|
filesDir := filepath.Join(dataDir, "files")
|
|
if err := os.MkdirAll(filesDir, 0o755); err != nil {
|
|
return nil, err
|
|
}
|
|
service := &StorageService{db: db, localFilesDir: filesDir}
|
|
err := db.Update(func(tx *bbolt.Tx) error {
|
|
if _, err := tx.CreateBucketIfNotExists(storageBackendsBucket); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.CreateBucketIfNotExists(storageBackendTestStatusBucket); err != nil {
|
|
return err
|
|
}
|
|
_, err := tx.CreateBucketIfNotExists(storageSpeedTestsBucket)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return service, nil
|
|
}
|
|
|
|
func (s *StorageService) LocalFilesDir() string {
|
|
return s.localFilesDir
|
|
}
|
|
|
|
func (s *StorageService) Backend(id string) (StorageBackend, error) {
|
|
cfg, err := s.BackendConfig(id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !cfg.Enabled {
|
|
return nil, fmt.Errorf("storage backend is disabled")
|
|
}
|
|
return s.backendFromConfig(cfg)
|
|
}
|
|
|
|
func (s *StorageService) BackendForMaintenance(id string) (StorageBackend, error) {
|
|
cfg, err := s.BackendConfig(id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return s.backendFromConfig(cfg)
|
|
}
|
|
|
|
func (s *StorageService) BackendConfig(id string) (StorageBackendConfig, error) {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" || id == StorageBackendLocal {
|
|
cfg := s.localConfig()
|
|
s.applyStoredTestStatus(&cfg)
|
|
return cfg, nil
|
|
}
|
|
var cfg StorageBackendConfig
|
|
err := s.db.View(func(tx *bbolt.Tx) error {
|
|
data := tx.Bucket(storageBackendsBucket).Get([]byte(id))
|
|
if data == nil {
|
|
return os.ErrNotExist
|
|
}
|
|
return json.Unmarshal(data, &cfg)
|
|
})
|
|
if err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
func (s *StorageService) ListBackendConfigs() ([]StorageBackendConfig, error) {
|
|
configs := []StorageBackendConfig{s.localConfig()}
|
|
err := s.db.View(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket(storageBackendsBucket).ForEach(func(_, value []byte) error {
|
|
var cfg StorageBackendConfig
|
|
if err := json.Unmarshal(value, &cfg); err != nil {
|
|
return err
|
|
}
|
|
configs = append(configs, cfg)
|
|
return nil
|
|
})
|
|
})
|
|
sort.Slice(configs, func(i, j int) bool {
|
|
if configs[i].ID == StorageBackendLocal {
|
|
return true
|
|
}
|
|
if configs[j].ID == StorageBackendLocal {
|
|
return false
|
|
}
|
|
return strings.ToLower(configs[i].Name) < strings.ToLower(configs[j].Name)
|
|
})
|
|
return configs, err
|
|
}
|
|
|
|
func (s *StorageService) CreateS3Backend(input StorageBackendConfig) (StorageBackendConfig, error) {
|
|
return s.CreateBackend(input)
|
|
}
|
|
|
|
func (s *StorageService) CreateBackend(input StorageBackendConfig) (StorageBackendConfig, error) {
|
|
input.ID = randomID(10)
|
|
input.Provider = normalizeStorageProvider(input.Provider)
|
|
input.Type = storageTypeForProvider(input.Provider)
|
|
if err := normalizeStorageBackendConfig(&input, true); err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
now := time.Now().UTC()
|
|
input.Enabled = true
|
|
input.CreatedAt = now
|
|
input.UpdatedAt = now
|
|
if err := s.SaveBackendConfig(input); err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
func (s *StorageService) UpdateS3Backend(id string, input StorageBackendConfig) (StorageBackendConfig, error) {
|
|
return s.UpdateBackend(id, input)
|
|
}
|
|
|
|
func (s *StorageService) UpdateBackend(id string, input StorageBackendConfig) (StorageBackendConfig, error) {
|
|
current, err := s.BackendConfig(id)
|
|
if err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
if current.ID == StorageBackendLocal {
|
|
return StorageBackendConfig{}, fmt.Errorf("local storage cannot be edited")
|
|
}
|
|
current.Provider = canonicalStorageProvider(current)
|
|
current.Type = storageTypeForProvider(current.Provider)
|
|
|
|
input.ID = current.ID
|
|
requestedProvider := normalizeStorageProvider(input.Provider)
|
|
requestedType := storageTypeForProvider(requestedProvider)
|
|
if input.Type != "" && input.Type != requestedType {
|
|
return StorageBackendConfig{}, fmt.Errorf("storage type cannot be changed after creation")
|
|
}
|
|
input.Provider = requestedProvider
|
|
input.Type = requestedType
|
|
if input.Provider != current.Provider || input.Type != current.Type {
|
|
return StorageBackendConfig{}, fmt.Errorf("storage provider cannot be changed after creation")
|
|
}
|
|
if strings.TrimSpace(input.SecretKey) == "" {
|
|
input.SecretKey = current.SecretKey
|
|
}
|
|
if strings.TrimSpace(input.Password) == "" {
|
|
input.Password = current.Password
|
|
}
|
|
if strings.TrimSpace(input.PrivateKey) == "" {
|
|
input.PrivateKey = current.PrivateKey
|
|
}
|
|
if strings.TrimSpace(input.HostKey) == "" {
|
|
input.HostKey = current.HostKey
|
|
}
|
|
input.Enabled = current.Enabled
|
|
input.CreatedAt = current.CreatedAt
|
|
input.LastTestedAt = current.LastTestedAt
|
|
input.LastTestError = current.LastTestError
|
|
input.LastTestSuccess = current.LastTestSuccess
|
|
if err := normalizeStorageBackendConfig(&input, false); err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
if err := s.SaveBackendConfig(input); err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
func normalizeStorageBackendConfig(input *StorageBackendConfig, creating bool) error {
|
|
input.Name = strings.TrimSpace(input.Name)
|
|
input.Provider = normalizeStorageProvider(input.Provider)
|
|
if input.Provider == StorageProviderSFTP {
|
|
input.Type = StorageBackendSFTP
|
|
input.Host = strings.TrimSpace(input.Host)
|
|
input.Username = strings.TrimSpace(input.Username)
|
|
input.Password = strings.TrimSpace(input.Password)
|
|
input.PrivateKey = strings.TrimSpace(input.PrivateKey)
|
|
input.HostKey = strings.TrimSpace(input.HostKey)
|
|
input.RemotePath = cleanRemoteRoot(input.RemotePath)
|
|
if input.Port <= 0 {
|
|
input.Port = 22
|
|
}
|
|
if input.Name == "" {
|
|
input.Name = input.Host
|
|
}
|
|
if input.Name == "" || input.Host == "" || input.Username == "" || (input.Password == "" && input.PrivateKey == "") {
|
|
return fmt.Errorf("name, host, username, and password or private key are required")
|
|
}
|
|
return nil
|
|
}
|
|
if input.Provider == StorageProviderSMB {
|
|
input.Type = StorageBackendSMB
|
|
input.Host = strings.TrimSpace(input.Host)
|
|
input.Username = strings.TrimSpace(input.Username)
|
|
input.Password = strings.TrimSpace(input.Password)
|
|
input.Share = strings.Trim(strings.TrimSpace(input.Share), `/\`)
|
|
input.Domain = strings.TrimSpace(input.Domain)
|
|
input.RemotePath = cleanRemoteRoot(input.RemotePath)
|
|
if input.Port <= 0 {
|
|
input.Port = 445
|
|
}
|
|
if input.Name == "" {
|
|
input.Name = input.Share
|
|
}
|
|
if input.Name == "" || input.Host == "" || input.Share == "" || input.Username == "" || input.Password == "" {
|
|
return fmt.Errorf("name, host, share, username, and password are required")
|
|
}
|
|
return nil
|
|
}
|
|
if input.Provider == StorageProviderWebDAV {
|
|
input.Type = StorageBackendWebDAV
|
|
input.Endpoint = strings.TrimSpace(input.Endpoint)
|
|
input.Username = strings.TrimSpace(input.Username)
|
|
input.Password = strings.TrimSpace(input.Password)
|
|
input.RemotePath = cleanRemoteRoot(input.RemotePath)
|
|
if input.Name == "" {
|
|
input.Name = input.Endpoint
|
|
}
|
|
if input.Name == "" || input.Endpoint == "" {
|
|
return fmt.Errorf("name and WebDAV URL are required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
input.Type = StorageBackendS3
|
|
if input.Provider == StorageProviderContabo {
|
|
input.UseSSL = true
|
|
input.PathStyle = true
|
|
}
|
|
input.Name = strings.TrimSpace(input.Name)
|
|
input.Endpoint = strings.TrimSpace(input.Endpoint)
|
|
input.Region = strings.TrimSpace(input.Region)
|
|
input.Bucket = strings.TrimSpace(input.Bucket)
|
|
input.AccessKey = strings.TrimSpace(input.AccessKey)
|
|
input.SecretKey = strings.TrimSpace(input.SecretKey)
|
|
if input.Name == "" {
|
|
input.Name = input.Bucket
|
|
}
|
|
if input.Name == "" || input.Endpoint == "" || input.Bucket == "" || input.AccessKey == "" || input.SecretKey == "" {
|
|
return fmt.Errorf("name, endpoint, bucket, access key, and secret key are required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *StorageService) SaveBackendConfig(cfg StorageBackendConfig) error {
|
|
if cfg.ID == "" || cfg.ID == StorageBackendLocal {
|
|
return fmt.Errorf("invalid storage backend id")
|
|
}
|
|
cfg.UpdatedAt = time.Now().UTC()
|
|
data, err := json.Marshal(cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket(storageBackendsBucket).Put([]byte(cfg.ID), data)
|
|
})
|
|
}
|
|
|
|
func (s *StorageService) DeleteBackend(id string, inUse bool) error {
|
|
if id == "" || id == StorageBackendLocal {
|
|
return fmt.Errorf("local storage cannot be deleted")
|
|
}
|
|
if inUse {
|
|
return fmt.Errorf("storage backend is in use")
|
|
}
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket(storageBackendsBucket).Delete([]byte(id))
|
|
})
|
|
}
|
|
|
|
func (s *StorageService) TestBackend(id string) (StorageBackendConfig, error) {
|
|
cfg, err := s.BackendConfig(id)
|
|
if err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
backend, err := s.backendFromConfig(cfg)
|
|
if err != nil {
|
|
return StorageBackendConfig{}, err
|
|
}
|
|
err = backend.Test(context.Background())
|
|
cfg.LastTestedAt = time.Now().UTC()
|
|
cfg.LastTestError = ""
|
|
cfg.LastTestSuccess = err == nil
|
|
if err != nil {
|
|
cfg.LastTestError = err.Error()
|
|
}
|
|
if cfg.ID != StorageBackendLocal {
|
|
_ = s.SaveBackendConfig(cfg)
|
|
} else {
|
|
_ = s.saveBackendTestStatus(cfg)
|
|
}
|
|
return cfg, err
|
|
}
|
|
|
|
func (s *StorageService) applyStoredTestStatus(cfg *StorageBackendConfig) {
|
|
_ = s.db.View(func(tx *bbolt.Tx) error {
|
|
bucket := tx.Bucket(storageBackendTestStatusBucket)
|
|
if bucket == nil {
|
|
return nil
|
|
}
|
|
data := bucket.Get([]byte(cfg.ID))
|
|
if data == nil {
|
|
return nil
|
|
}
|
|
var status struct {
|
|
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
|
LastTestError string `json:"lastTestError,omitempty"`
|
|
LastTestSuccess bool `json:"lastTestSuccess,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(data, &status); err != nil {
|
|
return nil
|
|
}
|
|
cfg.LastTestedAt = status.LastTestedAt
|
|
cfg.LastTestError = status.LastTestError
|
|
cfg.LastTestSuccess = status.LastTestSuccess
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (s *StorageService) saveBackendTestStatus(cfg StorageBackendConfig) error {
|
|
status := struct {
|
|
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
|
LastTestError string `json:"lastTestError,omitempty"`
|
|
LastTestSuccess bool `json:"lastTestSuccess,omitempty"`
|
|
}{
|
|
LastTestedAt: cfg.LastTestedAt,
|
|
LastTestError: cfg.LastTestError,
|
|
LastTestSuccess: cfg.LastTestSuccess,
|
|
}
|
|
data, err := json.Marshal(status)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.db.Update(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket(storageBackendTestStatusBucket).Put([]byte(cfg.ID), data)
|
|
})
|
|
}
|
|
|
|
func (s *StorageService) backendFromConfig(cfg StorageBackendConfig) (StorageBackend, error) {
|
|
switch cfg.Type {
|
|
case StorageBackendLocal:
|
|
return localStorageBackend{id: cfg.ID, root: cfg.LocalPath}, nil
|
|
case StorageBackendS3:
|
|
return newS3StorageBackend(cfg)
|
|
case StorageBackendSFTP:
|
|
return sftpStorageBackend{cfg: cfg}, nil
|
|
case StorageBackendSMB:
|
|
return smbStorageBackend{cfg: cfg}, nil
|
|
case StorageBackendWebDAV:
|
|
return newWebDAVStorageBackend(cfg), nil
|
|
default:
|
|
return nil, fmt.Errorf("unsupported storage backend type %q", cfg.Type)
|
|
}
|
|
}
|
|
|
|
func (s *StorageService) localConfig() StorageBackendConfig {
|
|
now := time.Now().UTC()
|
|
return StorageBackendConfig{
|
|
ID: StorageBackendLocal,
|
|
Name: "Local files",
|
|
Type: StorageBackendLocal,
|
|
Provider: StorageBackendLocal,
|
|
Enabled: true,
|
|
LocalPath: s.localFilesDir,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
}
|
|
|
|
func normalizeStorageProvider(provider string) string {
|
|
switch strings.TrimSpace(provider) {
|
|
case StorageProviderContabo:
|
|
return StorageProviderContabo
|
|
case StorageProviderSFTP:
|
|
return StorageProviderSFTP
|
|
case StorageProviderSMB:
|
|
return StorageProviderSMB
|
|
case StorageProviderWebDAV:
|
|
return StorageProviderWebDAV
|
|
default:
|
|
return StorageProviderS3
|
|
}
|
|
}
|
|
|
|
func canonicalStorageProvider(cfg StorageBackendConfig) string {
|
|
if cfg.Provider != "" && cfg.Provider != StorageBackendLocal {
|
|
return normalizeStorageProvider(cfg.Provider)
|
|
}
|
|
switch cfg.Type {
|
|
case StorageBackendSFTP:
|
|
return StorageProviderSFTP
|
|
case StorageBackendSMB:
|
|
return StorageProviderSMB
|
|
case StorageBackendWebDAV:
|
|
return StorageProviderWebDAV
|
|
default:
|
|
return StorageProviderS3
|
|
}
|
|
}
|
|
|
|
func storageTypeForProvider(provider string) string {
|
|
switch normalizeStorageProvider(provider) {
|
|
case StorageProviderSFTP:
|
|
return StorageBackendSFTP
|
|
case StorageProviderSMB:
|
|
return StorageBackendSMB
|
|
case StorageProviderWebDAV:
|
|
return StorageBackendWebDAV
|
|
default:
|
|
return StorageBackendS3
|
|
}
|
|
}
|
|
|
|
func cleanObjectKey(key string) string {
|
|
return strings.TrimPrefix(filepath.ToSlash(filepath.Clean(strings.TrimPrefix(key, "/"))), "./")
|
|
}
|
|
|
|
func cleanRemoteRoot(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "."
|
|
}
|
|
cleaned := path.Clean(strings.ReplaceAll(value, "\\", "/"))
|
|
if cleaned == "/" {
|
|
return "/"
|
|
}
|
|
return strings.TrimSuffix(cleaned, "/")
|
|
}
|