fix(auth): reject invalid bearer tokens instead of falling back
Modify the authentication handler to return an unauthorized error when an invalid or disabled bearer token is provided, rather than silently falling back to an anonymous request. This ensures that clients attempting to authenticate but failing (due to expired, malformed, or disabled tokens) are explicitly notified of the auth failure instead of proceeding anonymously. True anonymous requests without any Authorization header remain supported.
This commit is contained in:
@@ -100,25 +100,51 @@ func TestBearerTokenUploadActsAsUser(t *testing.T) {
|
||||
t.Fatalf("OwnerID = %q, want %q", box.OwnerID, user.ID)
|
||||
}
|
||||
|
||||
// An invalid bearer token must not authenticate as the user.
|
||||
// An invalid bearer token is an authentication failure, not an anonymous upload.
|
||||
badRequest := multipartUploadRequest(t, "/api/v1/upload", "file", "x.txt", "x")
|
||||
badRequest.Header.Set("Accept", "application/json")
|
||||
badRequest.Header.Set("Authorization", "Bearer wbx_bogus.secret")
|
||||
badResponse := httptest.NewRecorder()
|
||||
app.Upload(badResponse, badRequest)
|
||||
if badResponse.Code != http.StatusCreated {
|
||||
t.Fatalf("anonymous fallback upload status = %d, body = %s", badResponse.Code, badResponse.Body.String())
|
||||
if badResponse.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("invalid token upload status = %d, body = %s", badResponse.Code, badResponse.Body.String())
|
||||
}
|
||||
var badPayload services.UploadResult
|
||||
if err := json.Unmarshal(badResponse.Body.Bytes(), &badPayload); err != nil {
|
||||
t.Fatalf("json.Unmarshal returned error: %v", err)
|
||||
}
|
||||
|
||||
func TestAnonymousUploadWithoutBearerStillWorks(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
|
||||
response := httptest.NewRecorder()
|
||||
app.Upload(response, multipartUploadRequest(t, "/api/v1/upload", "file", "anonymous.txt", "anonymous"))
|
||||
if response.Code != http.StatusCreated {
|
||||
t.Fatalf("anonymous upload status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
badBox, err := app.uploadService.GetBox(badPayload.BoxID)
|
||||
}
|
||||
|
||||
func TestDisabledUserBearerTokenCannotUpload(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
|
||||
user, err := app.authService.CreateBootstrapUser("daniel", "daniel@example.test", "password123")
|
||||
if err != nil {
|
||||
t.Fatalf("GetBox returned error: %v", err)
|
||||
t.Fatalf("CreateBootstrapUser returned error: %v", err)
|
||||
}
|
||||
if badBox.OwnerID != "" {
|
||||
t.Fatalf("invalid token OwnerID = %q, want empty", badBox.OwnerID)
|
||||
tokenResult, err := app.authService.CreateAPIToken(user.ID, "cli")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateAPIToken returned error: %v", err)
|
||||
}
|
||||
if err := app.authService.DisableUser(user.ID, true); err != nil {
|
||||
t.Fatalf("DisableUser returned error: %v", err)
|
||||
}
|
||||
|
||||
request := multipartUploadRequest(t, "/api/v1/upload", "file", "blocked.txt", "blocked")
|
||||
request.Header.Set("Accept", "application/json")
|
||||
request.Header.Set("Authorization", "Bearer "+tokenResult.Plaintext)
|
||||
response := httptest.NewRecorder()
|
||||
app.Upload(response, request)
|
||||
if response.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("disabled bearer upload status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -251,23 +251,32 @@ func (a *App) loginAndRedirect(w http.ResponseWriter, r *http.Request, email, pa
|
||||
}
|
||||
|
||||
func (a *App) currentUser(r *http.Request) (services.User, bool) {
|
||||
user, ok, _ := a.currentUserWithAuthError(r)
|
||||
return user, ok
|
||||
}
|
||||
|
||||
func (a *App) currentUserWithAuthError(r *http.Request) (services.User, bool, error) {
|
||||
// Personal access tokens via Authorization: Bearer act as their owning user.
|
||||
// A bearer header is never set by browsers cross-site, so this path is not
|
||||
// subject to CSRF and intentionally bypasses the session cookie.
|
||||
if header := r.Header.Get("Authorization"); header != "" {
|
||||
if raw, ok := strings.CutPrefix(header, "Bearer "); ok {
|
||||
if user, err := a.authService.UserForAPIToken(raw); err == nil {
|
||||
return user, true
|
||||
user, err := a.authService.UserForAPIToken(raw)
|
||||
if err != nil {
|
||||
return services.User{}, false, err
|
||||
}
|
||||
return services.User{}, false
|
||||
return user, true, nil
|
||||
}
|
||||
}
|
||||
cookie, err := r.Cookie(userSessionCookieName)
|
||||
if err != nil {
|
||||
return services.User{}, false
|
||||
return services.User{}, false, nil
|
||||
}
|
||||
user, _, err := a.authService.UserForSession(cookie.Value)
|
||||
return user, err == nil
|
||||
if err != nil {
|
||||
return services.User{}, false, nil
|
||||
}
|
||||
return user, true, nil
|
||||
}
|
||||
|
||||
func (a *App) requireUser(w http.ResponseWriter, r *http.Request) (services.User, bool) {
|
||||
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
|
||||
func TestSetStaticCacheHeaders(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"/static/css/app.css": "public, max-age=86400",
|
||||
"/static/js/app.js": "public, max-age=86400",
|
||||
"/static/css/00-base.css": "public, max-age=86400",
|
||||
"/static/js/00-utils.js": "public, max-age=86400",
|
||||
"/static/img/preview.webp": "public, max-age=31536000, immutable",
|
||||
"/static/fonts/ui.woff2": "public, max-age=31536000, immutable",
|
||||
"/static/videos/intro.mp4": "public, max-age=31536000, immutable",
|
||||
|
||||
@@ -16,7 +16,11 @@ import (
|
||||
)
|
||||
|
||||
func (a *App) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
user, loggedIn := a.currentUser(r)
|
||||
user, loggedIn, authErr := a.currentUserWithAuthError(r)
|
||||
if authErr != nil {
|
||||
helpers.WriteJSONError(w, http.StatusUnauthorized, "invalid bearer token")
|
||||
return
|
||||
}
|
||||
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
|
||||
settings, err := a.settingsService.UploadPolicy()
|
||||
if err != nil {
|
||||
|
||||
@@ -1,29 +1,18 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/pkg/sftp"
|
||||
"go.etcd.io/bbolt"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
var storageBackendsBucket = []byte("storage_backends")
|
||||
@@ -400,7 +389,7 @@ func (s *StorageService) backendFromConfig(cfg StorageBackendConfig) (StorageBac
|
||||
case StorageBackendSMB:
|
||||
return smbStorageBackend{cfg: cfg}, nil
|
||||
case StorageBackendWebDAV:
|
||||
return webDAVStorageBackend{cfg: cfg, client: http.DefaultClient}, nil
|
||||
return newWebDAVStorageBackend(cfg), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage backend type %q", cfg.Type)
|
||||
}
|
||||
@@ -420,758 +409,6 @@ func (s *StorageService) localConfig() StorageBackendConfig {
|
||||
}
|
||||
}
|
||||
|
||||
type localStorageBackend struct {
|
||||
id string
|
||||
root string
|
||||
}
|
||||
|
||||
func (b localStorageBackend) ID() string { return b.id }
|
||||
func (b localStorageBackend) Type() string { return StorageBackendLocal }
|
||||
|
||||
func (b localStorageBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Get(_ context.Context, key string) (StorageObject, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
source, err := os.Open(path)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: source}, nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Delete(_ context.Context, key string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) DeletePrefix(_ context.Context, prefix string) error {
|
||||
path, err := b.path(prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Usage(_ context.Context) (int64, error) {
|
||||
var total int64
|
||||
err := filepath.WalkDir(b.root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
total += info.Size()
|
||||
return nil
|
||||
})
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return total, err
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b localStorageBackend) path(key string) (string, error) {
|
||||
key = filepath.Clean(strings.TrimPrefix(key, "/"))
|
||||
if key == "." || strings.HasPrefix(key, "..") || filepath.IsAbs(key) {
|
||||
return "", fmt.Errorf("invalid storage key")
|
||||
}
|
||||
path := filepath.Join(b.root, key)
|
||||
root, err := filepath.Abs(b.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if abs != root && !strings.HasPrefix(abs, root+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("invalid storage key")
|
||||
}
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
type s3StorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func newS3StorageBackend(cfg StorageBackendConfig) (*s3StorageBackend, error) {
|
||||
endpoint := normalizeS3Endpoint(cfg.Endpoint)
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
BucketLookup: s3BucketLookup(cfg.PathStyle),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3StorageBackend{cfg: cfg, client: client}, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b *s3StorageBackend) Type() string { return StorageBackendS3 }
|
||||
|
||||
func (b *s3StorageBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
opts := minio.PutObjectOptions{ContentType: contentType}
|
||||
_, err := b.client.PutObject(ctx, b.cfg.Bucket, cleanObjectKey(key), body, size, opts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
object, err := b.client.GetObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
info, err := object.Stat()
|
||||
if err != nil {
|
||||
object.Close()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: info.Size, ContentType: info.ContentType, ModTime: info.LastModified, Body: object}, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.client.RemoveObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
prefix = strings.TrimSuffix(cleanObjectKey(prefix), "/") + "/"
|
||||
objects := b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
|
||||
for object := range objects {
|
||||
if object.Err != nil {
|
||||
return object.Err
|
||||
}
|
||||
if err := b.Delete(ctx, object.Key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
var total int64
|
||||
for object := range b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Recursive: true}) {
|
||||
if object.Err != nil {
|
||||
return 0, object.Err
|
||||
}
|
||||
total += object.Size
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Test(ctx context.Context) error {
|
||||
exists, err := b.client.BucketExists(ctx, b.cfg.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", b.cfg.Bucket)
|
||||
}
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, bytes.NewReader([]byte("ok")), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
type sftpStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b sftpStorageBackend) Type() string { return StorageBackendSFTP }
|
||||
|
||||
func (b sftpStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
if err := client.MkdirAll(path.Dir(remotePath)); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := client.OpenFile(remotePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
source, err := client.Open(remotePath)
|
||||
if err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: closeWith(source, closer)}, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.Remove(b.remotePath(key)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(prefix)
|
||||
if err := client.RemoveDirectory(remotePath); err == nil || os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
walker := client.Walk(remotePath)
|
||||
paths := make([]string, 0)
|
||||
for walker.Step() {
|
||||
if walker.Err() != nil {
|
||||
return walker.Err()
|
||||
}
|
||||
paths = append(paths, walker.Path())
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool { return len(paths[i]) > len(paths[j]) })
|
||||
for _, item := range paths {
|
||||
if err := client.Remove(item); err != nil {
|
||||
_ = client.RemoveDirectory(item)
|
||||
}
|
||||
}
|
||||
_ = client.RemoveDirectory(remotePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
walker := client.Walk(cleanRemoteRoot(b.cfg.RemotePath))
|
||||
for walker.Step() {
|
||||
if walker.Err() != nil {
|
||||
return 0, walker.Err()
|
||||
}
|
||||
info := walker.Stat()
|
||||
if info != nil && !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) client() (*sftp.Client, func(), error) {
|
||||
auth := make([]ssh.AuthMethod, 0, 2)
|
||||
if b.cfg.PrivateKey != "" {
|
||||
signer, err := ssh.ParsePrivateKey([]byte(b.cfg.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
auth = append(auth, ssh.PublicKeys(signer))
|
||||
}
|
||||
if b.cfg.Password != "" {
|
||||
auth = append(auth, ssh.Password(b.cfg.Password))
|
||||
}
|
||||
if len(auth) == 0 {
|
||||
return nil, nil, fmt.Errorf("sftp password or private key is required")
|
||||
}
|
||||
hostKeyCallback, err := b.hostKeyCallback()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sshClient, err := ssh.Dial("tcp", b.cfg.Host+":"+strconv.Itoa(b.cfg.Port), &ssh.ClientConfig{
|
||||
User: b.cfg.Username,
|
||||
Auth: auth,
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 15 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, err := sftp.NewClient(sshClient)
|
||||
if err != nil {
|
||||
sshClient.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return client, func() {
|
||||
client.Close()
|
||||
sshClient.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) hostKeyCallback() (ssh.HostKeyCallback, error) {
|
||||
if strings.TrimSpace(b.cfg.HostKey) == "" {
|
||||
return ssh.InsecureIgnoreHostKey(), nil
|
||||
}
|
||||
key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(strings.TrimSpace(b.cfg.HostKey)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid sftp host public key: %w", err)
|
||||
}
|
||||
return ssh.FixedHostKey(key), nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) remotePath(key string) string {
|
||||
return path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key))
|
||||
}
|
||||
|
||||
type joinedReadCloser struct {
|
||||
io.ReadCloser
|
||||
close func()
|
||||
}
|
||||
|
||||
func closeWith(source io.ReadCloser, close func()) io.ReadCloser {
|
||||
return joinedReadCloser{ReadCloser: source, close: close}
|
||||
}
|
||||
|
||||
func (c joinedReadCloser) Close() error {
|
||||
err := c.ReadCloser.Close()
|
||||
c.close()
|
||||
return err
|
||||
}
|
||||
|
||||
type smbStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b smbStorageBackend) Type() string { return StorageBackendSMB }
|
||||
|
||||
func (b smbStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
if err := share.MkdirAll(path.Dir(remotePath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := share.OpenFile(remotePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
source, err := share.Open(b.remotePath(key))
|
||||
if err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: closeWith(source, closer)}, nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := share.Remove(b.remotePath(key)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
err = share.RemoveAll(b.remotePath(prefix))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return smbUsage(share, cleanRemoteRoot(b.cfg.RemotePath))
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) share() (*smb2.Share, func(), error) {
|
||||
conn, err := net.DialTimeout("tcp", b.cfg.Host+":"+strconv.Itoa(b.cfg.Port), 15*time.Second)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dialer := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: b.cfg.Username,
|
||||
Password: b.cfg.Password,
|
||||
Domain: b.cfg.Domain,
|
||||
},
|
||||
}
|
||||
session, err := dialer.Dial(conn)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
share, err := session.Mount(b.cfg.Share)
|
||||
if err != nil {
|
||||
session.Logoff()
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return share, func() {
|
||||
share.Umount()
|
||||
session.Logoff()
|
||||
conn.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) remotePath(key string) string {
|
||||
return strings.TrimPrefix(path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key)), "/")
|
||||
}
|
||||
|
||||
func smbUsage(share *smb2.Share, root string) (int64, error) {
|
||||
root = strings.TrimPrefix(root, "/")
|
||||
entries, err := share.ReadDir(root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, entry := range entries {
|
||||
item := path.Join(root, entry.Name())
|
||||
if entry.IsDir() {
|
||||
size, err := smbUsage(share, item)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += size
|
||||
continue
|
||||
}
|
||||
total += entry.Size()
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
type webDAVStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b webDAVStorageBackend) Type() string { return StorageBackendWebDAV }
|
||||
|
||||
func (b webDAVStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, contentType string) error {
|
||||
if err := b.mkcolParents(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := b.request(ctx, http.MethodPut, key, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if contentType != "" {
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return fmt.Errorf("webdav put failed: %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
request, err := b.request(ctx, http.MethodGet, key, nil)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
response.Body.Close()
|
||||
return StorageObject{}, fmt.Errorf("webdav get failed: %s", response.Status)
|
||||
}
|
||||
modTime, _ := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified"))
|
||||
return StorageObject{Key: key, Size: response.ContentLength, ContentType: response.Header.Get("Content-Type"), ModTime: modTime, Body: response.Body}, nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.deletePath(ctx, key)
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
return b.deletePath(ctx, strings.TrimSuffix(prefix, "/")+"/")
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
request, err := b.request(ctx, "PROPFIND", "", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
request.Header.Set("Depth", "infinity")
|
||||
request.Header.Set("Content-Type", "application/xml")
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return 0, fmt.Errorf("webdav usage failed: %s", response.Status)
|
||||
}
|
||||
var multi webDAVMultiStatus
|
||||
if err := xml.NewDecoder(response.Body).Decode(&multi); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, item := range multi.Responses {
|
||||
if item.PropStat.Prop.ResourceType.Collection != nil {
|
||||
continue
|
||||
}
|
||||
total += item.PropStat.Prop.ContentLength
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) deletePath(ctx context.Context, key string) error {
|
||||
request, err := b.request(ctx, http.MethodDelete, key, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
return nil
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return fmt.Errorf("webdav delete failed: %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) mkcolParents(ctx context.Context, key string) error {
|
||||
dir := path.Dir(cleanObjectKey(key))
|
||||
if dir == "." || dir == "/" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(strings.Trim(dir, "/"), "/")
|
||||
current := ""
|
||||
for _, part := range parts {
|
||||
current = path.Join(current, part)
|
||||
request, err := b.request(ctx, "MKCOL", strings.TrimSuffix(current, "/")+"/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusMethodNotAllowed && response.StatusCode != http.StatusConflict {
|
||||
return fmt.Errorf("webdav mkcol failed: %s", response.Status)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) request(ctx context.Context, method, key string, body io.Reader) (*http.Request, error) {
|
||||
endpoint := strings.TrimRight(b.cfg.Endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("webdav url is required")
|
||||
}
|
||||
remote := path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key))
|
||||
if strings.HasSuffix(key, "/") && !strings.HasSuffix(remote, "/") {
|
||||
remote += "/"
|
||||
}
|
||||
target := endpoint + "/" + strings.TrimLeft(remote, "/")
|
||||
request, err := http.NewRequestWithContext(ctx, method, target, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.cfg.Username != "" || b.cfg.Password != "" {
|
||||
request.SetBasicAuth(b.cfg.Username, b.cfg.Password)
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
type webDAVMultiStatus struct {
|
||||
Responses []webDAVResponse `xml:"response"`
|
||||
}
|
||||
|
||||
type webDAVResponse struct {
|
||||
PropStat webDAVPropStat `xml:"propstat"`
|
||||
}
|
||||
|
||||
type webDAVPropStat struct {
|
||||
Prop webDAVProp `xml:"prop"`
|
||||
}
|
||||
|
||||
type webDAVProp struct {
|
||||
ContentLength int64 `xml:"getcontentlength"`
|
||||
ResourceType webDAVResourceType `xml:"resourcetype"`
|
||||
}
|
||||
|
||||
type webDAVResourceType struct {
|
||||
Collection *struct{} `xml:"collection"`
|
||||
}
|
||||
|
||||
func s3BucketLookup(pathStyle bool) minio.BucketLookupType {
|
||||
if pathStyle {
|
||||
return minio.BucketLookupPath
|
||||
}
|
||||
return minio.BucketLookupAuto
|
||||
}
|
||||
|
||||
func normalizeS3Endpoint(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if parsed, err := url.Parse(endpoint); err == nil && parsed.Host != "" {
|
||||
return parsed.Host
|
||||
}
|
||||
return strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
|
||||
}
|
||||
|
||||
func normalizeStorageProvider(provider string) string {
|
||||
switch strings.TrimSpace(provider) {
|
||||
case StorageProviderContabo:
|
||||
|
||||
124
backend/libs/services/storage_local.go
Normal file
124
backend/libs/services/storage_local.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type localStorageBackend struct {
|
||||
id string
|
||||
root string
|
||||
}
|
||||
|
||||
func (b localStorageBackend) ID() string { return b.id }
|
||||
func (b localStorageBackend) Type() string { return StorageBackendLocal }
|
||||
|
||||
func (b localStorageBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Get(_ context.Context, key string) (StorageObject, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
source, err := os.Open(path)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: source}, nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Delete(_ context.Context, key string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) DeletePrefix(_ context.Context, prefix string) error {
|
||||
path, err := b.path(prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Usage(_ context.Context) (int64, error) {
|
||||
var total int64
|
||||
err := filepath.WalkDir(b.root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
total += info.Size()
|
||||
return nil
|
||||
})
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return total, err
|
||||
}
|
||||
|
||||
func (b localStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b localStorageBackend) path(key string) (string, error) {
|
||||
key = filepath.Clean(strings.TrimPrefix(key, "/"))
|
||||
if key == "." || strings.HasPrefix(key, "..") || filepath.IsAbs(key) {
|
||||
return "", fmt.Errorf("invalid storage key")
|
||||
}
|
||||
path := filepath.Join(b.root, key)
|
||||
root, err := filepath.Abs(b.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if abs != root && !strings.HasPrefix(abs, root+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("invalid storage key")
|
||||
}
|
||||
return abs, nil
|
||||
}
|
||||
18
backend/libs/services/storage_readcloser.go
Normal file
18
backend/libs/services/storage_readcloser.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package services
|
||||
|
||||
import "io"
|
||||
|
||||
type joinedReadCloser struct {
|
||||
io.ReadCloser
|
||||
close func()
|
||||
}
|
||||
|
||||
func closeWith(source io.ReadCloser, close func()) io.ReadCloser {
|
||||
return joinedReadCloser{ReadCloser: source, close: close}
|
||||
}
|
||||
|
||||
func (c joinedReadCloser) Close() error {
|
||||
err := c.ReadCloser.Close()
|
||||
c.close()
|
||||
return err
|
||||
}
|
||||
113
backend/libs/services/storage_s3.go
Normal file
113
backend/libs/services/storage_s3.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type s3StorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func newS3StorageBackend(cfg StorageBackendConfig) (*s3StorageBackend, error) {
|
||||
endpoint := normalizeS3Endpoint(cfg.Endpoint)
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
BucketLookup: s3BucketLookup(cfg.PathStyle),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3StorageBackend{cfg: cfg, client: client}, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b *s3StorageBackend) Type() string { return StorageBackendS3 }
|
||||
|
||||
func (b *s3StorageBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
opts := minio.PutObjectOptions{ContentType: contentType}
|
||||
_, err := b.client.PutObject(ctx, b.cfg.Bucket, cleanObjectKey(key), body, size, opts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
object, err := b.client.GetObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
info, err := object.Stat()
|
||||
if err != nil {
|
||||
object.Close()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: info.Size, ContentType: info.ContentType, ModTime: info.LastModified, Body: object}, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.client.RemoveObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
prefix = strings.TrimSuffix(cleanObjectKey(prefix), "/") + "/"
|
||||
objects := b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
|
||||
for object := range objects {
|
||||
if object.Err != nil {
|
||||
return object.Err
|
||||
}
|
||||
if err := b.Delete(ctx, object.Key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
var total int64
|
||||
for object := range b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Recursive: true}) {
|
||||
if object.Err != nil {
|
||||
return 0, object.Err
|
||||
}
|
||||
total += object.Size
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b *s3StorageBackend) Test(ctx context.Context) error {
|
||||
exists, err := b.client.BucketExists(ctx, b.cfg.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", b.cfg.Bucket)
|
||||
}
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, bytes.NewReader([]byte("ok")), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func s3BucketLookup(pathStyle bool) minio.BucketLookupType {
|
||||
if pathStyle {
|
||||
return minio.BucketLookupPath
|
||||
}
|
||||
return minio.BucketLookupAuto
|
||||
}
|
||||
|
||||
func normalizeS3Endpoint(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if parsed, err := url.Parse(endpoint); err == nil && parsed.Host != "" {
|
||||
return parsed.Host
|
||||
}
|
||||
return strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
|
||||
}
|
||||
200
backend/libs/services/storage_sftp.go
Normal file
200
backend/libs/services/storage_sftp.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type sftpStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b sftpStorageBackend) Type() string { return StorageBackendSFTP }
|
||||
|
||||
func (b sftpStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
if err := client.MkdirAll(path.Dir(remotePath)); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := client.OpenFile(remotePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
source, err := client.Open(remotePath)
|
||||
if err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: closeWith(source, closer)}, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.Remove(b.remotePath(key)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(prefix)
|
||||
if err := client.RemoveDirectory(remotePath); err == nil || os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
walker := client.Walk(remotePath)
|
||||
paths := make([]string, 0)
|
||||
for walker.Step() {
|
||||
if walker.Err() != nil {
|
||||
return walker.Err()
|
||||
}
|
||||
paths = append(paths, walker.Path())
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool { return len(paths[i]) > len(paths[j]) })
|
||||
for _, item := range paths {
|
||||
if err := client.Remove(item); err != nil {
|
||||
_ = client.RemoveDirectory(item)
|
||||
}
|
||||
}
|
||||
_ = client.RemoveDirectory(remotePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
walker := client.Walk(cleanRemoteRoot(b.cfg.RemotePath))
|
||||
for walker.Step() {
|
||||
if walker.Err() != nil {
|
||||
return 0, walker.Err()
|
||||
}
|
||||
info := walker.Stat()
|
||||
if info != nil && !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) client() (*sftp.Client, func(), error) {
|
||||
auth := make([]ssh.AuthMethod, 0, 2)
|
||||
if b.cfg.PrivateKey != "" {
|
||||
signer, err := ssh.ParsePrivateKey([]byte(b.cfg.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
auth = append(auth, ssh.PublicKeys(signer))
|
||||
}
|
||||
if b.cfg.Password != "" {
|
||||
auth = append(auth, ssh.Password(b.cfg.Password))
|
||||
}
|
||||
if len(auth) == 0 {
|
||||
return nil, nil, fmt.Errorf("sftp password or private key is required")
|
||||
}
|
||||
hostKeyCallback, err := b.hostKeyCallback()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sshClient, err := ssh.Dial("tcp", b.cfg.Host+":"+strconv.Itoa(b.cfg.Port), &ssh.ClientConfig{
|
||||
User: b.cfg.Username,
|
||||
Auth: auth,
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 15 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, err := sftp.NewClient(sshClient)
|
||||
if err != nil {
|
||||
sshClient.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return client, func() {
|
||||
client.Close()
|
||||
sshClient.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) hostKeyCallback() (ssh.HostKeyCallback, error) {
|
||||
if strings.TrimSpace(b.cfg.HostKey) == "" {
|
||||
return ssh.InsecureIgnoreHostKey(), nil
|
||||
}
|
||||
key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(strings.TrimSpace(b.cfg.HostKey)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid sftp host public key: %w", err)
|
||||
}
|
||||
return ssh.FixedHostKey(key), nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) remotePath(key string) string {
|
||||
return path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key))
|
||||
}
|
||||
176
backend/libs/services/storage_smb.go
Normal file
176
backend/libs/services/storage_smb.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
)
|
||||
|
||||
type smbStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b smbStorageBackend) Type() string { return StorageBackendSMB }
|
||||
|
||||
func (b smbStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
if err := share.MkdirAll(path.Dir(remotePath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := share.OpenFile(remotePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
source, err := share.Open(b.remotePath(key))
|
||||
if err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: closeWith(source, closer)}, nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := share.Remove(b.remotePath(key)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
err = share.RemoveAll(b.remotePath(prefix))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return smbUsage(share, cleanRemoteRoot(b.cfg.RemotePath))
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) share() (*smb2.Share, func(), error) {
|
||||
conn, err := net.DialTimeout("tcp", b.cfg.Host+":"+strconv.Itoa(b.cfg.Port), 15*time.Second)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dialer := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: b.cfg.Username,
|
||||
Password: b.cfg.Password,
|
||||
Domain: b.cfg.Domain,
|
||||
},
|
||||
}
|
||||
session, err := dialer.Dial(conn)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
share, err := session.Mount(b.cfg.Share)
|
||||
if err != nil {
|
||||
session.Logoff()
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return share, func() {
|
||||
share.Umount()
|
||||
session.Logoff()
|
||||
conn.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) remotePath(key string) string {
|
||||
return strings.TrimPrefix(path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key)), "/")
|
||||
}
|
||||
|
||||
func smbUsage(share *smb2.Share, root string) (int64, error) {
|
||||
root = strings.TrimPrefix(root, "/")
|
||||
entries, err := share.ReadDir(root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, entry := range entries {
|
||||
item := path.Join(root, entry.Name())
|
||||
if entry.IsDir() {
|
||||
size, err := smbUsage(share, item)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += size
|
||||
continue
|
||||
}
|
||||
total += entry.Size()
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
193
backend/libs/services/storage_webdav.go
Normal file
193
backend/libs/services/storage_webdav.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type webDAVStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b webDAVStorageBackend) Type() string { return StorageBackendWebDAV }
|
||||
|
||||
func (b webDAVStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, contentType string) error {
|
||||
if err := b.mkcolParents(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := b.request(ctx, http.MethodPut, key, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if contentType != "" {
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return fmt.Errorf("webdav put failed: %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
request, err := b.request(ctx, http.MethodGet, key, nil)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
response.Body.Close()
|
||||
return StorageObject{}, fmt.Errorf("webdav get failed: %s", response.Status)
|
||||
}
|
||||
modTime, _ := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified"))
|
||||
return StorageObject{Key: key, Size: response.ContentLength, ContentType: response.Header.Get("Content-Type"), ModTime: modTime, Body: response.Body}, nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.deletePath(ctx, key)
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
return b.deletePath(ctx, strings.TrimSuffix(prefix, "/")+"/")
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
request, err := b.request(ctx, "PROPFIND", "", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
request.Header.Set("Depth", "infinity")
|
||||
request.Header.Set("Content-Type", "application/xml")
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return 0, fmt.Errorf("webdav usage failed: %s", response.Status)
|
||||
}
|
||||
var multi webDAVMultiStatus
|
||||
if err := xml.NewDecoder(response.Body).Decode(&multi); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, item := range multi.Responses {
|
||||
if item.PropStat.Prop.ResourceType.Collection != nil {
|
||||
continue
|
||||
}
|
||||
total += item.PropStat.Prop.ContentLength
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) deletePath(ctx context.Context, key string) error {
|
||||
request, err := b.request(ctx, http.MethodDelete, key, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
return nil
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return fmt.Errorf("webdav delete failed: %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) mkcolParents(ctx context.Context, key string) error {
|
||||
dir := path.Dir(cleanObjectKey(key))
|
||||
if dir == "." || dir == "/" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(strings.Trim(dir, "/"), "/")
|
||||
current := ""
|
||||
for _, part := range parts {
|
||||
current = path.Join(current, part)
|
||||
request, err := b.request(ctx, "MKCOL", strings.TrimSuffix(current, "/")+"/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusMethodNotAllowed && response.StatusCode != http.StatusConflict {
|
||||
return fmt.Errorf("webdav mkcol failed: %s", response.Status)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) request(ctx context.Context, method, key string, body io.Reader) (*http.Request, error) {
|
||||
endpoint := strings.TrimRight(b.cfg.Endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("webdav url is required")
|
||||
}
|
||||
remote := path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key))
|
||||
if strings.HasSuffix(key, "/") && !strings.HasSuffix(remote, "/") {
|
||||
remote += "/"
|
||||
}
|
||||
target := endpoint + "/" + strings.TrimLeft(remote, "/")
|
||||
request, err := http.NewRequestWithContext(ctx, method, target, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.cfg.Username != "" || b.cfg.Password != "" {
|
||||
request.SetBasicAuth(b.cfg.Username, b.cfg.Password)
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
type webDAVMultiStatus struct {
|
||||
Responses []webDAVResponse `xml:"response"`
|
||||
}
|
||||
|
||||
type webDAVResponse struct {
|
||||
PropStat webDAVPropStat `xml:"propstat"`
|
||||
}
|
||||
|
||||
type webDAVPropStat struct {
|
||||
Prop webDAVProp `xml:"prop"`
|
||||
}
|
||||
|
||||
type webDAVProp struct {
|
||||
ContentLength int64 `xml:"getcontentlength"`
|
||||
ResourceType webDAVResourceType `xml:"resourcetype"`
|
||||
}
|
||||
|
||||
type webDAVResourceType struct {
|
||||
Collection *struct{} `xml:"collection"`
|
||||
}
|
||||
|
||||
func newWebDAVStorageBackend(cfg StorageBackendConfig) webDAVStorageBackend {
|
||||
return webDAVStorageBackend{cfg: cfg, client: http.DefaultClient}
|
||||
}
|
||||
Reference in New Issue
Block a user