Update
This commit is contained in:
@@ -431,15 +431,24 @@ func (a *App) AdminCreateS3Storage(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
_, err := a.uploadService.Storage().CreateS3Backend(services.StorageBackendConfig{
|
||||
Provider: r.FormValue("provider"),
|
||||
Name: r.FormValue("name"),
|
||||
Endpoint: r.FormValue("endpoint"),
|
||||
Region: r.FormValue("region"),
|
||||
Bucket: r.FormValue("bucket"),
|
||||
AccessKey: r.FormValue("access_key"),
|
||||
SecretKey: r.FormValue("secret_key"),
|
||||
UseSSL: r.FormValue("use_ssl") == "on",
|
||||
PathStyle: r.FormValue("path_style") == "on",
|
||||
Provider: r.FormValue("provider"),
|
||||
Name: r.FormValue("name"),
|
||||
Endpoint: r.FormValue("endpoint"),
|
||||
Region: r.FormValue("region"),
|
||||
Bucket: r.FormValue("bucket"),
|
||||
AccessKey: r.FormValue("access_key"),
|
||||
SecretKey: r.FormValue("secret_key"),
|
||||
UseSSL: r.FormValue("use_ssl") == "on",
|
||||
PathStyle: r.FormValue("path_style") == "on",
|
||||
Host: r.FormValue("host"),
|
||||
Port: parsePositiveInt(r.FormValue("port")),
|
||||
Username: r.FormValue("username"),
|
||||
Password: r.FormValue("password"),
|
||||
PrivateKey: r.FormValue("private_key"),
|
||||
HostKey: r.FormValue("host_key"),
|
||||
RemotePath: r.FormValue("remote_path"),
|
||||
Share: r.FormValue("share"),
|
||||
Domain: r.FormValue("domain"),
|
||||
})
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
@@ -457,15 +466,24 @@ func (a *App) AdminEditStorage(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
_, err := a.uploadService.Storage().UpdateS3Backend(r.PathValue("backendID"), services.StorageBackendConfig{
|
||||
Provider: r.FormValue("provider"),
|
||||
Name: r.FormValue("name"),
|
||||
Endpoint: r.FormValue("endpoint"),
|
||||
Region: r.FormValue("region"),
|
||||
Bucket: r.FormValue("bucket"),
|
||||
AccessKey: r.FormValue("access_key"),
|
||||
SecretKey: r.FormValue("secret_key"),
|
||||
UseSSL: r.FormValue("use_ssl") == "on",
|
||||
PathStyle: r.FormValue("path_style") == "on",
|
||||
Provider: r.FormValue("provider"),
|
||||
Name: r.FormValue("name"),
|
||||
Endpoint: r.FormValue("endpoint"),
|
||||
Region: r.FormValue("region"),
|
||||
Bucket: r.FormValue("bucket"),
|
||||
AccessKey: r.FormValue("access_key"),
|
||||
SecretKey: r.FormValue("secret_key"),
|
||||
UseSSL: r.FormValue("use_ssl") == "on",
|
||||
PathStyle: r.FormValue("path_style") == "on",
|
||||
Host: r.FormValue("host"),
|
||||
Port: parsePositiveInt(r.FormValue("port")),
|
||||
Username: r.FormValue("username"),
|
||||
Password: r.FormValue("password"),
|
||||
PrivateKey: r.FormValue("private_key"),
|
||||
HostKey: r.FormValue("host_key"),
|
||||
RemotePath: r.FormValue("remote_path"),
|
||||
Share: r.FormValue("share"),
|
||||
Domain: r.FormValue("domain"),
|
||||
})
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
|
||||
@@ -4,28 +4,42 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/pkg/sftp"
|
||||
"go.etcd.io/bbolt"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
var storageBackendsBucket = []byte("storage_backends")
|
||||
|
||||
const (
|
||||
StorageBackendLocal = "local"
|
||||
StorageBackendS3 = "s3"
|
||||
StorageBackendLocal = "local"
|
||||
StorageBackendS3 = "s3"
|
||||
StorageBackendSFTP = "sftp"
|
||||
StorageBackendSMB = "smb"
|
||||
StorageBackendWebDAV = "webdav"
|
||||
|
||||
StorageProviderS3 = "s3"
|
||||
StorageProviderContabo = "contabo"
|
||||
StorageProviderSFTP = "sftp"
|
||||
StorageProviderSMB = "smb"
|
||||
StorageProviderWebDAV = "webdav"
|
||||
)
|
||||
|
||||
type StorageObject struct {
|
||||
@@ -61,6 +75,15 @@ type StorageBackendConfig struct {
|
||||
SecretKey string `json:"secretKey,omitempty"`
|
||||
UseSSL bool `json:"useSsl,omitempty"`
|
||||
PathStyle bool `json:"pathStyle,omitempty"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Username string `json:"username,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
PrivateKey string `json:"privateKey,omitempty"`
|
||||
HostKey string `json:"hostKey,omitempty"`
|
||||
RemotePath string `json:"remotePath,omitempty"`
|
||||
Share string `json:"share,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
||||
@@ -156,23 +179,19 @@ func (s *StorageService) ListBackendConfigs() ([]StorageBackendConfig, error) {
|
||||
|
||||
func (s *StorageService) CreateS3Backend(input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
input.ID = randomID(10)
|
||||
input.Type = StorageBackendS3
|
||||
input.Provider = normalizeStorageProvider(input.Provider)
|
||||
if input.Provider == StorageProviderContabo {
|
||||
input.UseSSL = true
|
||||
input.PathStyle = true
|
||||
switch input.Provider {
|
||||
case StorageProviderSFTP:
|
||||
input.Type = StorageBackendSFTP
|
||||
case StorageProviderSMB:
|
||||
input.Type = StorageBackendSMB
|
||||
case StorageProviderWebDAV:
|
||||
input.Type = StorageBackendWebDAV
|
||||
default:
|
||||
input.Type = StorageBackendS3
|
||||
}
|
||||
input.Name = strings.TrimSpace(input.Name)
|
||||
input.Endpoint = strings.TrimSpace(input.Endpoint)
|
||||
input.Region = strings.TrimSpace(input.Region)
|
||||
input.Bucket = strings.TrimSpace(input.Bucket)
|
||||
input.AccessKey = strings.TrimSpace(input.AccessKey)
|
||||
input.SecretKey = strings.TrimSpace(input.SecretKey)
|
||||
if input.Name == "" {
|
||||
input.Name = input.Bucket
|
||||
}
|
||||
if input.Name == "" || input.Endpoint == "" || input.Bucket == "" || input.AccessKey == "" || input.SecretKey == "" {
|
||||
return StorageBackendConfig{}, fmt.Errorf("name, endpoint, bucket, access key, and secret key are required")
|
||||
if err := normalizeStorageBackendConfig(&input, true); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
input.Enabled = true
|
||||
@@ -189,35 +208,122 @@ func (s *StorageService) UpdateS3Backend(id string, input StorageBackendConfig)
|
||||
if err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
if current.ID == StorageBackendLocal || current.Type != StorageBackendS3 {
|
||||
return StorageBackendConfig{}, fmt.Errorf("only S3-compatible storage can be edited")
|
||||
if current.ID == StorageBackendLocal {
|
||||
return StorageBackendConfig{}, fmt.Errorf("local storage cannot be edited")
|
||||
}
|
||||
input.ID = current.ID
|
||||
input.Type = current.Type
|
||||
input.Provider = normalizeStorageProvider(input.Provider)
|
||||
switch input.Provider {
|
||||
case StorageProviderSFTP:
|
||||
input.Type = StorageBackendSFTP
|
||||
case StorageProviderSMB:
|
||||
input.Type = StorageBackendSMB
|
||||
case StorageProviderWebDAV:
|
||||
input.Type = StorageBackendWebDAV
|
||||
default:
|
||||
input.Type = StorageBackendS3
|
||||
}
|
||||
if strings.TrimSpace(input.SecretKey) == "" {
|
||||
input.SecretKey = current.SecretKey
|
||||
}
|
||||
if strings.TrimSpace(input.Password) == "" {
|
||||
input.Password = current.Password
|
||||
}
|
||||
if strings.TrimSpace(input.PrivateKey) == "" {
|
||||
input.PrivateKey = current.PrivateKey
|
||||
}
|
||||
if strings.TrimSpace(input.HostKey) == "" {
|
||||
input.HostKey = current.HostKey
|
||||
}
|
||||
input.Enabled = current.Enabled
|
||||
input.CreatedAt = current.CreatedAt
|
||||
input.LastTestedAt = current.LastTestedAt
|
||||
input.LastTestError = current.LastTestError
|
||||
input.LastTestSuccess = current.LastTestSuccess
|
||||
if err := normalizeStorageBackendConfig(&input, false); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
if err := s.SaveBackendConfig(input); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func normalizeStorageBackendConfig(input *StorageBackendConfig, creating bool) error {
|
||||
input.Name = strings.TrimSpace(input.Name)
|
||||
input.Provider = normalizeStorageProvider(input.Provider)
|
||||
if input.Provider == StorageProviderSFTP {
|
||||
input.Type = StorageBackendSFTP
|
||||
input.Host = strings.TrimSpace(input.Host)
|
||||
input.Username = strings.TrimSpace(input.Username)
|
||||
input.Password = strings.TrimSpace(input.Password)
|
||||
input.PrivateKey = strings.TrimSpace(input.PrivateKey)
|
||||
input.HostKey = strings.TrimSpace(input.HostKey)
|
||||
input.RemotePath = cleanRemoteRoot(input.RemotePath)
|
||||
if input.Port <= 0 {
|
||||
input.Port = 22
|
||||
}
|
||||
if input.Name == "" {
|
||||
input.Name = input.Host
|
||||
}
|
||||
if input.Name == "" || input.Host == "" || input.Username == "" || (input.Password == "" && input.PrivateKey == "") {
|
||||
return fmt.Errorf("name, host, username, and password or private key are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if input.Provider == StorageProviderSMB {
|
||||
input.Type = StorageBackendSMB
|
||||
input.Host = strings.TrimSpace(input.Host)
|
||||
input.Username = strings.TrimSpace(input.Username)
|
||||
input.Password = strings.TrimSpace(input.Password)
|
||||
input.Share = strings.Trim(strings.TrimSpace(input.Share), `/\`)
|
||||
input.Domain = strings.TrimSpace(input.Domain)
|
||||
input.RemotePath = cleanRemoteRoot(input.RemotePath)
|
||||
if input.Port <= 0 {
|
||||
input.Port = 445
|
||||
}
|
||||
if input.Name == "" {
|
||||
input.Name = input.Share
|
||||
}
|
||||
if input.Name == "" || input.Host == "" || input.Share == "" || input.Username == "" || input.Password == "" {
|
||||
return fmt.Errorf("name, host, share, username, and password are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if input.Provider == StorageProviderWebDAV {
|
||||
input.Type = StorageBackendWebDAV
|
||||
input.Endpoint = strings.TrimSpace(input.Endpoint)
|
||||
input.Username = strings.TrimSpace(input.Username)
|
||||
input.Password = strings.TrimSpace(input.Password)
|
||||
input.RemotePath = cleanRemoteRoot(input.RemotePath)
|
||||
if input.Name == "" {
|
||||
input.Name = input.Endpoint
|
||||
}
|
||||
if input.Name == "" || input.Endpoint == "" {
|
||||
return fmt.Errorf("name and WebDAV URL are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
current.Provider = normalizeStorageProvider(input.Provider)
|
||||
if current.Provider == StorageProviderContabo {
|
||||
input.Type = StorageBackendS3
|
||||
if input.Provider == StorageProviderContabo {
|
||||
input.UseSSL = true
|
||||
input.PathStyle = true
|
||||
}
|
||||
current.Name = strings.TrimSpace(input.Name)
|
||||
current.Endpoint = strings.TrimSpace(input.Endpoint)
|
||||
current.Region = strings.TrimSpace(input.Region)
|
||||
current.Bucket = strings.TrimSpace(input.Bucket)
|
||||
current.AccessKey = strings.TrimSpace(input.AccessKey)
|
||||
if strings.TrimSpace(input.SecretKey) != "" {
|
||||
current.SecretKey = strings.TrimSpace(input.SecretKey)
|
||||
input.Name = strings.TrimSpace(input.Name)
|
||||
input.Endpoint = strings.TrimSpace(input.Endpoint)
|
||||
input.Region = strings.TrimSpace(input.Region)
|
||||
input.Bucket = strings.TrimSpace(input.Bucket)
|
||||
input.AccessKey = strings.TrimSpace(input.AccessKey)
|
||||
input.SecretKey = strings.TrimSpace(input.SecretKey)
|
||||
if input.Name == "" {
|
||||
input.Name = input.Bucket
|
||||
}
|
||||
current.UseSSL = input.UseSSL
|
||||
current.PathStyle = input.PathStyle
|
||||
if current.Name == "" {
|
||||
current.Name = current.Bucket
|
||||
if input.Name == "" || input.Endpoint == "" || input.Bucket == "" || input.AccessKey == "" || input.SecretKey == "" {
|
||||
return fmt.Errorf("name, endpoint, bucket, access key, and secret key are required")
|
||||
}
|
||||
if current.Name == "" || current.Endpoint == "" || current.Bucket == "" || current.AccessKey == "" || current.SecretKey == "" {
|
||||
return StorageBackendConfig{}, fmt.Errorf("name, endpoint, bucket, access key, and secret key are required")
|
||||
}
|
||||
if err := s.SaveBackendConfig(current); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
return current, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *StorageService) SaveBackendConfig(cfg StorageBackendConfig) error {
|
||||
@@ -289,6 +395,12 @@ func (s *StorageService) backendFromConfig(cfg StorageBackendConfig) (StorageBac
|
||||
return localStorageBackend{id: cfg.ID, root: cfg.LocalPath}, nil
|
||||
case StorageBackendS3:
|
||||
return newS3StorageBackend(cfg)
|
||||
case StorageBackendSFTP:
|
||||
return sftpStorageBackend{cfg: cfg}, nil
|
||||
case StorageBackendSMB:
|
||||
return smbStorageBackend{cfg: cfg}, nil
|
||||
case StorageBackendWebDAV:
|
||||
return webDAVStorageBackend{cfg: cfg, client: http.DefaultClient}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage backend type %q", cfg.Type)
|
||||
}
|
||||
@@ -507,6 +619,544 @@ func (b *s3StorageBackend) Test(ctx context.Context) error {
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
type sftpStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b sftpStorageBackend) Type() string { return StorageBackendSFTP }
|
||||
|
||||
func (b sftpStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
if err := client.MkdirAll(path.Dir(remotePath)); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := client.OpenFile(remotePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
source, err := client.Open(remotePath)
|
||||
if err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: closeWith(source, closer)}, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.Remove(b.remotePath(key)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(prefix)
|
||||
if err := client.RemoveDirectory(remotePath); err == nil || os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
walker := client.Walk(remotePath)
|
||||
paths := make([]string, 0)
|
||||
for walker.Step() {
|
||||
if walker.Err() != nil {
|
||||
return walker.Err()
|
||||
}
|
||||
paths = append(paths, walker.Path())
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool { return len(paths[i]) > len(paths[j]) })
|
||||
for _, item := range paths {
|
||||
if err := client.Remove(item); err != nil {
|
||||
_ = client.RemoveDirectory(item)
|
||||
}
|
||||
}
|
||||
_ = client.RemoveDirectory(remotePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
client, closer, err := b.client()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
walker := client.Walk(cleanRemoteRoot(b.cfg.RemotePath))
|
||||
for walker.Step() {
|
||||
if walker.Err() != nil {
|
||||
return 0, walker.Err()
|
||||
}
|
||||
info := walker.Stat()
|
||||
if info != nil && !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) client() (*sftp.Client, func(), error) {
|
||||
auth := make([]ssh.AuthMethod, 0, 2)
|
||||
if b.cfg.PrivateKey != "" {
|
||||
signer, err := ssh.ParsePrivateKey([]byte(b.cfg.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
auth = append(auth, ssh.PublicKeys(signer))
|
||||
}
|
||||
if b.cfg.Password != "" {
|
||||
auth = append(auth, ssh.Password(b.cfg.Password))
|
||||
}
|
||||
if len(auth) == 0 {
|
||||
return nil, nil, fmt.Errorf("sftp password or private key is required")
|
||||
}
|
||||
hostKeyCallback, err := b.hostKeyCallback()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sshClient, err := ssh.Dial("tcp", b.cfg.Host+":"+strconv.Itoa(b.cfg.Port), &ssh.ClientConfig{
|
||||
User: b.cfg.Username,
|
||||
Auth: auth,
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 15 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, err := sftp.NewClient(sshClient)
|
||||
if err != nil {
|
||||
sshClient.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return client, func() {
|
||||
client.Close()
|
||||
sshClient.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) hostKeyCallback() (ssh.HostKeyCallback, error) {
|
||||
if strings.TrimSpace(b.cfg.HostKey) == "" {
|
||||
return ssh.InsecureIgnoreHostKey(), nil
|
||||
}
|
||||
key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(strings.TrimSpace(b.cfg.HostKey)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid sftp host public key: %w", err)
|
||||
}
|
||||
return ssh.FixedHostKey(key), nil
|
||||
}
|
||||
|
||||
func (b sftpStorageBackend) remotePath(key string) string {
|
||||
return path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key))
|
||||
}
|
||||
|
||||
type joinedReadCloser struct {
|
||||
io.ReadCloser
|
||||
close func()
|
||||
}
|
||||
|
||||
func closeWith(source io.ReadCloser, close func()) io.ReadCloser {
|
||||
return joinedReadCloser{ReadCloser: source, close: close}
|
||||
}
|
||||
|
||||
func (c joinedReadCloser) Close() error {
|
||||
err := c.ReadCloser.Close()
|
||||
c.close()
|
||||
return err
|
||||
}
|
||||
|
||||
type smbStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b smbStorageBackend) Type() string { return StorageBackendSMB }
|
||||
|
||||
func (b smbStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
remotePath := b.remotePath(key)
|
||||
if err := share.MkdirAll(path.Dir(remotePath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := share.OpenFile(remotePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
source, err := share.Open(b.remotePath(key))
|
||||
if err != nil {
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
stat, err := source.Stat()
|
||||
if err != nil {
|
||||
source.Close()
|
||||
closer()
|
||||
return StorageObject{}, err
|
||||
}
|
||||
return StorageObject{Key: key, Size: stat.Size(), ModTime: stat.ModTime(), Body: closeWith(source, closer)}, nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := share.Remove(b.remotePath(key)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
err = share.RemoveAll(b.remotePath(prefix))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
share, closer, err := b.share()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer closer()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return smbUsage(share, cleanRemoteRoot(b.cfg.RemotePath))
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) share() (*smb2.Share, func(), error) {
|
||||
conn, err := net.DialTimeout("tcp", b.cfg.Host+":"+strconv.Itoa(b.cfg.Port), 15*time.Second)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dialer := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: b.cfg.Username,
|
||||
Password: b.cfg.Password,
|
||||
Domain: b.cfg.Domain,
|
||||
},
|
||||
}
|
||||
session, err := dialer.Dial(conn)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
share, err := session.Mount(b.cfg.Share)
|
||||
if err != nil {
|
||||
session.Logoff()
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return share, func() {
|
||||
share.Umount()
|
||||
session.Logoff()
|
||||
conn.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b smbStorageBackend) remotePath(key string) string {
|
||||
return strings.TrimPrefix(path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key)), "/")
|
||||
}
|
||||
|
||||
func smbUsage(share *smb2.Share, root string) (int64, error) {
|
||||
root = strings.TrimPrefix(root, "/")
|
||||
entries, err := share.ReadDir(root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, entry := range entries {
|
||||
item := path.Join(root, entry.Name())
|
||||
if entry.IsDir() {
|
||||
size, err := smbUsage(share, item)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += size
|
||||
continue
|
||||
}
|
||||
total += entry.Size()
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
type webDAVStorageBackend struct {
|
||||
cfg StorageBackendConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) ID() string { return b.cfg.ID }
|
||||
func (b webDAVStorageBackend) Type() string { return StorageBackendWebDAV }
|
||||
|
||||
func (b webDAVStorageBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, contentType string) error {
|
||||
if err := b.mkcolParents(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := b.request(ctx, http.MethodPut, key, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if contentType != "" {
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return fmt.Errorf("webdav put failed: %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
|
||||
request, err := b.request(ctx, http.MethodGet, key, nil)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return StorageObject{}, err
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
response.Body.Close()
|
||||
return StorageObject{}, fmt.Errorf("webdav get failed: %s", response.Status)
|
||||
}
|
||||
modTime, _ := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified"))
|
||||
return StorageObject{Key: key, Size: response.ContentLength, ContentType: response.Header.Get("Content-Type"), ModTime: modTime, Body: response.Body}, nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.deletePath(ctx, key)
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
return b.deletePath(ctx, strings.TrimSuffix(prefix, "/")+"/")
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Usage(ctx context.Context) (int64, error) {
|
||||
request, err := b.request(ctx, "PROPFIND", "", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
request.Header.Set("Depth", "infinity")
|
||||
request.Header.Set("Content-Type", "application/xml")
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return 0, fmt.Errorf("webdav usage failed: %s", response.Status)
|
||||
}
|
||||
var multi webDAVMultiStatus
|
||||
if err := xml.NewDecoder(response.Body).Decode(&multi); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, item := range multi.Responses {
|
||||
if item.PropStat.Prop.ResourceType.Collection != nil {
|
||||
continue
|
||||
}
|
||||
total += item.PropStat.Prop.ContentLength
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) Test(ctx context.Context) error {
|
||||
key := ".warpbox-storage-test-" + randomID(6)
|
||||
if err := b.Put(ctx, key, strings.NewReader("ok"), 2, "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) deletePath(ctx context.Context, key string) error {
|
||||
request, err := b.request(ctx, http.MethodDelete, key, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
return nil
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return fmt.Errorf("webdav delete failed: %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) mkcolParents(ctx context.Context, key string) error {
|
||||
dir := path.Dir(cleanObjectKey(key))
|
||||
if dir == "." || dir == "/" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(strings.Trim(dir, "/"), "/")
|
||||
current := ""
|
||||
for _, part := range parts {
|
||||
current = path.Join(current, part)
|
||||
request, err := b.request(ctx, "MKCOL", strings.TrimSuffix(current, "/")+"/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := b.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusMethodNotAllowed && response.StatusCode != http.StatusConflict {
|
||||
return fmt.Errorf("webdav mkcol failed: %s", response.Status)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b webDAVStorageBackend) request(ctx context.Context, method, key string, body io.Reader) (*http.Request, error) {
|
||||
endpoint := strings.TrimRight(b.cfg.Endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("webdav url is required")
|
||||
}
|
||||
remote := path.Join(cleanRemoteRoot(b.cfg.RemotePath), cleanObjectKey(key))
|
||||
if strings.HasSuffix(key, "/") && !strings.HasSuffix(remote, "/") {
|
||||
remote += "/"
|
||||
}
|
||||
target := endpoint + "/" + strings.TrimLeft(remote, "/")
|
||||
request, err := http.NewRequestWithContext(ctx, method, target, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.cfg.Username != "" || b.cfg.Password != "" {
|
||||
request.SetBasicAuth(b.cfg.Username, b.cfg.Password)
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
type webDAVMultiStatus struct {
|
||||
Responses []webDAVResponse `xml:"response"`
|
||||
}
|
||||
|
||||
type webDAVResponse struct {
|
||||
PropStat webDAVPropStat `xml:"propstat"`
|
||||
}
|
||||
|
||||
type webDAVPropStat struct {
|
||||
Prop webDAVProp `xml:"prop"`
|
||||
}
|
||||
|
||||
type webDAVProp struct {
|
||||
ContentLength int64 `xml:"getcontentlength"`
|
||||
ResourceType webDAVResourceType `xml:"resourcetype"`
|
||||
}
|
||||
|
||||
type webDAVResourceType struct {
|
||||
Collection *struct{} `xml:"collection"`
|
||||
}
|
||||
|
||||
func s3BucketLookup(pathStyle bool) minio.BucketLookupType {
|
||||
if pathStyle {
|
||||
return minio.BucketLookupPath
|
||||
@@ -526,6 +1176,12 @@ func normalizeStorageProvider(provider string) string {
|
||||
switch strings.TrimSpace(provider) {
|
||||
case StorageProviderContabo:
|
||||
return StorageProviderContabo
|
||||
case StorageProviderSFTP:
|
||||
return StorageProviderSFTP
|
||||
case StorageProviderSMB:
|
||||
return StorageProviderSMB
|
||||
case StorageProviderWebDAV:
|
||||
return StorageProviderWebDAV
|
||||
default:
|
||||
return StorageProviderS3
|
||||
}
|
||||
@@ -534,3 +1190,15 @@ func normalizeStorageProvider(provider string) string {
|
||||
func cleanObjectKey(key string) string {
|
||||
return strings.TrimPrefix(filepath.ToSlash(filepath.Clean(strings.TrimPrefix(key, "/"))), "./")
|
||||
}
|
||||
|
||||
func cleanRemoteRoot(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "."
|
||||
}
|
||||
cleaned := path.Clean(strings.ReplaceAll(value, "\\", "/"))
|
||||
if cleaned == "/" {
|
||||
return "/"
|
||||
}
|
||||
return strings.TrimSuffix(cleaned, "/")
|
||||
}
|
||||
|
||||
@@ -148,6 +148,67 @@ func TestContaboStorageConfigAllowsDisplayNamesWithSpaces(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFTPStorageConfigValidation(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
cfg, err := service.Storage().CreateS3Backend(StorageBackendConfig{
|
||||
Provider: StorageProviderSFTP,
|
||||
Name: "NAS storage",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
RemotePath: "/srv/warpbox//",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateS3Backend returned error: %v", err)
|
||||
}
|
||||
if cfg.Type != StorageBackendSFTP || cfg.Provider != StorageProviderSFTP {
|
||||
t.Fatalf("sftp config type/provider = %+v", cfg)
|
||||
}
|
||||
if cfg.Port != 22 {
|
||||
t.Fatalf("port = %d, want 22", cfg.Port)
|
||||
}
|
||||
if cfg.RemotePath != "/srv/warpbox" {
|
||||
t.Fatalf("remote path = %q", cfg.RemotePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSMBAndWebDAVStorageConfigValidation(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
smb, err := service.Storage().CreateS3Backend(StorageBackendConfig{
|
||||
Provider: StorageProviderSMB,
|
||||
Name: "Office NAS",
|
||||
Host: "nas.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
Share: "uploads",
|
||||
RemotePath: "/warpbox//",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateS3Backend smb returned error: %v", err)
|
||||
}
|
||||
if smb.Type != StorageBackendSMB || smb.Provider != StorageProviderSMB || smb.Port != 445 {
|
||||
t.Fatalf("smb config was not normalized: %+v", smb)
|
||||
}
|
||||
if smb.RemotePath != "/warpbox" {
|
||||
t.Fatalf("smb remote path = %q", smb.RemotePath)
|
||||
}
|
||||
|
||||
webdav, err := service.Storage().CreateS3Backend(StorageBackendConfig{
|
||||
Provider: StorageProviderWebDAV,
|
||||
Name: "Nextcloud",
|
||||
Endpoint: "https://files.example.test/webdav",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
RemotePath: "/warpbox",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateS3Backend webdav returned error: %v", err)
|
||||
}
|
||||
if webdav.Type != StorageBackendWebDAV || webdav.Provider != StorageProviderWebDAV {
|
||||
t.Fatalf("webdav config was not normalized: %+v", webdav)
|
||||
}
|
||||
}
|
||||
|
||||
func testContext() context.Context {
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user