feat(admin): implement provider-specific storage configuration pages
Some checks failed
Build and Publish Docker Image / deploy (push) Has been cancelled
Some checks failed
Build and Publish Docker Image / deploy (push) Has been cancelled
Refactor the admin storage backend creation and editing flows to use provider-specific pages (e.g., `/admin/storage/new/sftp`) instead of a single generic form. This ensures only relevant fields are rendered for each storage provider (such as SFTP, S3, or WebDAV). Additionally: - Prevent mutation of the storage provider type during backend edits. - Add comprehensive unit tests for provider-specific rendering, edit validation, and CSRF/admin route protection.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -613,6 +614,190 @@ func TestAPIDocsHeaderReflectsLoggedOutUser(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStorageProviderPagesOnlyRenderRelevantFields(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
adminToken := createAdminSession(t, app)
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/admin/storage/new/sftp", nil)
|
||||
request.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
response := httptest.NewRecorder()
|
||||
app.AdminNewStorageProvider(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("AdminNewStorageProvider status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, "Private key") || !strings.Contains(body, "SSH host key") {
|
||||
t.Fatalf("sftp page did not render sftp fields: %s", body)
|
||||
}
|
||||
for _, unwanted := range []string{"Bucket display name", "WebDAV URL", "Share</span>", "Access key"} {
|
||||
if strings.Contains(body, unwanted) {
|
||||
t.Fatalf("sftp page rendered irrelevant field %q: %s", unwanted, body)
|
||||
}
|
||||
}
|
||||
|
||||
cfg, err := app.uploadService.Storage().CreateBackend(services.StorageBackendConfig{
|
||||
Provider: services.StorageProviderSFTP,
|
||||
Name: "NAS",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateBackend returned error: %v", err)
|
||||
}
|
||||
editRequest := httptest.NewRequest(http.MethodGet, "/admin/storage/"+cfg.ID+"/edit", nil)
|
||||
editRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
editRequest.SetPathValue("backendID", cfg.ID)
|
||||
editResponse := httptest.NewRecorder()
|
||||
app.AdminEditStorageForm(editResponse, editRequest)
|
||||
if editResponse.Code != http.StatusOK {
|
||||
t.Fatalf("AdminEditStorageForm status = %d, body = %s", editResponse.Code, editResponse.Body.String())
|
||||
}
|
||||
editBody := editResponse.Body.String()
|
||||
if !strings.Contains(editBody, "Immutable provider") || strings.Contains(editBody, "Bucket display name") || strings.Contains(editBody, "WebDAV URL") {
|
||||
t.Fatalf("edit page did not stay provider-specific: %s", editBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStorageEditRejectsProviderMutation(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
adminToken := createAdminSession(t, app)
|
||||
cfg, err := app.uploadService.Storage().CreateBackend(services.StorageBackendConfig{
|
||||
Provider: services.StorageProviderSFTP,
|
||||
Name: "NAS",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateBackend returned error: %v", err)
|
||||
}
|
||||
|
||||
form := strings.NewReader("provider=s3&name=Changed&endpoint=https://s3.example.test&bucket=bucket&access_key=access&secret_key=secret&use_ssl=on&csrf_token=test-csrf")
|
||||
request := httptest.NewRequest(http.MethodPost, "/admin/storage/"+cfg.ID+"/edit", form)
|
||||
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
request.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
request.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"})
|
||||
request.SetPathValue("backendID", cfg.ID)
|
||||
response := httptest.NewRecorder()
|
||||
app.AdminEditStorage(response, request)
|
||||
if response.Code != http.StatusSeeOther {
|
||||
t.Fatalf("AdminEditStorage status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
stored, err := app.uploadService.Storage().BackendConfig(cfg.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("BackendConfig returned error: %v", err)
|
||||
}
|
||||
if stored.Provider != services.StorageProviderSFTP || stored.Type != services.StorageBackendSFTP || stored.Name != "NAS" {
|
||||
t.Fatalf("storage backend mutated despite rejected provider change: %+v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStorageJobRoutesRequireAdminAndCSRF(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
|
||||
unauthorized := httptest.NewRecorder()
|
||||
app.AdminRunStorageCleanup(unauthorized, httptest.NewRequest(http.MethodPost, "/admin/storage/jobs/cleanup", nil))
|
||||
if unauthorized.Code != http.StatusSeeOther {
|
||||
t.Fatalf("unauthorized cleanup status = %d", unauthorized.Code)
|
||||
}
|
||||
|
||||
adminToken := createAdminSession(t, app)
|
||||
missingCSRFRequest := httptest.NewRequest(http.MethodPost, "/admin/storage/jobs/cleanup", nil)
|
||||
missingCSRFRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
missingCSRFResponse := httptest.NewRecorder()
|
||||
app.AdminRunStorageCleanup(missingCSRFResponse, missingCSRFRequest)
|
||||
if missingCSRFResponse.Code != http.StatusForbidden {
|
||||
t.Fatalf("missing csrf cleanup status = %d", missingCSRFResponse.Code)
|
||||
}
|
||||
|
||||
request := httptest.NewRequest(http.MethodPost, "/admin/storage/jobs/cleanup", strings.NewReader("csrf_token=test-csrf"))
|
||||
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
request.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
request.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"})
|
||||
response := httptest.NewRecorder()
|
||||
app.AdminRunStorageCleanup(response, request)
|
||||
if response.Code != http.StatusSeeOther {
|
||||
t.Fatalf("authorized cleanup status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStorageSpeedTestStartsBackgroundJob(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
adminToken := createAdminSession(t, app)
|
||||
if _, err := app.uploadService.Storage().TestBackend(services.StorageBackendLocal); err != nil {
|
||||
t.Fatalf("TestBackend returned error: %v", err)
|
||||
}
|
||||
|
||||
request := httptest.NewRequest(http.MethodPost, "/admin/storage/local/speed-test", strings.NewReader("mode=custom&custom_file_count=2&custom_file_size_mb=0.001&csrf_token=test-csrf"))
|
||||
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
request.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
request.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "test-csrf"})
|
||||
request.SetPathValue("backendID", services.StorageBackendLocal)
|
||||
response := httptest.NewRecorder()
|
||||
app.AdminStartStorageSpeedTest(response, request)
|
||||
if response.Code != http.StatusSeeOther {
|
||||
t.Fatalf("AdminStartStorageSpeedTest status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
tests, err := app.uploadService.Storage().ListSpeedTests(services.StorageBackendLocal, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpeedTests returned error: %v", err)
|
||||
}
|
||||
if len(tests) != 1 {
|
||||
t.Fatalf("speed tests len = %d, want 1", len(tests))
|
||||
}
|
||||
if tests[0].Mode != services.StorageSpeedModeCustom || tests[0].CustomFileCount != 2 || tests[0].CustomFileSizeMB != 0.001 {
|
||||
t.Fatalf("custom speed test options were not stored: %+v", tests[0])
|
||||
}
|
||||
location := response.Header().Get("Location")
|
||||
if !strings.Contains(location, "/admin/storage/local/tests") {
|
||||
t.Fatalf("speed test redirect location = %q", location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStorageTestingPageRendersHistory(t *testing.T) {
|
||||
app, cleanup := newTestApp(t)
|
||||
defer cleanup()
|
||||
adminToken := createAdminSession(t, app)
|
||||
if _, err := app.uploadService.Storage().TestBackend(services.StorageBackendLocal); err != nil {
|
||||
t.Fatalf("TestBackend returned error: %v", err)
|
||||
}
|
||||
test, err := app.uploadService.Storage().StartSpeedTest(services.StorageBackendLocal, services.StorageSpeedModeSmall)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSpeedTest returned error: %v", err)
|
||||
}
|
||||
app.uploadService.Storage().RunSpeedTest(context.Background(), test.ID)
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/admin/storage/local/tests", nil)
|
||||
request.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
request.SetPathValue("backendID", services.StorageBackendLocal)
|
||||
response := httptest.NewRecorder()
|
||||
app.AdminStorageTests(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("AdminStorageTests status = %d, body = %s", response.Code, response.Body.String())
|
||||
}
|
||||
body := response.Body.String()
|
||||
if !strings.Contains(body, "New Test") || !strings.Contains(body, "Many small files") || strings.Contains(body, "storage-test-menu") {
|
||||
t.Fatalf("testing page missing expected page-based controls: %s", body)
|
||||
}
|
||||
|
||||
jsonRequest := httptest.NewRequest(http.MethodGet, "/admin/storage/local/tests.json", nil)
|
||||
jsonRequest.AddCookie(&http.Cookie{Name: userSessionCookieName, Value: adminToken})
|
||||
jsonRequest.SetPathValue("backendID", services.StorageBackendLocal)
|
||||
jsonResponse := httptest.NewRecorder()
|
||||
app.AdminStorageTestsJSON(jsonResponse, jsonRequest)
|
||||
if jsonResponse.Code != http.StatusOK {
|
||||
t.Fatalf("AdminStorageTestsJSON status = %d, body = %s", jsonResponse.Code, jsonResponse.Body.String())
|
||||
}
|
||||
if !strings.Contains(jsonResponse.Body.String(), `"progress":100`) || !strings.Contains(jsonResponse.Body.String(), `"stage":"complete"`) {
|
||||
t.Fatalf("tests json missing progress fields: %s", jsonResponse.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func createOwnedBoxThroughApp(t *testing.T, app *App, userID string) services.UploadResult {
|
||||
t.Helper()
|
||||
user, err := app.authService.UserByID(userID)
|
||||
@@ -638,6 +823,19 @@ func createOwnedBoxThroughApp(t *testing.T, app *App, userID string) services.Up
|
||||
return payload
|
||||
}
|
||||
|
||||
func createAdminSession(t *testing.T, app *App) string {
|
||||
t.Helper()
|
||||
_, err := app.authService.CreateBootstrapUser("admin", "admin@example.test", "password123")
|
||||
if err != nil && !strings.Contains(err.Error(), "registration is closed") {
|
||||
t.Fatalf("CreateBootstrapUser returned error: %v", err)
|
||||
}
|
||||
_, token, err := app.authService.Login("admin@example.test", "password123")
|
||||
if err != nil {
|
||||
t.Fatalf("Login returned error: %v", err)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
func testPolicy(t *testing.T, app *App) services.UploadPolicySettings {
|
||||
t.Helper()
|
||||
policy, err := app.settingsService.UploadPolicy()
|
||||
|
||||
@@ -4,11 +4,15 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"warpbox.dev/backend/libs/jobs"
|
||||
"warpbox.dev/backend/libs/services"
|
||||
"warpbox.dev/backend/libs/web"
|
||||
)
|
||||
@@ -22,12 +26,56 @@ type adminPageData struct {
|
||||
Settings services.UploadPolicySettings
|
||||
Storage []services.StorageBackendView
|
||||
UserEdit adminUserEditView
|
||||
StorageForm adminStorageFormView
|
||||
StorageTest adminStorageTestView
|
||||
StorageTypes []adminStorageProviderView
|
||||
Section string
|
||||
PageTitle string
|
||||
LastInviteURL string
|
||||
Notice string
|
||||
Error string
|
||||
}
|
||||
|
||||
type adminStorageFormView struct {
|
||||
Mode string
|
||||
Provider string
|
||||
ProviderLabel string
|
||||
Action string
|
||||
BackHref string
|
||||
Config services.StorageBackendConfig
|
||||
}
|
||||
|
||||
type adminStorageTestView struct {
|
||||
Config services.StorageBackendConfig
|
||||
UsageLabel string
|
||||
Tests []services.StorageSpeedTest
|
||||
CanRun bool
|
||||
}
|
||||
|
||||
type adminStorageSpeedTestJSON struct {
|
||||
ID string `json:"id"`
|
||||
Mode string `json:"mode"`
|
||||
ModeLabel string `json:"modeLabel"`
|
||||
Status string `json:"status"`
|
||||
Stage string `json:"stage"`
|
||||
Progress int `json:"progress"`
|
||||
CustomLabel string `json:"customLabel,omitempty"`
|
||||
StartedLabel string `json:"startedLabel"`
|
||||
FinishedLabel string `json:"finishedLabel"`
|
||||
Files int `json:"files"`
|
||||
SizeLabel string `json:"sizeLabel"`
|
||||
WriteSpeed string `json:"writeSpeed"`
|
||||
ReadSpeed string `json:"readSpeed"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type adminStorageProviderView struct {
|
||||
Provider string
|
||||
Label string
|
||||
Description string
|
||||
Icon string
|
||||
}
|
||||
|
||||
type adminBoxView struct {
|
||||
ID string
|
||||
Owner string
|
||||
@@ -417,44 +465,167 @@ func (a *App) AdminStorage(w http.ResponseWriter, r *http.Request) {
|
||||
Storage: views,
|
||||
Section: "storage",
|
||||
PageTitle: "Storage",
|
||||
Notice: r.URL.Query().Get("notice"),
|
||||
Error: r.URL.Query().Get("error"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) AdminCreateS3Storage(w http.ResponseWriter, r *http.Request) {
|
||||
func (a *App) AdminNewStorage(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) {
|
||||
return
|
||||
}
|
||||
a.renderPage(w, r, http.StatusOK, "admin_storage_new.html", web.PageData{
|
||||
Title: "Add storage",
|
||||
Description: "Choose a Warpbox storage provider.",
|
||||
CurrentUser: a.currentPublicUser(r),
|
||||
Data: adminPageData{
|
||||
Section: "storage",
|
||||
PageTitle: "Add storage",
|
||||
StorageTypes: adminStorageProviderOptions(),
|
||||
Error: r.URL.Query().Get("error"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) AdminNewStorageProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) {
|
||||
return
|
||||
}
|
||||
rawProvider := adminStorageProviderFromRequest(r)
|
||||
if !validAdminStorageProvider(rawProvider) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
provider := normalizeAdminStorageProvider(rawProvider)
|
||||
a.renderStorageForm(w, r, http.StatusOK, adminStorageFormView{
|
||||
Mode: "create",
|
||||
Provider: provider,
|
||||
ProviderLabel: adminStorageProviderLabel(provider),
|
||||
Action: "/admin/storage/new/" + provider,
|
||||
BackHref: "/admin/storage/new",
|
||||
Config: services.StorageBackendConfig{
|
||||
Provider: provider,
|
||||
Type: adminStorageTypeForProvider(provider),
|
||||
UseSSL: true,
|
||||
},
|
||||
}, r.URL.Query().Get("error"))
|
||||
}
|
||||
|
||||
func (a *App) AdminEditStorageForm(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) {
|
||||
return
|
||||
}
|
||||
cfg, err := a.uploadService.Storage().BackendConfig(r.PathValue("backendID"))
|
||||
if err != nil || cfg.ID == services.StorageBackendLocal {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
provider := normalizeAdminStorageProvider(cfg.Provider)
|
||||
a.renderStorageForm(w, r, http.StatusOK, adminStorageFormView{
|
||||
Mode: "edit",
|
||||
Provider: provider,
|
||||
ProviderLabel: adminStorageProviderLabel(provider),
|
||||
Action: "/admin/storage/" + cfg.ID + "/edit",
|
||||
BackHref: "/admin/storage",
|
||||
Config: cfg,
|
||||
}, r.URL.Query().Get("error"))
|
||||
}
|
||||
|
||||
func (a *App) AdminStorageTests(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) {
|
||||
return
|
||||
}
|
||||
cfg, err := a.uploadService.Storage().BackendConfig(r.PathValue("backendID"))
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
tests, err := a.uploadService.Storage().ListSpeedTests(cfg.ID, 100)
|
||||
if err != nil {
|
||||
http.Error(w, "unable to load storage tests", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var usage int64
|
||||
if cfg.Enabled {
|
||||
if backend, err := a.uploadService.Storage().Backend(cfg.ID); err == nil {
|
||||
usage, _ = backend.Usage(context.Background())
|
||||
}
|
||||
}
|
||||
a.renderPage(w, r, http.StatusOK, "admin_storage_tests.html", web.PageData{
|
||||
Title: cfg.Name + " tests",
|
||||
Description: "Storage speed-test history.",
|
||||
CurrentUser: a.currentPublicUser(r),
|
||||
Data: adminPageData{
|
||||
Section: "storage",
|
||||
PageTitle: cfg.Name + " tests",
|
||||
StorageTest: adminStorageTestView{
|
||||
Config: cfg,
|
||||
UsageLabel: services.FormatMegabytesFromBytes(usage),
|
||||
Tests: tests,
|
||||
CanRun: cfg.Enabled && cfg.LastTestSuccess,
|
||||
},
|
||||
Notice: r.URL.Query().Get("notice"),
|
||||
Error: r.URL.Query().Get("error"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) AdminStorageTestsJSON(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) {
|
||||
return
|
||||
}
|
||||
cfg, err := a.uploadService.Storage().BackendConfig(r.PathValue("backendID"))
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
tests, err := a.uploadService.Storage().ListSpeedTests(cfg.ID, 100)
|
||||
if err != nil {
|
||||
http.Error(w, "unable to load storage tests", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
payload := struct {
|
||||
Tests []adminStorageSpeedTestJSON `json:"tests"`
|
||||
}{Tests: adminStorageSpeedTestsJSON(tests)}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func (a *App) renderStorageForm(w http.ResponseWriter, r *http.Request, status int, form adminStorageFormView, message string) {
|
||||
a.renderPage(w, r, status, "admin_storage_form.html", web.PageData{
|
||||
Title: form.ProviderLabel + " storage",
|
||||
Description: "Configure Warpbox storage.",
|
||||
CurrentUser: a.currentPublicUser(r),
|
||||
Data: adminPageData{
|
||||
Section: "storage",
|
||||
PageTitle: form.ProviderLabel + " storage",
|
||||
StorageForm: form,
|
||||
Error: message,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) AdminCreateStorage(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
}
|
||||
rawProvider := adminStorageProviderFromRequest(r)
|
||||
if !validAdminStorageProvider(rawProvider) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
provider := normalizeAdminStorageProvider(rawProvider)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Redirect(w, r, "/admin/storage", http.StatusSeeOther)
|
||||
http.Redirect(w, r, "/admin/storage/new/"+provider, http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
_, err := a.uploadService.Storage().CreateS3Backend(services.StorageBackendConfig{
|
||||
Provider: r.FormValue("provider"),
|
||||
Name: r.FormValue("name"),
|
||||
Endpoint: r.FormValue("endpoint"),
|
||||
Region: r.FormValue("region"),
|
||||
Bucket: r.FormValue("bucket"),
|
||||
AccessKey: r.FormValue("access_key"),
|
||||
SecretKey: r.FormValue("secret_key"),
|
||||
UseSSL: r.FormValue("use_ssl") == "on",
|
||||
PathStyle: r.FormValue("path_style") == "on",
|
||||
Host: r.FormValue("host"),
|
||||
Port: parsePositiveInt(r.FormValue("port")),
|
||||
Username: r.FormValue("username"),
|
||||
Password: r.FormValue("password"),
|
||||
PrivateKey: r.FormValue("private_key"),
|
||||
HostKey: r.FormValue("host_key"),
|
||||
RemotePath: r.FormValue("remote_path"),
|
||||
Share: r.FormValue("share"),
|
||||
Domain: r.FormValue("domain"),
|
||||
})
|
||||
_, err := a.uploadService.Storage().CreateBackend(a.storageConfigFromForm(r, provider))
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
http.Redirect(w, r, "/admin/storage/new/"+provider+"?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/admin/storage", http.StatusSeeOther)
|
||||
http.Redirect(w, r, "/admin/storage?notice="+url.QueryEscape("Storage backend added."), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminEditStorage(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -465,42 +636,50 @@ func (a *App) AdminEditStorage(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/admin/storage", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
_, err := a.uploadService.Storage().UpdateS3Backend(r.PathValue("backendID"), services.StorageBackendConfig{
|
||||
Provider: r.FormValue("provider"),
|
||||
Name: r.FormValue("name"),
|
||||
Endpoint: r.FormValue("endpoint"),
|
||||
Region: r.FormValue("region"),
|
||||
Bucket: r.FormValue("bucket"),
|
||||
AccessKey: r.FormValue("access_key"),
|
||||
SecretKey: r.FormValue("secret_key"),
|
||||
UseSSL: r.FormValue("use_ssl") == "on",
|
||||
PathStyle: r.FormValue("path_style") == "on",
|
||||
Host: r.FormValue("host"),
|
||||
Port: parsePositiveInt(r.FormValue("port")),
|
||||
Username: r.FormValue("username"),
|
||||
Password: r.FormValue("password"),
|
||||
PrivateKey: r.FormValue("private_key"),
|
||||
HostKey: r.FormValue("host_key"),
|
||||
RemotePath: r.FormValue("remote_path"),
|
||||
Share: r.FormValue("share"),
|
||||
Domain: r.FormValue("domain"),
|
||||
})
|
||||
provider := normalizeAdminStorageProvider(r.FormValue("provider"))
|
||||
_, err := a.uploadService.Storage().UpdateBackend(r.PathValue("backendID"), a.storageConfigFromForm(r, provider))
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
http.Redirect(w, r, "/admin/storage/"+r.PathValue("backendID")+"/edit?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/admin/storage", http.StatusSeeOther)
|
||||
http.Redirect(w, r, "/admin/storage?notice="+url.QueryEscape("Storage backend updated."), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminTestStorage(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
}
|
||||
next := "/admin/storage"
|
||||
if r.FormValue("next") == "tests" {
|
||||
next = "/admin/storage/" + r.PathValue("backendID") + "/tests"
|
||||
}
|
||||
if _, err := a.uploadService.Storage().TestBackend(r.PathValue("backendID")); err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
http.Redirect(w, r, next+"?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/admin/storage", http.StatusSeeOther)
|
||||
http.Redirect(w, r, next+"?notice="+url.QueryEscape("Storage connection test passed."), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminStartStorageSpeedTest(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape("Unable to read speed test form."), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
options := services.StorageSpeedTestOptions{
|
||||
Mode: r.FormValue("mode"),
|
||||
CustomFileCount: parsePositiveInt(r.FormValue("custom_file_count")),
|
||||
CustomFileSizeMB: parsePositiveFloat(r.FormValue("custom_file_size_mb")),
|
||||
}
|
||||
test, err := a.uploadService.Storage().StartSpeedTestWithOptions(r.PathValue("backendID"), options)
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage/"+r.PathValue("backendID")+"/tests?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
go a.uploadService.Storage().RunSpeedTest(context.Background(), test.ID)
|
||||
http.Redirect(w, r, "/admin/storage/"+r.PathValue("backendID")+"/tests?notice="+url.QueryEscape("Storage speed test started in the background."), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminDisableStorage(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -529,6 +708,56 @@ func (a *App) AdminDeleteStorage(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/admin/storage", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminRunStorageCleanup(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
}
|
||||
cleaned, err := jobs.RunCleanupNow(a.uploadService, a.logger)
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/admin/storage?notice="+url.QueryEscape(fmt.Sprintf("Cleanup finished. Removed %d unavailable boxes.", cleaned)), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminRunStorageThumbnails(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
}
|
||||
result, err := jobs.RunThumbnailsNow(a.uploadService, a.logger)
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
message := fmt.Sprintf("Thumbnail pass finished. Scanned %d files, generated %d, failed %d.", result.Scanned, result.Generated, result.Failed)
|
||||
http.Redirect(w, r, "/admin/storage?notice="+url.QueryEscape(message), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminVerifyStorageBackends(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
}
|
||||
configs, err := a.uploadService.Storage().ListBackendConfigs()
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/admin/storage?error="+url.QueryEscape(err.Error()), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
passed := 0
|
||||
failed := 0
|
||||
for _, cfg := range configs {
|
||||
if !cfg.Enabled {
|
||||
continue
|
||||
}
|
||||
if _, err := a.uploadService.Storage().TestBackend(cfg.ID); err != nil {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
passed++
|
||||
}
|
||||
message := fmt.Sprintf("Storage verification finished. %d passed, %d failed.", passed, failed)
|
||||
http.Redirect(w, r, "/admin/storage?notice="+url.QueryEscape(message), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) AdminUpdateUserQuota(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.requireAdmin(w, r) || !a.validateCSRF(w, r) {
|
||||
return
|
||||
@@ -872,6 +1101,152 @@ func formatMB(value float64) string {
|
||||
return strconv.FormatFloat(value, 'f', -1, 64) + " MB"
|
||||
}
|
||||
|
||||
func adminStorageSpeedTestsJSON(tests []services.StorageSpeedTest) []adminStorageSpeedTestJSON {
|
||||
rows := make([]adminStorageSpeedTestJSON, 0, len(tests))
|
||||
for _, test := range tests {
|
||||
rows = append(rows, adminStorageSpeedTestJSON{
|
||||
ID: test.ID,
|
||||
Mode: test.Mode,
|
||||
ModeLabel: test.ModeLabel(),
|
||||
Status: test.Status,
|
||||
Stage: test.Stage,
|
||||
Progress: test.ProgressPercent,
|
||||
CustomLabel: storageSpeedCustomLabel(test),
|
||||
StartedLabel: test.StartedLabel(),
|
||||
FinishedLabel: test.FinishedLabel(),
|
||||
Files: test.FilesWritten,
|
||||
SizeLabel: test.TotalSizeLabel(),
|
||||
WriteSpeed: test.WriteSpeedLabel(),
|
||||
ReadSpeed: test.ReadSpeedLabel(),
|
||||
Error: test.Error,
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func storageSpeedCustomLabel(test services.StorageSpeedTest) string {
|
||||
if test.Mode != services.StorageSpeedModeCustom {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%d files × %s each", test.CustomFileCount, services.FormatMegabytesLabel(test.CustomFileSizeMB))
|
||||
}
|
||||
|
||||
func (a *App) storageConfigFromForm(r *http.Request, provider string) services.StorageBackendConfig {
|
||||
cfg := services.StorageBackendConfig{
|
||||
Provider: provider,
|
||||
Name: r.FormValue("name"),
|
||||
}
|
||||
switch provider {
|
||||
case services.StorageProviderSFTP:
|
||||
cfg.Host = r.FormValue("host")
|
||||
cfg.Port = parsePositiveInt(r.FormValue("port"))
|
||||
cfg.Username = r.FormValue("username")
|
||||
cfg.Password = r.FormValue("password")
|
||||
cfg.PrivateKey = r.FormValue("private_key")
|
||||
cfg.HostKey = r.FormValue("host_key")
|
||||
cfg.RemotePath = r.FormValue("remote_path")
|
||||
case services.StorageProviderSMB:
|
||||
cfg.Host = r.FormValue("host")
|
||||
cfg.Port = parsePositiveInt(r.FormValue("port"))
|
||||
cfg.Share = r.FormValue("share")
|
||||
cfg.Domain = r.FormValue("domain")
|
||||
cfg.Username = r.FormValue("username")
|
||||
cfg.Password = r.FormValue("password")
|
||||
cfg.RemotePath = r.FormValue("remote_path")
|
||||
case services.StorageProviderWebDAV:
|
||||
cfg.Endpoint = r.FormValue("endpoint")
|
||||
cfg.Username = r.FormValue("username")
|
||||
cfg.Password = r.FormValue("password")
|
||||
cfg.RemotePath = r.FormValue("remote_path")
|
||||
case services.StorageProviderContabo:
|
||||
cfg.Endpoint = r.FormValue("endpoint")
|
||||
cfg.Region = r.FormValue("region")
|
||||
cfg.Bucket = r.FormValue("bucket")
|
||||
cfg.AccessKey = r.FormValue("access_key")
|
||||
cfg.SecretKey = r.FormValue("secret_key")
|
||||
cfg.UseSSL = true
|
||||
cfg.PathStyle = true
|
||||
default:
|
||||
cfg.Endpoint = r.FormValue("endpoint")
|
||||
cfg.Region = r.FormValue("region")
|
||||
cfg.Bucket = r.FormValue("bucket")
|
||||
cfg.AccessKey = r.FormValue("access_key")
|
||||
cfg.SecretKey = r.FormValue("secret_key")
|
||||
cfg.UseSSL = r.FormValue("use_ssl") == "on"
|
||||
cfg.PathStyle = r.FormValue("path_style") == "on"
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func adminStorageProviderOptions() []adminStorageProviderView {
|
||||
return []adminStorageProviderView{
|
||||
{Provider: services.StorageProviderS3, Label: "S3 Bucket", Description: "Generic S3-compatible object storage.", Icon: "cloud"},
|
||||
{Provider: services.StorageProviderContabo, Label: "Contabo Object Storage", Description: "Contabo COS with TLS and path-style lookup locked on.", Icon: "cloud"},
|
||||
{Provider: services.StorageProviderSFTP, Label: "SFTP", Description: "SSH file transfer to a server or NAS.", Icon: "database"},
|
||||
{Provider: services.StorageProviderSMB, Label: "Samba / SMB", Description: "Windows share or network attached storage.", Icon: "folder"},
|
||||
{Provider: services.StorageProviderWebDAV, Label: "WebDAV", Description: "Nextcloud, ownCloud, or any WebDAV endpoint.", Icon: "sync"},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeAdminStorageProvider(provider string) string {
|
||||
switch provider {
|
||||
case services.StorageProviderContabo:
|
||||
return services.StorageProviderContabo
|
||||
case services.StorageProviderSFTP:
|
||||
return services.StorageProviderSFTP
|
||||
case services.StorageProviderSMB:
|
||||
return services.StorageProviderSMB
|
||||
case services.StorageProviderWebDAV:
|
||||
return services.StorageProviderWebDAV
|
||||
default:
|
||||
return services.StorageProviderS3
|
||||
}
|
||||
}
|
||||
|
||||
func adminStorageProviderFromRequest(r *http.Request) string {
|
||||
if provider := r.PathValue("provider"); provider != "" {
|
||||
return provider
|
||||
}
|
||||
return path.Base(r.URL.Path)
|
||||
}
|
||||
|
||||
func validAdminStorageProvider(provider string) bool {
|
||||
switch provider {
|
||||
case services.StorageProviderS3, services.StorageProviderContabo, services.StorageProviderSFTP, services.StorageProviderSMB, services.StorageProviderWebDAV:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func adminStorageProviderLabel(provider string) string {
|
||||
switch normalizeAdminStorageProvider(provider) {
|
||||
case services.StorageProviderContabo:
|
||||
return "Contabo Object Storage"
|
||||
case services.StorageProviderSFTP:
|
||||
return "SFTP"
|
||||
case services.StorageProviderSMB:
|
||||
return "Samba / SMB"
|
||||
case services.StorageProviderWebDAV:
|
||||
return "WebDAV"
|
||||
default:
|
||||
return "S3 Bucket"
|
||||
}
|
||||
}
|
||||
|
||||
func adminStorageTypeForProvider(provider string) string {
|
||||
switch normalizeAdminStorageProvider(provider) {
|
||||
case services.StorageProviderSFTP:
|
||||
return services.StorageBackendSFTP
|
||||
case services.StorageProviderSMB:
|
||||
return services.StorageBackendSMB
|
||||
case services.StorageProviderWebDAV:
|
||||
return services.StorageBackendWebDAV
|
||||
default:
|
||||
return services.StorageBackendS3
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) storageBackendViews() ([]services.StorageBackendView, error) {
|
||||
configs, err := a.uploadService.Storage().ListBackendConfigs()
|
||||
if err != nil {
|
||||
@@ -886,11 +1261,14 @@ func (a *App) storageBackendViews() ([]services.StorageBackendView, error) {
|
||||
}
|
||||
}
|
||||
inUse, _ := a.storageBackendInUse(cfg.ID)
|
||||
speedTests, _ := a.uploadService.Storage().ListSpeedTests(cfg.ID, 25)
|
||||
views = append(views, services.StorageBackendView{
|
||||
Config: cfg,
|
||||
UsageBytes: usage,
|
||||
UsageLabel: services.FormatMegabytesFromBytes(usage),
|
||||
InUse: inUse,
|
||||
Config: cfg,
|
||||
UsageBytes: usage,
|
||||
UsageLabel: services.FormatMegabytesFromBytes(usage),
|
||||
InUse: inUse,
|
||||
SpeedTests: speedTests,
|
||||
CanSpeedTest: cfg.LastTestSuccess,
|
||||
})
|
||||
}
|
||||
return views, nil
|
||||
|
||||
@@ -68,11 +68,28 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /admin/settings", a.AdminSettings)
|
||||
mux.HandleFunc("POST /admin/settings", a.AdminSettingsPost)
|
||||
mux.HandleFunc("GET /admin/storage", a.AdminStorage)
|
||||
mux.HandleFunc("POST /admin/storage/s3", a.AdminCreateS3Storage)
|
||||
mux.HandleFunc("GET /admin/storage/new", a.AdminNewStorage)
|
||||
mux.HandleFunc("GET /admin/storage/new/s3", a.AdminNewStorageProvider)
|
||||
mux.HandleFunc("GET /admin/storage/new/contabo", a.AdminNewStorageProvider)
|
||||
mux.HandleFunc("GET /admin/storage/new/sftp", a.AdminNewStorageProvider)
|
||||
mux.HandleFunc("GET /admin/storage/new/smb", a.AdminNewStorageProvider)
|
||||
mux.HandleFunc("GET /admin/storage/new/webdav", a.AdminNewStorageProvider)
|
||||
mux.HandleFunc("POST /admin/storage/new/s3", a.AdminCreateStorage)
|
||||
mux.HandleFunc("POST /admin/storage/new/contabo", a.AdminCreateStorage)
|
||||
mux.HandleFunc("POST /admin/storage/new/sftp", a.AdminCreateStorage)
|
||||
mux.HandleFunc("POST /admin/storage/new/smb", a.AdminCreateStorage)
|
||||
mux.HandleFunc("POST /admin/storage/new/webdav", a.AdminCreateStorage)
|
||||
mux.HandleFunc("GET /admin/storage/{backendID}/edit", a.AdminEditStorageForm)
|
||||
mux.HandleFunc("GET /admin/storage/{backendID}/tests", a.AdminStorageTests)
|
||||
mux.HandleFunc("GET /admin/storage/{backendID}/tests.json", a.AdminStorageTestsJSON)
|
||||
mux.HandleFunc("POST /admin/storage/{backendID}/edit", a.AdminEditStorage)
|
||||
mux.HandleFunc("POST /admin/storage/{backendID}/test", a.AdminTestStorage)
|
||||
mux.HandleFunc("POST /admin/storage/{backendID}/speed-test", a.AdminStartStorageSpeedTest)
|
||||
mux.HandleFunc("POST /admin/storage/{backendID}/disable", a.AdminDisableStorage)
|
||||
mux.HandleFunc("POST /admin/storage/{backendID}/delete", a.AdminDeleteStorage)
|
||||
mux.HandleFunc("POST /admin/storage/jobs/cleanup", a.AdminRunStorageCleanup)
|
||||
mux.HandleFunc("POST /admin/storage/jobs/thumbnails", a.AdminRunStorageThumbnails)
|
||||
mux.HandleFunc("POST /admin/storage/jobs/verify", a.AdminVerifyStorageBackends)
|
||||
mux.HandleFunc("POST /admin/invites", a.AdminCreateInvite)
|
||||
mux.HandleFunc("POST /admin/users/{userID}/disable", a.AdminDisableUser)
|
||||
mux.HandleFunc("POST /admin/users/{userID}/reset", a.AdminResetUser)
|
||||
|
||||
@@ -26,6 +26,10 @@ func newCleanupJob(cfg config.Config, logger *slog.Logger, uploadService *servic
|
||||
}
|
||||
}
|
||||
|
||||
func RunCleanupNow(uploadService *services.UploadService, logger *slog.Logger) (int, error) {
|
||||
return cleanupUnavailableBoxes(uploadService, logger)
|
||||
}
|
||||
|
||||
func cleanupUnavailableBoxes(uploadService *services.UploadService, logger *slog.Logger) (int, error) {
|
||||
boxes, err := uploadService.ListBoxes(0)
|
||||
if err != nil {
|
||||
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
"warpbox.dev/backend/libs/services"
|
||||
)
|
||||
|
||||
type thumbnailJobResult struct {
|
||||
type ThumbnailJobResult struct {
|
||||
Scanned int
|
||||
Generated int
|
||||
Failed int
|
||||
@@ -63,13 +63,17 @@ func newThumbnailsJob(cfg config.Config, logger *slog.Logger, uploadService *ser
|
||||
}
|
||||
}
|
||||
|
||||
func generateMissingThumbnails(uploadService *services.UploadService, logger *slog.Logger) (thumbnailJobResult, error) {
|
||||
func RunThumbnailsNow(uploadService *services.UploadService, logger *slog.Logger) (ThumbnailJobResult, error) {
|
||||
return generateMissingThumbnails(uploadService, logger)
|
||||
}
|
||||
|
||||
func generateMissingThumbnails(uploadService *services.UploadService, logger *slog.Logger) (ThumbnailJobResult, error) {
|
||||
boxes, err := uploadService.ListBoxes(0)
|
||||
if err != nil {
|
||||
return thumbnailJobResult{}, err
|
||||
return ThumbnailJobResult{}, err
|
||||
}
|
||||
|
||||
var result thumbnailJobResult
|
||||
var result ThumbnailJobResult
|
||||
now := time.Now().UTC()
|
||||
for _, box := range boxes {
|
||||
if !box.ExpiresAt.After(now) {
|
||||
@@ -88,8 +92,8 @@ func generateMissingThumbnails(uploadService *services.UploadService, logger *sl
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func generateMissingThumbnailsForBox(uploadService *services.UploadService, logger *slog.Logger, box services.Box) (thumbnailJobResult, error) {
|
||||
var result thumbnailJobResult
|
||||
func generateMissingThumbnailsForBox(uploadService *services.UploadService, logger *slog.Logger, box services.Box) (ThumbnailJobResult, error) {
|
||||
var result ThumbnailJobResult
|
||||
if !box.ExpiresAt.After(time.Now().UTC()) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
)
|
||||
|
||||
var storageBackendsBucket = []byte("storage_backends")
|
||||
var storageBackendTestStatusBucket = []byte("storage_backend_test_status")
|
||||
|
||||
const (
|
||||
StorageBackendLocal = "local"
|
||||
@@ -81,10 +82,12 @@ type StorageBackendConfig struct {
|
||||
}
|
||||
|
||||
type StorageBackendView struct {
|
||||
Config StorageBackendConfig
|
||||
UsageBytes int64
|
||||
UsageLabel string
|
||||
InUse bool
|
||||
Config StorageBackendConfig
|
||||
UsageBytes int64
|
||||
UsageLabel string
|
||||
InUse bool
|
||||
SpeedTests []StorageSpeedTest
|
||||
CanSpeedTest bool
|
||||
}
|
||||
|
||||
type StorageService struct {
|
||||
@@ -99,7 +102,13 @@ func NewStorageService(db *bbolt.DB, dataDir string) (*StorageService, error) {
|
||||
}
|
||||
service := &StorageService{db: db, localFilesDir: filesDir}
|
||||
err := db.Update(func(tx *bbolt.Tx) error {
|
||||
_, err := tx.CreateBucketIfNotExists(storageBackendsBucket)
|
||||
if _, err := tx.CreateBucketIfNotExists(storageBackendsBucket); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.CreateBucketIfNotExists(storageBackendTestStatusBucket); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.CreateBucketIfNotExists(storageSpeedTestsBucket)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
@@ -126,7 +135,9 @@ func (s *StorageService) Backend(id string) (StorageBackend, error) {
|
||||
func (s *StorageService) BackendConfig(id string) (StorageBackendConfig, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == StorageBackendLocal {
|
||||
return s.localConfig(), nil
|
||||
cfg := s.localConfig()
|
||||
s.applyStoredTestStatus(&cfg)
|
||||
return cfg, nil
|
||||
}
|
||||
var cfg StorageBackendConfig
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
@@ -167,18 +178,13 @@ func (s *StorageService) ListBackendConfigs() ([]StorageBackendConfig, error) {
|
||||
}
|
||||
|
||||
func (s *StorageService) CreateS3Backend(input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
return s.CreateBackend(input)
|
||||
}
|
||||
|
||||
func (s *StorageService) CreateBackend(input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
input.ID = randomID(10)
|
||||
input.Provider = normalizeStorageProvider(input.Provider)
|
||||
switch input.Provider {
|
||||
case StorageProviderSFTP:
|
||||
input.Type = StorageBackendSFTP
|
||||
case StorageProviderSMB:
|
||||
input.Type = StorageBackendSMB
|
||||
case StorageProviderWebDAV:
|
||||
input.Type = StorageBackendWebDAV
|
||||
default:
|
||||
input.Type = StorageBackendS3
|
||||
}
|
||||
input.Type = storageTypeForProvider(input.Provider)
|
||||
if err := normalizeStorageBackendConfig(&input, true); err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
}
|
||||
@@ -193,6 +199,10 @@ func (s *StorageService) CreateS3Backend(input StorageBackendConfig) (StorageBac
|
||||
}
|
||||
|
||||
func (s *StorageService) UpdateS3Backend(id string, input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
return s.UpdateBackend(id, input)
|
||||
}
|
||||
|
||||
func (s *StorageService) UpdateBackend(id string, input StorageBackendConfig) (StorageBackendConfig, error) {
|
||||
current, err := s.BackendConfig(id)
|
||||
if err != nil {
|
||||
return StorageBackendConfig{}, err
|
||||
@@ -200,18 +210,19 @@ func (s *StorageService) UpdateS3Backend(id string, input StorageBackendConfig)
|
||||
if current.ID == StorageBackendLocal {
|
||||
return StorageBackendConfig{}, fmt.Errorf("local storage cannot be edited")
|
||||
}
|
||||
current.Provider = canonicalStorageProvider(current)
|
||||
current.Type = storageTypeForProvider(current.Provider)
|
||||
|
||||
input.ID = current.ID
|
||||
input.Type = current.Type
|
||||
input.Provider = normalizeStorageProvider(input.Provider)
|
||||
switch input.Provider {
|
||||
case StorageProviderSFTP:
|
||||
input.Type = StorageBackendSFTP
|
||||
case StorageProviderSMB:
|
||||
input.Type = StorageBackendSMB
|
||||
case StorageProviderWebDAV:
|
||||
input.Type = StorageBackendWebDAV
|
||||
default:
|
||||
input.Type = StorageBackendS3
|
||||
requestedProvider := normalizeStorageProvider(input.Provider)
|
||||
requestedType := storageTypeForProvider(requestedProvider)
|
||||
if input.Type != "" && input.Type != requestedType {
|
||||
return StorageBackendConfig{}, fmt.Errorf("storage type cannot be changed after creation")
|
||||
}
|
||||
input.Provider = requestedProvider
|
||||
input.Type = requestedType
|
||||
if input.Provider != current.Provider || input.Type != current.Type {
|
||||
return StorageBackendConfig{}, fmt.Errorf("storage provider cannot be changed after creation")
|
||||
}
|
||||
if strings.TrimSpace(input.SecretKey) == "" {
|
||||
input.SecretKey = current.SecretKey
|
||||
@@ -374,10 +385,56 @@ func (s *StorageService) TestBackend(id string) (StorageBackendConfig, error) {
|
||||
}
|
||||
if cfg.ID != StorageBackendLocal {
|
||||
_ = s.SaveBackendConfig(cfg)
|
||||
} else {
|
||||
_ = s.saveBackendTestStatus(cfg)
|
||||
}
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
func (s *StorageService) applyStoredTestStatus(cfg *StorageBackendConfig) {
|
||||
_ = s.db.View(func(tx *bbolt.Tx) error {
|
||||
bucket := tx.Bucket(storageBackendTestStatusBucket)
|
||||
if bucket == nil {
|
||||
return nil
|
||||
}
|
||||
data := bucket.Get([]byte(cfg.ID))
|
||||
if data == nil {
|
||||
return nil
|
||||
}
|
||||
var status struct {
|
||||
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
||||
LastTestError string `json:"lastTestError,omitempty"`
|
||||
LastTestSuccess bool `json:"lastTestSuccess,omitempty"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &status); err != nil {
|
||||
return nil
|
||||
}
|
||||
cfg.LastTestedAt = status.LastTestedAt
|
||||
cfg.LastTestError = status.LastTestError
|
||||
cfg.LastTestSuccess = status.LastTestSuccess
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageService) saveBackendTestStatus(cfg StorageBackendConfig) error {
|
||||
status := struct {
|
||||
LastTestedAt time.Time `json:"lastTestedAt,omitempty"`
|
||||
LastTestError string `json:"lastTestError,omitempty"`
|
||||
LastTestSuccess bool `json:"lastTestSuccess,omitempty"`
|
||||
}{
|
||||
LastTestedAt: cfg.LastTestedAt,
|
||||
LastTestError: cfg.LastTestError,
|
||||
LastTestSuccess: cfg.LastTestSuccess,
|
||||
}
|
||||
data, err := json.Marshal(status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.db.Update(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(storageBackendTestStatusBucket).Put([]byte(cfg.ID), data)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageService) backendFromConfig(cfg StorageBackendConfig) (StorageBackend, error) {
|
||||
switch cfg.Type {
|
||||
case StorageBackendLocal:
|
||||
@@ -424,6 +481,35 @@ func normalizeStorageProvider(provider string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalStorageProvider(cfg StorageBackendConfig) string {
|
||||
if cfg.Provider != "" && cfg.Provider != StorageBackendLocal {
|
||||
return normalizeStorageProvider(cfg.Provider)
|
||||
}
|
||||
switch cfg.Type {
|
||||
case StorageBackendSFTP:
|
||||
return StorageProviderSFTP
|
||||
case StorageBackendSMB:
|
||||
return StorageProviderSMB
|
||||
case StorageBackendWebDAV:
|
||||
return StorageProviderWebDAV
|
||||
default:
|
||||
return StorageProviderS3
|
||||
}
|
||||
}
|
||||
|
||||
func storageTypeForProvider(provider string) string {
|
||||
switch normalizeStorageProvider(provider) {
|
||||
case StorageProviderSFTP:
|
||||
return StorageBackendSFTP
|
||||
case StorageProviderSMB:
|
||||
return StorageBackendSMB
|
||||
case StorageProviderWebDAV:
|
||||
return StorageBackendWebDAV
|
||||
default:
|
||||
return StorageBackendS3
|
||||
}
|
||||
}
|
||||
|
||||
func cleanObjectKey(key string) string {
|
||||
return strings.TrimPrefix(filepath.ToSlash(filepath.Clean(strings.TrimPrefix(key, "/"))), "./")
|
||||
}
|
||||
|
||||
424
backend/libs/services/storage_speed.go
Normal file
424
backend/libs/services/storage_speed.go
Normal file
@@ -0,0 +1,424 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
var storageSpeedTestsBucket = []byte("storage_speed_tests")
|
||||
|
||||
const (
|
||||
StorageSpeedModeSmall = "small"
|
||||
StorageSpeedModeBig = "big"
|
||||
StorageSpeedModeMixed = "mixed"
|
||||
StorageSpeedModeCustom = "custom"
|
||||
|
||||
StorageSpeedStatusRunning = "running"
|
||||
StorageSpeedStatusDone = "done"
|
||||
StorageSpeedStatusFailed = "failed"
|
||||
)
|
||||
|
||||
type StorageSpeedTest struct {
|
||||
ID string `json:"id"`
|
||||
BackendID string `json:"backendId"`
|
||||
BackendName string `json:"backendName"`
|
||||
Mode string `json:"mode"`
|
||||
Status string `json:"status"`
|
||||
Stage string `json:"stage"`
|
||||
ProgressPercent int `json:"progressPercent"`
|
||||
CustomFileCount int `json:"customFileCount,omitempty"`
|
||||
CustomFileSizeMB float64 `json:"customFileSizeMb,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
FinishedAt time.Time `json:"finishedAt,omitempty"`
|
||||
BytesWritten int64 `json:"bytesWritten"`
|
||||
BytesRead int64 `json:"bytesRead"`
|
||||
FilesWritten int `json:"filesWritten"`
|
||||
WriteDurationMS int64 `json:"writeDurationMs"`
|
||||
ReadDurationMS int64 `json:"readDurationMs"`
|
||||
DeleteDurationMS int64 `json:"deleteDurationMs"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (t StorageSpeedTest) ModeLabel() string {
|
||||
switch t.Mode {
|
||||
case StorageSpeedModeSmall:
|
||||
return "Many small files"
|
||||
case StorageSpeedModeBig:
|
||||
return "One big file"
|
||||
case StorageSpeedModeMixed:
|
||||
return "Average mix"
|
||||
case StorageSpeedModeCustom:
|
||||
return "Custom"
|
||||
default:
|
||||
return t.Mode
|
||||
}
|
||||
}
|
||||
|
||||
func (t StorageSpeedTest) StartedLabel() string {
|
||||
if t.StartedAt.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.StartedAt.Format("Jan 2, 15:04:05")
|
||||
}
|
||||
|
||||
func (t StorageSpeedTest) FinishedLabel() string {
|
||||
if t.FinishedAt.IsZero() {
|
||||
return "Still running"
|
||||
}
|
||||
return t.FinishedAt.Format("Jan 2, 15:04:05")
|
||||
}
|
||||
|
||||
func (t StorageSpeedTest) TotalSizeLabel() string {
|
||||
return FormatMegabytesFromBytes(max(t.BytesWritten, t.BytesRead))
|
||||
}
|
||||
|
||||
func (t StorageSpeedTest) WriteSpeedLabel() string {
|
||||
return speedLabel(t.BytesWritten, t.WriteDurationMS)
|
||||
}
|
||||
|
||||
func (t StorageSpeedTest) ReadSpeedLabel() string {
|
||||
return speedLabel(t.BytesRead, t.ReadDurationMS)
|
||||
}
|
||||
|
||||
func speedLabel(bytes int64, durationMS int64) string {
|
||||
if bytes <= 0 || durationMS <= 0 {
|
||||
return "n/a"
|
||||
}
|
||||
mb := float64(bytes) / 1024 / 1024
|
||||
seconds := float64(durationMS) / 1000
|
||||
value := math.Round((mb/seconds)*100) / 100
|
||||
return fmt.Sprintf("%.2f MB/s", value)
|
||||
}
|
||||
|
||||
func (s *StorageService) StartSpeedTest(backendID, mode string) (StorageSpeedTest, error) {
|
||||
return s.StartSpeedTestWithOptions(backendID, StorageSpeedTestOptions{Mode: mode})
|
||||
}
|
||||
|
||||
type StorageSpeedTestOptions struct {
|
||||
Mode string
|
||||
CustomFileCount int
|
||||
CustomFileSizeMB float64
|
||||
}
|
||||
|
||||
func (s *StorageService) StartSpeedTestWithOptions(backendID string, options StorageSpeedTestOptions) (StorageSpeedTest, error) {
|
||||
cfg, err := s.BackendConfig(backendID)
|
||||
if err != nil {
|
||||
return StorageSpeedTest{}, err
|
||||
}
|
||||
if !cfg.Enabled {
|
||||
return StorageSpeedTest{}, fmt.Errorf("storage backend is disabled")
|
||||
}
|
||||
if !cfg.LastTestSuccess {
|
||||
return StorageSpeedTest{}, fmt.Errorf("run a successful connection test before testing speed")
|
||||
}
|
||||
mode := normalizeSpeedTestMode(options.Mode)
|
||||
if mode == StorageSpeedModeCustom {
|
||||
if err := validateCustomSpeedTest(options.CustomFileCount, options.CustomFileSizeMB); err != nil {
|
||||
return StorageSpeedTest{}, err
|
||||
}
|
||||
}
|
||||
test := StorageSpeedTest{
|
||||
ID: randomID(10),
|
||||
BackendID: cfg.ID,
|
||||
BackendName: cfg.Name,
|
||||
Mode: mode,
|
||||
Status: StorageSpeedStatusRunning,
|
||||
Stage: "queued",
|
||||
CustomFileCount: options.CustomFileCount,
|
||||
CustomFileSizeMB: options.CustomFileSizeMB,
|
||||
StartedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := s.saveSpeedTest(test); err != nil {
|
||||
return StorageSpeedTest{}, err
|
||||
}
|
||||
return test, nil
|
||||
}
|
||||
|
||||
func (s *StorageService) RunSpeedTest(ctx context.Context, testID string) {
|
||||
test, err := s.speedTest(testID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := s.runSpeedTest(ctx, &test); err != nil {
|
||||
test.Status = StorageSpeedStatusFailed
|
||||
test.Error = err.Error()
|
||||
test.FinishedAt = time.Now().UTC()
|
||||
if test.Stage == "" || test.Stage == "queued" {
|
||||
test.Stage = "failed"
|
||||
}
|
||||
_ = s.saveSpeedTest(test)
|
||||
return
|
||||
}
|
||||
test.Status = StorageSpeedStatusDone
|
||||
test.Stage = "complete"
|
||||
test.ProgressPercent = 100
|
||||
test.FinishedAt = time.Now().UTC()
|
||||
_ = s.saveSpeedTest(test)
|
||||
}
|
||||
|
||||
func (s *StorageService) ListSpeedTests(backendID string, limit int) ([]StorageSpeedTest, error) {
|
||||
var tests []StorageSpeedTest
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
bucket := tx.Bucket(storageSpeedTestsBucket)
|
||||
if bucket == nil {
|
||||
return nil
|
||||
}
|
||||
return bucket.ForEach(func(_, value []byte) error {
|
||||
var test StorageSpeedTest
|
||||
if err := json.Unmarshal(value, &test); err != nil {
|
||||
return err
|
||||
}
|
||||
if backendID == "" || test.BackendID == backendID {
|
||||
tests = append(tests, test)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(tests, func(i, j int) bool {
|
||||
return tests[i].StartedAt.After(tests[j].StartedAt)
|
||||
})
|
||||
if limit > 0 && len(tests) > limit {
|
||||
tests = tests[:limit]
|
||||
}
|
||||
return tests, nil
|
||||
}
|
||||
|
||||
func (s *StorageService) speedTest(id string) (StorageSpeedTest, error) {
|
||||
var test StorageSpeedTest
|
||||
err := s.db.View(func(tx *bbolt.Tx) error {
|
||||
data := tx.Bucket(storageSpeedTestsBucket).Get([]byte(id))
|
||||
if data == nil {
|
||||
return fmt.Errorf("speed test not found")
|
||||
}
|
||||
return json.Unmarshal(data, &test)
|
||||
})
|
||||
return test, err
|
||||
}
|
||||
|
||||
func (s *StorageService) saveSpeedTest(test StorageSpeedTest) error {
|
||||
data, err := json.Marshal(test)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.db.Update(func(tx *bbolt.Tx) error {
|
||||
return tx.Bucket(storageSpeedTestsBucket).Put([]byte(test.ID), data)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageService) runSpeedTest(ctx context.Context, test *StorageSpeedTest) error {
|
||||
backend, err := s.Backend(test.BackendID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files, err := createSpeedTestFiles(test)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(files.Root)
|
||||
keys := make([]string, 0, len(files.Files))
|
||||
defer func() {
|
||||
for _, key := range keys {
|
||||
_ = backend.Delete(context.Background(), key)
|
||||
}
|
||||
}()
|
||||
|
||||
writeStart := time.Now()
|
||||
for i, file := range files.Files {
|
||||
key := fmt.Sprintf(".warpbox-speed-test/%s/%03d.bin", test.ID, i)
|
||||
source, err := os.Open(file.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = backend.Put(ctx, key, source, file.Size, "application/octet-stream")
|
||||
source.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys = append(keys, key)
|
||||
test.BytesWritten += file.Size
|
||||
test.FilesWritten++
|
||||
updateSpeedProgress(test, "writing", i+1, len(files.Files), 0, 45)
|
||||
_ = s.saveSpeedTest(*test)
|
||||
}
|
||||
test.WriteDurationMS = time.Since(writeStart).Milliseconds()
|
||||
_ = s.saveSpeedTest(*test)
|
||||
|
||||
readStart := time.Now()
|
||||
for i, key := range keys {
|
||||
object, err := backend.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
read, err := io.Copy(io.Discard, object.Body)
|
||||
object.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
test.BytesRead += read
|
||||
updateSpeedProgress(test, "reading", i+1, len(keys), 45, 90)
|
||||
_ = s.saveSpeedTest(*test)
|
||||
}
|
||||
test.ReadDurationMS = time.Since(readStart).Milliseconds()
|
||||
_ = s.saveSpeedTest(*test)
|
||||
|
||||
deleteStart := time.Now()
|
||||
for i, key := range keys {
|
||||
if err := backend.Delete(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
updateSpeedProgress(test, "cleaning up", i+1, len(keys), 90, 100)
|
||||
_ = s.saveSpeedTest(*test)
|
||||
}
|
||||
test.DeleteDurationMS = time.Since(deleteStart).Milliseconds()
|
||||
keys = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateSpeedProgress(test *StorageSpeedTest, stage string, done, total, start, end int) {
|
||||
test.Stage = stage
|
||||
if total <= 0 {
|
||||
test.ProgressPercent = start
|
||||
return
|
||||
}
|
||||
span := end - start
|
||||
progress := start + int(math.Round(float64(span)*float64(done)/float64(total)))
|
||||
if progress < 0 {
|
||||
progress = 0
|
||||
}
|
||||
if progress > 100 {
|
||||
progress = 100
|
||||
}
|
||||
test.ProgressPercent = progress
|
||||
}
|
||||
|
||||
type speedTestFile struct {
|
||||
Path string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type speedTestFiles struct {
|
||||
Root string
|
||||
Files []speedTestFile
|
||||
}
|
||||
|
||||
func createSpeedTestFiles(test *StorageSpeedTest) (speedTestFiles, error) {
|
||||
plan, err := speedTestPlan(test)
|
||||
if err != nil {
|
||||
return speedTestFiles{}, err
|
||||
}
|
||||
root, err := os.MkdirTemp("", "warpbox-speed-test-*")
|
||||
if err != nil {
|
||||
return speedTestFiles{}, err
|
||||
}
|
||||
files := speedTestFiles{Root: root, Files: make([]speedTestFile, 0, len(plan))}
|
||||
for i, size := range plan {
|
||||
path := filepath.Join(root, fmt.Sprintf("%03d.bin", i))
|
||||
if err := writeMockFile(path, size, byte(65+(i%23))); err != nil {
|
||||
os.RemoveAll(root)
|
||||
return speedTestFiles{}, err
|
||||
}
|
||||
files.Files = append(files.Files, speedTestFile{Path: path, Size: size})
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func speedTestPlan(test *StorageSpeedTest) ([]int64, error) {
|
||||
mode := normalizeSpeedTestMode(test.Mode)
|
||||
if mode == StorageSpeedModeCustom {
|
||||
if err := validateCustomSpeedTest(test.CustomFileCount, test.CustomFileSizeMB); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := MegabytesToBytes(test.CustomFileSizeMB)
|
||||
plan := make([]int64, test.CustomFileCount)
|
||||
for i := range plan {
|
||||
plan[i] = size
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
return speedTestPlanForMode(mode), nil
|
||||
}
|
||||
|
||||
func speedTestPlanForMode(mode string) []int64 {
|
||||
mode = normalizeSpeedTestMode(mode)
|
||||
switch mode {
|
||||
case StorageSpeedModeSmall:
|
||||
return repeatedSizes(24, 32*1024)
|
||||
case StorageSpeedModeBig:
|
||||
return repeatedSizes(1, 8*1024*1024)
|
||||
default:
|
||||
sizes := repeatedSizes(8, 64*1024)
|
||||
return append(sizes, repeatedSizes(1, 4*1024*1024)...)
|
||||
}
|
||||
}
|
||||
|
||||
func repeatedSizes(count int, size int64) []int64 {
|
||||
sizes := make([]int64, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
sizes = append(sizes, size)
|
||||
}
|
||||
return sizes
|
||||
}
|
||||
|
||||
func writeMockFile(path string, size int64, seed byte) error {
|
||||
target, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
chunk := make([]byte, 64*1024)
|
||||
for i := range chunk {
|
||||
chunk[i] = seed
|
||||
}
|
||||
remaining := size
|
||||
for remaining > 0 {
|
||||
writeSize := int64(len(chunk))
|
||||
if remaining < writeSize {
|
||||
writeSize = remaining
|
||||
}
|
||||
if _, err := target.Write(chunk[:int(writeSize)]); err != nil {
|
||||
return err
|
||||
}
|
||||
remaining -= writeSize
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCustomSpeedTest(count int, sizeMB float64) error {
|
||||
if count <= 0 || count > 500 {
|
||||
return fmt.Errorf("custom speed test file count must be between 1 and 500")
|
||||
}
|
||||
if sizeMB <= 0 {
|
||||
return fmt.Errorf("custom speed test file size must be positive")
|
||||
}
|
||||
totalMB := float64(count) * sizeMB
|
||||
if totalMB > 4096 {
|
||||
return fmt.Errorf("custom speed test total size cannot exceed 4096 MB")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeSpeedTestMode(mode string) string {
|
||||
switch strings.TrimSpace(mode) {
|
||||
case StorageSpeedModeSmall:
|
||||
return StorageSpeedModeSmall
|
||||
case StorageSpeedModeBig:
|
||||
return StorageSpeedModeBig
|
||||
case StorageSpeedModeCustom:
|
||||
return StorageSpeedModeCustom
|
||||
default:
|
||||
return StorageSpeedModeMixed
|
||||
}
|
||||
}
|
||||
@@ -172,6 +172,157 @@ func TestSFTPStorageConfigValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageUpdateRejectsProviderMutation(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
cfg, err := service.Storage().CreateBackend(StorageBackendConfig{
|
||||
Provider: StorageProviderSFTP,
|
||||
Name: "SFTP",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateBackend returned error: %v", err)
|
||||
}
|
||||
if _, err := service.Storage().UpdateBackend(cfg.ID, StorageBackendConfig{
|
||||
Provider: StorageProviderS3,
|
||||
Name: "Mutated",
|
||||
Endpoint: "https://s3.example.test",
|
||||
Bucket: "bucket",
|
||||
AccessKey: "access",
|
||||
SecretKey: "secret",
|
||||
UseSSL: true,
|
||||
}); err == nil {
|
||||
t.Fatalf("UpdateBackend allowed provider mutation")
|
||||
}
|
||||
stored, err := service.Storage().BackendConfig(cfg.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("BackendConfig returned error: %v", err)
|
||||
}
|
||||
if stored.Provider != StorageProviderSFTP || stored.Type != StorageBackendSFTP {
|
||||
t.Fatalf("provider/type mutated despite error: %+v", stored)
|
||||
}
|
||||
if _, err := service.Storage().UpdateBackend(cfg.ID, StorageBackendConfig{
|
||||
Provider: StorageProviderSFTP,
|
||||
Type: StorageBackendS3,
|
||||
Name: "Mutated",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
}); err == nil {
|
||||
t.Fatalf("UpdateBackend allowed type mutation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageUpdatePreservesSecretsWhenBlank(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
cfg, err := service.Storage().CreateBackend(StorageBackendConfig{
|
||||
Provider: StorageProviderSFTP,
|
||||
Name: "SFTP",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
Password: "secret",
|
||||
PrivateKey: "private-key",
|
||||
HostKey: "host-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateBackend returned error: %v", err)
|
||||
}
|
||||
updated, err := service.Storage().UpdateBackend(cfg.ID, StorageBackendConfig{
|
||||
Provider: StorageProviderSFTP,
|
||||
Name: "SFTP renamed",
|
||||
Host: "files.example.test",
|
||||
Username: "warpbox",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateBackend returned error: %v", err)
|
||||
}
|
||||
if updated.Password != "secret" || updated.PrivateKey != "private-key" || updated.HostKey != "host-key" {
|
||||
t.Fatalf("blank secret fields were not preserved: %+v", updated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContaboUpdateKeepsTLSAndPathStyleLocked(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
cfg, err := service.Storage().CreateBackend(StorageBackendConfig{
|
||||
Provider: StorageProviderContabo,
|
||||
Name: "Contabo",
|
||||
Endpoint: "https://eu2.contabostorage.com",
|
||||
Bucket: "My Main Bucket",
|
||||
AccessKey: "access",
|
||||
SecretKey: "secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateBackend returned error: %v", err)
|
||||
}
|
||||
updated, err := service.Storage().UpdateBackend(cfg.ID, StorageBackendConfig{
|
||||
Provider: StorageProviderContabo,
|
||||
Name: "Contabo",
|
||||
Endpoint: "https://eu2.contabostorage.com",
|
||||
Bucket: "My Main Bucket",
|
||||
AccessKey: "access",
|
||||
SecretKey: "secret",
|
||||
UseSSL: false,
|
||||
PathStyle: false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateBackend returned error: %v", err)
|
||||
}
|
||||
if !updated.UseSSL || !updated.PathStyle {
|
||||
t.Fatalf("contabo TLS/path-style were not locked: %+v", updated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageSpeedTestRequiresConnectionAndRuns(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
if _, err := service.Storage().StartSpeedTest(StorageBackendLocal, StorageSpeedModeSmall); err == nil {
|
||||
t.Fatalf("StartSpeedTest allowed speed test before connection test")
|
||||
}
|
||||
if _, err := service.Storage().TestBackend(StorageBackendLocal); err != nil {
|
||||
t.Fatalf("TestBackend local returned error: %v", err)
|
||||
}
|
||||
test, err := service.Storage().StartSpeedTest(StorageBackendLocal, StorageSpeedModeSmall)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSpeedTest returned error: %v", err)
|
||||
}
|
||||
service.Storage().RunSpeedTest(testContext(), test.ID)
|
||||
tests, err := service.Storage().ListSpeedTests(StorageBackendLocal, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpeedTests returned error: %v", err)
|
||||
}
|
||||
if len(tests) != 1 {
|
||||
t.Fatalf("speed tests len = %d, want 1", len(tests))
|
||||
}
|
||||
got := tests[0]
|
||||
if got.Status != StorageSpeedStatusDone || got.ProgressPercent != 100 || got.Stage != "complete" || got.BytesWritten == 0 || got.BytesRead == 0 || got.FilesWritten == 0 {
|
||||
t.Fatalf("speed test did not complete with metrics: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomStorageSpeedTestUsesRequestedFiles(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
if _, err := service.Storage().TestBackend(StorageBackendLocal); err != nil {
|
||||
t.Fatalf("TestBackend local returned error: %v", err)
|
||||
}
|
||||
test, err := service.Storage().StartSpeedTestWithOptions(StorageBackendLocal, StorageSpeedTestOptions{
|
||||
Mode: StorageSpeedModeCustom,
|
||||
CustomFileCount: 3,
|
||||
CustomFileSizeMB: 0.001,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StartSpeedTestWithOptions returned error: %v", err)
|
||||
}
|
||||
service.Storage().RunSpeedTest(testContext(), test.ID)
|
||||
tests, err := service.Storage().ListSpeedTests(StorageBackendLocal, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpeedTests returned error: %v", err)
|
||||
}
|
||||
got := tests[0]
|
||||
if got.Mode != StorageSpeedModeCustom || got.CustomFileCount != 3 || got.CustomFileSizeMB != 0.001 || got.FilesWritten != 3 || got.Status != StorageSpeedStatusDone {
|
||||
t.Fatalf("custom speed test did not use requested files: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSMBAndWebDAVStorageConfigValidation(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
smb, err := service.Storage().CreateS3Backend(StorageBackendConfig{
|
||||
|
||||
Reference in New Issue
Block a user