kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backend): handle processing errors and add PWA routes

Browse Source 
- Block file downloads and previews with a 424 StatusFailedDependency if file processing failed or the box has issues.
- Register routes for `/service-worker.js` and `/share-target` to support PWA features.
- Update README.md with an AI usage disclosure.
This commit is contained in:
Daniel Legt
2026-06-08 11:53:37 +03:00
parent dbfdacc396
commit d11aec96e5
26 changed files with 1186 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/libs/handlers/app.go
View File

@@ -54,6 +54,8 @@ func (a *App) renderPage(w http.ResponseWriter, r *http.Request, status int, pag
func (a *App) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /", a.Home)
mux.HandleFunc("GET /api", a.APIDocs)
mux.HandleFunc("GET /service-worker.js", a.ServiceWorker)
mux.HandleFunc("POST /share-target", a.ShareTargetFallback)
mux.HandleFunc("GET /register", a.Register)
mux.HandleFunc("POST /register", a.RegisterPost)
mux.HandleFunc("GET /login", a.Login)

84
backend/libs/handlers/download.go
View File

@@ -59,6 +59,8 @@ type fileView struct {
ReactionMore int
Reacted bool
Processing bool
Failed bool
Error string
}
type reactionView struct {
@@ -242,12 +244,32 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
http.Error(w, "file is still processing", http.StatusAccepted)
return
}
if file.ProcessingError != "" {
a.logger.Warn("failed file preview blocked for social bot", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4241, "box_id", box.ID, "file_id", file.ID, "error", file.ProcessingError)...)
http.Error(w, "file processing failed: "+file.ProcessingError, http.StatusFailedDependency)
return
}
if services.BoxHasTrouble(box) {
a.logger.Warn("failed box preview blocked for social bot", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4245, "box_id", box.ID, "file_id", file.ID, "error", services.BoxTroubleReason(box))...)
http.Error(w, "box processing failed: "+services.BoxTroubleReason(box), http.StatusFailedDependency)
return
}
if shouldServeRawSocialMedia(file) {
a.serveFileContent(w, r, box, file, false)
a.logger.Info("media file served inline for social preview", withRequestLogAttrs(r, "source", "download", "severity", "user_activity", "code", 2009, "box_id", box.ID, "file_id", file.ID)...)
return
}
}
if file.ProcessingError != "" && !locked {
a.logger.Warn("failed file preview blocked", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4242, "box_id", box.ID, "file_id", file.ID, "error", file.ProcessingError)...)
http.Error(w, "file processing failed: "+file.ProcessingError, http.StatusFailedDependency)
return
}
if services.BoxHasTrouble(box) && !locked {
a.logger.Warn("failed box preview blocked", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4246, "box_id", box.ID, "file_id", file.ID, "error", services.BoxTroubleReason(box))...)
http.Error(w, "box processing failed: "+services.BoxTroubleReason(box), http.StatusFailedDependency)
return
}
view := a.fileView(box, file)
fileSize := helpers.FormatBytes(file.Size)
title := file.Name
@@ -306,6 +328,16 @@ func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
http.Error(w, "file is still processing", http.StatusAccepted)
return
}
if file.ProcessingError != "" {
a.logger.Warn("failed file download blocked", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4243, "box_id", box.ID, "file_id", file.ID, "error", file.ProcessingError)...)
http.Error(w, "file processing failed: "+file.ProcessingError, http.StatusFailedDependency)
return
}
if services.BoxHasTrouble(box) {
a.logger.Warn("failed box download blocked", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4247, "box_id", box.ID, "file_id", file.ID, "error", services.BoxTroubleReason(box))...)
http.Error(w, "box processing failed: "+services.BoxTroubleReason(box), http.StatusFailedDependency)
return
}
a.serveFileContent(w, r, box, file, r.URL.Query().Get("inline") != "1")
a.logger.Info("file content served", withRequestLogAttrs(r, "source", "download", "severity", "user_activity", "code", 2005, "box_id", box.ID, "file_id", file.ID, "attachment", r.URL.Query().Get("inline") != "1")...)
@@ -321,6 +353,11 @@ func (a *App) Thumbnail(w http.ResponseWriter, r *http.Request) {
a.servePlaceholderThumbnail(w, r)
return
}
if services.BoxHasTrouble(box) || services.FileHasTrouble(file) {
a.logger.Warn("thumbnail request skipped trouble file", withRequestLogAttrs(r, "source", "thumbnail", "severity", "warn", "code", 4110, "box_id", box.ID, "file_id", file.ID, "error", troubleReasonForLog(box, file))...)
a.servePlaceholderThumbnail(w, r)
return
}
object, err := a.uploadService.OpenThumbnailObject(r.Context(), box, file)
if err != nil {
@@ -363,6 +400,11 @@ func (a *App) VideoScenesPreview(w http.ResponseWriter, r *http.Request) {
a.servePlaceholderThumbnail(w, r)
return
}
if services.BoxHasTrouble(box) || services.FileHasTrouble(file) {
a.logger.Warn("video scenes preview request skipped trouble file", withRequestLogAttrs(r, "source", "thumbnail", "severity", "warn", "code", 4111, "box_id", box.ID, "file_id", file.ID, "error", troubleReasonForLog(box, file))...)
a.servePlaceholderThumbnail(w, r)
return
}
object, err := a.uploadService.OpenSceneThumbnailObject(r.Context(), box, file)
if err != nil {
@@ -400,6 +442,11 @@ func (a *App) ArchiveListing(w http.ResponseWriter, r *http.Request) {
http.Error(w, "password required", http.StatusUnauthorized)
return
}
if services.BoxHasTrouble(box) || services.FileHasTrouble(file) {
a.logger.Warn("archive listing request skipped trouble file", withRequestLogAttrs(r, "source", "thumbnail", "severity", "warn", "code", 4112, "box_id", box.ID, "file_id", file.ID, "error", troubleReasonForLog(box, file))...)
http.Error(w, "archive preview unavailable: file processing failed", http.StatusFailedDependency)
return
}
if strings.ToLower(filepath.Ext(file.ArchiveListing)) != ".json" {
if listing := a.generateMissingArchiveListingForRequest(r, box, file); listing != "" {
@@ -432,7 +479,7 @@ func (a *App) ArchiveListing(w http.ResponseWriter, r *http.Request) {
}
func (a *App) generateMissingThumbnailForRequest(r *http.Request, box services.Box, file services.File) string {
if file.Thumbnail != "" || !jobs.NeedsThumbnail(file) {
if file.Thumbnail != "" || !jobs.NeedsThumbnail(file) || services.BoxHasTrouble(box) || services.FileHasTrouble(file) || file.Processing {
return ""
}
thumbnail, err := jobs.GenerateThumbnailForFile(a.uploadService, box, file)
@@ -456,7 +503,7 @@ func (a *App) generateMissingThumbnailForRequest(r *http.Request, box services.B
}
func (a *App) generateMissingVideoScenesForRequest(r *http.Request, box services.Box, file services.File) string {
if file.SceneThumbnail != "" || !jobs.NeedsVideoScenes(file) {
if file.SceneThumbnail != "" || !jobs.NeedsVideoScenes(file) || services.BoxHasTrouble(box) || services.FileHasTrouble(file) || file.Processing {
return ""
}
scene, err := jobs.GenerateVideoScenesForFile(a.uploadService, box, file)
@@ -480,7 +527,7 @@ func (a *App) generateMissingVideoScenesForRequest(r *http.Request, box services
}
func (a *App) generateMissingArchiveListingForRequest(r *http.Request, box services.Box, file services.File) string {
if strings.ToLower(filepath.Ext(file.ArchiveListing)) == ".json" || !jobs.NeedsArchiveListing(file) {
if strings.ToLower(filepath.Ext(file.ArchiveListing)) == ".json" || !jobs.NeedsArchiveListing(file) || services.BoxHasTrouble(box) || services.FileHasTrouble(file) || file.Processing {
return ""
}
listing, err := jobs.GenerateArchiveListingForFile(a.uploadService, box, file)
@@ -504,6 +551,13 @@ func (a *App) generateMissingArchiveListingForRequest(r *http.Request, box servi
return listing
}
func troubleReasonForLog(box services.Box, file services.File) string {
if services.FileHasTrouble(file) {
return file.ProcessingError
}
return services.BoxTroubleReason(box)
}
// servePlaceholderThumbnail serves the fallback image with no-store so the
// browser re-requests on the next load and picks up the real thumbnail as soon
// as it has been generated.
@@ -651,6 +705,22 @@ func (a *App) DownloadZip(w http.ResponseWriter, r *http.Request) {
http.Error(w, "password required", http.StatusUnauthorized)
return
}
for _, file := range box.Files {
if file.Processing {
http.Error(w, "file is still processing", http.StatusAccepted)
return
}
if file.ProcessingError != "" {
a.logger.Warn("zip download blocked by failed file", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4244, "box_id", box.ID, "file_id", file.ID, "error", file.ProcessingError)...)
http.Error(w, "file processing failed: "+file.ProcessingError, http.StatusFailedDependency)
return
}
}
if services.BoxHasTrouble(box) {
a.logger.Warn("zip download blocked by failed box", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4248, "box_id", box.ID, "error", services.BoxTroubleReason(box))...)
http.Error(w, "box processing failed: "+services.BoxTroubleReason(box), http.StatusFailedDependency)
return
}
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", contentDisposition("attachment", "warpbox-"+box.ID+".zip"))
@@ -685,9 +755,9 @@ func (a *App) fileViewWithReactions(box services.Box, file services.File, reacti
ThumbnailURL: fmt.Sprintf("/d/%s/thumb/%s", box.ID, file.ID),
SceneURL: fmt.Sprintf("/d/%s/scene/%s", box.ID, file.ID),
ArchiveURL: fmt.Sprintf("/d/%s/archive/%s", box.ID, file.ID),
HasThumbnail: file.Thumbnail != "" || jobs.NeedsThumbnail(file),
HasScene: file.SceneThumbnail != "" || jobs.NeedsVideoScenes(file),
HasArchive: file.ArchiveListing != "" || jobs.NeedsArchiveListing(file),
HasThumbnail: !services.BoxHasTrouble(box) && !services.FileHasTrouble(file) && (file.Thumbnail != "" || jobs.NeedsThumbnail(file)),
HasScene: !services.BoxHasTrouble(box) && !services.FileHasTrouble(file) && (file.SceneThumbnail != "" || jobs.NeedsVideoScenes(file)),
HasArchive: !services.BoxHasTrouble(box) && !services.FileHasTrouble(file) && (file.ArchiveListing != "" || jobs.NeedsArchiveListing(file)),
IconURL: fileIconURL("standard", icon.Standard),
IconRetroURL: fileIconURL("retro", icon.Retro),
ReactURL: fmt.Sprintf("/d/%s/f/%s/react", box.ID, file.ID),
@@ -695,6 +765,8 @@ func (a *App) fileViewWithReactions(box services.Box, file services.File, reacti
ReactionMore: reactionOverflowCount(reactionViews),
Reacted: reacted,
Processing: file.Processing,
Failed: services.BoxHasTrouble(box) || services.FileHasTrouble(file),
Error: troubleReasonForLog(box, file),
}
}

17
backend/libs/handlers/pages.go
View File

@@ -10,6 +10,7 @@ import (
type homeData struct {
MaxUploadSize string
MaxUploadBytes int64
LimitSummary string
Collections []collectionView
IsAdmin bool
@@ -57,7 +58,7 @@ func (a *App) Home(w http.ResponseWriter, r *http.Request) {
"actor", actor,
"user_id", user.ID,
)...)
maxUploadSize, limitSummary := a.homeUploadPolicyLabels(settings, user, loggedIn, isAdmin)
maxUploadSize, maxUploadBytes, limitSummary := a.homeUploadPolicyLabels(settings, user, loggedIn, isAdmin)
expiryOptions, defaultExpiry := a.homeExpiryOptions(settings, user, loggedIn, isAdmin)
a.renderPage(w, r, http.StatusOK, "home.html", web.PageData{
Title: "Upload your files",
@@ -68,6 +69,7 @@ func (a *App) Home(w http.ResponseWriter, r *http.Request) {
CurrentUser: currentUser,
Data: homeData{
MaxUploadSize: maxUploadSize,
MaxUploadBytes: maxUploadBytes,
LimitSummary: limitSummary,
Collections: collections,
IsAdmin: isAdmin,
@@ -155,22 +157,25 @@ func expiryLabel(minutes int) string {
}
}
func (a *App) homeUploadPolicyLabels(settings services.UploadPolicySettings, user services.User, loggedIn, isAdmin bool) (string, string) {
func (a *App) homeUploadPolicyLabels(settings services.UploadPolicySettings, user services.User, loggedIn, isAdmin bool) (string, int64, string) {
if isAdmin {
return "No file size limit", "Admin uploads bypass storage and daily caps."
return "No file size limit", -1, "Admin uploads bypass storage and daily caps."
}
if !loggedIn {
if !settings.AnonymousUploadsEnabled {
return "Anonymous uploads disabled", "Sign in to upload files."
return "Anonymous uploads disabled", 0, "Sign in to upload files."
}
return services.FormatMegabytesLabel(settings.AnonymousMaxUploadMB), "Daily anonymous cap: " + services.FormatMegabytesLabel(settings.AnonymousDailyUploadMB) + " per IP · " + strconv.Itoa(settings.AnonymousMaxDays) + " day max."
return services.FormatMegabytesLabel(settings.AnonymousMaxUploadMB), services.MegabytesToBytes(settings.AnonymousMaxUploadMB), "Daily anonymous cap: " + services.FormatMegabytesLabel(settings.AnonymousDailyUploadMB) + " per IP · " + strconv.Itoa(settings.AnonymousMaxDays) + " day max."
}
policy := a.settingsService.EffectivePolicyForUser(settings, user)
maxUpload := a.uploadService.MaxUploadSizeLabel()
maxUploadBytes := a.uploadService.MaxUploadSize()
if policy.MaxUploadMB < 0 {
maxUpload = "unlimited"
maxUploadBytes = -1
} else if policy.MaxUploadMB > 0 {
maxUpload = services.FormatMegabytesLabel(policy.MaxUploadMB)
maxUploadBytes = services.MegabytesToBytes(policy.MaxUploadMB)
}
quota := "unlimited"
if policy.StorageQuotaSet {
@@ -180,5 +185,5 @@ func (a *App) homeUploadPolicyLabels(settings services.UploadPolicySettings, use
if policy.MaxDays < 0 {
expiryLimit = "no expiry limit."
}
return maxUpload, "Daily cap: " + services.FormatMegabytesLabel(policy.DailyUploadMB) + " · Storage quota: " + quota + " · " + expiryLimit
return maxUpload, maxUploadBytes, "Daily cap: " + services.FormatMegabytesLabel(policy.DailyUploadMB) + " · Storage quota: " + quota + " · " + expiryLimit
}

13
backend/libs/handlers/resumable.go
View File

@@ -180,7 +180,7 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
if !ok {
return
}
if session.Status == services.ResumableStatusCompleted || session.Status == services.ResumableStatusProcessing {
if session.Status == services.ResumableStatusCompleted {
result, completed, err := a.uploadService.CompleteResumableSession(r.Context(), session.ID)
if err != nil {
a.logger.Warn("resumable upload completion replay failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
@@ -191,6 +191,17 @@ func (a *App) CompleteResumableUpload(w http.ResponseWriter, r *http.Request) {
helpers.WriteJSON(w, http.StatusOK, result)
return
}
if session.Status == services.ResumableStatusProcessing {
result, err := a.uploadService.FinalizeProcessingResumableSession(r.Context(), session.ID)
if err != nil {
a.logger.Warn("resumable upload completion replay failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4004, "session_id", session.ID, "error", err.Error())...)
helpers.WriteJSONError(w, http.StatusBadRequest, err.Error())
return
}
a.logger.Info("resumable upload completion replayed", withRequestLogAttrs(r, "source", "user-upload", "severity", "user_activity", "code", 2004, "session_id", session.ID, "box_id", result.BoxID, "files", len(result.Files))...)
helpers.WriteJSON(w, http.StatusOK, result)
return
}
user, loggedIn, _ := a.currentUserWithAuthError(r)
isAdminUpload := loggedIn && user.Role == services.UserRoleAdmin
settings, policy, ok := a.loadUploadPolicyForAPI(w, r, user, loggedIn)

11
backend/libs/handlers/static.go
View File

@@ -34,6 +34,17 @@ func (a *App) EmojiAsset(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, path)
}
func (a *App) ServiceWorker(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
w.Header().Set("Cache-Control", "public, max-age=86400")
w.Header().Set("Service-Worker-Allowed", "/")
http.ServeFile(w, r, filepath.Join(a.cfg.StaticDir, "js", "service-worker.js"))
}
func (a *App) ShareTargetFallback(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/?share-target=unsupported", http.StatusSeeOther)
}
func setStaticCacheHeaders(w http.ResponseWriter, path string) {
ext := strings.ToLower(filepath.Ext(path))

77
backend/libs/handlers/static_test.go
View File

@@ -1,7 +1,11 @@
package handlers
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
)
@@ -24,3 +28,76 @@ func TestSetStaticCacheHeaders(t *testing.T) {
}
}
}
func TestWebManifestIncludesShareTarget(t *testing.T) {
data, err := os.ReadFile(filepath.Join("..", "..", "static", "site.webmanifest"))
if err != nil {
t.Fatalf("ReadFile returned error: %v", err)
}
var manifest struct {
ShareTarget struct {
Action string `json:"action"`
Method string `json:"method"`
EncType string `json:"enctype"`
Params struct {
Title string `json:"title"`
Text string `json:"text"`
URL string `json:"url"`
Files []struct {
Name string `json:"name"`
Accept []string `json:"accept"`
} `json:"files"`
} `json:"params"`
} `json:"share_target"`
}
if err := json.Unmarshal(data, &manifest); err != nil {
t.Fatalf("json.Unmarshal returned error: %v", err)
}
if manifest.ShareTarget.Action != "/share-target" || manifest.ShareTarget.Method != "POST" || manifest.ShareTarget.EncType != "multipart/form-data" {
t.Fatalf("unexpected share_target config: %+v", manifest.ShareTarget)
}
if manifest.ShareTarget.Params.Title != "title" || manifest.ShareTarget.Params.Text != "text" || manifest.ShareTarget.Params.URL != "url" {
t.Fatalf("unexpected share_target params: %+v", manifest.ShareTarget.Params)
}
if len(manifest.ShareTarget.Params.Files) != 1 || manifest.ShareTarget.Params.Files[0].Name != "files" || len(manifest.ShareTarget.Params.Files[0].Accept) != 1 || manifest.ShareTarget.Params.Files[0].Accept[0] != "*/*" {
t.Fatalf("unexpected share_target files: %+v", manifest.ShareTarget.Params.Files)
}
}
func TestServiceWorkerServedFromRootScope(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
request := httptest.NewRequest(http.MethodGet, "/service-worker.js", nil)
response := httptest.NewRecorder()
app.ServiceWorker(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
}
if got := response.Header().Get("Service-Worker-Allowed"); got != "/" {
t.Fatalf("Service-Worker-Allowed = %q, want /", got)
}
if got := response.Header().Get("Content-Type"); got != "text/javascript; charset=utf-8" {
t.Fatalf("Content-Type = %q", got)
}
if response.Body.Len() == 0 {
t.Fatalf("service worker body missing")
}
}
func TestShareTargetFallbackRedirectsHome(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
request := httptest.NewRequest(http.MethodPost, "/share-target", nil)
response := httptest.NewRecorder()
app.ShareTargetFallback(response, request)
if response.Code != http.StatusSeeOther {
t.Fatalf("status = %d, want %d", response.Code, http.StatusSeeOther)
}
if got := response.Header().Get("Location"); got != "/?share-target=unsupported" {
t.Fatalf("Location = %q", got)
}
}

7
backend/libs/handlers/upload.go
View File

@@ -53,6 +53,11 @@ func (a *App) Upload(w http.ResponseWriter, r *http.Request) {
}
if err := r.ParseMultipartForm(parseLimit); err != nil {
a.logger.Warn("upload form parse failed", withRequestLogAttrs(r, "source", "user-upload", "severity", "warn", "code", 4000, "user_id", user.ID, "error", err.Error())...)
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
helpers.WriteJSONError(w, http.StatusRequestEntityTooLarge, "upload exceeds the configured upload limit")
return
}
helpers.WriteJSONError(w, http.StatusBadRequest, "upload form could not be read")
return
}
@@ -244,7 +249,7 @@ func (a *App) checkUploadPolicyForSizes(r *http.Request, user services.User, log
maxBytes := services.MegabytesToBytes(policy.MaxUploadMB)
for _, fileSize := range fileSizes {
if fileSize > maxBytes {
return http.StatusRequestEntityTooLarge, "file exceeds upload size limit"
return http.StatusRequestEntityTooLarge, "file exceeds upload size limit of " + services.FormatMegabytesLabel(policy.MaxUploadMB)
}
}
}

38
backend/libs/handlers/upload_stage3_test.go
View File

@@ -218,6 +218,44 @@ func TestFilePreviewPageIncludesPreviewMetadata(t *testing.T) {
}
}
func TestDownloadPageShowsProcessingFailure(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()
payload := uploadThroughApp(t, app)
box, err := app.uploadService.GetBox(payload.BoxID)
if err != nil {
t.Fatalf("GetBox returned error: %v", err)
}
box.Files[0].Processing = false
box.Files[0].ProcessingError = "Access Denied."
if err := app.uploadService.SaveBox(box); err != nil {
t.Fatalf("SaveBox returned error: %v", err)
}
request := httptest.NewRequest(http.MethodGet, "/d/"+payload.BoxID, nil)
request.SetPathValue("boxID", payload.BoxID)
response := httptest.NewRecorder()
app.DownloadPage(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{
"Upload processing failed",
"Access Denied.",
"is-failed",
"Failed",
} {
if !strings.Contains(body, want) {
t.Fatalf("download page missing %q: %s", want, body)
}
}
if strings.Contains(body, `data-download-url="/d/`+payload.BoxID+`/f/`+payload.Files[0].ID+`/download"`) {
t.Fatalf("failed file still exposed download context: %s", body)
}
}
func TestFileDownloadUsesOriginalFilename(t *testing.T) {
app, cleanup := newTestApp(t)
defer cleanup()

40
backend/libs/jobs/thumbnails.go
View File

@@ -45,6 +45,10 @@ func GenerateThumbnailsForBoxAsync(uploadService *services.UploadService, logger
logger.Warn("thumbnail box lookup failed", "source", "thumbnail", "severity", "warn", "code", 4204, "box_id", boxID, "error", err.Error())
return
}
if services.BoxHasTrouble(box) {
logger.Warn("thumbnail one-shot skipped trouble box", "source", "thumbnail", "severity", "warn", "code", 4206, "box_id", boxID, "error", services.BoxTroubleReason(box))
return
}
result, err := generateMissingThumbnailsForBox(uploadService, logger, box)
if err != nil {
@@ -91,6 +95,9 @@ func generateMissingThumbnails(uploadService *services.UploadService, logger *sl
if !box.ExpiresAt.After(now) {
continue
}
if services.BoxHasTrouble(box) {
continue
}
boxResult, err := generateMissingThumbnailsForBox(uploadService, logger, box)
result.Scanned += boxResult.Scanned
@@ -109,10 +116,16 @@ func generateMissingThumbnailsForBox(uploadService *services.UploadService, logg
if !box.ExpiresAt.After(time.Now().UTC()) {
return result, nil
}
if services.BoxHasTrouble(box) {
return result, nil
}
changed := false
for i := range box.Files {
file := &box.Files[i]
if file.Processing || services.FileHasTrouble(*file) {
continue
}
needsPrimary := file.Thumbnail == "" && needsThumbnail(*file)
needsScenes := file.SceneThumbnail == "" && needsVideoScenes(*file)
needsArchive := !archiveListingCurrent(*file) && needsArchiveListing(*file)
@@ -206,6 +219,15 @@ func GenerateArchiveListingForFile(uploadService *services.UploadService, box se
}
func generateThumbnail(uploadService *services.UploadService, box services.Box, file services.File) (string, error) {
if services.BoxHasTrouble(box) {
return "", fmt.Errorf("box is marked as trouble: %s", services.BoxTroubleReason(box))
}
if file.Processing {
return "", fmt.Errorf("file is still processing")
}
if services.FileHasTrouble(file) {
return "", fmt.Errorf("file processing failed: %s", file.ProcessingError)
}
thumbnailName := "@thumb@" + file.ID + ".jpg"
object, err := uploadService.OpenFileObject(context.Background(), box, file)
if err != nil {
@@ -244,6 +266,15 @@ func generateVideoScenesThumbnail(uploadService *services.UploadService, box ser
if !needsVideoScenes(file) {
return "", nil
}
if services.BoxHasTrouble(box) {
return "", fmt.Errorf("box is marked as trouble: %s", services.BoxTroubleReason(box))
}
if file.Processing {
return "", fmt.Errorf("file is still processing")
}
if services.FileHasTrouble(file) {
return "", fmt.Errorf("file processing failed: %s", file.ProcessingError)
}
sceneName := "@scene@" + file.ID + ".jpg"
object, err := uploadService.OpenFileObject(context.Background(), box, file)
if err != nil {
@@ -263,6 +294,15 @@ func generateArchiveListing(uploadService *services.UploadService, box services.
if !needsArchiveListing(file) {
return "", nil
}
if services.BoxHasTrouble(box) {
return "", fmt.Errorf("box is marked as trouble: %s", services.BoxTroubleReason(box))
}
if file.Processing {
return "", fmt.Errorf("file is still processing")
}
if services.FileHasTrouble(file) {
return "", fmt.Errorf("file processing failed: %s", file.ProcessingError)
}
listingName := "@archive@" + file.ID + ".json"
object, err := uploadService.OpenFileObject(context.Background(), box, file)
if err != nil {

30
backend/libs/jobs/thumbnails_test.go
View File

@@ -50,6 +50,36 @@ func TestGenerateMissingThumbnailsForBox(t *testing.T) {
}
}
func TestGenerateMissingThumbnailsForTroubleBoxSkipsWork(t *testing.T) {
service := newThumbnailTestUploadService(t)
result := createThumbnailTestBox(t, service)
box, err := service.GetBox(result.BoxID)
if err != nil {
t.Fatalf("GetBox returned error: %v", err)
}
box.Trouble = true
box.TroubleReason = "storage backend failed"
if err := service.SaveBox(box); err != nil {
t.Fatalf("SaveBox returned error: %v", err)
}
jobResult, err := generateMissingThumbnailsForBox(service, slog.New(slog.NewTextHandler(io.Discard, nil)), box)
if err != nil {
t.Fatalf("generateMissingThumbnailsForBox returned error: %v", err)
}
if jobResult != (ThumbnailJobResult{}) {
t.Fatalf("job result = %+v, want no work for trouble box", jobResult)
}
updated, err := service.GetBox(result.BoxID)
if err != nil {
t.Fatalf("GetBox after job returned error: %v", err)
}
if updated.Files[0].Thumbnail != "" {
t.Fatalf("thumbnail was generated for trouble box: %+v", updated.Files[0])
}
}
func TestCreateTextThumbnailRendersMarkdownAsJPEG(t *testing.T) {
data, err := createTextThumbnail(services.File{
Name: "notes.md",

34
backend/libs/services/resumable.go
View File

@@ -369,19 +369,20 @@ func (s *UploadService) FinalizeProcessingResumableSession(ctx context.Context,
}
backend, err := s.storage.Backend(box.StorageBackendID)
if err != nil {
_ = s.markProcessingBoxFailed(box, err)
return UploadResult{}, err
}
for i, incoming := range staged {
source, err := incoming.Open()
if err != nil {
_ = s.markProcessingBoxFailed(box, err)
return UploadResult{}, err
}
file := box.Files[i]
if err := s.writeUploadedObject(ctx, backend, file.ObjectKey, source, incoming.Size(), 0, incoming.ContentType()); err != nil {
source.Close()
_ = backend.Delete(context.Background(), file.ObjectKey)
box.Files[i].ProcessingError = err.Error()
_ = s.saveBoxRecord(box)
_ = s.markProcessingBoxFailed(box, err)
return UploadResult{}, err
}
source.Close()
@@ -406,6 +407,35 @@ func (s *UploadService) FinalizeProcessingResumableSession(ctx context.Context,
return s.resultForBox(box, ""), nil
}
func (s *UploadService) markProcessingBoxFailed(box Box, cause error) error {
message := "upload processing failed"
if cause != nil && strings.TrimSpace(cause.Error()) != "" {
message = cause.Error()
}
s.logger.Warn("resumable upload box marked failed", "source", "user-upload", "severity", "warn", "code", 4021, "box_id", box.ID, "backend_id", s.BoxStorageBackendID(box), "files", len(box.Files), "error", message)
now := time.Now().UTC()
box.Trouble = true
box.TroubleReason = message
for i := range box.Files {
if box.Files[i].Processing || box.Files[i].ProcessingError == "" {
box.Files[i].Processing = false
box.Files[i].ProcessingError = message
if box.Files[i].UploadedAt.IsZero() {
box.Files[i].UploadedAt = now
}
}
}
if err := s.saveBoxRecord(box); err != nil {
s.logger.Warn("failed to save failed upload box state", "source", "user-upload", "severity", "warn", "code", 4022, "box_id", box.ID, "backend_id", s.BoxStorageBackendID(box), "error", err.Error())
return err
}
if err := s.writeBoxMetadata(box); err != nil {
s.logger.Warn("failed to write failed upload box metadata", "source", "user-upload", "severity", "warn", "code", 4023, "box_id", box.ID, "backend_id", s.BoxStorageBackendID(box), "error", err.Error())
return err
}
return nil
}
func (s *UploadService) CompleteUploadedResumableSession(ctx context.Context, sessionID string) (UploadResult, ResumableSession, error) {
session, err := s.GetResumableSession(sessionID)
if err != nil {

27
backend/libs/services/storage_s3.go
View File

@@ -35,26 +35,35 @@ func (b *s3StorageBackend) ID() string { return b.cfg.ID }
func (b *s3StorageBackend) Type() string { return StorageBackendS3 }
func (b *s3StorageBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
cleanKey := cleanObjectKey(key)
opts := minio.PutObjectOptions{ContentType: contentType}
_, err := b.client.PutObject(ctx, b.cfg.Bucket, cleanObjectKey(key), body, size, opts)
return err
_, err := b.client.PutObject(ctx, b.cfg.Bucket, cleanKey, body, size, opts)
if err != nil {
return fmt.Errorf("s3 put object %q in bucket %q failed: %w", cleanKey, b.cfg.Bucket, err)
}
return nil
}
func (b *s3StorageBackend) Get(ctx context.Context, key string) (StorageObject, error) {
object, err := b.client.GetObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.GetObjectOptions{})
cleanKey := cleanObjectKey(key)
object, err := b.client.GetObject(ctx, b.cfg.Bucket, cleanKey, minio.GetObjectOptions{})
if err != nil {
return StorageObject{}, err
return StorageObject{}, fmt.Errorf("s3 get object %q from bucket %q failed: %w", cleanKey, b.cfg.Bucket, err)
}
info, err := object.Stat()
if err != nil {
object.Close()
return StorageObject{}, err
return StorageObject{}, fmt.Errorf("s3 stat object %q in bucket %q failed: %w", cleanKey, b.cfg.Bucket, err)
}
return StorageObject{Key: key, Size: info.Size, ContentType: info.ContentType, ModTime: info.LastModified, Body: object}, nil
}
func (b *s3StorageBackend) Delete(ctx context.Context, key string) error {
return b.client.RemoveObject(ctx, b.cfg.Bucket, cleanObjectKey(key), minio.RemoveObjectOptions{})
cleanKey := cleanObjectKey(key)
if err := b.client.RemoveObject(ctx, b.cfg.Bucket, cleanKey, minio.RemoveObjectOptions{}); err != nil {
return fmt.Errorf("s3 delete object %q from bucket %q failed: %w", cleanKey, b.cfg.Bucket, err)
}
return nil
}
func (b *s3StorageBackend) DeletePrefix(ctx context.Context, prefix string) error {
@@ -62,7 +71,7 @@ func (b *s3StorageBackend) DeletePrefix(ctx context.Context, prefix string) erro
objects := b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
for object := range objects {
if object.Err != nil {
return object.Err
return fmt.Errorf("s3 list prefix %q in bucket %q failed: %w", prefix, b.cfg.Bucket, object.Err)
}
if err := b.Delete(ctx, object.Key); err != nil {
return err
@@ -75,7 +84,7 @@ func (b *s3StorageBackend) Usage(ctx context.Context) (int64, error) {
var total int64
for object := range b.client.ListObjects(ctx, b.cfg.Bucket, minio.ListObjectsOptions{Recursive: true}) {
if object.Err != nil {
return 0, object.Err
return 0, fmt.Errorf("s3 usage list bucket %q failed: %w", b.cfg.Bucket, object.Err)
}
total += object.Size
}
@@ -85,7 +94,7 @@ func (b *s3StorageBackend) Usage(ctx context.Context) (int64, error) {
func (b *s3StorageBackend) Test(ctx context.Context) error {
exists, err := b.client.BucketExists(ctx, b.cfg.Bucket)
if err != nil {
return err
return fmt.Errorf("s3 bucket check for %q failed: %w", b.cfg.Bucket, err)
}
if !exists {
return fmt.Errorf("bucket %q does not exist", b.cfg.Bucket)

33
backend/libs/services/upload.go
View File

@@ -117,6 +117,8 @@ type Box struct {
Obfuscate bool `json:"obfuscate"`
CreatorIP string `json:"creatorIp,omitempty"`
StorageBackendID string `json:"storageBackendId,omitempty"`
Trouble bool `json:"trouble,omitempty"`
TroubleReason string `json:"troubleReason,omitempty"`
Files []File `json:"files"`
}
@@ -139,6 +141,37 @@ type File struct {
UploadedAt time.Time `json:"uploadedAt"`
}
func BoxHasTrouble(box Box) bool {
if box.Trouble || strings.TrimSpace(box.TroubleReason) != "" {
return true
}
for _, file := range box.Files {
if FileHasTrouble(file) {
return true
}
}
return false
}
func BoxTroubleReason(box Box) string {
if strings.TrimSpace(box.TroubleReason) != "" {
return box.TroubleReason
}
for _, file := range box.Files {
if strings.TrimSpace(file.ProcessingError) != "" {
return file.ProcessingError
}
}
if box.Trouble {
return "box has failed processing"
}
return ""
}
func FileHasTrouble(file File) bool {
return strings.TrimSpace(file.ProcessingError) != ""
}
type UploadResult struct {
BoxID string `json:"boxId"`
BoxURL string `json:"boxUrl"`

41
backend/libs/services/upload_test.go
View File

@@ -230,6 +230,47 @@ func TestResumableCompleteRejectsMissingChunks(t *testing.T) {
}
}
func TestProcessingResumableFailureMarksBoxFailed(t *testing.T) {
service := newTestUploadService(t)
session, err := service.CreateResumableSession([]ResumableFileInput{{
Name: "note.txt",
Size: 4,
ContentType: "text/plain",
}}, UploadOptions{MaxDays: 1, StorageBackendID: "missing"}, 4, time.Hour, "")
if err != nil {
t.Fatalf("CreateResumableSession returned error: %v", err)
}
if _, err := service.PutResumableChunk(testContext(), session.ID, session.Files[0].ID, 0, strings.NewReader("note")); err != nil {
t.Fatalf("PutResumableChunk returned error: %v", err)
}
result, processing, err := service.CreateProcessingBoxFromResumable(session.ID)
if err != nil {
t.Fatalf("CreateProcessingBoxFromResumable returned error: %v", err)
}
if processing.Status != ResumableStatusProcessing {
t.Fatalf("session status = %q, want processing", processing.Status)
}
if _, err := service.FinalizeProcessingResumableSession(testContext(), session.ID); err == nil {
t.Fatalf("FinalizeProcessingResumableSession accepted missing backend")
}
box := getTestBox(t, service, result.BoxID)
if len(box.Files) != 1 {
t.Fatalf("box files = %+v", box.Files)
}
if box.Files[0].Processing {
t.Fatalf("failed file is still marked processing: %+v", box.Files[0])
}
if box.Files[0].ProcessingError == "" {
t.Fatalf("failed file did not store processing error: %+v", box.Files[0])
}
if !box.Trouble {
t.Fatalf("failed box was not marked as trouble: %+v", box)
}
if box.TroubleReason == "" {
t.Fatalf("failed box did not store trouble reason: %+v", box)
}
}
func TestResumablePartialCompleteKeepsOnlyFinishedFiles(t *testing.T) {
service := newTestUploadService(t)
session, err := service.CreateResumableSession([]ResumableFileInput{