feat(upload): add resumable chunk configuration and file validation

- Add `WARPBOX_RESUMABLE_CHUNK_MODE` and `WARPBOX_RESUMABLE_CHUNK_PATH` environment variables to configure temporary chunk storage.
- Implement strict file validation for resuming uploads to ensure selected files match the pending session's metadata.
- Add `PLANS.md` to document development stages, roadmap, and API specifications (including batching and resumable flows).

backend/libs/services/resumable.go

@@ -2,6 +2,9 @@ package services
 
 import (
 	"context"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -17,9 +20,10 @@ import (
 var resumableUploadsBucket = []byte("resumable_uploads")
 
 const (
-	ResumableStatusUploading = "uploading"
-	ResumableStatusCompleted = "completed"
-	ResumableStatusCancelled = "cancelled"
+	ResumableStatusUploading  = "uploading"
+	ResumableStatusProcessing = "processing"
+	ResumableStatusCompleted  = "completed"
+	ResumableStatusCancelled  = "cancelled"
 )
 
 type ResumableFileInput struct {
@@ -30,15 +34,18 @@ type ResumableFileInput struct {
 }
 
 type ResumableSession struct {
-	ID        string          `json:"id"`
-	Options   UploadOptions   `json:"options"`
-	Files     []ResumableFile `json:"files"`
-	ChunkSize int64           `json:"chunkSize"`
-	Status    string          `json:"status"`
-	BoxID     string          `json:"boxId,omitempty"`
-	CreatedAt time.Time       `json:"createdAt"`
-	UpdatedAt time.Time       `json:"updatedAt"`
-	ExpiresAt time.Time       `json:"expiresAt"`
+	ID              string          `json:"id"`
+	Options         UploadOptions   `json:"options"`
+	Files           []ResumableFile `json:"files"`
+	ChunkSize       int64           `json:"chunkSize"`
+	Status          string          `json:"status"`
+	BoxID           string          `json:"boxId,omitempty"`
+	ResumeTokenHash string          `json:"resumeTokenHash,omitempty"`
+	ResumeToken     string          `json:"-"`
+	ChunkRoot       string          `json:"chunkRoot,omitempty"`
+	CreatedAt       time.Time       `json:"createdAt"`
+	UpdatedAt       time.Time       `json:"updatedAt"`
+	ExpiresAt       time.Time       `json:"expiresAt"`
 }
 
 type ResumableFile struct {
@@ -58,7 +65,7 @@ func (s *UploadService) ensureResumableBucket() error {
 	})
 }
 
-func (s *UploadService) CreateResumableSession(files []ResumableFileInput, opts UploadOptions, chunkSize int64, retention time.Duration) (ResumableSession, error) {
+func (s *UploadService) CreateResumableSession(files []ResumableFileInput, opts UploadOptions, chunkSize int64, retention time.Duration, chunkRoot string) (ResumableSession, error) {
 	if len(files) == 0 {
 		return ResumableSession{}, fmt.Errorf("no files were uploaded")
 	}
@@ -77,15 +84,20 @@ func (s *UploadService) CreateResumableSession(files []ResumableFileInput, opts
 		return ResumableSession{}, err
 	}
 	now := time.Now().UTC()
+	resumeToken := randomID(32)
+	sessionID := randomID(12)
 	session := ResumableSession{
-		ID:        randomID(12),
-		Options:   opts,
-		Files:     sessionFiles,
-		ChunkSize: chunkSize,
-		Status:    ResumableStatusUploading,
-		CreatedAt: now,
-		UpdatedAt: now,
-		ExpiresAt: now.Add(retention),
+		ID:              sessionID,
+		Options:         opts,
+		Files:           sessionFiles,
+		ChunkSize:       chunkSize,
+		Status:          ResumableStatusUploading,
+		ResumeTokenHash: resumableTokenHash(sessionID, resumeToken),
+		ResumeToken:     resumeToken,
+		ChunkRoot:       strings.TrimSpace(chunkRoot),
+		CreatedAt:       now,
+		UpdatedAt:       now,
+		ExpiresAt:       now.Add(retention),
 	}
 	if err := s.saveResumableSession(session); err != nil {
 		return ResumableSession{}, err
@@ -93,6 +105,14 @@ func (s *UploadService) CreateResumableSession(files []ResumableFileInput, opts
 	return session, nil
 }
 
+func (s *UploadService) VerifyResumableToken(session ResumableSession, token string) bool {
+	if session.ResumeTokenHash == "" || strings.TrimSpace(token) == "" {
+		return false
+	}
+	hash := resumableTokenHash(session.ID, token)
+	return subtle.ConstantTimeCompare([]byte(hash), []byte(session.ResumeTokenHash)) == 1
+}
+
 func (s *UploadService) AddResumableFiles(sessionID string, files []ResumableFileInput) (ResumableSession, error) {
 	if len(files) == 0 {
 		return s.GetResumableSession(sessionID)
@@ -165,11 +185,11 @@ func (s *UploadService) PutResumableChunk(ctx context.Context, sessionID, fileID
 		return ResumableSession{}, fmt.Errorf("chunk index is invalid")
 	}
 	expectedSize := expectedChunkSize(file.Size, session.ChunkSize, index)
-	chunkDir := s.resumableFileDir(session.ID, file.ID)
+	chunkDir := s.resumableFileDirFor(session, file.ID)
 	if err := os.MkdirAll(chunkDir, 0o755); err != nil {
 		return ResumableSession{}, err
 	}
-	chunkPath := s.resumableChunkPath(session.ID, file.ID, index)
+	chunkPath := s.resumableChunkPathFor(session, file.ID, index)
 	tempPath := chunkPath + ".tmp"
 	target, err := os.OpenFile(tempPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
 	if err != nil {
@@ -206,20 +226,26 @@ func (s *UploadService) CompleteResumableSession(ctx context.Context, sessionID
 	if err != nil {
 		return UploadResult{}, ResumableSession{}, err
 	}
+	if (session.Status == ResumableStatusCompleted || session.Status == ResumableStatusProcessing) && session.BoxID != "" {
+		box, err := s.GetBox(session.BoxID)
+		if err != nil {
+			return UploadResult{}, ResumableSession{}, err
+		}
+		return s.resultForBox(box, ""), session, nil
+	}
 	if err := resumableSessionWritable(session); err != nil {
 		return UploadResult{}, ResumableSession{}, err
 	}
-	staged, cleanup, err := s.assembleResumableFiles(ctx, session)
+	staged, err := s.resumableIncomingFiles(session)
 	if err != nil {
 		return UploadResult{}, ResumableSession{}, err
 	}
-	defer cleanup()
 
-	result, err := s.CreateBoxFromIncoming(staged, session.Options)
+	result, err := s.CreateBoxFromIncomingContext(ctx, staged, session.Options)
 	if err != nil {
 		return UploadResult{}, ResumableSession{}, err
 	}
-	if err := os.RemoveAll(s.resumableSessionDir(session.ID)); err != nil {
+	if err := os.RemoveAll(s.resumableSessionDirFor(session)); err != nil {
 		return UploadResult{}, ResumableSession{}, err
 	}
 	session.Status = ResumableStatusCompleted
@@ -231,17 +257,205 @@ func (s *UploadService) CompleteResumableSession(ctx context.Context, sessionID
 	return result, session, nil
 }
 
+func (s *UploadService) CreateProcessingBoxFromResumable(sessionID string) (UploadResult, ResumableSession, error) {
+	session, err := s.GetResumableSession(sessionID)
+	if err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	if (session.Status == ResumableStatusCompleted || session.Status == ResumableStatusProcessing) && session.BoxID != "" {
+		box, err := s.GetBox(session.BoxID)
+		if err != nil {
+			return UploadResult{}, ResumableSession{}, err
+		}
+		return s.resultForBox(box, ""), session, nil
+	}
+	if err := resumableSessionWritable(session); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	if _, err := s.resumableIncomingFiles(session); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+
+	now := time.Now().UTC()
+	expiresAt := now.AddDate(0, 0, 7)
+	if session.Options.ExpiresInMinutes < 0 || session.Options.MaxDays < 0 {
+		expiresAt = now.AddDate(100, 0, 0)
+	} else if session.Options.ExpiresInMinutes > 0 {
+		expiresAt = now.Add(time.Duration(session.Options.ExpiresInMinutes) * time.Minute)
+	} else if session.Options.MaxDays > 0 {
+		expiresAt = now.Add(time.Duration(session.Options.MaxDays) * 24 * time.Hour)
+	}
+
+	box := Box{
+		ID:               randomID(10),
+		OwnerID:          strings.TrimSpace(session.Options.OwnerID),
+		CollectionID:     strings.TrimSpace(session.Options.CollectionID),
+		CreatorIP:        strings.TrimSpace(session.Options.CreatorIP),
+		StorageBackendID: normalizeBackendID(session.Options.StorageBackendID),
+		CreatedAt:        now,
+		ExpiresAt:        expiresAt,
+		MaxDownloads:     session.Options.MaxDownloads,
+		Obfuscate:        session.Options.ObfuscateMetadata && (strings.TrimSpace(session.Options.Password) != "" || strings.TrimSpace(session.Options.PasswordHash) != ""),
+		Files:            make([]File, 0, len(session.Files)),
+	}
+	deleteToken := randomID(32)
+	box.DeleteTokenHash = deleteTokenHash(box.ID, deleteToken)
+	if strings.TrimSpace(session.Options.PasswordHash) != "" {
+		box.PasswordSalt = session.Options.PasswordSalt
+		box.PasswordHash = session.Options.PasswordHash
+	} else if strings.TrimSpace(session.Options.Password) != "" {
+		salt, hash := hashPassword(session.Options.Password)
+		box.PasswordSalt = salt
+		box.PasswordHash = hash
+	}
+
+	for _, incoming := range session.Files {
+		fileID := randomID(8)
+		storedName := "@each@" + fileID + strings.ToLower(filepath.Ext(incoming.Name))
+		objectKey := boxObjectKey(box.ID, storedName)
+		contentType := incoming.ContentType
+		if contentType == "" {
+			contentType = "application/octet-stream"
+		}
+		box.Files = append(box.Files, File{
+			ID:          fileID,
+			Name:        filepath.Base(incoming.Name),
+			StoredName:  storedName,
+			Size:        incoming.Size,
+			ContentType: contentType,
+			PreviewKind: previewKind(contentType),
+			ObjectKey:   objectKey,
+			Processing:  true,
+			UploadedAt:  now,
+		})
+	}
+	if err := s.saveBoxRecord(box); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	session.Status = ResumableStatusProcessing
+	session.BoxID = box.ID
+	session.UpdatedAt = time.Now().UTC()
+	if err := s.saveResumableSession(session); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	return s.resultForBox(box, deleteToken), session, nil
+}
+
+func (s *UploadService) FinalizeProcessingResumableSession(ctx context.Context, sessionID string) (UploadResult, error) {
+	session, err := s.GetResumableSession(sessionID)
+	if err != nil {
+		return UploadResult{}, err
+	}
+	if session.Status == ResumableStatusCompleted && session.BoxID != "" {
+		box, err := s.GetBox(session.BoxID)
+		if err != nil {
+			return UploadResult{}, err
+		}
+		return s.resultForBox(box, ""), nil
+	}
+	if session.Status != ResumableStatusProcessing || session.BoxID == "" {
+		return UploadResult{}, fmt.Errorf("upload session is not processing")
+	}
+	box, err := s.GetBox(session.BoxID)
+	if err != nil {
+		return UploadResult{}, err
+	}
+	staged, err := s.resumableIncomingFiles(session)
+	if err != nil {
+		return UploadResult{}, err
+	}
+	if len(staged) != len(box.Files) {
+		return UploadResult{}, fmt.Errorf("processing file count mismatch")
+	}
+	backend, err := s.storage.Backend(box.StorageBackendID)
+	if err != nil {
+		return UploadResult{}, err
+	}
+	for i, incoming := range staged {
+		source, err := incoming.Open()
+		if err != nil {
+			return UploadResult{}, err
+		}
+		file := box.Files[i]
+		if err := s.writeUploadedObject(ctx, backend, file.ObjectKey, source, incoming.Size(), 0, incoming.ContentType()); err != nil {
+			source.Close()
+			_ = backend.Delete(context.Background(), file.ObjectKey)
+			box.Files[i].ProcessingError = err.Error()
+			_ = s.saveBoxRecord(box)
+			return UploadResult{}, err
+		}
+		source.Close()
+		box.Files[i].Processing = false
+		box.Files[i].ProcessingError = ""
+		box.Files[i].UploadedAt = time.Now().UTC()
+		if err := s.saveBoxRecord(box); err != nil {
+			return UploadResult{}, err
+		}
+	}
+	if err := s.writeBoxMetadata(box); err != nil {
+		s.logger.Warn("box metadata write failed after resumable processing", "source", "storage", "severity", "warn", "code", 4020, "box_id", box.ID, "error", err.Error())
+	}
+	if err := os.RemoveAll(s.resumableSessionDirFor(session)); err != nil {
+		return UploadResult{}, err
+	}
+	session.Status = ResumableStatusCompleted
+	session.UpdatedAt = time.Now().UTC()
+	if err := s.saveResumableSession(session); err != nil {
+		return UploadResult{}, err
+	}
+	return s.resultForBox(box, ""), nil
+}
+
+func (s *UploadService) CompleteUploadedResumableSession(ctx context.Context, sessionID string) (UploadResult, ResumableSession, error) {
+	session, err := s.GetResumableSession(sessionID)
+	if err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	if err := resumableSessionWritable(session); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	completeFiles := make([]ResumableFile, 0, len(session.Files))
+	for _, file := range session.Files {
+		if resumableFileComplete(file) {
+			completeFiles = append(completeFiles, file)
+		}
+	}
+	if len(completeFiles) == 0 {
+		return UploadResult{}, ResumableSession{}, fmt.Errorf("no fully uploaded files to finish")
+	}
+	partial := session
+	partial.Files = completeFiles
+	staged, err := s.resumableIncomingFiles(partial)
+	if err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+
+	result, err := s.CreateBoxFromIncomingContext(ctx, staged, session.Options)
+	if err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	if err := os.RemoveAll(s.resumableSessionDirFor(session)); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	session.Status = ResumableStatusCompleted
+	session.BoxID = result.BoxID
+	session.Files = completeFiles
+	session.UpdatedAt = time.Now().UTC()
+	if err := s.deleteResumableSession(session.ID); err != nil {
+		return UploadResult{}, ResumableSession{}, err
+	}
+	return result, session, nil
+}
+
 func (s *UploadService) CancelResumableSession(sessionID string) error {
 	session, err := s.GetResumableSession(sessionID)
 	if err != nil {
 		return err
 	}
-	session.Status = ResumableStatusCancelled
-	session.UpdatedAt = time.Now().UTC()
-	if err := s.saveResumableSession(session); err != nil {
+	if err := os.RemoveAll(s.resumableSessionDirFor(session)); err != nil {
 		return err
 	}
-	return os.RemoveAll(s.resumableSessionDir(session.ID))
+	return s.deleteResumableSession(session.ID)
 }
 
 func (s *UploadService) CleanupExpiredResumableSessions(now time.Time) (int, error) {
@@ -256,7 +470,9 @@ func (s *UploadService) CleanupExpiredResumableSessions(now time.Time) (int, err
 			if err := json.Unmarshal(value, &session); err != nil {
 				return err
 			}
-			if !session.ExpiresAt.After(now) || session.Status != ResumableStatusUploading {
+			if session.Status == ResumableStatusCompleted ||
+				session.Status == ResumableStatusCancelled ||
+				(session.Status == ResumableStatusUploading && !session.ExpiresAt.After(now)) {
 				candidates = append(candidates, session)
 			}
 			return nil
@@ -266,7 +482,7 @@ func (s *UploadService) CleanupExpiredResumableSessions(now time.Time) (int, err
 		return 0, err
 	}
 	for _, session := range candidates {
-		if err := os.RemoveAll(s.resumableSessionDir(session.ID)); err != nil {
+		if err := os.RemoveAll(s.resumableSessionDirFor(session)); err != nil {
 			return 0, err
 		}
 	}
@@ -285,6 +501,16 @@ func (s *UploadService) CleanupExpiredResumableSessions(now time.Time) (int, err
 	return len(candidates), err
 }
 
+func (s *UploadService) deleteResumableSession(sessionID string) error {
+	return s.db.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(resumableUploadsBucket)
+		if bucket == nil {
+			return nil
+		}
+		return bucket.Delete([]byte(sessionID))
+	})
+}
+
 func (s *UploadService) saveResumableSession(session ResumableSession) error {
 	if err := s.ensureResumableBucket(); err != nil {
 		return err
@@ -341,71 +567,102 @@ func resumableFileKey(name string, size int64, fingerprint string) string {
 	return strings.TrimSpace(fingerprint) + "|" + filepath.Base(strings.TrimSpace(name)) + "|" + fmt.Sprintf("%d", size)
 }
 
-func (s *UploadService) assembleResumableFiles(ctx context.Context, session ResumableSession) ([]IncomingFile, func(), error) {
-	assembledDir := filepath.Join(s.resumableSessionDir(session.ID), "assembled")
-	if err := os.MkdirAll(assembledDir, 0o755); err != nil {
-		return nil, func() {}, err
+type resumableIncomingFile struct {
+	service *UploadService
+	session ResumableSession
+	file    ResumableFile
+}
+
+func (f resumableIncomingFile) Name() string {
+	return f.file.Name
+}
+
+func (f resumableIncomingFile) Size() int64 {
+	return f.file.Size
+}
+
+func (f resumableIncomingFile) ContentType() string {
+	return f.file.ContentType
+}
+
+func (f resumableIncomingFile) Open() (io.ReadCloser, error) {
+	return &resumableChunkReader{
+		service: f.service,
+		session: f.session,
+		file:    f.file,
+	}, nil
+}
+
+type resumableChunkReader struct {
+	service *UploadService
+	session ResumableSession
+	file    ResumableFile
+	index   int
+	current *os.File
+}
+
+func (r *resumableChunkReader) Read(p []byte) (int, error) {
+	for {
+		if r.current == nil {
+			if r.index >= r.file.ChunkCount {
+				return 0, io.EOF
+			}
+			chunk, err := os.Open(r.service.resumableChunkPathFor(r.session, r.file.ID, r.index))
+			if err != nil {
+				return 0, err
+			}
+			r.current = chunk
+		}
+		n, err := r.current.Read(p)
+		if err == io.EOF {
+			if closeErr := r.current.Close(); closeErr != nil {
+				r.current = nil
+				return n, closeErr
+			}
+			r.current = nil
+			r.index++
+			if n > 0 {
+				return n, nil
+			}
+			continue
+		}
+		return n, err
 	}
-	cleanup := func() {
-		_ = os.RemoveAll(assembledDir)
+}
+
+func (r *resumableChunkReader) Close() error {
+	if r.current == nil {
+		return nil
 	}
+	err := r.current.Close()
+	r.current = nil
+	return err
+}
+
+func (s *UploadService) resumableIncomingFiles(session ResumableSession) ([]IncomingFile, error) {
 	staged := make([]IncomingFile, 0, len(session.Files))
 	for _, file := range session.Files {
 		if len(file.UploadedChunks) != file.ChunkCount {
-			cleanup()
-			return nil, func() {}, fmt.Errorf("file %s is missing chunks", file.Name)
-		}
-		assembledPath := filepath.Join(assembledDir, file.ID)
-		target, err := os.OpenFile(assembledPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
-		if err != nil {
-			cleanup()
-			return nil, func() {}, err
+			return nil, fmt.Errorf("file %s is missing chunks", file.Name)
 		}
 		var written int64
 		for i := 0; i < file.ChunkCount; i++ {
-			select {
-			case <-ctx.Done():
-				_ = target.Close()
-				cleanup()
-				return nil, func() {}, ctx.Err()
-			default:
-			}
-			chunk, err := os.Open(s.resumableChunkPath(session.ID, file.ID, i))
+			info, err := os.Stat(s.resumableChunkPathFor(session, file.ID, i))
 			if err != nil {
-				_ = target.Close()
-				cleanup()
-				return nil, func() {}, fmt.Errorf("file %s is missing chunks", file.Name)
+				return nil, fmt.Errorf("file %s is missing chunks", file.Name)
 			}
-			n, copyErr := io.Copy(target, chunk)
-			closeErr := chunk.Close()
-			if copyErr != nil {
-				_ = target.Close()
-				cleanup()
-				return nil, func() {}, copyErr
-			}
-			if closeErr != nil {
-				_ = target.Close()
-				cleanup()
-				return nil, func() {}, closeErr
-			}
-			written += n
-		}
-		if err := target.Close(); err != nil {
-			cleanup()
-			return nil, func() {}, err
+			written += info.Size()
 		}
 		if written != file.Size {
-			cleanup()
-			return nil, func() {}, fmt.Errorf("assembled file size mismatch")
+			return nil, fmt.Errorf("chunk size total mismatch")
 		}
-		staged = append(staged, StagedUploadFile{
-			Filename: file.Name,
-			FileSize: file.Size,
-			MIMEType: file.ContentType,
-			Path:     assembledPath,
+		staged = append(staged, resumableIncomingFile{
+			service: s,
+			session: session,
+			file:    file,
 		})
 	}
-	return staged, cleanup, nil
+	return staged, nil
 }
 
 func resumableSessionWritable(session ResumableSession) error {
@@ -418,6 +675,10 @@ func resumableSessionWritable(session ResumableSession) error {
 	return nil
 }
 
+func resumableFileComplete(file ResumableFile) bool {
+	return file.ChunkCount > 0 && len(file.UploadedChunks) == file.ChunkCount
+}
+
 func expectedChunkSize(fileSize, chunkSize int64, index int) int64 {
 	offset := int64(index) * chunkSize
 	remaining := fileSize - offset
@@ -441,14 +702,34 @@ func addChunkIndex(chunks []int, index int) []int {
 	return chunks
 }
 
+func resumableTokenHash(sessionID, token string) string {
+	sum := sha256.Sum256([]byte("warpbox-resumable:" + sessionID + ":" + token))
+	return hex.EncodeToString(sum[:])
+}
+
 func (s *UploadService) resumableSessionDir(sessionID string) string {
 	return filepath.Join(s.dataDir, "tmp", "uploads", sessionID)
 }
 
+func (s *UploadService) resumableSessionDirFor(session ResumableSession) string {
+	if strings.TrimSpace(session.ChunkRoot) != "" {
+		return filepath.Join(session.ChunkRoot, session.ID)
+	}
+	return s.resumableSessionDir(session.ID)
+}
+
 func (s *UploadService) resumableFileDir(sessionID, fileID string) string {
 	return filepath.Join(s.resumableSessionDir(sessionID), fileID)
 }
 
+func (s *UploadService) resumableFileDirFor(session ResumableSession, fileID string) string {
+	return filepath.Join(s.resumableSessionDirFor(session), fileID)
+}
+
 func (s *UploadService) resumableChunkPath(sessionID, fileID string, index int) string {
 	return filepath.Join(s.resumableFileDir(sessionID, fileID), fmt.Sprintf("%06d.part", index))
 }
+
+func (s *UploadService) resumableChunkPathFor(session ResumableSession, fileID string, index int) string {
+	return filepath.Join(s.resumableFileDirFor(session, fileID), fmt.Sprintf("%06d.part", index))
+}