feat(uploads): add native resumable upload support
Implement a native chunked resumable upload API and frontend integration to support reliable large file uploads. Changes include: - Added a 3-step resumable upload API flow (create session, upload chunks, complete session). - Introduced configuration options for chunk size, retention hours, and toggling the feature. - Updated the frontend to utilize resumable uploads with progress tracking. - Configured temporary chunk storage under `data/tmp/uploads` with automatic cleanup. - Documented the API flow and configuration in the README.
This commit is contained in:
@@ -126,6 +126,166 @@ func TestLocalStorageBackendAndLegacyFallback(t *testing.T) {
|
||||
object.Body.Close()
|
||||
}
|
||||
|
||||
func TestResumableSessionUploadOutOfOrderAndComplete(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
session, err := service.CreateResumableSession([]ResumableFileInput{{
|
||||
Name: "note.txt",
|
||||
Size: 11,
|
||||
ContentType: "text/plain",
|
||||
Fingerprint: "sha256:first-chunk",
|
||||
}}, UploadOptions{MaxDays: 1, Password: "secret"}, 4, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateResumableSession returned error: %v", err)
|
||||
}
|
||||
if session.Options.Password != "" || session.Options.PasswordHash == "" || session.Options.PasswordSalt == "" {
|
||||
t.Fatalf("resumable session did not hash password before storage: %+v", session.Options)
|
||||
}
|
||||
if session.Files[0].ChunkCount != 3 {
|
||||
t.Fatalf("ChunkCount = %d, want 3", session.Files[0].ChunkCount)
|
||||
}
|
||||
if session.Files[0].Fingerprint != "sha256:first-chunk" {
|
||||
t.Fatalf("Fingerprint = %q", session.Files[0].Fingerprint)
|
||||
}
|
||||
for index, body := range map[int]string{2: "rld", 0: "hell", 1: "o wo"} {
|
||||
updated, err := service.PutResumableChunk(testContext(), session.ID, session.Files[0].ID, index, strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("PutResumableChunk(%d) returned error: %v", index, err)
|
||||
}
|
||||
if len(updated.Files[0].UploadedChunks) == 0 {
|
||||
t.Fatalf("UploadedChunks was not updated")
|
||||
}
|
||||
}
|
||||
result, completed, err := service.CompleteResumableSession(testContext(), session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteResumableSession returned error: %v", err)
|
||||
}
|
||||
if completed.Status != ResumableStatusCompleted || completed.BoxID != result.BoxID {
|
||||
t.Fatalf("completed session = %+v, result = %+v", completed, result)
|
||||
}
|
||||
box := getTestBox(t, service, result.BoxID)
|
||||
if box.PasswordHash == "" || box.PasswordSalt == "" || box.PasswordHash != session.Options.PasswordHash {
|
||||
t.Fatalf("completed box did not preserve hashed password")
|
||||
}
|
||||
object, err := service.OpenFileObject(testContext(), box, box.Files[0])
|
||||
if err != nil {
|
||||
t.Fatalf("OpenFileObject returned error: %v", err)
|
||||
}
|
||||
data, err := io.ReadAll(object.Body)
|
||||
object.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll returned error: %v", err)
|
||||
}
|
||||
if string(data) != "hello world" {
|
||||
t.Fatalf("object body = %q", string(data))
|
||||
}
|
||||
if _, err := os.Stat(service.resumableSessionDir(session.ID)); !os.IsNotExist(err) {
|
||||
t.Fatalf("resumable temp dir after complete error = %v, want os.ErrNotExist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableCompleteRejectsMissingChunks(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
session, err := service.CreateResumableSession([]ResumableFileInput{{
|
||||
Name: "note.txt",
|
||||
Size: 8,
|
||||
ContentType: "text/plain",
|
||||
}}, UploadOptions{MaxDays: 1}, 4, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateResumableSession returned error: %v", err)
|
||||
}
|
||||
if _, err := service.PutResumableChunk(testContext(), session.ID, session.Files[0].ID, 0, strings.NewReader("hell")); err != nil {
|
||||
t.Fatalf("PutResumableChunk returned error: %v", err)
|
||||
}
|
||||
if _, _, err := service.CompleteResumableSession(testContext(), session.ID); err == nil {
|
||||
t.Fatalf("CompleteResumableSession accepted missing chunks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableSessionCanAddFilesBeforeComplete(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
session, err := service.CreateResumableSession([]ResumableFileInput{{
|
||||
Name: "one.txt",
|
||||
Size: 4,
|
||||
ContentType: "text/plain",
|
||||
Fingerprint: "one",
|
||||
}}, UploadOptions{MaxDays: 1}, 4, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateResumableSession returned error: %v", err)
|
||||
}
|
||||
if _, err := service.PutResumableChunk(testContext(), session.ID, session.Files[0].ID, 0, strings.NewReader("one!")); err != nil {
|
||||
t.Fatalf("PutResumableChunk one returned error: %v", err)
|
||||
}
|
||||
updated, err := service.AddResumableFiles(session.ID, []ResumableFileInput{{
|
||||
Name: "two.txt",
|
||||
Size: 4,
|
||||
ContentType: "text/plain",
|
||||
Fingerprint: "two",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("AddResumableFiles returned error: %v", err)
|
||||
}
|
||||
if len(updated.Files) != 2 {
|
||||
t.Fatalf("files after add = %d, want 2", len(updated.Files))
|
||||
}
|
||||
if updated.Files[0].UploadedChunks[0] != 0 {
|
||||
t.Fatalf("existing uploaded chunk was not preserved: %+v", updated.Files[0])
|
||||
}
|
||||
if _, err := service.AddResumableFiles(session.ID, []ResumableFileInput{{
|
||||
Name: "two.txt",
|
||||
Size: 4,
|
||||
ContentType: "text/plain",
|
||||
Fingerprint: "two",
|
||||
}}); err != nil {
|
||||
t.Fatalf("duplicate AddResumableFiles returned error: %v", err)
|
||||
}
|
||||
updated, err = service.GetResumableSession(session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetResumableSession returned error: %v", err)
|
||||
}
|
||||
if len(updated.Files) != 2 {
|
||||
t.Fatalf("duplicate add changed file count to %d", len(updated.Files))
|
||||
}
|
||||
if _, err := service.PutResumableChunk(testContext(), session.ID, updated.Files[1].ID, 0, strings.NewReader("two!")); err != nil {
|
||||
t.Fatalf("PutResumableChunk two returned error: %v", err)
|
||||
}
|
||||
result, _, err := service.CompleteResumableSession(testContext(), session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteResumableSession returned error: %v", err)
|
||||
}
|
||||
box := getTestBox(t, service, result.BoxID)
|
||||
if len(box.Files) != 2 {
|
||||
t.Fatalf("completed box file count = %d, want 2", len(box.Files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableCleanupRemovesExpiredSessionsAndChunks(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
session, err := service.CreateResumableSession([]ResumableFileInput{{
|
||||
Name: "note.txt",
|
||||
Size: 4,
|
||||
ContentType: "text/plain",
|
||||
}}, UploadOptions{MaxDays: 1}, 4, time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateResumableSession returned error: %v", err)
|
||||
}
|
||||
if _, err := service.PutResumableChunk(testContext(), session.ID, session.Files[0].ID, 0, strings.NewReader("hell")); err != nil {
|
||||
t.Fatalf("PutResumableChunk returned error: %v", err)
|
||||
}
|
||||
cleaned, err := service.CleanupExpiredResumableSessions(time.Now().UTC().Add(time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupExpiredResumableSessions returned error: %v", err)
|
||||
}
|
||||
if cleaned != 1 {
|
||||
t.Fatalf("cleaned = %d, want 1", cleaned)
|
||||
}
|
||||
if _, err := service.GetResumableSession(session.ID); !os.IsNotExist(err) {
|
||||
t.Fatalf("GetResumableSession after cleanup error = %v, want os.ErrNotExist", err)
|
||||
}
|
||||
if _, err := os.Stat(service.resumableSessionDir(session.ID)); !os.IsNotExist(err) {
|
||||
t.Fatalf("resumable temp dir after cleanup error = %v, want os.ErrNotExist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContaboStorageConfigAllowsDisplayNamesWithSpaces(t *testing.T) {
|
||||
service := newTestUploadService(t)
|
||||
cfg, err := service.Storage().CreateS3Backend(StorageBackendConfig{
|
||||
|
||||
Reference in New Issue
Block a user