kato/warpbox-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(uploads): add native resumable upload support

Browse Source 
Implement a native chunked resumable upload API and frontend integration
to support reliable large file uploads.

Changes include:
- Added a 3-step resumable upload API flow (create session, upload chunks, complete session).
- Introduced configuration options for chunk size, retention hours, and toggling the feature.
- Updated the frontend to utilize resumable uploads with progress tracking.
- Configured temporary chunk storage under `data/tmp/uploads` with automatic cleanup.
- Documented the API flow and configuration in the README.
This commit is contained in:
Daniel Legt
2026-06-02 17:41:41 +03:00
parent d3b6a86753
commit 5cd476e7f3
16 changed files with 1805 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
backend/libs/services/upload.go
View File

@@ -42,6 +42,8 @@ type UploadOptions struct {
ExpiresInMinutes int
MaxDownloads int
Password string
PasswordSalt string
PasswordHash string
ObfuscateMetadata bool
OwnerID string
CollectionID string
@@ -50,6 +52,56 @@ type UploadOptions struct {
StorageBackendID string
}
type IncomingFile interface {
Name() string
Size() int64
ContentType() string
Open() (io.ReadCloser, error)
}
type multipartIncomingFile struct {
header *multipart.FileHeader
}
func (f multipartIncomingFile) Name() string {
return f.header.Filename
}
func (f multipartIncomingFile) Size() int64 {
return f.header.Size
}
func (f multipartIncomingFile) ContentType() string {
return f.header.Header.Get("Content-Type")
}
func (f multipartIncomingFile) Open() (io.ReadCloser, error) {
return f.header.Open()
}
type StagedUploadFile struct {
Filename string
FileSize int64
MIMEType string
Path string
}
func (f StagedUploadFile) Name() string {
return f.Filename
}
func (f StagedUploadFile) Size() int64 {
return f.FileSize
}
func (f StagedUploadFile) ContentType() string {
return f.MIMEType
}
func (f StagedUploadFile) Open() (io.ReadCloser, error) {
return os.Open(f.Path)
}
type Box struct {
ID string `json:"id"`
OwnerID string `json:"ownerId,omitempty"`
@@ -198,6 +250,10 @@ func (s *UploadService) ValidateSize(size int64) error {
}
func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOptions) (UploadResult, error) {
return s.CreateBoxFromIncoming(multipartIncomingFiles(files), opts)
}
func (s *UploadService) CreateBoxFromIncoming(files []IncomingFile, opts UploadOptions) (UploadResult, error) {
if len(files) == 0 {
return UploadResult{}, fmt.Errorf("no files were uploaded")
}
@@ -232,13 +288,16 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
}
deleteToken := randomID(32)
box.DeleteTokenHash = deleteTokenHash(box.ID, deleteToken)
if strings.TrimSpace(opts.Password) != "" {
if strings.TrimSpace(opts.PasswordHash) != "" {
box.PasswordSalt = opts.PasswordSalt
box.PasswordHash = opts.PasswordHash
} else if strings.TrimSpace(opts.Password) != "" {
salt, hash := hashPassword(opts.Password)
box.PasswordSalt = salt
box.PasswordHash = hash
}
if err := s.writeFilesToBox(&box, files, opts); err != nil {
if err := s.writeIncomingFilesToBox(&box, files, opts); err != nil {
return UploadResult{}, err
}
@@ -261,6 +320,10 @@ func (s *UploadService) CreateBox(files []*multipart.FileHeader, opts UploadOpti
// selection into a single box). The box keeps its original expiry, password and
// other settings; only the new files are written.
func (s *UploadService) AppendFiles(boxID string, files []*multipart.FileHeader, opts UploadOptions) (UploadResult, error) {
return s.AppendIncomingFiles(boxID, multipartIncomingFiles(files), opts)
}
func (s *UploadService) AppendIncomingFiles(boxID string, files []IncomingFile, opts UploadOptions) (UploadResult, error) {
if len(files) == 0 {
return UploadResult{}, fmt.Errorf("no files were uploaded")
}
@@ -268,7 +331,7 @@ func (s *UploadService) AppendFiles(boxID string, files []*multipart.FileHeader,
if err != nil {
return UploadResult{}, err
}
if err := s.writeFilesToBox(&box, files, opts); err != nil {
if err := s.writeIncomingFilesToBox(&box, files, opts); err != nil {
return UploadResult{}, err
}
if err := s.SaveBox(box); err != nil {
@@ -289,14 +352,26 @@ func (s *UploadService) AppendFiles(boxID string, files []*multipart.FileHeader,
// appends the file metadata to box.Files. The box's StorageBackendID determines
// where files land, so it works for both new and existing boxes.
func (s *UploadService) writeFilesToBox(box *Box, files []*multipart.FileHeader, opts UploadOptions) error {
return s.writeIncomingFilesToBox(box, multipartIncomingFiles(files), opts)
}
func multipartIncomingFiles(files []*multipart.FileHeader) []IncomingFile {
incoming := make([]IncomingFile, 0, len(files))
for _, file := range files {
incoming = append(incoming, multipartIncomingFile{header: file})
}
return incoming
}
func (s *UploadService) writeIncomingFilesToBox(box *Box, files []IncomingFile, opts UploadOptions) error {
backend, err := s.storage.Backend(box.StorageBackendID)
if err != nil {
return err
}
for _, header := range files {
for _, incoming := range files {
if !opts.SkipSizeLimit {
if err := s.ValidateSize(header.Size); err != nil {
if err := s.ValidateSize(incoming.Size()); err != nil {
return err
}
}
@@ -306,15 +381,15 @@ func (s *UploadService) writeFilesToBox(box *Box, files []*multipart.FileHeader,
maxSize = 0
}
file, err := header.Open()
file, err := incoming.Open()
if err != nil {
return err
}
fileID := randomID(8)
storedName := "@each@" + fileID + strings.ToLower(filepath.Ext(header.Filename))
storedName := "@each@" + fileID + strings.ToLower(filepath.Ext(incoming.Name()))
objectKey := boxObjectKey(box.ID, storedName)
contentType := header.Header.Get("Content-Type")
contentType := incoming.ContentType()
if contentType == "" {
buffer := make([]byte, 512)
n, _ := file.Read(buffer)
@@ -324,7 +399,7 @@ func (s *UploadService) writeFilesToBox(box *Box, files []*multipart.FileHeader,
}
}
if err := s.writeUploadedObject(context.Background(), backend, objectKey, file, header.Size, maxSize, contentType); err != nil {
if err := s.writeUploadedObject(context.Background(), backend, objectKey, file, incoming.Size(), maxSize, contentType); err != nil {
file.Close()
return err
}
@@ -332,9 +407,9 @@ func (s *UploadService) writeFilesToBox(box *Box, files []*multipart.FileHeader,
box.Files = append(box.Files, File{
ID: fileID,
Name: filepath.Base(header.Filename),
Name: filepath.Base(incoming.Name()),
StoredName: storedName,
Size: header.Size,
Size: incoming.Size(),
ContentType: contentType,
PreviewKind: previewKind(contentType),
ObjectKey: objectKey,
@@ -931,21 +1006,17 @@ func writeUploadedFile(path string, source multipart.File, maxSize int64) error
return nil
}
func (s *UploadService) writeUploadedObject(ctx context.Context, backend StorageBackend, key string, source multipart.File, size, maxSize int64, contentType string) error {
func (s *UploadService) writeUploadedObject(ctx context.Context, backend StorageBackend, key string, source io.Reader, size, maxSize int64, contentType string) error {
var reader io.Reader = source
putSize := size
if maxSize > 0 {
reader = io.LimitReader(source, maxSize+1)
var buffer bytes.Buffer
written, err := io.Copy(&buffer, reader)
if err != nil {
return err
}
if written > maxSize {
if size > maxSize {
return fmt.Errorf("file exceeds max upload size")
}
return backend.Put(ctx, key, bytes.NewReader(buffer.Bytes()), written, contentType)
reader = io.LimitReader(source, maxSize)
putSize = size
}
return backend.Put(ctx, key, reader, size, contentType)
return backend.Put(ctx, key, reader, putSize, contentType)
}
func boxObjectKey(boxID, name string) string {