feat(uploads): add native resumable upload support
Implement a native chunked resumable upload API and frontend integration to support reliable large file uploads. Changes include: - Added a 3-step resumable upload API flow (create session, upload chunks, complete session). - Introduced configuration options for chunk size, retention hours, and toggling the feature. - Updated the frontend to utilize resumable uploads with progress tracking. - Configured temporary chunk storage under `data/tmp/uploads` with automatic cleanup. - Documented the API flow and configuration in the README.
This commit is contained in:
@@ -13,6 +13,7 @@
|
||||
const copyURL = document.querySelector("#copy-url");
|
||||
const openBox = document.querySelector("#open-box");
|
||||
const manageLink = document.querySelector("#manage-link");
|
||||
const RESUMABLE_SESSIONS_KEY = "warpbox-resumable-sessions";
|
||||
|
||||
if (!form || !dropZone || !fileInput) {
|
||||
return;
|
||||
@@ -42,6 +43,7 @@
|
||||
|
||||
let latestBoxURL = "";
|
||||
let selectedFiles = [];
|
||||
let uploadLocked = false;
|
||||
|
||||
["dragenter", "dragover"].forEach((eventName) => {
|
||||
dropZone.addEventListener(eventName, (event) => {
|
||||
@@ -57,33 +59,52 @@
|
||||
});
|
||||
});
|
||||
|
||||
dropZone.addEventListener("drop", (event) => {
|
||||
if (event.dataTransfer && event.dataTransfer.files.length > 0) {
|
||||
fileInput.files = event.dataTransfer.files;
|
||||
updateSelectedState(event.dataTransfer.files);
|
||||
document.addEventListener("dragover", (event) => {
|
||||
if (event.dataTransfer && Array.from(event.dataTransfer.types || []).includes("Files")) {
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
|
||||
fileInput.addEventListener("change", () => updateSelectedState(fileInput.files));
|
||||
document.addEventListener("drop", (event) => {
|
||||
if (!event.dataTransfer || !event.dataTransfer.files.length) {
|
||||
return;
|
||||
}
|
||||
event.preventDefault();
|
||||
if (!dropZone.contains(event.target)) {
|
||||
addSelectedFiles(event.dataTransfer.files);
|
||||
}
|
||||
});
|
||||
|
||||
dropZone.addEventListener("drop", (event) => {
|
||||
if (event.dataTransfer && event.dataTransfer.files.length > 0) {
|
||||
addSelectedFiles(event.dataTransfer.files);
|
||||
}
|
||||
});
|
||||
|
||||
fileInput.addEventListener("change", () => {
|
||||
addSelectedFiles(fileInput.files);
|
||||
fileInput.value = "";
|
||||
});
|
||||
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (!fileInput.files || fileInput.files.length === 0) {
|
||||
if (selectedFiles.length === 0) {
|
||||
updateStatus("Choose at least one file first.");
|
||||
return;
|
||||
}
|
||||
|
||||
const submit = form.querySelector("button[type='submit']");
|
||||
const formData = new FormData(form);
|
||||
selectedFiles = Array.from(fileInput.files);
|
||||
const formData = uploadFormData();
|
||||
renderQueue(selectedFiles, "queued");
|
||||
setLoading(true, submit);
|
||||
|
||||
try {
|
||||
const payload = await uploadWithProgress(form.action, formData, selectedFiles);
|
||||
const payload = await uploadResumable(form.action, formData, selectedFiles);
|
||||
renderResult(payload);
|
||||
form.reset();
|
||||
updateSelectedState([]);
|
||||
selectedFiles = [];
|
||||
fileInput.value = "";
|
||||
updateSelectedState();
|
||||
} catch (error) {
|
||||
updateStatus(error.message || "Upload failed");
|
||||
} finally {
|
||||
@@ -97,8 +118,27 @@
|
||||
});
|
||||
}
|
||||
|
||||
function updateSelectedState(files) {
|
||||
selectedFiles = Array.from(files || []);
|
||||
function addSelectedFiles(files) {
|
||||
if (uploadLocked) {
|
||||
return;
|
||||
}
|
||||
Array.from(files || []).forEach((file) => {
|
||||
if (!selectedFiles.some((existing) => fileIdentity(existing) === fileIdentity(file))) {
|
||||
selectedFiles.push(file);
|
||||
}
|
||||
});
|
||||
updateSelectedState();
|
||||
}
|
||||
|
||||
function removeSelectedFile(index) {
|
||||
if (uploadLocked) {
|
||||
return;
|
||||
}
|
||||
selectedFiles.splice(index, 1);
|
||||
updateSelectedState();
|
||||
}
|
||||
|
||||
function updateSelectedState() {
|
||||
const count = selectedFiles.length || 0;
|
||||
const title = dropZone.querySelector(".drop-title");
|
||||
if (title) {
|
||||
@@ -116,6 +156,7 @@
|
||||
}
|
||||
|
||||
function setLoading(isLoading, submit) {
|
||||
uploadLocked = isLoading;
|
||||
if (progress) {
|
||||
progress.hidden = !isLoading;
|
||||
}
|
||||
@@ -201,18 +242,294 @@
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadResumable(fallbackUrl, formData, files) {
|
||||
if (!window.fetch || typeof Blob === "undefined") {
|
||||
return uploadWithProgress(fallbackUrl, formData, files);
|
||||
}
|
||||
|
||||
updateStatus("Fingerprinting files...");
|
||||
const fingerprints = await Promise.all(files.map((file) => fileFingerprint(file)));
|
||||
const createPayload = {
|
||||
files: files.map((file, index) => ({
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
contentType: file.type || "application/octet-stream",
|
||||
fingerprint: fingerprints[index],
|
||||
})),
|
||||
expiresMinutes: parseInt(formData.get("expires_minutes") || "0", 10) || 0,
|
||||
maxDownloads: parseInt(formData.get("max_downloads") || "0", 10) || 0,
|
||||
password: formData.get("password") || "",
|
||||
obfuscateMetadata: formData.get("obfuscate_metadata") === "on",
|
||||
collectionId: formData.get("collection_id") || "",
|
||||
};
|
||||
const persistable = !createPayload.password;
|
||||
let session = persistable ? await findResumableSession(createPayload) : null;
|
||||
if (session) {
|
||||
session = await addMissingResumableFiles(session, createPayload);
|
||||
}
|
||||
if (!session || session.status !== "uploading") {
|
||||
try {
|
||||
session = await createResumableSession(createPayload);
|
||||
} catch (error) {
|
||||
if ((error.message || "").toLowerCase().includes("resumable uploads are disabled")) {
|
||||
return uploadWithProgress(fallbackUrl, formData, files);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (persistable) {
|
||||
saveResumableSession(session, createPayload);
|
||||
}
|
||||
}
|
||||
const sessionFiles = files.map((file, index) => matchSessionFile(session, createPayload.files[index]));
|
||||
if (sessionFiles.some((file) => !file)) {
|
||||
throw new Error("Upload session could not match the selected files");
|
||||
}
|
||||
|
||||
updateStatus("Uploading...");
|
||||
const totalBytes = files.reduce((sum, file) => sum + file.size, 0);
|
||||
const completedByFile = new Array(files.length).fill(0);
|
||||
sessionFiles.forEach((sessionFile, index) => {
|
||||
completedByFile[index] = uploadedBytesForSessionFile(sessionFile, session.chunkSize);
|
||||
setSingleFileProgress(index, files[index], percentForBytes(completedByFile[index], files[index].size));
|
||||
});
|
||||
setTotalProgress(percentForBytes(completedByFile.reduce((sum, bytes) => sum + bytes, 0), totalBytes));
|
||||
|
||||
for (let fileIndex = 0; fileIndex < files.length; fileIndex++) {
|
||||
const file = files[fileIndex];
|
||||
const sessionFile = sessionFiles[fileIndex];
|
||||
const uploaded = new Set(sessionFile.uploadedChunks || []);
|
||||
for (let chunkIndex = 0; chunkIndex < sessionFile.chunkCount; chunkIndex++) {
|
||||
if (uploaded.has(chunkIndex)) {
|
||||
continue;
|
||||
}
|
||||
const start = chunkIndex * session.chunkSize;
|
||||
const end = Math.min(file.size, start + session.chunkSize);
|
||||
await uploadChunk(session.sessionId, sessionFile.id, chunkIndex, file.slice(start, end), (loaded) => {
|
||||
const currentTotal = completedByFile.reduce((sum, bytes) => sum + bytes, 0) + loaded;
|
||||
setTotalProgress(percentForBytes(currentTotal, totalBytes));
|
||||
setSingleFileProgress(fileIndex, file, percentForBytes(completedByFile[fileIndex] + loaded, file.size));
|
||||
updateStatus(`${percentForBytes(currentTotal, totalBytes)}%`);
|
||||
});
|
||||
completedByFile[fileIndex] += end - start;
|
||||
uploaded.add(chunkIndex);
|
||||
sessionFile.uploadedChunks = Array.from(uploaded).sort((a, b) => a - b);
|
||||
if (persistable) {
|
||||
saveResumableSession(session, createPayload);
|
||||
}
|
||||
}
|
||||
setSingleFileProgress(fileIndex, file, 100);
|
||||
}
|
||||
|
||||
const resultPayload = await completeResumableSession(session.sessionId);
|
||||
if (persistable) {
|
||||
removeResumableSession(session.sessionId);
|
||||
}
|
||||
setTotalProgress(100);
|
||||
setFileProgress(files, 100);
|
||||
return resultPayload;
|
||||
}
|
||||
|
||||
async function createResumableSession(payload) {
|
||||
const response = await fetch("/api/v1/uploads/resumable", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
return readUploadJSON(response, "Upload session could not be created");
|
||||
}
|
||||
|
||||
async function fetchResumableStatus(sessionID) {
|
||||
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}`, {
|
||||
headers: { "Accept": "application/json" },
|
||||
});
|
||||
return readUploadJSON(response, "Upload session could not be resumed");
|
||||
}
|
||||
|
||||
async function addResumableFiles(sessionID, files) {
|
||||
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}/files`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ files }),
|
||||
});
|
||||
return readUploadJSON(response, "Upload session files could not be added");
|
||||
}
|
||||
|
||||
function uploadChunk(sessionID, fileID, chunkIndex, chunk, onProgress) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = new XMLHttpRequest();
|
||||
request.open("PUT", `/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}/files/${encodeURIComponent(fileID)}/chunks/${chunkIndex}`);
|
||||
request.setRequestHeader("Accept", "application/json");
|
||||
request.upload.addEventListener("progress", (event) => {
|
||||
if (event.lengthComputable && onProgress) {
|
||||
onProgress(event.loaded);
|
||||
}
|
||||
});
|
||||
request.addEventListener("load", () => {
|
||||
if (request.status < 200 || request.status >= 300) {
|
||||
let payload = {};
|
||||
try {
|
||||
payload = JSON.parse(request.responseText || "{}");
|
||||
} catch (error) {
|
||||
payload = {};
|
||||
}
|
||||
reject(new Error(payload.error || "Chunk upload failed"));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
request.addEventListener("error", () => reject(new Error("Network error during chunk upload")));
|
||||
request.addEventListener("abort", () => reject(new Error("Chunk upload aborted")));
|
||||
request.send(chunk);
|
||||
});
|
||||
}
|
||||
|
||||
async function completeResumableSession(sessionID) {
|
||||
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}/complete`, {
|
||||
method: "POST",
|
||||
headers: { "Accept": "application/json" },
|
||||
});
|
||||
return readUploadJSON(response, "Upload could not be completed");
|
||||
}
|
||||
|
||||
async function readUploadJSON(response, fallback) {
|
||||
let payload = {};
|
||||
try {
|
||||
payload = await response.json();
|
||||
} catch (error) {
|
||||
payload = {};
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(payload.error || fallback);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function findResumableSession(payload) {
|
||||
const records = loadResumableSessions();
|
||||
const optionKey = resumableOptionKey(payload);
|
||||
const selectedKeys = new Set(payload.files.map((file) => resumableFileKey(file)));
|
||||
for (const record of records) {
|
||||
if (record.optionKey !== optionKey) {
|
||||
continue;
|
||||
}
|
||||
if (!record.files || !record.files.some((file) => selectedKeys.has(resumableFileKey(file)))) {
|
||||
continue;
|
||||
}
|
||||
const session = await fetchResumableStatus(record.sessionId).catch(() => null);
|
||||
if (!session || session.status !== "uploading") {
|
||||
removeResumableSession(record.sessionId);
|
||||
continue;
|
||||
}
|
||||
const sessionKeys = new Set(session.files.map((file) => resumableFileKey(file)));
|
||||
const sessionHasOnlySelectedFiles = session.files.every((file) => selectedKeys.has(resumableFileKey(file)));
|
||||
const selectedContainsSessionFile = Array.from(sessionKeys).some((key) => selectedKeys.has(key));
|
||||
if (sessionHasOnlySelectedFiles && selectedContainsSessionFile) {
|
||||
return session;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function addMissingResumableFiles(session, payload) {
|
||||
const existing = new Set(session.files.map((file) => resumableFileKey(file)));
|
||||
const missing = payload.files.filter((file) => !existing.has(resumableFileKey(file)));
|
||||
if (missing.length === 0) {
|
||||
return session;
|
||||
}
|
||||
return addResumableFiles(session.sessionId, missing);
|
||||
}
|
||||
|
||||
function matchSessionFile(session, file) {
|
||||
const key = resumableFileKey(file);
|
||||
return session.files.find((sessionFile) => resumableFileKey(sessionFile) === key) || null;
|
||||
}
|
||||
|
||||
function resumableOptionKey(payload) {
|
||||
return [
|
||||
payload.expiresMinutes,
|
||||
payload.maxDownloads,
|
||||
payload.obfuscateMetadata ? "1" : "0",
|
||||
payload.collectionId || "",
|
||||
].join(":");
|
||||
}
|
||||
|
||||
function resumableFileKey(file) {
|
||||
return [file.name, file.size, file.fingerprint || ""].join(":");
|
||||
}
|
||||
|
||||
function loadResumableSessions() {
|
||||
try {
|
||||
const value = localStorage.getItem(RESUMABLE_SESSIONS_KEY);
|
||||
const records = value ? JSON.parse(value) : [];
|
||||
return Array.isArray(records) ? records : [];
|
||||
} catch (error) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function saveResumableSession(session, payload) {
|
||||
try {
|
||||
const records = loadResumableSessions().filter((record) => record.sessionId !== session.sessionId);
|
||||
records.push({
|
||||
sessionId: session.sessionId,
|
||||
optionKey: resumableOptionKey(payload),
|
||||
files: session.files.map((file) => ({
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
fingerprint: file.fingerprint || "",
|
||||
})),
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
localStorage.setItem(RESUMABLE_SESSIONS_KEY, JSON.stringify(records.slice(-25)));
|
||||
} catch (error) {
|
||||
/* ignore persistence failures */
|
||||
}
|
||||
}
|
||||
|
||||
function removeResumableSession(sessionID) {
|
||||
try {
|
||||
const records = loadResumableSessions().filter((record) => record.sessionId !== sessionID);
|
||||
localStorage.setItem(RESUMABLE_SESSIONS_KEY, JSON.stringify(records));
|
||||
} catch (error) {
|
||||
/* ignore persistence failures */
|
||||
}
|
||||
}
|
||||
|
||||
function uploadedBytesForSessionFile(file, chunkSize) {
|
||||
return (file.uploadedChunks || []).reduce((sum, index) => {
|
||||
const start = index * chunkSize;
|
||||
const end = Math.min(file.size, start + chunkSize);
|
||||
return sum + Math.max(0, end - start);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
function percentForBytes(bytes, total) {
|
||||
if (!total) {
|
||||
return 100;
|
||||
}
|
||||
return Math.max(0, Math.min(100, Math.round((bytes / total) * 100)));
|
||||
}
|
||||
|
||||
function renderQueue(files, status) {
|
||||
if (!uploadQueue) {
|
||||
return;
|
||||
}
|
||||
uploadQueue.hidden = files.length === 0;
|
||||
uploadQueue.replaceChildren();
|
||||
files.forEach((file) => {
|
||||
files.forEach((file, index) => {
|
||||
uploadQueue.append(createFileRow({
|
||||
name: file.name,
|
||||
meta: window.Warpbox.formatBytes(file.size),
|
||||
progress: status === "queued" ? 0 : 100,
|
||||
status,
|
||||
index,
|
||||
removable: status === "queued",
|
||||
}));
|
||||
});
|
||||
}
|
||||
@@ -221,6 +538,7 @@
|
||||
const row = document.createElement("div");
|
||||
row.className = "result-item upload-file-row";
|
||||
row.dataset.fileName = file.name;
|
||||
row.dataset.fileIndex = file.index || 0;
|
||||
|
||||
const body = document.createElement("span");
|
||||
const name = document.createElement("strong");
|
||||
@@ -242,11 +560,47 @@
|
||||
fill.style.transform = `scaleX(${file.progress / 100})`;
|
||||
bar.append(fill);
|
||||
side.append(percent, bar);
|
||||
if (file.removable) {
|
||||
const remove = document.createElement("button");
|
||||
remove.className = "upload-file-remove";
|
||||
remove.type = "button";
|
||||
remove.setAttribute("aria-label", `Remove ${file.name}`);
|
||||
remove.textContent = "×";
|
||||
remove.addEventListener("click", () => removeSelectedFile(file.index || 0));
|
||||
side.append(remove);
|
||||
}
|
||||
|
||||
row.append(body, side);
|
||||
return row;
|
||||
}
|
||||
|
||||
function uploadFormData() {
|
||||
const formData = new FormData(form);
|
||||
formData.delete("file");
|
||||
selectedFiles.forEach((file) => {
|
||||
formData.append("file", file, file.name);
|
||||
});
|
||||
return formData;
|
||||
}
|
||||
|
||||
function fileIdentity(file) {
|
||||
return [file.name, file.size, file.lastModified || 0].join(":");
|
||||
}
|
||||
|
||||
async function fileFingerprint(file) {
|
||||
if (!window.crypto || !window.crypto.subtle || !file.slice || typeof TextEncoder === "undefined") {
|
||||
return fileIdentity(file);
|
||||
}
|
||||
const sampleSize = Math.min(file.size, 1024 * 1024);
|
||||
const sample = await file.slice(0, sampleSize).arrayBuffer();
|
||||
const metadata = new TextEncoder().encode([file.name, file.size, file.lastModified || 0, sampleSize].join(":"));
|
||||
const combined = new Uint8Array(metadata.byteLength + sample.byteLength);
|
||||
combined.set(metadata, 0);
|
||||
combined.set(new Uint8Array(sample), metadata.byteLength);
|
||||
const digest = await window.crypto.subtle.digest("SHA-256", combined);
|
||||
return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
function setTotalProgress(percent) {
|
||||
if (totalProgressBar) {
|
||||
totalProgressBar.style.transform = `scaleX(${Math.max(0, Math.min(100, percent)) / 100})`;
|
||||
@@ -271,4 +625,23 @@
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function setSingleFileProgress(index, file, progress) {
|
||||
if (!uploadQueue) {
|
||||
return;
|
||||
}
|
||||
const row = uploadQueue.querySelector(`.upload-file-row[data-file-index="${index}"]`);
|
||||
if (!row) {
|
||||
return;
|
||||
}
|
||||
const percent = row.querySelector(".file-progress-percent");
|
||||
const fill = row.querySelector(".file-progress span");
|
||||
const normalized = Math.max(0, Math.min(100, progress));
|
||||
if (percent) {
|
||||
percent.textContent = `${normalized}%`;
|
||||
}
|
||||
if (fill) {
|
||||
fill.style.transform = `scaleX(${normalized / 100})`;
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
Reference in New Issue
Block a user