feat(upload): add resumable chunk configuration and file validation
Some checks failed
Build and Publish Docker Image / deploy (push) Failing after 56s

- Add `WARPBOX_RESUMABLE_CHUNK_MODE` and `WARPBOX_RESUMABLE_CHUNK_PATH` environment variables to configure temporary chunk storage.
- Implement strict file validation for resuming uploads to ensure selected files match the pending session's metadata.
- Add `PLANS.md` to document development stages, roadmap, and API specifications (including batching and resumable flows).
This commit is contained in:
2026-06-02 22:13:54 +03:00
parent 5cd476e7f3
commit 313c89483c
22 changed files with 1809 additions and 324 deletions

View File

@@ -13,6 +13,7 @@
const copyURL = document.querySelector("#copy-url");
const openBox = document.querySelector("#open-box");
const manageLink = document.querySelector("#manage-link");
const newUpload = document.querySelector("#new-upload");
const RESUMABLE_SESSIONS_KEY = "warpbox-resumable-sessions";
if (!form || !dropZone || !fileInput) {
@@ -44,6 +45,8 @@
let latestBoxURL = "";
let selectedFiles = [];
let uploadLocked = false;
let recoveredDraft = null;
let resumeMode = false;
["dragenter", "dragover"].forEach((eventName) => {
dropZone.addEventListener(eventName, (event) => {
@@ -95,7 +98,11 @@
const submit = form.querySelector("button[type='submit']");
const formData = uploadFormData();
renderQueue(selectedFiles, "queued");
if (resumeMode && recoveredDraft) {
renderResumeQueue(recoveredDraft.session, selectedFiles);
} else {
renderQueue(selectedFiles, "queued");
}
setLoading(true, submit);
try {
@@ -103,8 +110,19 @@
renderResult(payload);
form.reset();
selectedFiles = [];
resumeMode = false;
recoveredDraft = null;
fileInput.value = "";
updateSelectedState();
if (uploadQueue) {
uploadQueue.hidden = true;
uploadQueue.replaceChildren();
}
if (newUpload) {
newUpload.hidden = true;
}
if (fileSummary) {
fileSummary.textContent = "Upload complete.";
}
} catch (error) {
updateStatus(error.message || "Upload failed");
} finally {
@@ -118,6 +136,16 @@
});
}
if (newUpload) {
newUpload.addEventListener("click", () => {
cancelRecoveredDraft().catch((error) => {
updateStatus(error.message || "Upload draft could not be deleted");
});
});
}
recoverResumableSessions();
function addSelectedFiles(files) {
if (uploadLocked) {
return;
@@ -145,14 +173,25 @@
title.textContent = count === 0 ? "Drop files to upload" : count === 1 ? "1 file selected" : `${count} files selected`;
}
if (fileSummary) {
fileSummary.textContent = count === 0 ? "Choose one or more files to begin." : `${count} file${count === 1 ? "" : "s"} ready.`;
if (resumeMode && recoveredDraft) {
fileSummary.textContent = count === 0
? "Reselect missing files to resume, or add extra files to this upload."
: `${count} local file${count === 1 ? "" : "s"} ready for the recovered upload.`;
} else {
fileSummary.textContent = count === 0 ? "Choose one or more files to begin." : `${count} file${count === 1 ? "" : "s"} ready.`;
}
}
if (count > 0) {
if (resumeMode && recoveredDraft) {
renderResumeQueue(recoveredDraft.session, selectedFiles);
} else if (count > 0) {
renderQueue(selectedFiles, "queued");
} else if (uploadQueue) {
uploadQueue.hidden = true;
uploadQueue.replaceChildren();
}
if (newUpload) {
newUpload.hidden = !(resumeMode && recoveredDraft);
}
}
function setLoading(isLoading, submit) {
@@ -164,6 +203,9 @@
submit.disabled = isLoading;
submit.textContent = isLoading ? "Uploading..." : "Upload files";
}
if (newUpload) {
newUpload.disabled = isLoading;
}
updateStatus(isLoading ? "Transferring files..." : "");
setTotalProgress(isLoading ? 0 : 100);
}
@@ -179,15 +221,15 @@
return;
}
latestBoxURL = payload.boxUrl;
latestBoxURL = window.Warpbox.absoluteURL(payload.boxUrl);
result.hidden = false;
openBox.href = payload.boxUrl;
openBox.href = latestBoxURL;
resultMeta.textContent = `${payload.files.length} file${payload.files.length === 1 ? "" : "s"} · expires ${window.Warpbox.formatDate(payload.expiresAt)}`;
if (manageLink) {
const anchor = manageLink.querySelector("a");
manageLink.hidden = !payload.manageUrl;
if (anchor && payload.manageUrl) {
anchor.href = payload.manageUrl;
anchor.href = window.Warpbox.absoluteURL(payload.manageUrl);
}
}
@@ -195,11 +237,12 @@
payload.files.forEach((file) => {
resultList.append(createFileRow({
name: file.name,
meta: `${file.size} · ${file.url}`,
meta: `${file.size} · ${window.Warpbox.absoluteURL(file.url)}`,
progress: 100,
status: "complete",
}));
});
result.scrollIntoView({ behavior: "smooth", block: "start" });
}
function uploadWithProgress(url, formData, files) {
@@ -263,9 +306,22 @@
collectionId: formData.get("collection_id") || "",
};
const persistable = !createPayload.password;
let session = persistable ? await findResumableSession(createPayload) : null;
let session = null;
if (persistable && resumeMode && recoveredDraft) {
session = await fetchResumableStatus(recoveredDraft.session.sessionId, recoveredDraft.session.resumeToken);
session.resumeToken = recoveredDraft.session.resumeToken;
} else if (persistable) {
session = await findResumableSession(createPayload);
}
if (session) {
validateResumeSelection(session, createPayload);
session = await addMissingResumableFiles(session, createPayload);
if (resumeMode && recoveredDraft && recoveredDraft.session.sessionId === session.sessionId) {
recoveredDraft.session = session;
}
if (persistable) {
saveResumableSession(session, createPayload);
}
}
if (!session || session.status !== "uploading") {
try {
@@ -304,7 +360,7 @@
}
const start = chunkIndex * session.chunkSize;
const end = Math.min(file.size, start + session.chunkSize);
await uploadChunk(session.sessionId, sessionFile.id, chunkIndex, file.slice(start, end), (loaded) => {
await uploadChunkWithRetry(session, sessionFile, chunkIndex, file.slice(start, end), (loaded) => {
const currentTotal = completedByFile.reduce((sum, bytes) => sum + bytes, 0) + loaded;
setTotalProgress(percentForBytes(currentTotal, totalBytes));
setSingleFileProgress(fileIndex, file, percentForBytes(completedByFile[fileIndex] + loaded, file.size));
@@ -320,12 +376,20 @@
setSingleFileProgress(fileIndex, file, 100);
}
const resultPayload = await completeResumableSession(session.sessionId);
updateStatus("Finalizing upload...");
const resultPayload = await completeResumableSession(session.sessionId, session.resumeToken);
const wasResumeMode = resumeMode;
if (persistable) {
removeResumableSession(session.sessionId);
}
if (resumeMode && recoveredDraft && recoveredDraft.session.sessionId === session.sessionId) {
resumeMode = false;
recoveredDraft = null;
}
setTotalProgress(100);
setFileProgress(files, 100);
if (!wasResumeMode) {
setFileProgress(files, 100);
}
return resultPayload;
}
@@ -341,17 +405,18 @@
return readUploadJSON(response, "Upload session could not be created");
}
async function fetchResumableStatus(sessionID) {
async function fetchResumableStatus(sessionID, resumeToken) {
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}`, {
headers: { "Accept": "application/json" },
headers: resumableHeaders(resumeToken),
});
return readUploadJSON(response, "Upload session could not be resumed");
}
async function addResumableFiles(sessionID, files) {
async function addResumableFiles(sessionID, resumeToken, files) {
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}/files`, {
method: "POST",
headers: {
...resumableHeaders(resumeToken),
"Accept": "application/json",
"Content-Type": "application/json",
},
@@ -360,11 +425,12 @@
return readUploadJSON(response, "Upload session files could not be added");
}
function uploadChunk(sessionID, fileID, chunkIndex, chunk, onProgress) {
function uploadChunk(sessionID, resumeToken, fileID, chunkIndex, chunk, onProgress) {
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest();
request.open("PUT", `/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}/files/${encodeURIComponent(fileID)}/chunks/${chunkIndex}`);
request.setRequestHeader("Accept", "application/json");
request.setRequestHeader("X-Warpbox-Resume-Token", resumeToken || "");
request.upload.addEventListener("progress", (event) => {
if (event.lengthComputable && onProgress) {
onProgress(event.loaded);
@@ -389,14 +455,54 @@
});
}
async function completeResumableSession(sessionID) {
async function uploadChunkWithRetry(session, sessionFile, chunkIndex, chunk, onProgress) {
const delays = [1000, 2000, 5000, 10000, 20000];
let lastError = null;
for (let attempt = 0; attempt <= delays.length; attempt++) {
try {
return await uploadChunk(session.sessionId, session.resumeToken, sessionFile.id, chunkIndex, chunk, onProgress);
} catch (error) {
lastError = error;
if (attempt >= delays.length) {
break;
}
const seconds = Math.round(delays[attempt] / 1000);
updateStatus(`Connection interrupted, retrying chunk ${chunkIndex + 1} in ${seconds}s`);
await wait(delays[attempt]);
}
}
throw lastError || new Error("Chunk upload failed");
}
async function completeResumableSession(sessionID, resumeToken) {
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}/complete`, {
method: "POST",
headers: { "Accept": "application/json" },
headers: resumableHeaders(resumeToken),
});
return readUploadJSON(response, "Upload could not be completed");
}
async function cancelResumableSession(sessionID, resumeToken) {
const response = await fetch(`/api/v1/uploads/resumable/${encodeURIComponent(sessionID)}`, {
method: "DELETE",
headers: resumableHeaders(resumeToken),
});
if (!response.ok && response.status !== 404) {
await readUploadJSON(response, "Upload draft could not be deleted");
}
}
function resumableHeaders(resumeToken) {
return {
"Accept": "application/json",
"X-Warpbox-Resume-Token": resumeToken || "",
};
}
function wait(ms) {
return new Promise((resolve) => window.setTimeout(resolve, ms));
}
async function readUploadJSON(response, fallback) {
let payload = {};
try {
@@ -421,15 +527,15 @@
if (!record.files || !record.files.some((file) => selectedKeys.has(resumableFileKey(file)))) {
continue;
}
const session = await fetchResumableStatus(record.sessionId).catch(() => null);
const session = await fetchResumableStatus(record.sessionId, record.resumeToken).catch(() => null);
if (!session || session.status !== "uploading") {
removeResumableSession(record.sessionId);
continue;
}
session.resumeToken = record.resumeToken;
const sessionKeys = new Set(session.files.map((file) => resumableFileKey(file)));
const sessionHasOnlySelectedFiles = session.files.every((file) => selectedKeys.has(resumableFileKey(file)));
const selectedContainsSessionFile = Array.from(sessionKeys).some((key) => selectedKeys.has(key));
if (sessionHasOnlySelectedFiles && selectedContainsSessionFile) {
if (selectedContainsSessionFile) {
return session;
}
}
@@ -442,7 +548,25 @@
if (missing.length === 0) {
return session;
}
return addResumableFiles(session.sessionId, missing);
const updated = await addResumableFiles(session.sessionId, session.resumeToken, missing);
updated.resumeToken = session.resumeToken;
return updated;
}
function validateResumeSelection(session, payload) {
if (!resumeMode || !recoveredDraft || session.sessionId !== recoveredDraft.session.sessionId) {
return;
}
const existingByNameSize = new Map();
(session.files || []).forEach((file) => {
existingByNameSize.set(`${file.name}:${file.size}`, resumableFileKey(file));
});
for (const file of payload.files || []) {
const expectedKey = existingByNameSize.get(`${file.name}:${file.size}`);
if (expectedKey && expectedKey !== resumableFileKey(file)) {
throw new Error(`"${file.name}" does not match the pending upload. Select the exact original file.`);
}
}
}
function matchSessionFile(session, file) {
@@ -478,11 +602,21 @@
const records = loadResumableSessions().filter((record) => record.sessionId !== session.sessionId);
records.push({
sessionId: session.sessionId,
resumeToken: session.resumeToken || "",
optionKey: resumableOptionKey(payload),
options: {
expiresMinutes: payload.expiresMinutes,
maxDownloads: payload.maxDownloads,
obfuscateMetadata: !!payload.obfuscateMetadata,
collectionId: payload.collectionId || "",
},
files: session.files.map((file) => ({
name: file.name,
size: file.size,
contentType: file.contentType || "application/octet-stream",
fingerprint: file.fingerprint || "",
uploadedChunks: file.uploadedChunks || [],
chunkCount: file.chunkCount || 0,
})),
updatedAt: new Date().toISOString(),
});
@@ -492,6 +626,38 @@
}
}
async function recoverResumableSessions() {
const records = loadResumableSessions()
.filter((record) => record.sessionId && record.resumeToken)
.sort((a, b) => new Date(b.updatedAt || 0).getTime() - new Date(a.updatedAt || 0).getTime());
if (records.length === 0) {
return;
}
for (const record of records) {
const session = await fetchResumableStatus(record.sessionId, record.resumeToken).catch(() => null);
if (!session || session.status !== "uploading") {
removeResumableSession(record.sessionId);
continue;
}
session.resumeToken = record.resumeToken;
recoveredDraft = { session, record };
selectedFiles = [];
renderRecoveredQueue([{ session, record }]);
updateRecoveredSummary(session);
showRecoveryModal(recoveredDraft);
return;
}
}
function updateRecoveredSummary(session) {
updateStatus("Unfinished upload found. Choose how to continue.");
if (fileSummary) {
const totalFiles = (session.files || []).length;
const completedFiles = completedSessionFiles(session).length;
fileSummary.textContent = `Recovered ${totalFiles} pending file${totalFiles === 1 ? "" : "s"}; ${completedFiles} fully uploaded.`;
}
}
function removeResumableSession(sessionID) {
try {
const records = loadResumableSessions().filter((record) => record.sessionId !== sessionID);
@@ -501,6 +667,105 @@
}
}
function completedSessionFiles(session) {
return (session.files || []).filter((file) => (file.uploadedChunks || []).length >= file.chunkCount);
}
function showRecoveryModal(draft) {
const old = document.querySelector(".upload-recovery-overlay");
if (old) {
old.remove();
}
const completeCount = completedSessionFiles(draft.session).length;
const totalCount = (draft.session.files || []).length;
const overlay = document.createElement("div");
overlay.className = "upload-recovery-overlay";
overlay.setAttribute("role", "dialog");
overlay.setAttribute("aria-modal", "true");
overlay.setAttribute("aria-labelledby", "upload-recovery-title");
const modal = document.createElement("div");
modal.className = "upload-recovery-modal card";
const content = document.createElement("div");
content.className = "card-content";
const title = document.createElement("h2");
title.id = "upload-recovery-title";
title.textContent = "Unfinished upload found";
const copy = document.createElement("p");
copy.textContent = `Warpbox found a private draft with ${totalCount} file${totalCount === 1 ? "" : "s"}. ${completeCount} file${completeCount === 1 ? " is" : "s are"} already fully uploaded.`;
const actions = document.createElement("div");
actions.className = "upload-recovery-actions";
const startOver = document.createElement("button");
startOver.type = "button";
startOver.className = "button button-danger";
startOver.textContent = "New Upload";
startOver.addEventListener("click", async () => {
startOver.disabled = true;
try {
await cancelRecoveredDraft();
overlay.remove();
} catch (error) {
startOver.disabled = false;
updateStatus(error.message || "Upload draft could not be deleted");
}
});
const resume = document.createElement("button");
resume.type = "button";
resume.className = "button button-primary";
resume.textContent = "Resume";
resume.addEventListener("click", () => {
resumeRecoveredDraft();
overlay.remove();
});
actions.append(startOver, resume);
content.append(title, copy, actions);
modal.append(content);
overlay.append(modal);
document.body.append(overlay);
}
async function cancelRecoveredDraft() {
if (!recoveredDraft) {
resetFreshUploadState();
return;
}
const draft = recoveredDraft;
updateStatus("Deleting unfinished upload...");
await cancelResumableSession(draft.session.sessionId, draft.session.resumeToken);
removeResumableSession(draft.session.sessionId);
resetFreshUploadState();
}
function resumeRecoveredDraft() {
if (!recoveredDraft) {
return;
}
resumeMode = true;
selectedFiles = [];
renderResumeQueue(recoveredDraft.session, selectedFiles);
updateSelectedState();
updateStatus("Drop or reselect missing files to continue. Extra files will be added to this upload.");
}
function resetFreshUploadState() {
selectedFiles = [];
resumeMode = false;
recoveredDraft = null;
fileInput.value = "";
result.hidden = true;
if (resultList) {
resultList.replaceChildren();
}
setTotalProgress(0);
updateStatus("");
updateSelectedState();
}
function uploadedBytesForSessionFile(file, chunkSize) {
return (file.uploadedChunks || []).reduce((sum, index) => {
const start = index * chunkSize;
@@ -509,6 +774,91 @@
}, 0);
}
function renderRecoveredQueue(items) {
if (!uploadQueue) {
return;
}
const rows = [];
items.forEach(({ session }) => {
(session.files || []).forEach((file) => {
const uploadedBytes = uploadedBytesForSessionFile(file, session.chunkSize);
const complete = (file.uploadedChunks || []).length >= file.chunkCount;
rows.push({
name: file.name,
size: file.size,
uploadedBytes,
meta: complete
? `${window.Warpbox.formatBytes(file.size)} · uploaded`
: `${window.Warpbox.formatBytes(uploadedBytes)} of ${window.Warpbox.formatBytes(file.size)} · Drop/reselect this file to continue`,
progress: percentForBytes(uploadedBytes, file.size),
status: complete ? "complete" : "waiting",
readonly: true,
});
});
});
uploadQueue.hidden = rows.length === 0;
uploadQueue.replaceChildren();
rows.forEach((row) => uploadQueue.append(createFileRow(row)));
const totalBytes = rows.reduce((sum, row) => sum + (row.size || 0), 0);
if (totalBytes > 0) {
setTotalProgress(percentForBytes(rows.reduce((sum, row) => sum + (row.uploadedBytes || 0), 0), totalBytes));
} else if (rows.length > 0) {
const completed = rows.filter((row) => row.status === "complete").length;
setTotalProgress(percentForBytes(completed, rows.length));
}
}
function renderResumeQueue(session, localFiles) {
if (!uploadQueue) {
return;
}
const rows = [];
const localByNameSize = new Map();
(localFiles || []).forEach((file, index) => {
localByNameSize.set(`${file.name}:${file.size}`, { file, index });
});
const usedLocalIndexes = new Set();
(session.files || []).forEach((file) => {
const uploadedBytes = uploadedBytesForSessionFile(file, session.chunkSize);
const complete = (file.uploadedChunks || []).length >= file.chunkCount;
const localMatch = localByNameSize.get(`${file.name}:${file.size}`) || null;
if (localMatch) {
usedLocalIndexes.add(localMatch.index);
}
rows.push({
name: file.name,
size: file.size,
uploadedBytes,
meta: complete
? `${window.Warpbox.formatBytes(file.size)} · uploaded`
: localMatch
? `${window.Warpbox.formatBytes(uploadedBytes)} of ${window.Warpbox.formatBytes(file.size)} · ready to resume`
: `${window.Warpbox.formatBytes(uploadedBytes)} of ${window.Warpbox.formatBytes(file.size)} · waiting for local file`,
progress: percentForBytes(uploadedBytes, file.size),
status: complete ? "complete" : localMatch ? "queued" : "waiting",
readonly: !localMatch,
index: localMatch ? localMatch.index : undefined,
removable: Boolean(localMatch && !complete),
});
});
(localFiles || []).forEach((file, index) => {
if (usedLocalIndexes.has(index)) {
return;
}
rows.push({
name: file.name,
meta: `${window.Warpbox.formatBytes(file.size)} · new file`,
progress: 0,
status: "queued",
index,
removable: true,
});
});
uploadQueue.hidden = rows.length === 0;
uploadQueue.replaceChildren();
rows.forEach((row) => uploadQueue.append(createFileRow(row)));
}
function percentForBytes(bytes, total) {
if (!total) {
return 100;
@@ -536,9 +886,11 @@
function createFileRow(file) {
const row = document.createElement("div");
row.className = "result-item upload-file-row";
row.className = `result-item upload-file-row upload-file-${file.status || "queued"}`;
row.dataset.fileName = file.name;
row.dataset.fileIndex = file.index || 0;
if (typeof file.index === "number") {
row.dataset.fileIndex = file.index;
}
const body = document.createElement("span");
const name = document.createElement("strong");
@@ -560,6 +912,12 @@
fill.style.transform = `scaleX(${file.progress / 100})`;
bar.append(fill);
side.append(percent, bar);
if (file.status === "waiting") {
const badge = document.createElement("small");
badge.className = "upload-file-state";
badge.textContent = "Needs local file";
side.append(badge);
}
if (file.removable) {
const remove = document.createElement("button");
remove.className = "upload-file-remove";