diff --git a/package-lock.json b/package-lock.json index 4ac0a6d..fd0d51d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "waterdrop", - "version": "0.1.9", + "version": "0.1.18", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "waterdrop", - "version": "0.1.9", + "version": "0.1.18", "license": "SEE LICENSE IN LICENSE", "dependencies": { "busboy": "^1.6.0", diff --git a/package.json b/package.json index bfddd87..2f35b33 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "waterdrop", - "version": "0.1.9", + "version": "0.1.18", "description": "Private AirDrop-style file shelf for Tailscale networks.", "main": "src/main/main.js", "scripts": { diff --git a/src/main/main.js b/src/main/main.js index 5940d66..5e7520b 100644 --- a/src/main/main.js +++ b/src/main/main.js @@ -127,6 +127,42 @@ async function autoPublishServe() { } } +let httpsRenewTimer = null; + +// Stand up the direct HTTPS listener with a Tailscale-issued cert so phones can +// open several parallel connections straight to this process. Best-effort: if +// the tailnet has no HTTPS certs, we quietly keep using Serve. `tailscale cert` +// reuses a still-valid cert, so re-running it is cheap and doubles as renewal. +async function autoEnableDirectHttps() { + try { + if (!dropServer?.startHttps) return; + const state = await tailscale.status(); + if (!state.running || !state.loggedIn || !state.dnsName) return; + const certDir = path.join(app.getPath("userData"), "certs"); + await fs.promises.mkdir(certDir, { recursive: true }); + const certPath = path.join(certDir, `${state.dnsName}.crt`); + const keyPath = path.join(certDir, `${state.dnsName}.key`); + const result = await tailscale.provisionCert(state.dnsName, certPath, keyPath); + if (!result.ok) { + logDiagnostic("direct https cert unavailable", result.message); + return; + } + const [cert, key] = await Promise.all([ + fs.promises.readFile(certPath), + fs.promises.readFile(keyPath), + ]); + await dropServer.startHttps({ cert, key }); + } catch (err) { + logDiagnostic("direct https enable error", err); + } +} + +function scheduleHttpsRenewal() { + if (httpsRenewTimer) return; + httpsRenewTimer = setInterval(() => autoEnableDirectHttps(), 12 * 60 * 60 * 1000); + httpsRenewTimer.unref?.(); +} + async function createWindow() { const preload = path.join(__dirname, "preload.js"); mainWindow = new BrowserWindow({ @@ -330,6 +366,8 @@ if (gotLock) { Menu.setApplicationMenu(null); await startServer(); autoPublishServe(); // best-effort, non-blocking: makes the QR a real phone link + autoEnableDirectHttps(); // best-effort: direct HTTPS unlocks full-speed parallel transfers + scheduleHttpsRenewal(); applyLoginSettings(); createTray(); wireIpc(); diff --git a/src/main/tailscale.js b/src/main/tailscale.js index 054fadd..61a14a4 100644 --- a/src/main/tailscale.js +++ b/src/main/tailscale.js @@ -112,13 +112,16 @@ function findServeHandler(config, pathName = "/drop") { return null; } -async function inspect(port) { +async function inspect(port, { httpsPort } = {}) { const ts = await status(); const serve = await serveStatus(); const handler = serve.ok ? findServeHandler(serve.config, "/drop") : null; const serveUrl = ts.dnsName ? `https://${ts.dnsName}/drop/` : null; const directUrl = ts.tailnetIp ? `http://${ts.tailnetIp}:${port}/drop/` : null; const localUrl = `http://127.0.0.1:${port}/drop/`; + // Direct HTTPS to this process (own Tailscale cert). Fastest and still a secure + // context, so it's the preferred link whenever it's up. + const httpsDirectUrl = ts.dnsName && httpsPort ? `https://${ts.dnsName}:${httpsPort}/drop/` : null; return { ...ts, @@ -128,12 +131,28 @@ async function inspect(port) { serveProxy: handler ? handler.proxy : null, serveUrl, directUrl, + // Advertised for the client to probe and route bulk transfers through when + // reachable, but NOT the QR link: a phone must be able to open the QR, and + // the direct port may be blocked by the Windows firewall. Serve always works. + httpsDirectUrl, localUrl, preferredUrl: handler && serveUrl ? serveUrl : directUrl || localUrl, serveCommand: `tailscale serve --bg --yes --set-path /drop http://127.0.0.1:${port}`, }; } +// Ask Tailscale to issue (and thereafter renew) a TLS cert for this node's +// MagicDNS name, written to the given files. Needs HTTPS certs enabled on the +// tailnet; best-effort — the caller falls back to Serve/local if it fails. +async function provisionCert(dnsName, certPath, keyPath) { + if (!dnsName) return { ok: false, message: "No MagicDNS name" }; + const result = await run(["cert", "--cert-file", certPath, "--key-file", keyPath, dnsName], 45000); + if (!result.ok) { + return { ok: false, message: result.message || "tailscale cert failed" }; + } + return { ok: true, message: result.stdout.trim() || "cert ready" }; +} + async function configureServe(port) { const target = `http://127.0.0.1:${port}`; const result = await run(["serve", "--bg", "--yes", "--set-path", "/drop", target], 15000); @@ -147,6 +166,7 @@ module.exports = { configureServe, exePath, inspect, + provisionCert, run, serveStatus, status, diff --git a/src/main/updater.js b/src/main/updater.js index 88a7caa..035695f 100644 --- a/src/main/updater.js +++ b/src/main/updater.js @@ -45,6 +45,10 @@ function wireAutoUpdaterEvents() { }); autoUpdater.on("error", (err) => { + if (isUpdateManifestNotReady(err)) { + send({ state: "up-to-date", message: "No update is ready yet." }); + return; + } send({ state: "error", message: (err && err.message) || String(err) }); }); } @@ -59,6 +63,16 @@ function normalizeNotes(notes) { return ""; } +function isUpdateManifestNotReady(err) { + const message = (err && err.message) || String(err); + const code = err && err.code; + + return ( + code === "ERR_UPDATER_CHANNEL_FILE_NOT_FOUND" || + /Cannot find (latest|testing)\.yml in the latest release artifacts/i.test(message) + ); +} + async function checkForUpdates({ silent = false } = {}) { if (!app.isPackaged) { send({ state: "dev" }); @@ -69,6 +83,10 @@ async function checkForUpdates({ silent = false } = {}) { try { await autoUpdater.checkForUpdates(); } catch (err) { + if (isUpdateManifestNotReady(err)) { + send({ state: "up-to-date", message: "No update is ready yet." }); + return lastStatus; + } if (!silent) send({ state: "error", message: (err && err.message) || String(err) }); } finally { checkInFlight = false; diff --git a/src/renderer/public/waterdrop-sw.js b/src/renderer/public/waterdrop-sw.js index c1dd000..77d101f 100644 --- a/src/renderer/public/waterdrop-sw.js +++ b/src/renderer/public/waterdrop-sw.js @@ -53,7 +53,7 @@ async function drainUploadQueue() { lockedUntil: now + WORKER_LOCK_MS, lastError: "", }; - await putUploadRecord(claimed); + if (!await putUploadRecord(claimed)) continue; broadcast({ type: "WATERDROP_UPLOAD_SYNCING", id: claimed.id }); try { @@ -79,6 +79,7 @@ async function uploadRecord(record) { "X-WaterDrop-File-Name": encodeURIComponent(record.name || "unnamed-file"), "X-WaterDrop-Mime-Type": record.mimeType || "application/octet-stream", "X-WaterDrop-Upload-Id": record.id, + ...(record.folderId ? { "X-WaterDrop-Folder-Id": record.folderId } : {}), }, }); if (!response.ok) throw new Error(`HTTP ${response.status}`); @@ -92,19 +93,59 @@ async function broadcast(message) { async function listQueuedUploads() { const db = await openDb(); const records = await requestAsPromise(db.transaction(STORE_NAME, "readonly").objectStore(STORE_NAME).getAll()); - return records.sort((a, b) => Number(a.createdAt || 0) - Number(b.createdAt || 0)); + const now = Date.now(); + const expiredCancelledIds = []; + const visible = []; + for (const record of records) { + if (isCancelledUploadRecord(record, now)) continue; + if (isExpiredCancelledUploadRecord(record, now)) { + expiredCancelledIds.push(record.id); + continue; + } + visible.push(record); + } + if (expiredCancelledIds.length) deleteUploadRecords(expiredCancelledIds).catch(() => {}); + return visible.sort((a, b) => Number(a.createdAt || 0) - Number(b.createdAt || 0)); } async function putUploadRecord(record) { const db = await openDb(); - await requestAsPromise(db.transaction(STORE_NAME, "readwrite").objectStore(STORE_NAME).put(record)); + return new Promise((resolve, reject) => { + const transaction = db.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + let stored = false; + transaction.oncomplete = () => resolve(stored); + transaction.onerror = () => reject(transaction.error || new Error("IndexedDB transaction failed")); + transaction.onabort = () => reject(transaction.error || new Error("IndexedDB transaction aborted")); + const request = store.get(record.id); + request.onsuccess = () => { + if (record.status !== "cancelled" && isCancelledUploadRecord(request.result)) return; + store.put(record); + stored = true; + }; + request.onerror = () => reject(request.error || new Error("IndexedDB request failed")); + }); } async function deleteUploadRecord(id) { + const record = await getUploadRecord(id); + if (isCancelledUploadRecord(record)) return; const db = await openDb(); await requestAsPromise(db.transaction(STORE_NAME, "readwrite").objectStore(STORE_NAME).delete(id)); } +async function deleteUploadRecords(ids) { + const db = await openDb(); + await new Promise((resolve, reject) => { + const transaction = db.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + transaction.oncomplete = resolve; + transaction.onerror = () => reject(transaction.error || new Error("IndexedDB transaction failed")); + transaction.onabort = () => reject(transaction.error || new Error("IndexedDB transaction aborted")); + ids.forEach((id) => store.delete(id)); + }); +} + async function getUploadRecord(id) { const db = await openDb(); return requestAsPromise(db.transaction(STORE_NAME, "readonly").objectStore(STORE_NAME).get(id)); @@ -113,7 +154,7 @@ async function getUploadRecord(id) { async function markUploadQueued(id, message, knownRecord = null) { const record = knownRecord || await getUploadRecord(id); if (!record) return; - await putUploadRecord({ + const queued = await putUploadRecord({ ...record, status: "queued", progress: 0, @@ -122,9 +163,22 @@ async function markUploadQueued(id, message, knownRecord = null) { lockedUntil: 0, lastError: message || "Upload paused", }); + if (!queued) return; await broadcast({ type: "WATERDROP_UPLOAD_QUEUED", id }); } +function isCancelledUploadRecord(record, now = Date.now()) { + if (record?.status !== "cancelled") return false; + const cancelledUntil = Number(record.cancelledUntil || 0); + return !cancelledUntil || cancelledUntil > now; +} + +function isExpiredCancelledUploadRecord(record, now = Date.now()) { + if (record?.status !== "cancelled") return false; + const cancelledUntil = Number(record.cancelledUntil || 0); + return Boolean(cancelledUntil && cancelledUntil <= now); +} + function openDb() { return new Promise((resolve, reject) => { const request = indexedDB.open(DB_NAME, DB_VERSION); diff --git a/src/renderer/src/App.jsx b/src/renderer/src/App.jsx index 432ece5..f57685c 100644 --- a/src/renderer/src/App.jsx +++ b/src/renderer/src/App.jsx @@ -18,6 +18,7 @@ import { Info, MonitorUp, MoreHorizontal, + Pencil, QrCode, RefreshCw, Server, @@ -43,12 +44,19 @@ import { markQueuedUploadSyncing, queueUpload, recoverInterruptedUploads, + releaseQueuedUpload, requestBackgroundFetchUpload, requestBackgroundUploadSync, touchQueuedUploadLock, updateQueuedUploadProgress, uploadRecordToView, } from "./uploadQueue"; +import { + canParallelDownload, + canParallelUpload, + parallelDownload, + parallelUpload, +} from "./transfer"; const TAILSCALE_URL = "https://tailscale.com/download"; const TAILSCALE_IOS_URL = "https://apps.apple.com/app/tailscale/id1470499037"; @@ -58,6 +66,9 @@ const TAILSCALE_ANDROID_URL = "https://play.google.com/store/apps/details?id=com const ONBOARDING_STEPS = ["Welcome", "This PC", "Phone", "Publish", "Finish"]; const LAST_ONBOARD_STEP = ONBOARDING_STEPS.length - 1; const UPLOAD_STALL_MS = Math.max(90 * 1000, STALE_UPLOAD_LOCK_MS); +// Drain several queued files at once instead of strictly one-at-a-time; large +// files also fan out internally into parallel chunks. +const MAX_CONCURRENT_UPLOADS = 3; const apiDefault = { getInfo: () => request("api/info"), @@ -67,6 +78,8 @@ const apiDefault = { rawUploadUrl: "api/files/raw", downloadUrl: (id) => `api/files/${id}/download`, previewUrl: (id) => `api/files/${id}/preview`, + createFolder: (name) => request("api/folders", { method: "POST", body: JSON.stringify({ name }) }), + renameFolder: (id, name) => request(`api/folders/${id}`, { method: "PATCH", body: JSON.stringify({ name }) }), clearFiles: () => request("api/files", { method: "DELETE" }), deleteFile: (id) => request(`api/files/${id}`, { method: "DELETE" }), configureServe: () => request("api/tailscale/serve", { method: "POST" }), @@ -83,6 +96,8 @@ function buildApi(baseUrl = "") { rawUploadUrl: join("api/files/raw"), downloadUrl: (id) => join(`api/files/${id}/download`), previewUrl: (id) => join(`api/files/${id}/preview`), + createFolder: (name) => request(join("api/folders"), { method: "POST", body: JSON.stringify({ name }) }), + renameFolder: (id, name) => request(join(`api/folders/${id}`), { method: "PATCH", body: JSON.stringify({ name }) }), clearFiles: () => request(join("api/files"), { method: "DELETE" }), deleteFile: (id) => request(join(`api/files/${id}`), { method: "DELETE" }), configureServe: () => request(join("api/tailscale/serve"), { method: "POST" }), @@ -97,6 +112,11 @@ export default function App() { const serviceWorkerRegistrationRef = useRef(null); const drainingUploadsRef = useRef(false); const lastUploadPauseNoticeRef = useRef(0); + // When the direct HTTPS origin is reachable, bulk transfers are routed through + // it (HTTP/1.1 -> real parallel connections). Null means "use this page's + // origin" (e.g. Tailscale Serve). Held in a ref so it can change mid-session + // without rebuilding upload callbacks. + const transferBaseRef = useRef(null); const [api, setApi] = useState(apiDefault); const [appInfo, setAppInfo] = useState(null); const [info, setInfo] = useState(null); @@ -104,6 +124,7 @@ export default function App() { const [pending, setPending] = useState([]); const [settings, setSettings] = useState({}); const [uploads, setUploads] = useState([]); + const [createFolderUpload, setCreateFolderUpload] = useState(false); const [dragging, setDragging] = useState(false); const [busy, setBusy] = useState(true); const [qrOpen, setQrOpen] = useState(false); @@ -137,8 +158,18 @@ export default function App() { const refreshFiles = useCallback(async ({ silent = false } = {}) => { try { const result = await api.getFiles(); - setFiles(result.files || []); - setPending(result.pending || []); + const nextFiles = result.files || []; + const landedIds = nextFiles.map((file) => file.id).filter(Boolean); + const deletedUploadIds = Array.isArray(result.deletedUploads) ? result.deletedUploads.filter(Boolean) : []; + const closedIds = Array.from(new Set([...landedIds, ...deletedUploadIds])); + if (closedIds.length && isUploadQueueSupported()) { + Promise.allSettled(closedIds.map((id) => clearQueuedUpload(id))).catch(() => {}); + setUploads((items) => + items.filter((item) => !item.queued || !closedIds.includes(item.id)) + ); + } + setFiles(nextFiles); + setPending((result.pending || []).filter((item) => !closedIds.includes(item.id))); setSettings(result.settings || {}); } catch (err) { if (!silent) notify(err.message || "Could not refresh files", "warn"); @@ -189,17 +220,21 @@ export default function App() { if (!registration?.backgroundFetch?.fetch) return false; try { const syncing = await markQueuedUploadSyncing(record.id); - const bgFetch = await requestBackgroundFetchUpload(registration, syncing || record); - if (!bgFetch) return false; + if (!syncing) return false; + const bgFetch = await requestBackgroundFetchUpload(registration, syncing); + if (!bgFetch) { + await releaseQueuedUpload(record.id, "Background fetch unavailable").catch(() => {}); + return false; + } setUploads((items) => - upsertUpload(items, { ...uploadRecordToView(syncing || record), status: "syncing", progress: 1 }) + upsertUpload(items, { ...uploadRecordToView(syncing), status: "syncing", progress: 1 }) ); bgFetch.addEventListener?.("progress", () => { const total = Number(bgFetch.uploadTotal || 0); if (!total) return; const progress = Math.max(1, Math.min(99, Math.round((bgFetch.uploaded / total) * 100))); setUploads((items) => - upsertUpload(items, { ...uploadRecordToView(syncing || record), status: "syncing", progress }) + upsertUpload(items, { ...uploadRecordToView(syncing), status: "syncing", progress }) ); }); return true; @@ -215,22 +250,38 @@ export default function App() { try { await recoverInterruptedUploads("Upload interrupted by app suspend"); const records = await listQueuedUploads(); - for (const record of records) { - const activeBackgroundFetch = await serviceWorkerRegistrationRef.current?.backgroundFetch - ?.get(record.id) - .catch(() => null); - if (activeBackgroundFetch) continue; - const claimed = await claimQueuedUpload(record.id); - if (!claimed) continue; - await uploadQueuedRecord({ - api, - record: claimed, - notifyUploadPaused, - refreshFiles, - registerBackgroundUpload, - setUploads, - }); - } + // Drain several files at once from a shared queue; claimQueuedUpload keeps + // any one record from being picked up twice. + const queue = [...records]; + const runNext = async () => { + while (queue.length) { + const record = queue.shift(); + const activeBackgroundFetch = await serviceWorkerRegistrationRef.current?.backgroundFetch + ?.get(record.id) + .catch(() => null); + if (activeBackgroundFetch) continue; + const forceClaim = record.status === "syncing"; + if (forceClaim) { + await releaseQueuedUpload(record.id, "Background fetch did not start").catch(() => {}); + } + const claimed = await claimQueuedUpload(record.id, { force: forceClaim }); + if (!claimed) continue; + await uploadQueuedRecord({ + api, + record: claimed, + transferBase: transferBaseRef.current, + notifyUploadPaused, + refreshFiles, + registerBackgroundUpload, + setUploads, + }); + } + }; + const pool = Array.from( + { length: Math.min(MAX_CONCURRENT_UPLOADS, queue.length) }, + () => runNext() + ); + await Promise.all(pool); } finally { drainingUploadsRef.current = false; await refreshQueuedUploads(); @@ -275,8 +326,7 @@ export default function App() { const onMessage = (event) => { const type = event.data?.type || ""; if (!type.startsWith("WATERDROP_UPLOAD_")) return; - refreshQueuedUploads(); - refreshFiles({ silent: true }); + refreshFiles({ silent: true }).then(refreshQueuedUploads); }; navigator.serviceWorker.addEventListener("message", onMessage); return () => { @@ -312,16 +362,53 @@ export default function App() { useEffect(() => { if (busy) return; refreshInfo(); - refreshFiles(); - refreshQueuedUploads().then(drainQueuedUploads); + refreshFiles().then(refreshQueuedUploads).then(drainQueuedUploads); }, [busy, drainQueuedUploads, refreshFiles, refreshInfo, refreshQueuedUploads]); + // Probe the direct HTTPS origin. If a phone can reach it, route bulk transfers + // there for real parallel (HTTP/1.1) connections; otherwise stay on this page's + // origin (e.g. Tailscale Serve) so nothing breaks. + const directTransferUrl = info?.network?.httpsDirectUrl || null; + useEffect(() => { + if (isDesktop || !directTransferUrl) { + transferBaseRef.current = null; + return undefined; + } + let directOrigin; + try { + directOrigin = new URL(directTransferUrl).origin; + } catch { + transferBaseRef.current = null; + return undefined; + } + if (directOrigin === window.location.origin) { + transferBaseRef.current = directTransferUrl; + return undefined; + } + let cancelled = false; + const controller = new AbortController(); + const timer = window.setTimeout(() => controller.abort(), 3500); + fetch(new URL("api/ping", directTransferUrl), { cache: "no-store", signal: controller.signal }) + .then((res) => { + if (!cancelled) transferBaseRef.current = res.ok ? directTransferUrl : null; + }) + .catch(() => { + if (!cancelled) transferBaseRef.current = null; + }) + .finally(() => window.clearTimeout(timer)); + return () => { + cancelled = true; + controller.abort(); + window.clearTimeout(timer); + }; + }, [directTransferUrl, isDesktop]); + useEffect(() => { if (busy) return undefined; const refreshQuietly = () => refreshFiles({ silent: true }); const retryUploadsQuietly = () => { registerBackgroundUpload(); - refreshQueuedUploads().then(drainQueuedUploads); + refreshFiles({ silent: true }).then(refreshQueuedUploads).then(drainQueuedUploads); }; const refreshInterval = window.setInterval(refreshQuietly, window.EventSource ? 15000 : 3000); const uploadRetryInterval = window.setInterval(retryUploadsQuietly, 15000); @@ -528,8 +615,19 @@ export default function App() { async function uploadFiles(picked) { const valid = picked.filter(Boolean); if (!valid.length) return; + let targetFolder = null; + if (createFolderUpload) { + try { + const result = await api.createFolder(); + targetFolder = result.folder; + await refreshFiles({ silent: true }); + } catch (err) { + notify(err.message || "Could not create folder", "warn"); + return; + } + } if (!api.rawUploadUrl || !isUploadQueueSupported()) { - valid.forEach((file) => uploadOneLegacy(file)); + valid.forEach((file) => uploadOneLegacy(file, targetFolder)); return; } @@ -537,27 +635,34 @@ export default function App() { let browserBackground = 0; for (const file of valid) { try { - const record = await queueUpload(file, api.rawUploadUrl); + const record = await queueUpload(file, api.rawUploadUrl, { + folderId: targetFolder?.id, + folderName: targetFolder?.name, + }); queued += 1; - setUploads((items) => upsertUpload(items, uploadRecordToView(record))); + setUploads((items) => + upsertUpload(items, { ...uploadRecordToView(record), progress: 1 }) + ); if (await startBackgroundFetchUpload(record)) { browserBackground += 1; } } catch (err) { notify(err.message || `Could not queue ${file.name}`, "warn"); - uploadOneLegacy(file); + uploadOneLegacy(file, targetFolder); } } if (queued > 0) { const hasBackgroundSync = await registerBackgroundUpload(); notify( - browserBackground > 0 + targetFolder + ? `${valid.length} files queued in ${targetFolder.name}.` + : browserBackground > 0 ? "Upload handed to the browser background task." : hasBackgroundSync ? "Upload queued for background retry." : "Upload queued. It resumes when this page is open.", - browserBackground > 0 || hasBackgroundSync ? "ok" : "info" + targetFolder || browserBackground > 0 || hasBackgroundSync ? "ok" : "info" ); if (browserBackground < queued) drainQueuedUploads(); } @@ -576,11 +681,11 @@ export default function App() { setUploads((items) => items.filter((item) => item.id !== upload.id)); } - function uploadOneLegacy(file) { + function uploadOneLegacy(file, targetFolder = null) { const id = crypto.randomUUID(); setUploads((items) => [ ...items, - { id, name: file.name, size: file.size, progress: 0, status: "uploading", createdAt: Date.now() }, + { id, name: file.name, size: file.size, progress: 1, status: "uploading", createdAt: Date.now() }, ]); const xhr = new XMLHttpRequest(); @@ -591,10 +696,11 @@ export default function App() { xhr.setRequestHeader("X-WaterDrop-File-Name", encodeURIComponent(file.name || "unnamed-file")); xhr.setRequestHeader("X-WaterDrop-Mime-Type", file.type || "application/octet-stream"); xhr.setRequestHeader("X-WaterDrop-Upload-Id", id); + if (targetFolder?.id) xhr.setRequestHeader("X-WaterDrop-Folder-Id", targetFolder.id); } xhr.upload.onprogress = (event) => { if (!event.lengthComputable) return; - const progress = Math.round((event.loaded / event.total) * 100); + const progress = Math.max(1, Math.min(99, Math.round((event.loaded / event.total) * 100))); setUploads((items) => items.map((item) => (item.id === id ? { ...item, progress } : item)) ); @@ -634,7 +740,7 @@ export default function App() { if (window.waterdrop) { const result = await window.waterdrop.saveFile(file.id); if (result.ok) { - notify("Saved to folder", "ok", { + notify(isFolderEntry(file) ? "ZIP saved to folder" : "Saved to folder", "ok", { label: "Reveal", run: () => window.waterdrop.revealPath(result.destination), }); @@ -644,12 +750,70 @@ export default function App() { notify(result.message || "Save failed", "warn"); return; } - window.location.href = api.downloadUrl(file.id); - window.setTimeout(refreshFiles, 1000); + await browserDownload(file); + } + + function shouldAccelerateDownload(file) { + return !isFolderEntry(file) && canParallelDownload(file); + } + + // Accelerated multi-connection download with a plain native download as the + // fallback for folders, small files, unsupported browsers, or any error. + async function browserDownload(file) { + if (shouldAccelerateDownload(file)) { + let lastPct = -100; + const base = transferBaseRef.current; + const url = base ? new URL(`api/files/${file.id}/download`, base).toString() : api.downloadUrl(file.id); + try { + notify(`Downloading ${file.name}…`, "info"); + await parallelDownload(file, url, { + onProgress: (pct) => { + if (pct - lastPct >= 10) { + lastPct = pct; + notify(`Downloading ${file.name} — ${pct}%`, "info"); + } + }, + }); + notify("Download complete", "ok"); + window.setTimeout(() => refreshFiles({ silent: true }), 500); + return; + } catch { + // Fall through to the native download below. + } + } + nativeDownload(file); + } + + function nativeDownload(file) { + const anchor = document.createElement("a"); + anchor.href = api.downloadUrl(file.id); + anchor.download = file.name || "download"; + anchor.rel = "noopener"; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + window.setTimeout(() => refreshFiles({ silent: true }), 1000); + } + + async function renameFolder(file) { + if (!isFolderEntry(file)) return; + const nextName = window.prompt("Folder name", file.name); + if (nextName === null) return; + const trimmed = nextName.trim(); + if (!trimmed || trimmed === file.name) return; + try { + const result = await api.renameFolder(file.id, trimmed); + setFiles((items) => items.map((item) => (item.id === file.id ? result.folder : item))); + if (previewFile?.id === file.id) setPreviewFile(result.folder); + notify("Folder renamed", "ok"); + } catch (err) { + notify(err.message || "Rename failed", "warn"); + } } function startExternalDrag(file, event) { if (!window.waterdrop?.startFileDrag) return; + if (isFolderEntry(file)) return; event.preventDefault(); event.stopPropagation(); if (event.dataTransfer) { @@ -662,7 +826,12 @@ export default function App() { async function clearFiles() { try { + const deletedIds = files.map((file) => file.id); const result = await api.clearFiles(); + if (isUploadQueueSupported()) { + await Promise.allSettled(deletedIds.map((id) => clearQueuedUpload(id))); + } + setUploads((items) => items.filter((item) => !deletedIds.includes(item.id))); setConfirmClear(false); setConfirmDelete(null); setPreviewFile(null); @@ -676,6 +845,10 @@ export default function App() { async function deleteFile(file) { try { await api.deleteFile(file.id); + if (isUploadQueueSupported()) { + await clearQueuedUpload(file.id).catch(() => {}); + } + setUploads((items) => items.filter((item) => item.id !== file.id)); setConfirmDelete(null); notify("Deleted", "ok"); await refreshFiles(); @@ -816,7 +989,7 @@ export default function App() {
- {stats.count} files + {stats.count} items {formatBytes(stats.totalBytes)} @@ -881,10 +1054,26 @@ export default function App() { onDragLeave={onDragLeave} title={uploadModeTitle} > - - Drop files + {createFolderUpload ? ( + + ) : ( + + )} + {createFolderUpload ? "Drop files into a new folder" : "Drop files"} + + {uploads.length > 0 && ( @@ -936,6 +1125,7 @@ export default function App() { onDelete={() => deleteFile(file)} onDownload={() => downloadFile(file)} onPreview={() => setPreviewFile(file)} + onRename={() => renameFolder(file)} onRequestDelete={() => setConfirmDelete(file.id)} onCancelDelete={() => setConfirmDelete(null)} onStartDrag={(event) => startExternalDrag(file, event)} @@ -1217,7 +1407,7 @@ function UpdateInline({ update, onDownload, onInstall }) { return (
{state === "checking" && Checking for updates...} - {state === "up-to-date" && You're on the latest version.} + {state === "up-to-date" && {update.message || "You're on the latest version."}} {state === "dev" && Updates run only in the installed app.} {state === "error" && {update.message || "Update check failed."}} {state === "available" && ( @@ -1284,14 +1474,16 @@ function FileCard({ onDelete, onDownload, onPreview, + onRename, onRequestDelete, onStartDrag, }) { const Icon = iconFor(file); + const isFolder = isFolderEntry(file); const isImage = isImageFile(file); const isVideo = isVideoFile(file); const isPdf = isPdfFile(file); - const title = isDesktop ? "Click to preview. Drag to copy." : "Click to preview."; + const title = isFolder ? "Open folder." : isDesktop ? "Click to preview. Drag to copy." : "Click to preview."; function onCardKeyDown(event) { if (event.key !== "Enter" && event.key !== " ") return; event.preventDefault(); @@ -1301,8 +1493,8 @@ function FileCard({ return (
- {isImage ? ( + {isFolder ? ( + <> + + ZIP + + ) : isImage ? ( ) : isVideo ? ( <> @@ -1356,7 +1553,8 @@ function FileCard({ {file.name}
- {formatBytes(file.size)} + {isFolder ? `${file.itemCount || 0} files` : formatBytes(file.size)} + {isFolder && {formatBytes(file.size)}} {timeLeft(file.expiresAt)} @@ -1364,17 +1562,31 @@ function FileCard({ {file.downloads || 0}
- + {isFolder ? ( + + ) : ( + + )}
Save ) : ( - + { + // Let the native download run for folders/small files; otherwise + // take over with the accelerated multi-connection download. + if (isFolder || !canParallelDownload(file)) return; + event.preventDefault(); + onDownload(); + }} + > Get )} @@ -1411,7 +1634,7 @@ function FileCard({ function PreviewModal({ api, file, isDesktop, onClose, onCopyImage, onDownload }) { const kind = previewKindFor(file); - const previewUrl = api.previewUrl(file.id); + const previewUrl = kind === "folder" ? "" : api.previewUrl(file.id); const [contextMenu, setContextMenu] = useState(null); const [closing, setClosing] = useState(false); const closeTimer = useRef(null); @@ -1482,7 +1705,7 @@ function PreviewModal({ api, file, isDesktop, onClose, onCopyImage, onDownload }
{formatBytes(file.size)} - {file.mimeType || "application/octet-stream"} + {kind === "folder" ? `${file.itemCount || 0} files` : file.mimeType || "application/octet-stream"} {timeLeft(file.expiresAt)}
@@ -1492,6 +1715,7 @@ function PreviewModal({ api, file, isDesktop, onClose, onCopyImage, onDownload }
+ {kind === "folder" && } {kind === "image" && ( : } {isDesktop ? "Save" : "Get"} - - Open - + {kind !== "folder" && ( + + Open + + )} {contextMenu && ( @@ -1588,6 +1814,34 @@ function TextPreview({ file, url }) { return 
{state.text || " "}
; } +function FolderPreview({ file }) { + const files = Array.isArray(file.files) ? file.files : []; + if (!files.length) { + return ( +
+ + This folder is empty. +
+ ); + } + return ( +
+ {files.map((child) => { + const Icon = iconFor(child); + return ( +
+ + + {child.name} + + {formatBytes(child.size)} +
+ ); + })} +
+ ); +} + async function request(url, options = {}) { const response = await fetch(url, { cache: "no-store", @@ -1608,28 +1862,36 @@ async function request(url, options = {}) { async function uploadQueuedRecord({ api, record, + transferBase = null, notifyUploadPaused, refreshFiles, registerBackgroundUpload, setUploads, }) { - const uploadUrl = record.uploadUrl || api.rawUploadUrl; + // Route through the direct HTTPS origin when it's reachable (real parallel + // HTTP/1.1 connections); otherwise use this page's origin. + const uploadUrl = transferBase + ? new URL("api/files/raw", transferBase).toString() + : record.uploadUrl || api.rawUploadUrl; const view = uploadRecordToView(record); - setUploads((items) => upsertUpload(items, { ...view, status: "uploading", progress: 0 })); + setUploads((items) => upsertUpload(items, { ...view, status: "uploading", progress: 1 })); - let lastPersistedProgress = 0; + let lastPersistedProgress = 1; const lockTimer = window.setInterval(() => { touchQueuedUploadLock(record.id).catch(() => {}); }, Math.max(5000, Math.floor(PAGE_UPLOAD_LOCK_MS / 3))); try { - await new Promise((resolve, reject) => { - const xhr = new XMLHttpRequest(); + const result = await new Promise((resolve, reject) => { let settled = false; let lastActivityAt = Date.now(); + let activeXhr = null; + let activeController = null; + let heartbeat = null; const cleanup = () => { window.clearInterval(stallTimer); + if (heartbeat) window.clearInterval(heartbeat); window.removeEventListener("offline", onOffline); window.removeEventListener("pagehide", onPageHide); document.removeEventListener("visibilitychange", onVisibilityChange); @@ -1643,13 +1905,24 @@ async function uploadQueuedRecord({ const fail = (message) => { if (settled) return; try { - xhr.abort(); + activeXhr?.abort(); + } catch {} + try { + activeController?.abort(); } catch {} settle(reject, new Error(message)); }; const markActivity = () => { lastActivityAt = Date.now(); }; + const applyProgress = (progress) => { + const visibleProgress = Math.max(1, Math.min(99, progress)); + setUploads((items) => upsertUpload(items, { ...view, status: "uploading", progress: visibleProgress })); + if (progress - lastPersistedProgress >= 10 || progress === 100) { + lastPersistedProgress = visibleProgress; + updateQueuedUploadProgress(record.id, visibleProgress).catch(() => {}); + } + }; const onOffline = () => fail("Upload paused while offline"); const onPageHide = () => fail("Upload paused while app was backgrounded"); const onVisibilityChange = () => { @@ -1665,35 +1938,65 @@ async function uploadQueuedRecord({ window.addEventListener("pagehide", onPageHide); document.addEventListener("visibilitychange", onVisibilityChange); - xhr.open("POST", uploadUrl, true); - xhr.setRequestHeader("Content-Type", record.mimeType || "application/octet-stream"); - xhr.setRequestHeader("X-WaterDrop-File-Name", encodeURIComponent(record.name || "unnamed-file")); - xhr.setRequestHeader("X-WaterDrop-Mime-Type", record.mimeType || "application/octet-stream"); - xhr.setRequestHeader("X-WaterDrop-Upload-Id", record.id); - xhr.upload.onprogress = (event) => { + if (canParallelUpload(record)) { + // Large file: fan out into parallel byte-range chunks. Server reassembles + // and hashes them, so the stored bytes are identical to the source. + const controller = new AbortController(); + activeController = controller; markActivity(); - if (!event.lengthComputable) return; - const progress = Math.round((event.loaded / event.total) * 100); - setUploads((items) => upsertUpload(items, { ...view, status: "uploading", progress })); - if (progress - lastPersistedProgress >= 10 || progress === 100) { - lastPersistedProgress = progress; - updateQueuedUploadProgress(record.id, progress).catch(() => {}); - } - }; - xhr.upload.onload = markActivity; - xhr.onload = () => { - if (xhr.status >= 200 && xhr.status < 300) { - settle(resolve); - return; - } - settle(reject, new Error(`HTTP ${xhr.status}`)); - }; - xhr.onerror = () => settle(reject, new Error("Network error")); - xhr.onabort = () => settle(reject, new Error("Upload aborted")); - xhr.send(record.blob); + // XHR reports sub-chunk upload progress, but the final server-side hash + // can still spend time after all bytes are sent. Keep the connection + // considered "alive" while the final response is pending. + heartbeat = window.setInterval(markActivity, 15000); + parallelUpload({ + blob: record.blob, + uploadUrl, + id: record.id, + name: record.name, + mimeType: record.mimeType, + folderId: record.folderId, + signal: controller.signal, + onProgress: applyProgress, + onActivity: markActivity, + }) + .then((result) => settle(resolve, result)) + .catch((err) => settle(reject, err instanceof Error ? err : new Error(String(err)))); + } else { + // Small file: a single streamed request is already optimal. + const xhr = new XMLHttpRequest(); + activeXhr = xhr; + xhr.open("POST", uploadUrl, true); + xhr.setRequestHeader("Content-Type", record.mimeType || "application/octet-stream"); + xhr.setRequestHeader("X-WaterDrop-File-Name", encodeURIComponent(record.name || "unnamed-file")); + xhr.setRequestHeader("X-WaterDrop-Mime-Type", record.mimeType || "application/octet-stream"); + xhr.setRequestHeader("X-WaterDrop-Upload-Id", record.id); + if (record.folderId) xhr.setRequestHeader("X-WaterDrop-Folder-Id", record.folderId); + xhr.upload.onprogress = (event) => { + markActivity(); + if (!event.lengthComputable) return; + applyProgress(Math.round((event.loaded / event.total) * 100)); + }; + xhr.upload.onload = markActivity; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + const data = parseJsonResponse(xhr.responseText); + settle(resolve, data); + return; + } + settle(reject, new Error(`HTTP ${xhr.status}`)); + }; + xhr.onerror = () => settle(reject, new Error("Network error")); + xhr.onabort = () => settle(reject, new Error("Upload aborted")); + xhr.send(record.blob); + } }); await clearQueuedUpload(record.id); + if (result?.deleted) { + setUploads((items) => items.filter((item) => item.id !== record.id)); + await refreshFiles({ silent: true }); + return; + } setUploads((items) => upsertUpload(items, { ...view, queued: false, status: "done", progress: 100 })); window.setTimeout(() => { setUploads((items) => items.filter((item) => item.id !== record.id)); @@ -1709,6 +2012,15 @@ async function uploadQueuedRecord({ } } +function parseJsonResponse(text) { + if (!text) return {}; + try { + return JSON.parse(text); + } catch { + return {}; + } +} + function upsertUpload(items, upload) { const next = items.filter((item) => item.id !== upload.id); next.push(upload); @@ -1736,6 +2048,7 @@ function isWaterDropDrag(event) { } function iconFor(file) { + if (isFolderEntry(file)) return Folder; const type = String(file.mimeType || ""); const name = String(file.name || "").toLowerCase(); if (type.startsWith("image/")) return FileImage; @@ -1749,25 +2062,34 @@ function iconFor(file) { return MoreHorizontal; } +function isFolderEntry(file) { + return file?.kind === "folder"; +} + function isImageFile(file) { + if (isFolderEntry(file)) return false; return String(file.mimeType || "").startsWith("image/"); } function isVideoFile(file) { + if (isFolderEntry(file)) return false; return String(file.mimeType || "").startsWith("video/"); } function isAudioFile(file) { + if (isFolderEntry(file)) return false; return String(file.mimeType || "").startsWith("audio/"); } function isPdfFile(file) { + if (isFolderEntry(file)) return false; const type = String(file.mimeType || "").toLowerCase(); const name = String(file.name || "").toLowerCase(); return type === "application/pdf" || name.endsWith(".pdf"); } function isTextFile(file) { + if (isFolderEntry(file)) return false; const type = String(file.mimeType || "").toLowerCase(); const name = String(file.name || "").toLowerCase(); return ( @@ -1778,6 +2100,7 @@ function isTextFile(file) { } function previewKindFor(file) { + if (isFolderEntry(file)) return "folder"; if (isImageFile(file)) return "image"; if (isVideoFile(file)) return "video"; if (isAudioFile(file)) return "audio"; diff --git a/src/renderer/src/styles.css b/src/renderer/src/styles.css index a227e6d..5713643 100644 --- a/src/renderer/src/styles.css +++ b/src/renderer/src/styles.css @@ -560,6 +560,52 @@ body::before { font-size: 1rem; } +.bulk-toggle { + display: flex; + align-items: center; + gap: 10px; + border: 1px solid var(--line); + border-radius: 8px; + background: var(--panel); + padding: 10px 12px; + cursor: pointer; +} +.bulk-toggle input { + appearance: none; + width: 18px; + height: 18px; + flex: none; + border-radius: 5px; + border: 1px solid var(--line-strong); + background: var(--panel-solid); + position: relative; +} +.bulk-toggle input:checked { + background: var(--ink); + border-color: var(--ink); +} +.bulk-toggle input:checked::after { + content: ""; + position: absolute; + left: 5px; + top: 2px; + width: 5px; + height: 9px; + border: solid #08080a; + border-width: 0 2px 2px 0; + transform: rotate(45deg); +} +.bulk-title { + display: block; + color: var(--text); + font-weight: 700; + line-height: 1.2; +} +.bulk-sub { + display: block; + margin-top: 2px; +} + /* ---- Upload progress ---- */ .uploads { display: flex; @@ -676,6 +722,12 @@ body::before { border-color: var(--line-strong); background: var(--panel-strong); } +.file-folder .file-tile { + color: var(--ink); + background: + linear-gradient(135deg, rgba(143, 184, 154, 0.13), transparent 45%), + var(--panel-strong); +} .file:focus-visible { outline: 2px solid var(--ink); outline-offset: 3px; @@ -991,6 +1043,39 @@ body::before { text-align: center; } +.folder-preview-list { + width: min(620px, calc(100% - 28px)); + max-height: min(58vh, 540px); + overflow: auto; + display: flex; + flex-direction: column; + border: 1px solid var(--line); + border-radius: 10px; + background: var(--panel); +} +.folder-preview-row { + display: grid; + grid-template-columns: 18px minmax(0, 1fr) auto; + align-items: center; + gap: 10px; + padding: 10px 12px; + border-bottom: 1px solid var(--line); +} +.folder-preview-row:last-child { + border-bottom: none; +} +.folder-preview-row svg { + color: var(--silver); +} +.folder-preview-name { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--text); + font-weight: 600; +} + .context-menu { position: fixed; z-index: 460; diff --git a/src/renderer/src/transfer.js b/src/renderer/src/transfer.js new file mode 100644 index 0000000..4a1f7f2 --- /dev/null +++ b/src/renderer/src/transfer.js @@ -0,0 +1,223 @@ +// Parallel, multi-connection transfers. +// +// A single TCP stream can't saturate a real network: throughput is capped by the +// bandwidth-delay product and by congestion control ramping up after every loss. +// Splitting a transfer across several connections at once sidesteps that limit — +// the same trick download accelerators and multipart cloud uploads use — and it +// is completely byte-identical: we move the exact same bytes, just in parallel. + +export const DOWNLOAD_CHUNK_SIZE = 8 * 1024 * 1024; +export const UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024; +export const MAX_PARALLEL_DOWNLOADS = 6; +export const MAX_PARALLEL_UPLOADS = 6; +// Below this the round-trip overhead of splitting isn't worth it. +export const DOWNLOAD_PARALLEL_MIN_BYTES = 8 * 1024 * 1024; +export const UPLOAD_PARALLEL_MIN_BYTES = 12 * 1024 * 1024; +// Accelerated downloads assemble the file in memory before saving, so cap the +// size we'll hold on a phone; larger files fall back to a streamed native save. +export const DOWNLOAD_PARALLEL_MAX_BYTES = 512 * 1024 * 1024; + +// iOS Safari's handling of programmatic Blob downloads is unreliable (it often +// opens the file inline instead of saving), so there we keep the native path untouched. Upload acceleration is unaffected and stays on. +function looksLikeIos() { + if (typeof navigator === "undefined") return false; + const ua = navigator.userAgent || ""; + const platform = navigator.platform || ""; + const iOsUa = /iPad|iPhone|iPod/.test(ua); + const iPadOs = platform === "MacIntel" && Number(navigator.maxTouchPoints || 0) > 1; + return iOsUa || iPadOs; +} + +export function canParallelDownload(file) { + const size = Number(file?.size || 0); + return ( + typeof fetch === "function" && + typeof Blob === "function" && + typeof URL !== "undefined" && + typeof URL.createObjectURL === "function" && + "download" in document.createElement("a") && + !looksLikeIos() && + size >= DOWNLOAD_PARALLEL_MIN_BYTES && + size <= DOWNLOAD_PARALLEL_MAX_BYTES + ); +} + +// Download one file over several parallel HTTP range requests, reassemble the +// chunks in order, and save the result. Rejects (without saving) if anything is +// off, so callers can fall back to a plain native download. +export async function parallelDownload(file, downloadUrl, { onProgress, signal } = {}) { + const size = Number(file.size || 0); + const chunkCount = Math.ceil(size / DOWNLOAD_CHUNK_SIZE); + const parts = new Array(chunkCount); + let received = 0; + let nextIndex = 0; + + async function worker() { + while (true) { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const index = nextIndex++; + if (index >= chunkCount) return; + const start = index * DOWNLOAD_CHUNK_SIZE; + const end = Math.min(size, start + DOWNLOAD_CHUNK_SIZE) - 1; + const headers = { Range: `bytes=${start}-${end}` }; + // Only the first part counts, so the whole accelerated download registers + // as a single download rather than one per chunk. + if (index > 0) headers["X-WaterDrop-No-Count"] = "1"; + const response = await fetch(downloadUrl, { headers, signal, cache: "no-store" }); + if (response.status !== 206) throw new Error(`Expected 206, got ${response.status}`); + const buffer = await response.arrayBuffer(); + if (buffer.byteLength !== end - start + 1) throw new Error("Chunk size mismatch"); + parts[index] = buffer; + received += buffer.byteLength; + onProgress?.(Math.min(100, Math.round((received / size) * 100))); + } + } + + const workers = []; + const parallel = Math.max(1, Math.min(MAX_PARALLEL_DOWNLOADS, chunkCount)); + for (let i = 0; i < parallel; i += 1) workers.push(worker()); + await Promise.all(workers); + + const blob = new Blob(parts, { type: file.mimeType || "application/octet-stream" }); + saveBlob(blob, file.name || "download"); +} + +function saveBlob(blob, filename) { + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = filename; + anchor.rel = "noopener"; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + window.setTimeout(() => URL.revokeObjectURL(url), 60 * 1000); +} + +export function canParallelUpload(record) { + const blob = record?.blob; + const size = Number(record?.size || blob?.size || 0); + return ( + typeof fetch === "function" && + typeof AbortController === "function" && + typeof XMLHttpRequest === "function" && + blob && + typeof blob.slice === "function" && + size >= UPLOAD_PARALLEL_MIN_BYTES + ); +} + +// Upload one file as many byte-range chunks over parallel connections. The server +// writes each chunk at its offset and hashes the reassembled file, so the stored +// bytes are identical to the source. Resolves once the file is committed; rejects +// (leaving the queued record intact) so the caller can retry or fall back. +export async function parallelUpload({ + blob, + uploadUrl, + id, + name, + mimeType, + folderId, + signal, + onProgress, + onActivity, +}) { + const size = blob.size; + const chunkCount = Math.ceil(size / UPLOAD_CHUNK_SIZE); + const chunkProgress = new Array(chunkCount).fill(0); + let nextIndex = 0; + let resultFile = null; + let deleted = false; + + const reportProgress = () => { + const uploadedBytes = chunkProgress.reduce((sum, bytes) => sum + bytes, 0); + // Reaching 100 here only means the browser has sent all request bodies. + // The caller marks 100/done after the server confirms the file is committed. + onProgress?.(Math.max(1, Math.min(99, Math.round((uploadedBytes / size) * 100)))); + }; + + const uploadChunk = (index, start, end) => + new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException("Aborted", "AbortError")); + return; + } + + const xhr = new XMLHttpRequest(); + const length = end - start; + const cleanup = () => { + signal?.removeEventListener?.("abort", onAbort); + }; + const onAbort = () => { + xhr.abort(); + }; + + xhr.open("POST", uploadUrl, true); + xhr.setRequestHeader("Content-Type", mimeType || "application/octet-stream"); + xhr.setRequestHeader("X-WaterDrop-File-Name", encodeURIComponent(name || "unnamed-file")); + xhr.setRequestHeader("X-WaterDrop-Mime-Type", mimeType || "application/octet-stream"); + xhr.setRequestHeader("X-WaterDrop-Upload-Id", id); + xhr.setRequestHeader("X-WaterDrop-Chunk-Offset", String(start)); + xhr.setRequestHeader("X-WaterDrop-Total-Size", String(size)); + if (folderId) xhr.setRequestHeader("X-WaterDrop-Folder-Id", folderId); + + xhr.upload.onprogress = (event) => { + onActivity?.(); + if (!event.lengthComputable) return; + chunkProgress[index] = Math.min(length, event.loaded); + reportProgress(); + }; + xhr.upload.onload = () => { + onActivity?.(); + chunkProgress[index] = length; + reportProgress(); + }; + xhr.onload = () => { + cleanup(); + onActivity?.(); + if (xhr.status < 200 || xhr.status >= 300) { + reject(new Error(`HTTP ${xhr.status}`)); + return; + } + try { + resolve(xhr.responseText ? JSON.parse(xhr.responseText) : {}); + } catch { + resolve({}); + } + }; + xhr.onerror = () => { + cleanup(); + reject(new Error("Network error")); + }; + xhr.onabort = () => { + cleanup(); + reject(new DOMException("Aborted", "AbortError")); + }; + signal?.addEventListener?.("abort", onAbort, { once: true }); + xhr.send(blob.slice(start, end)); + }); + + onProgress?.(1); + + async function worker() { + while (true) { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const index = nextIndex++; + if (index >= chunkCount) return; + const start = index * UPLOAD_CHUNK_SIZE; + const end = Math.min(size, start + UPLOAD_CHUNK_SIZE); + const data = await uploadChunk(index, start, end); + if (data?.deleted) deleted = true; + if (data && Array.isArray(data.files) && data.files.length) resultFile = data.files[0]; + } + } + + const workers = []; + const parallel = Math.max(1, Math.min(MAX_PARALLEL_UPLOADS, chunkCount)); + for (let i = 0; i < parallel; i += 1) workers.push(worker()); + await Promise.all(workers); + + if (deleted) return { deleted: true }; + return resultFile; +} diff --git a/src/renderer/src/uploadQueue.js b/src/renderer/src/uploadQueue.js index cacd45d..d76fb8b 100644 --- a/src/renderer/src/uploadQueue.js +++ b/src/renderer/src/uploadQueue.js @@ -1,6 +1,7 @@ const DB_NAME = "waterdrop-upload-queue"; const DB_VERSION = 1; const STORE_NAME = "uploads"; +const CANCELLED_UPLOAD_TOMBSTONE_MS = 7 * 24 * 60 * 60 * 1000; export const PAGE_UPLOAD_LOCK_MS = 45 * 1000; export const STALE_UPLOAD_LOCK_MS = 2 * PAGE_UPLOAD_LOCK_MS; @@ -12,7 +13,7 @@ export function isUploadQueueSupported() { return typeof indexedDB !== "undefined" && typeof Blob !== "undefined"; } -export async function queueUpload(file, uploadUrl) { +export async function queueUpload(file, uploadUrl, options = {}) { if (!isUploadQueueSupported()) throw new Error("Upload queue is not available"); const now = Date.now(); const record = { @@ -23,22 +24,37 @@ export async function queueUpload(file, uploadUrl) { blob: file, uploadUrl, status: "queued", - progress: 0, + progress: 1, attempts: 0, createdAt: now, updatedAt: now, lockedUntil: 0, lastError: "", }; - await putUploadRecord(record); + if (options.folderId) record.folderId = options.folderId; + if (options.folderName) record.folderName = options.folderName; + const stored = await putUploadRecord(record); + if (!stored) throw new Error("Upload was cancelled"); return record; } export async function listQueuedUploads() { if (!isUploadQueueSupported()) return []; const db = await openDb(); - return requestAsPromise(db.transaction(STORE_NAME, "readonly").objectStore(STORE_NAME).getAll()) - .then((items) => items.sort((a, b) => Number(a.createdAt || 0) - Number(b.createdAt || 0))); + const items = await requestAsPromise(db.transaction(STORE_NAME, "readonly").objectStore(STORE_NAME).getAll()); + const now = Date.now(); + const expiredCancelledIds = []; + const visible = []; + for (const item of items) { + if (isCancelledUploadRecord(item, now)) continue; + if (isExpiredCancelledUploadRecord(item, now)) { + expiredCancelledIds.push(item.id); + continue; + } + visible.push(item); + } + if (expiredCancelledIds.length) deleteUploadRecords(expiredCancelledIds).catch(() => {}); + return visible.sort((a, b) => Number(a.createdAt || 0) - Number(b.createdAt || 0)); } export async function recoverInterruptedUploads(message = "Upload interrupted") { @@ -59,23 +75,36 @@ export async function recoverInterruptedUploads(message = "Upload interrupted") return recovered; } -export async function claimQueuedUpload(id) { +export async function claimQueuedUpload(id, { force = false } = {}) { const record = await getUploadRecord(id); if (!record) return null; const now = Date.now(); - if (record.lockedUntil > now && ["uploading", "syncing"].includes(record.status)) { + if (!force && record.lockedUntil > now && ["uploading", "syncing"].includes(record.status)) { return null; } const claimed = { ...record, status: "uploading", - progress: 0, + progress: Math.max(1, Number(record.progress || 0)), updatedAt: now, lockedUntil: now + PAGE_UPLOAD_LOCK_MS, lastError: "", }; - await putUploadRecord(claimed); - return claimed; + return await putUploadRecord(claimed) ? claimed : null; +} + +export async function releaseQueuedUpload(id, message = "") { + const record = await getUploadRecord(id); + if (!record) return null; + const released = { + ...record, + status: "queued", + progress: Math.max(1, Number(record.progress || 0)), + updatedAt: Date.now(), + lockedUntil: 0, + lastError: message || record.lastError || "", + }; + return await putUploadRecord(released) ? released : null; } export async function touchQueuedUploadLock(id) { @@ -86,8 +115,7 @@ export async function touchQueuedUploadLock(id) { updatedAt: Date.now(), lockedUntil: Date.now() + PAGE_UPLOAD_LOCK_MS, }; - await putUploadRecord(updated); - return updated; + return await putUploadRecord(updated) ? updated : null; } export async function updateQueuedUploadProgress(id, progress) { @@ -95,11 +123,10 @@ export async function updateQueuedUploadProgress(id, progress) { if (!record) return null; const updated = { ...record, - progress: Math.max(0, Math.min(100, Number(progress || 0))), + progress: Math.max(1, Math.min(99, Number(progress || 0))), updatedAt: Date.now(), }; - await putUploadRecord(updated); - return updated; + return await putUploadRecord(updated) ? updated : null; } export async function markQueuedUploadFailed(id, message) { @@ -114,8 +141,7 @@ export async function markQueuedUploadFailed(id, message) { lockedUntil: 0, lastError: message || "Upload paused", }; - await putUploadRecord(updated); - return updated; + return await putUploadRecord(updated) ? updated : null; } export async function markQueuedUploadSyncing(id) { @@ -129,13 +155,28 @@ export async function markQueuedUploadSyncing(id) { lockedUntil: Date.now() + 2 * PAGE_UPLOAD_LOCK_MS, lastError: "", }; - await putUploadRecord(updated); - return updated; + return await putUploadRecord(updated) ? updated : null; } export async function clearQueuedUpload(id) { - const db = await openDb(); - await requestAsPromise(db.transaction(STORE_NAME, "readwrite").objectStore(STORE_NAME).delete(id)); + if (!id) return; + const now = Date.now(); + await putUploadRecord({ + id, + name: "", + size: 0, + mimeType: "application/octet-stream", + blob: null, + uploadUrl: "", + status: "cancelled", + progress: 0, + attempts: 0, + createdAt: now, + updatedAt: now, + lockedUntil: 0, + cancelledUntil: now + CANCELLED_UPLOAD_TOMBSTONE_MS, + lastError: "", + }); } export async function requestBackgroundFetchUpload(registration, record) { @@ -151,6 +192,7 @@ export async function requestBackgroundFetchUpload(registration, record) { "X-WaterDrop-File-Name": encodeURIComponent(record.name || "unnamed-file"), "X-WaterDrop-Mime-Type": record.mimeType || "application/octet-stream", "X-WaterDrop-Upload-Id": record.id, + ...(record.folderId ? { "X-WaterDrop-Folder-Id": record.folderId } : {}), }, }); @@ -182,18 +224,65 @@ export function uploadRecordToView(record) { status: record.status || "queued", attempts: Number(record.attempts || 0), createdAt: Number(record.createdAt || 0), + folderId: record.folderId || "", + folderName: record.folderName || "", queued: true, }; } async function getUploadRecord(id) { const db = await openDb(); - return requestAsPromise(db.transaction(STORE_NAME, "readonly").objectStore(STORE_NAME).get(id)); + const record = await requestAsPromise(db.transaction(STORE_NAME, "readonly").objectStore(STORE_NAME).get(id)); + if (!isAnyCancelledUploadRecord(record)) return record; + if (isExpiredCancelledUploadRecord(record)) deleteUploadRecords([id]).catch(() => {}); + return null; } async function putUploadRecord(record) { const db = await openDb(); - await requestAsPromise(db.transaction(STORE_NAME, "readwrite").objectStore(STORE_NAME).put(record)); + return new Promise((resolve, reject) => { + const transaction = db.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + let stored = false; + transaction.oncomplete = () => resolve(stored); + transaction.onerror = () => reject(transaction.error || new Error("IndexedDB transaction failed")); + transaction.onabort = () => reject(transaction.error || new Error("IndexedDB transaction aborted")); + const request = store.get(record.id); + request.onsuccess = () => { + if (record.status !== "cancelled" && isCancelledUploadRecord(request.result)) return; + store.put(record); + stored = true; + }; + request.onerror = () => reject(request.error || new Error("IndexedDB request failed")); + }); +} + +async function deleteUploadRecords(ids) { + const db = await openDb(); + await new Promise((resolve, reject) => { + const transaction = db.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + transaction.oncomplete = resolve; + transaction.onerror = () => reject(transaction.error || new Error("IndexedDB transaction failed")); + transaction.onabort = () => reject(transaction.error || new Error("IndexedDB transaction aborted")); + ids.forEach((id) => store.delete(id)); + }); +} + +function isAnyCancelledUploadRecord(record) { + return record?.status === "cancelled"; +} + +function isCancelledUploadRecord(record, now = Date.now()) { + if (!isAnyCancelledUploadRecord(record)) return false; + const cancelledUntil = Number(record.cancelledUntil || 0); + return !cancelledUntil || cancelledUntil > now; +} + +function isExpiredCancelledUploadRecord(record, now = Date.now()) { + if (!isAnyCancelledUploadRecord(record)) return false; + const cancelledUntil = Number(record.cancelledUntil || 0); + return Boolean(cancelledUntil && cancelledUntil <= now); } function openDb() { diff --git a/src/server/dropServer.js b/src/server/dropServer.js index 30b178b..dafb84d 100644 --- a/src/server/dropServer.js +++ b/src/server/dropServer.js @@ -6,6 +6,7 @@ const crypto = require("crypto"); const EventEmitter = require("events"); const fs = require("fs"); const http = require("http"); +const https = require("https"); const mime = require("mime-types"); const path = require("path"); const { Transform, pipeline } = require("stream"); @@ -16,8 +17,13 @@ const streamPipeline = promisify(pipeline); const fsp = fs.promises; const BASE_PATH = "/drop"; const DEFAULT_PORT = 41737; +// A second, direct HTTPS listener (HTTP/1.1) served with a Tailscale-issued cert. +// It lets phones open several parallel connections straight to this process for +// full-speed transfers, skipping the Tailscale Serve proxy, while staying a +// secure context. TLS uses AES-NI, so it's cheap — and Serve already did TLS. +const DEFAULT_HTTPS_PORT = 41843; const RETENTION_MS = 7 * 24 * 60 * 60 * 1000; -const UPLOAD_HIGH_WATER_MARK = 1024 * 1024; +const UPLOAD_HIGH_WATER_MARK = 4 * 1024 * 1024; const DOWNLOAD_HIGH_WATER_MARK = 8 * 1024 * 1024; const SSE_HEARTBEAT_MS = 25 * 1000; const DOWNLOAD_RECORD_DELAY_MS = 25; @@ -25,6 +31,12 @@ const DOWNLOAD_METADATA_DEBOUNCE_MS = 1000; // Backstop so a dropped connection mid-upload can't leave a shimmer placeholder // stuck on every device forever. const PENDING_UPLOAD_TIMEOUT_MS = 60 * 60 * 1000; +// A stalled chunked-upload assembly (some chunks never arrive) is torn down after +// this long so its temp file and placeholder don't linger. +const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; +// If a queued/background upload retries after the user has deleted its completed +// file, acknowledge the retry without recreating the deleted item. +const DELETED_UPLOAD_TOMBSTONE_MS = RETENTION_MS; class FileStore extends EventEmitter { constructor({ dataDir, defaultDownloadDir }) { @@ -34,15 +46,22 @@ class FileStore extends EventEmitter { this.tmpDir = path.join(dataDir, "tmp"); this.dragDir = path.join(dataDir, "drag"); this.metaPath = path.join(dataDir, "files.json"); + this.deletedUploadsPath = path.join(dataDir, "deleted-uploads.json"); this.settingsPath = path.join(dataDir, "settings.json"); this.defaultDownloadDir = defaultDownloadDir; this.filesSaveQueue = Promise.resolve(); + this.deletedUploadsSaveQueue = Promise.resolve(); this.pendingDownloadRecords = new Set(); this.pendingDownloadSaveTimer = null; // In-flight uploads, keyed by the id the finished file will receive, so every // connected device can show a live "loading" placeholder while bytes arrive. this.pendingUploads = new Map(); + // In-flight chunked uploads (parallel multi-connection transfers), keyed by + // the same id. Each session assembles positioned chunk writes into one temp + // file, then hashes + commits it exactly like a single-shot upload. + this.uploadSessions = new Map(); this.files = []; + this.deletedUploads = {}; this.settings = { downloadDir: defaultDownloadDir, retentionDays: 7, @@ -63,6 +82,7 @@ class FileStore extends EventEmitter { async load() { this.files = await readJson(this.metaPath, []); + this.deletedUploads = normalizeDeletedUploads(await readJson(this.deletedUploadsPath, {})); this.settings = { ...this.settings, ...(await readJson(this.settingsPath, {})), @@ -76,6 +96,13 @@ class FileStore extends EventEmitter { await this.filesSaveQueue; } + async saveDeletedUploads() { + this.deletedUploadsSaveQueue = this.deletedUploadsSaveQueue + .catch(() => {}) + .then(() => writeJsonAtomic(this.deletedUploadsPath, this.deletedUploads)); + await this.deletedUploadsSaveQueue; + } + async saveSettings() { await writeJsonAtomic(this.settingsPath, this.settings); } @@ -90,11 +117,13 @@ class FileStore extends EventEmitter { const now = Date.now(); return this.files .filter((file) => file.expiresAt > now) + .filter((file) => !file.folderId) .sort((a, b) => b.createdAt - a.createdAt) - .map((file) => toPublicFile(file)); + .map((file) => toPublicFile(file, this)); } addPendingUpload({ id, name, size }) { + if (this.isDeletedUpload(id)) return null; this.clearPendingUpload(id); const entry = { id, @@ -126,6 +155,7 @@ class FileStore extends EventEmitter { listPendingUploads() { return Array.from(this.pendingUploads.values()) + .filter((entry) => !this.isDeletedUpload(entry.id)) .sort((a, b) => b.createdAt - a.createdAt) .map((entry) => ({ id: entry.id, @@ -142,7 +172,67 @@ class FileStore extends EventEmitter { return file; } - async addFromTemp({ id, tempPath, originalName, mimeType, size, sha256 }) { + getFolder(id) { + const folder = this.get(id); + if (!folder || folder.kind !== "folder") return null; + return folder; + } + + getFolderChildren(folderId) { + const now = Date.now(); + return this.files + .filter((entry) => entry.folderId === folderId && entry.expiresAt > now) + .sort((a, b) => a.createdAt - b.createdAt); + } + + async createFolder({ name } = {}) { + const now = Date.now(); + const entry = { + id: crypto.randomUUID(), + kind: "folder", + name: safeDisplayName(name || this.nextFolderName()), + size: 0, + createdAt: now, + expiresAt: now + RETENTION_MS, + downloads: 0, + lastDownloadedAt: null, + }; + this.files.push(entry); + await this.saveFiles(); + this.notifyFilesChanged("folder-create"); + return toPublicFile(entry, this); + } + + async renameFolder(id, name) { + const folder = this.getFolder(id); + if (!folder) return null; + folder.name = safeDisplayName(name || folder.name); + await this.saveFiles(); + this.notifyFilesChanged("folder-rename"); + return toPublicFile(folder, this); + } + + nextFolderName() { + const used = new Set( + this.files + .filter((entry) => entry.kind === "folder" && entry.expiresAt > Date.now()) + .map((entry) => String(entry.name || "").toLowerCase()) + ); + for (let i = 1; i < 1000; i += 1) { + const name = `Folder${i}`; + if (!used.has(name.toLowerCase())) return name; + } + return `Folder${Date.now()}`; + } + + async addFromTemp({ id, tempPath, originalName, mimeType, size, sha256, folderId }) { + if (this.isDeletedUpload(id) || (folderId && this.isDeletedUpload(folderId))) { + await removeIfExists(tempPath); + this.clearPendingUpload(id); + this.notifyFilesChanged("upload-deleted"); + return null; + } + const folder = folderId ? this.getFolder(folderId) : null; const ext = safeExtension(originalName); const storedName = `${id}${ext}`; const finalPath = path.join(this.filesDir, storedName); @@ -156,17 +246,173 @@ class FileStore extends EventEmitter { size, sha256, createdAt: now, - expiresAt: now + RETENTION_MS, + expiresAt: folder ? folder.expiresAt : now + RETENTION_MS, downloads: 0, lastDownloadedAt: null, }; + if (folder) entry.folderId = folder.id; this.files.push(entry); // The finished file carries the same id as its placeholder, so the single // "upload" broadcast swaps shimmer for the real card everywhere at once. this.clearPendingUpload(id); await this.saveFiles(); this.notifyFilesChanged("upload"); - return toPublicFile(entry); + return toPublicFile(entry, this); + } + + isDeletedUpload(id) { + if (!isValidFileId(id)) return false; + const expiresAt = Number(this.deletedUploads[id] || 0); + if (!expiresAt) return false; + if (expiresAt > Date.now()) return true; + delete this.deletedUploads[id]; + this.saveDeletedUploads().catch((err) => console.error("Could not save deleted upload metadata", err)); + return false; + } + + async rememberDeletedUploads(ids) { + const now = Date.now(); + let changed = false; + for (const id of ids) { + if (!isValidFileId(id)) continue; + this.deletedUploads[id] = now + DELETED_UPLOAD_TOMBSTONE_MS; + changed = true; + } + changed = this.cleanupDeletedUploads(now) || changed; + if (changed) await this.saveDeletedUploads(); + } + + cleanupDeletedUploads(now = Date.now()) { + let changed = false; + for (const [id, expiresAt] of Object.entries(this.deletedUploads)) { + if (!isValidFileId(id) || Number(expiresAt || 0) <= now) { + delete this.deletedUploads[id]; + changed = true; + } + } + return changed; + } + + listDeletedUploadIds() { + if (this.cleanupDeletedUploads()) { + this.saveDeletedUploads().catch((err) => console.error("Could not save deleted upload metadata", err)); + } + return Object.keys(this.deletedUploads); + } + + async cancelInFlightUploads(ids, reason = "upload-deleted") { + const uniqueIds = Array.from(new Set(ids.filter(isValidFileId))); + let changed = false; + for (const id of uniqueIds) { + changed = this.clearPendingUpload(id) || changed; + const session = this.uploadSessions.get(id); + if (session) { + clearTimeout(session.timer); + this.uploadSessions.delete(id); + try { + await session.ready; + } catch {} + try { + await removeIfExists(session.tempPath); + } catch (err) { + console.error("Could not remove cancelled upload temp file", err); + } + changed = true; + } + } + if (changed) this.notifyFilesChanged(reason); + return changed; + } + + // --- Chunked (parallel) upload assembly --------------------------------- + // A large file is uploaded as many chunks over separate connections at once. + // Each chunk is written straight to its byte offset in a preallocated temp + // file, so the reassembled bytes are identical to the original — the finished + // file is hashed and committed through the very same addFromTemp path. + + getOrCreateUploadSession({ id, name, totalSize, mimeType, folderId }) { + let session = this.uploadSessions.get(id); + if (session) return session; + const tempPath = path.join(this.tmpDir, `${id}.chunked.upload`); + session = { + id, + name: safeDisplayName(name), + mimeType, + folderId: folderId || "", + totalSize, + tempPath, + received: new Map(), // offset -> bytes, so retried chunks aren't double-counted + receivedBytes: 0, + finalizing: false, + timer: null, + // Preallocate the file up front so every parallel chunk can open its own + // positioned write stream without racing to create it. + ready: (async () => { + const handle = await fsp.open(tempPath, "w"); + try { + if (totalSize > 0) await handle.truncate(totalSize); + } finally { + await handle.close(); + } + })(), + }; + session.timer = setTimeout(() => { + this.abortUploadSession(id, "upload-timeout").catch(() => {}); + }, UPLOAD_SESSION_TIMEOUT_MS); + session.timer.unref?.(); + this.uploadSessions.set(id, session); + // Same shimmer placeholder a single-shot upload shows. + this.addPendingUpload({ id, name, size: totalSize }); + return session; + } + + async writeUploadChunk({ id, name, mimeType, folderId, totalSize, offset, length, req }) { + const session = this.getOrCreateUploadSession({ id, name, totalSize, mimeType, folderId }); + await session.ready; + const written = await writeRequestAtOffset(session.tempPath, offset, req); + if (written !== length) { + throw new Error(`Chunk length mismatch (expected ${length}, wrote ${written})`); + } + if (!session.received.has(offset)) { + session.received.set(offset, written); + session.receivedBytes += written; + } + return session; + } + + async finalizeUploadSession({ id, originalName, mimeType }) { + const session = this.uploadSessions.get(id); + if (!session) return null; + clearTimeout(session.timer); + this.uploadSessions.delete(id); + const sha256 = await hashFile(session.tempPath); + return this.addFromTemp({ + id, + tempPath: session.tempPath, + originalName: originalName || session.name, + mimeType: mimeType || session.mimeType, + size: session.totalSize, + sha256, + folderId: session.folderId, + }); + } + + async abortUploadSession(id, reason = "upload-cancelled") { + const session = this.uploadSessions.get(id); + if (!session) return false; + clearTimeout(session.timer); + this.uploadSessions.delete(id); + try { + await session.ready; + } catch {} + await removeIfExists(session.tempPath); + this.removePendingUpload(id, reason); + return true; + } + + async abortAllUploadSessions(reason = "shutdown") { + const ids = Array.from(this.uploadSessions.keys()); + await Promise.allSettled(ids.map((id) => this.abortUploadSession(id, reason))); } async recordDownload(id) { @@ -225,27 +471,47 @@ class FileStore extends EventEmitter { const index = this.files.findIndex((entry) => entry.id === id); if (index === -1) return false; const [file] = this.files.splice(index, 1); - await removeIfExists(this.pathFor(file)); + const payloads = file.kind === "folder" ? this.files.filter((entry) => entry.folderId === file.id) : [file]; + const deletedIds = [file.id, ...payloads.map((entry) => entry.id)]; + if (file.kind === "folder") { + this.files = this.files.filter((entry) => entry.folderId !== file.id); + } + await Promise.all( + payloads.flatMap((entry) => [ + removeIfExists(this.pathFor(entry)), + removeDirIfExists(path.join(this.dragDir, entry.id)), + ]) + ); await removeDirIfExists(path.join(this.dragDir, file.id)); await this.saveFiles(); + await this.rememberDeletedUploads(deletedIds); + await this.cancelInFlightUploads(deletedIds); this.notifyFilesChanged("delete"); return true; } async clear() { const files = this.files; + const deletedIds = [ + ...files.map((file) => file.id), + ...Array.from(this.pendingUploads.keys()), + ...Array.from(this.uploadSessions.keys()), + ]; this.files = []; - await Promise.all(files.map((file) => removeIfExists(this.pathFor(file)))); + await Promise.all(files.filter((file) => file.kind !== "folder").map((file) => removeIfExists(this.pathFor(file)))); await removeDirIfExists(this.dragDir); await fsp.mkdir(this.dragDir, { recursive: true }); await this.saveFiles(); - if (files.length) this.notifyFilesChanged("clear"); - return files.length; + await this.rememberDeletedUploads(deletedIds); + await this.cancelInFlightUploads(deletedIds); + if (files.length || deletedIds.length) this.notifyFilesChanged("clear"); + return files.filter((file) => !file.folderId).length; } prepareDragFile(id) { const file = this.get(id); if (!file) throw new Error("File not found"); + if (file.kind === "folder") throw new Error("Drag folders from the saved ZIP."); const source = this.pathFor(file); const dragFileRoot = path.join(this.dragDir, file.id); @@ -274,23 +540,38 @@ class FileStore extends EventEmitter { if (!file) throw new Error("File not found"); const destinationDir = targetDir || this.settings.downloadDir || this.defaultDownloadDir; await fsp.mkdir(destinationDir, { recursive: true }); - const destination = await uniqueDestination(destinationDir, file.name); - await fsp.copyFile(this.pathFor(file), destination); + const destinationName = file.kind === "folder" ? ensureZipName(file.name) : file.name; + const destination = await uniqueDestination(destinationDir, destinationName); + if (file.kind === "folder") { + await writeFolderZipToPath(this, file, destination); + } else { + await fsp.copyFile(this.pathFor(file), destination); + } return destination; } async cleanupExpired() { const now = Date.now(); - const expired = this.files.filter((file) => file.expiresAt <= now); - if (!expired.length) return 0; - this.files = this.files.filter((file) => file.expiresAt > now); + const deletedUploadsChanged = this.cleanupDeletedUploads(now); + const expiredFolderIds = new Set( + this.files.filter((file) => file.kind === "folder" && file.expiresAt <= now).map((file) => file.id) + ); + const expired = this.files.filter((file) => file.expiresAt <= now || expiredFolderIds.has(file.folderId)); + if (!expired.length) { + if (deletedUploadsChanged) await this.saveDeletedUploads(); + return 0; + } + const expiredIds = new Set(expired.map((file) => file.id)); + this.files = this.files.filter((file) => !expiredIds.has(file.id)); await Promise.all( - expired.flatMap((file) => [ - removeIfExists(this.pathFor(file)), - removeDirIfExists(path.join(this.dragDir, file.id)), - ]) + expired.flatMap((file) => { + const tasks = [removeDirIfExists(path.join(this.dragDir, file.id))]; + if (file.kind !== "folder") tasks.push(removeIfExists(this.pathFor(file))); + return tasks; + }) ); await this.saveFiles(); + await this.rememberDeletedUploads(expired.map((file) => file.id)); this.notifyFilesChanged("expire"); return expired.length; } @@ -309,35 +590,84 @@ async function createDropServer({ dataDir, defaultDownloadDir, rendererDir, port await store.init(); const eventClients = new Set(); - const server = http.createServer((req, res) => { - handleRequest({ req, res, store, rendererDir, getPort: () => actualPort, eventClients }).catch((err) => { + let actualPort; + let actualHttpsPort = null; + let httpsServer = null; + + const requestListener = (req, res) => { + handleRequest({ + req, + res, + store, + rendererDir, + getPort: () => actualPort, + getHttpsPort: () => actualHttpsPort, + eventClients, + }).catch((err) => { if (err?.code === "UPLOAD_ABORTED") return; console.error(err); sendJson(res, 500, { error: "Internal server error" }); }); - }); + }; + + const server = http.createServer(requestListener); + + // Disable Nagle so the many small round-trips of parallel range/chunk transfers + // aren't delayed by TCP coalescing. + server.on("connection", (socket) => socket.setNoDelay(true)); + // Big transfers can legitimately hold a single request open for a long time on + // a slow link; the client's own stall detection handles truly dead uploads. + server.requestTimeout = 0; - let actualPort = await listenWithFallback(server, port); + actualPort = await listenWithFallback(server, port); const cleanupTimer = setInterval(() => { store.cleanupExpired().catch((err) => console.error("Cleanup failed", err)); }, 60 * 60 * 1000); cleanupTimer.unref(); + // Bring up (or hot-swap the cert of) the direct HTTPS listener. Plain HTTP/1.1 + // over TLS — deliberately not HTTP/2 — so browsers open several parallel + // connections and the chunked transfers actually run in parallel. + async function startHttps({ cert, key, preferredPort = DEFAULT_HTTPS_PORT }) { + if (!cert || !key) throw new Error("HTTPS needs a cert and key"); + if (httpsServer) { + httpsServer.setSecureContext({ cert, key }); + return actualHttpsPort; + } + const secure = https.createServer({ cert, key }, requestListener); + secure.on("secureConnection", (socket) => socket.setNoDelay(true)); + secure.requestTimeout = 0; + actualHttpsPort = await listenWithFallback(secure, preferredPort); + httpsServer = secure; + return actualHttpsPort; + } + return { port: actualPort, basePath: BASE_PATH, localUrl: `http://127.0.0.1:${actualPort}${BASE_PATH}/`, store, + startHttps, + getHttpsPort: () => actualHttpsPort, async close() { clearInterval(cleanupTimer); for (const client of eventClients) client.end(); - await new Promise((resolve) => server.close(resolve)); + await store.abortAllUploadSessions("shutdown"); + const closers = [new Promise((resolve) => server.close(resolve))]; + server.closeIdleConnections?.(); + if (httpsServer) { + closers.push(new Promise((resolve) => httpsServer.close(resolve))); + httpsServer.closeIdleConnections?.(); + } + // Don't wait out keep-alive timeouts on idle sockets; active transfers + // still get to finish. + await Promise.all(closers); await store.flushPendingDownloadRecords(); }, }; } -async function handleRequest({ req, res, store, rendererDir, getPort, eventClients }) { +async function handleRequest({ req, res, store, rendererDir, getPort, getHttpsPort, eventClients }) { if (!isAllowedRemote(req.socket.remoteAddress)) { sendJson(res, 403, { error: "WaterDrop only accepts loopback or Tailscale clients." }); return; @@ -358,14 +688,14 @@ async function handleRequest({ req, res, store, rendererDir, getPort, eventClien } if (pathname.startsWith("/api/")) { - await handleApi({ req, res, url, relative: pathname, store, getPort, eventClients }); + await handleApi({ req, res, url, relative: pathname, store, getPort, getHttpsPort, eventClients }); return; } if (pathname.startsWith(`${BASE_PATH}/`)) { const relative = pathname.slice(BASE_PATH.length); if (relative.startsWith("/api/")) { - await handleApi({ req, res, url, relative, store, getPort, eventClients }); + await handleApi({ req, res, url, relative, store, getPort, getHttpsPort, eventClients }); return; } await serveStatic({ req, res, pathname: relative, rendererDir }); @@ -380,13 +710,13 @@ async function handleRequest({ req, res, store, rendererDir, getPort, eventClien sendJson(res, 404, { error: "Not found" }); } -async function handleApi({ req, res, relative, store, getPort, eventClients }) { +async function handleApi({ req, res, relative, store, getPort, getHttpsPort, eventClients }) { if (relative.startsWith(BASE_PATH + "/api/")) { relative = relative.slice(BASE_PATH.length); } if (req.method === "GET" && relative === "/api/info") { - const network = await tailscale.inspect(getPort()); + const network = await tailscale.inspect(getPort(), { httpsPort: getHttpsPort?.() }); const qrSvg = await QRCode.toString(network.preferredUrl, { type: "svg", margin: 1, @@ -402,6 +732,13 @@ async function handleApi({ req, res, relative, store, getPort, eventClients }) { return; } + if ((req.method === "GET" || req.method === "HEAD") && relative === "/api/ping") { + // Cheap, side-effect-free reachability probe (used to test the direct HTTPS + // origin before routing bulk transfers through it). + sendJson(res, 200, { ok: true }); + return; + } + if (req.method === "POST" && relative === "/api/tailscale/serve") { const result = await tailscale.configureServe(getPort()); sendJson(res, result.ok ? 200 : 500, result); @@ -410,7 +747,12 @@ async function handleApi({ req, res, relative, store, getPort, eventClients }) { if (req.method === "GET" && relative === "/api/files") { await store.cleanupExpired(); - sendJson(res, 200, { files: store.list(), pending: store.listPendingUploads(), settings: store.settings }); + sendJson(res, 200, { + files: store.list(), + pending: store.listPendingUploads(), + deletedUploads: store.listDeletedUploadIds(), + settings: store.settings, + }); return; } @@ -429,12 +771,30 @@ async function handleApi({ req, res, relative, store, getPort, eventClients }) { return; } + if (req.method === "POST" && relative === "/api/folders") { + const body = await readRequestJson(req); + const folder = await store.createFolder({ name: body.name }); + sendJson(res, 201, { folder }); + return; + } + if (req.method === "DELETE" && relative === "/api/files") { const deleted = await store.clear(); sendJson(res, 200, { ok: true, deleted }); return; } + const folderMatch = relative.match(/^\/api\/folders\/([a-f0-9-]+)$/); + if (folderMatch) { + const id = folderMatch[1]; + if (req.method === "PATCH") { + const body = await readRequestJson(req); + const folder = await store.renameFolder(id, body.name); + sendJson(res, folder ? 200 : 404, folder ? { folder } : { error: "Folder not found" }); + return; + } + } + const fileMatch = relative.match(/^\/api\/files\/([a-f0-9-]+)(?:\/(download|preview))?$/); if (fileMatch) { const id = fileMatch[1]; @@ -461,11 +821,40 @@ async function handleRawUpload(req, res, store) { const originalName = decodeHeaderValue(req.headers["x-waterdrop-file-name"]) || "unnamed-file"; const mimeType = cleanHeaderValue(req.headers["x-waterdrop-mime-type"]) || cleanMimeType(req.headers["content-type"]); const requestedId = cleanHeaderValue(req.headers["x-waterdrop-upload-id"]); + const requestedFolderId = cleanHeaderValue(req.headers["x-waterdrop-folder-id"]); + const folderId = isValidFileId(requestedFolderId) ? requestedFolderId : ""; const id = isValidFileId(requestedId) ? requestedId : crypto.randomUUID(); + if (store.isDeletedUpload(id) || (folderId && store.isDeletedUpload(folderId))) { + req.resume(); + sendJson(res, 200, { files: [], deleted: true, duplicate: true }); + return; + } + if (folderId && !store.getFolder(folderId)) { + req.resume(); + sendJson(res, 404, { error: "Folder not found" }); + return; + } const existingFile = store.get(id); if (existingFile) { req.resume(); - sendJson(res, 200, { files: [toPublicFile(existingFile)], duplicate: true }); + sendJson(res, 200, { files: [toPublicFile(existingFile, store)], duplicate: true }); + return; + } + + // Parallel multi-connection upload: each request carries one byte-range chunk. + // The whole-file path below stays the fallback for small files and clients that + // don't chunk. + const chunkOffsetHeader = cleanHeaderValue(req.headers["x-waterdrop-chunk-offset"]); + const totalSizeHeader = cleanHeaderValue(req.headers["x-waterdrop-total-size"]); + if (chunkOffsetHeader !== "" && totalSizeHeader !== "") { + await handleChunkUpload(req, res, store, { + id, + originalName, + mimeType, + folderId, + chunkOffsetHeader, + totalSizeHeader, + }); return; } @@ -494,7 +883,7 @@ async function handleRawUpload(req, res, store) { const finishedElsewhere = store.get(id); if (finishedElsewhere) { await removeIfExists(tempPath); - sendJson(res, 200, { files: [toPublicFile(finishedElsewhere)], duplicate: true }); + sendJson(res, 200, { files: [toPublicFile(finishedElsewhere, store)], duplicate: true }); return; } const file = await store.addFromTemp({ @@ -504,7 +893,12 @@ async function handleRawUpload(req, res, store) { mimeType, size, sha256: hash.digest("hex"), + folderId, }); + if (!file) { + sendJson(res, 200, { files: [], deleted: true, duplicate: true }); + return; + } sendJson(res, 201, { files: [file] }); } catch (err) { store.removePendingUpload(id, "upload-failed"); @@ -517,6 +911,64 @@ async function handleRawUpload(req, res, store) { } } +async function handleChunkUpload(req, res, store, { id, originalName, mimeType, folderId, chunkOffsetHeader, totalSizeHeader }) { + const offset = Number(chunkOffsetHeader); + const totalSize = Number(totalSizeHeader); + const length = Number(cleanHeaderValue(req.headers["content-length"])); + if ( + !Number.isInteger(offset) || offset < 0 || + !Number.isInteger(totalSize) || totalSize <= 0 || + !Number.isInteger(length) || length <= 0 || + offset + length > totalSize + ) { + req.resume(); + sendJson(res, 400, { error: "Invalid upload chunk" }); + return; + } + + try { + const session = await store.writeUploadChunk({ + id, + name: originalName, + mimeType, + folderId, + totalSize, + offset, + length, + req, + }); + + // The chunk whose write completes the file finalizes it. JS is single + // threaded, so exactly one chunk sees the byte count reach the total with + // finalizing still false. + if (session.receivedBytes >= session.totalSize && !session.finalizing) { + session.finalizing = true; + try { + const file = await store.finalizeUploadSession({ id, originalName, mimeType }); + if (!file) { + sendJson(res, 200, { files: [], deleted: true, duplicate: true }); + return; + } + sendJson(res, 201, { files: [file] }); + } catch (err) { + session.finalizing = false; + await store.abortUploadSession(id, "upload-failed"); + sendJson(res, 500, { error: `Upload failed: ${err.message}` }); + } + return; + } + + sendJson(res, 200, { ok: true, received: session.receivedBytes, total: session.totalSize }); + } catch (err) { + if (req.aborted) { + err.code = "UPLOAD_ABORTED"; + throw err; + } + // One failed chunk doesn't kill the file — the client just retries it. + sendJson(res, 500, { error: `Chunk failed: ${err.message}` }); + } +} + async function handleUpload(req, res, store) { const contentType = req.headers["content-type"] || ""; if (!contentType.includes("multipart/form-data")) { @@ -611,6 +1063,15 @@ async function handleUpload(req, res, store) { } async function handleDownload(req, res, store, id) { + const file = store.get(id); + if (file?.kind === "folder") { + await sendFolderZip({ req, res, store, folder: file }); + return; + } + // An accelerated download issues many parallel range requests for one file; + // it tags every request but the first with X-WaterDrop-No-Count so the file + // still registers as a single logical download. + const noCount = cleanHeaderValue(req.headers["x-waterdrop-no-count"]); await sendStoredFile({ req, res, @@ -618,11 +1079,16 @@ async function handleDownload(req, res, store, id) { id, disposition: "attachment", cacheControl: "private, max-age=0, no-transform", - recordDownload: true, + recordDownload: !noCount, }); } async function handlePreview(req, res, store, id) { + const file = store.get(id); + if (file?.kind === "folder") { + sendJson(res, 415, { error: "Folders can be downloaded as ZIP files." }); + return; + } await sendStoredFile({ req, res, @@ -634,6 +1100,28 @@ async function handlePreview(req, res, store, id) { }); } +async function sendFolderZip({ req, res, store, folder }) { + const entries = await buildZipEntries(store, folder); + const encodedName = encodeRFC5987(ensureZipName(folder.name)); + res.writeHead(200, { + ...corsHeaders(), + "Cache-Control": "private, max-age=0, no-transform", + "Content-Disposition": `attachment; filename="${asciiFileName(ensureZipName(folder.name))}"; filename*=UTF-8''${encodedName}`, + "Content-Type": "application/zip", + "X-Content-Type-Options": "nosniff", + }); + if (req.method === "HEAD") { + res.end(); + return; + } + recordDownloadAfterResponse(res, store, folder.id); + try { + await writeZipToStream(entries, res); + } catch (err) { + if (!res.destroyed) res.destroy(err); + } +} + async function sendStoredFile({ req, res, store, id, disposition, cacheControl, recordDownload }) { const file = store.get(id); if (!file) { @@ -710,12 +1198,185 @@ function pipeFileToResponse(res, filePath, options) { fileStream.pipe(res); } +// Stream one upload chunk straight to its byte offset in the (preallocated) temp +// file. Each chunk uses an independent positioned write stream, so many can run +// at once without disturbing each other's file position. +function writeRequestAtOffset(tempPath, offset, req) { + let bytes = 0; + const counter = new Transform({ + transform(chunk, _encoding, callback) { + bytes += chunk.length; + callback(null, chunk); + }, + }); + return streamPipeline( + req, + counter, + fs.createWriteStream(tempPath, { flags: "r+", start: offset, highWaterMark: UPLOAD_HIGH_WATER_MARK }) + ).then(() => bytes); +} + +// SHA-256 the reassembled file. Chunks arrive out of order, so the hash is taken +// once at the end over the finished bytes — identical to hashing the original. +function hashFile(filePath) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash("sha256"); + const stream = fs.createReadStream(filePath, { highWaterMark: DOWNLOAD_HIGH_WATER_MARK }); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("error", reject); + stream.on("end", () => resolve(hash.digest("hex"))); + }); +} + function recordDownloadAfterResponse(res, store, id) { res.once("finish", () => { store.scheduleDownloadRecord(id); }); } +async function writeFolderZipToPath(store, folder, destination) { + const entries = await buildZipEntries(store, folder); + const output = fs.createWriteStream(destination, { highWaterMark: DOWNLOAD_HIGH_WATER_MARK }); + try { + await writeZipToStream(entries, output); + } catch (err) { + output.destroy(); + await removeIfExists(destination); + throw err; + } +} + +async function buildZipEntries(store, folder) { + const children = store.getFolderChildren(folder.id); + const usedNames = new Set(); + const entries = []; + for (const child of children) { + const filePath = store.pathFor(child); + const stat = await fsp.stat(filePath); + if (stat.size > 0xffffffff) throw new Error("ZIP download does not support files over 4 GB"); + const name = uniqueArchiveName(usedNames, safeDisplayName(child.name)); + entries.push({ + name, + path: filePath, + size: stat.size, + crc32: await crc32File(filePath), + modifiedAt: child.createdAt || Date.now(), + }); + } + return entries; +} + +async function writeZipToStream(entries, stream) { + if (entries.length > 0xffff) throw new Error("ZIP download does not support more than 65535 files"); + let offset = 0; + const centralRecords = []; + + for (const entry of entries) { + const nameBuffer = Buffer.from(entry.name, "utf8"); + const { time, date } = dosDateTime(entry.modifiedAt); + const localOffset = offset; + const localHeader = Buffer.alloc(30); + localHeader.writeUInt32LE(0x04034b50, 0); + localHeader.writeUInt16LE(20, 4); + localHeader.writeUInt16LE(0, 6); + localHeader.writeUInt16LE(0, 8); + localHeader.writeUInt16LE(time, 10); + localHeader.writeUInt16LE(date, 12); + localHeader.writeUInt32LE(entry.crc32 >>> 0, 14); + localHeader.writeUInt32LE(entry.size, 18); + localHeader.writeUInt32LE(entry.size, 22); + localHeader.writeUInt16LE(nameBuffer.length, 26); + localHeader.writeUInt16LE(0, 28); + await writeBuffer(stream, localHeader); + await writeBuffer(stream, nameBuffer); + offset += localHeader.length + nameBuffer.length; + + for await (const chunk of fs.createReadStream(entry.path, { highWaterMark: DOWNLOAD_HIGH_WATER_MARK })) { + await writeBuffer(stream, chunk); + offset += chunk.length; + } + + centralRecords.push({ entry, nameBuffer, time, date, localOffset }); + } + + const centralStart = offset; + for (const record of centralRecords) { + const header = Buffer.alloc(46); + header.writeUInt32LE(0x02014b50, 0); + header.writeUInt16LE(20, 4); + header.writeUInt16LE(20, 6); + header.writeUInt16LE(0, 8); + header.writeUInt16LE(0, 10); + header.writeUInt16LE(record.time, 12); + header.writeUInt16LE(record.date, 14); + header.writeUInt32LE(record.entry.crc32 >>> 0, 16); + header.writeUInt32LE(record.entry.size, 20); + header.writeUInt32LE(record.entry.size, 24); + header.writeUInt16LE(record.nameBuffer.length, 28); + header.writeUInt16LE(0, 30); + header.writeUInt16LE(0, 32); + header.writeUInt16LE(0, 34); + header.writeUInt16LE(0, 36); + header.writeUInt32LE(0, 38); + header.writeUInt32LE(record.localOffset, 42); + await writeBuffer(stream, header); + await writeBuffer(stream, record.nameBuffer); + offset += header.length + record.nameBuffer.length; + } + + const centralSize = offset - centralStart; + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); + end.writeUInt16LE(0, 6); + end.writeUInt16LE(centralRecords.length, 8); + end.writeUInt16LE(centralRecords.length, 10); + end.writeUInt32LE(centralSize, 12); + end.writeUInt32LE(centralStart, 16); + end.writeUInt16LE(0, 20); + await writeBuffer(stream, end); + await endStream(stream); +} + +function writeBuffer(stream, buffer) { + return new Promise((resolve, reject) => { + if (stream.destroyed) { + reject(new Error("Stream closed")); + return; + } + const onError = (err) => { + cleanup(); + reject(err); + }; + const onDrain = () => { + cleanup(); + resolve(); + }; + const cleanup = () => { + stream.off("error", onError); + stream.off("drain", onDrain); + }; + stream.once("error", onError); + if (stream.write(buffer)) { + cleanup(); + resolve(); + return; + } + stream.once("drain", onDrain); + }); +} + +function endStream(stream) { + return new Promise((resolve, reject) => { + if (stream.writableEnded || stream.destroyed) { + resolve(); + return; + } + stream.once("error", reject); + stream.end(resolve); + }); +} + async function serveStatic({ req, res, pathname, rendererDir }) { if (req.method !== "GET" && req.method !== "HEAD") { sendJson(res, 405, { error: "Method not allowed" }); @@ -868,9 +1529,28 @@ function isValidFileId(id) { return typeof id === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id); } -function toPublicFile(file) { +function toPublicFile(file, store = null) { + if (file.kind === "folder") { + const children = store ? store.getFolderChildren(file.id) : []; + const size = children.reduce((sum, child) => sum + Number(child.size || 0), 0); + return { + id: file.id, + kind: "folder", + name: file.name, + mimeType: "application/zip", + size, + itemCount: children.length, + files: children.map((child) => toPublicFolderChild(child)), + createdAt: file.createdAt, + expiresAt: file.expiresAt, + expiresInMs: Math.max(0, file.expiresAt - Date.now()), + downloads: file.downloads || 0, + lastDownloadedAt: file.lastDownloadedAt || null, + }; + } return { id: file.id, + kind: "file", name: file.name, mimeType: file.mimeType, size: file.size, @@ -883,6 +1563,19 @@ function toPublicFile(file) { }; } +function toPublicFolderChild(file) { + return { + id: file.id, + kind: "file", + name: file.name, + mimeType: file.mimeType, + size: file.size, + sha256: file.sha256, + createdAt: file.createdAt, + expiresAt: file.expiresAt, + }; +} + function sendJson(res, status, payload) { const body = Buffer.from(JSON.stringify(payload)); res.writeHead(status, { @@ -896,9 +1589,12 @@ function sendJson(res, status, payload) { function corsHeaders() { return { - "Access-Control-Allow-Headers": "Content-Type, Range, X-WaterDrop-File-Name, X-WaterDrop-Mime-Type, X-WaterDrop-Upload-Id", - "Access-Control-Allow-Methods": "GET, HEAD, POST, DELETE, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, Range, X-WaterDrop-File-Name, X-WaterDrop-Mime-Type, X-WaterDrop-Upload-Id, X-WaterDrop-Folder-Id, X-WaterDrop-Chunk-Offset, X-WaterDrop-Total-Size, X-WaterDrop-No-Count", + "Access-Control-Allow-Methods": "GET, HEAD, POST, PATCH, DELETE, OPTIONS", "Access-Control-Allow-Origin": "*", + // Cache preflights so cross-origin chunk/range transfers to the direct HTTPS + // origin don't pay an OPTIONS round-trip per request. + "Access-Control-Max-Age": "600", "Access-Control-Expose-Headers": "Accept-Ranges, Content-Disposition, Content-Length, Content-Range, ETag, Last-Modified, X-WaterDrop-SHA256", }; } @@ -916,6 +1612,28 @@ async function readJson(filePath, fallback) { } } +function normalizeDeletedUploads(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + const normalized = {}; + for (const [id, expiresAt] of Object.entries(value)) { + if (isValidFileId(id) && Number(expiresAt || 0) > Date.now()) { + normalized[id] = Number(expiresAt); + } + } + return normalized; +} + +async function readRequestJson(req) { + const chunks = []; + for await (const chunk of req) { + chunks.push(chunk); + if (Buffer.concat(chunks).length > 64 * 1024) throw new Error("Request body too large"); + } + const text = Buffer.concat(chunks).toString("utf8").trim(); + if (!text) return {}; + return JSON.parse(text); +} + async function writeJsonAtomic(filePath, value) { const tmpPath = `${filePath}.${process.pid}.${crypto.randomUUID()}.tmp`; await fsp.mkdir(path.dirname(filePath), { recursive: true }); @@ -945,6 +1663,62 @@ function safeExtension(name) { return ext.replace(/[^a-zA-Z0-9.]/g, ""); } +function ensureZipName(name) { + const safeName = safeDisplayName(name || "folder"); + return /\.zip$/i.test(safeName) ? safeName : `${safeName}.zip`; +} + +function uniqueArchiveName(usedNames, name) { + const safeName = safeDisplayName(name); + const parsed = path.parse(safeName); + for (let i = 0; i < 1000; i += 1) { + const candidate = i === 0 ? safeName : `${parsed.name} (${i})${parsed.ext}`; + const key = candidate.toLowerCase(); + if (!usedNames.has(key)) { + usedNames.add(key); + return candidate; + } + } + throw new Error("Could not find a unique ZIP entry name"); +} + +async function crc32File(filePath) { + let crc = 0xffffffff; + for await (const chunk of fs.createReadStream(filePath, { highWaterMark: DOWNLOAD_HIGH_WATER_MARK })) { + crc = crc32Update(crc, chunk); + } + return (crc ^ 0xffffffff) >>> 0; +} + +const CRC32_TABLE = (() => { + const table = new Uint32Array(256); + for (let i = 0; i < 256; i += 1) { + let value = i; + for (let bit = 0; bit < 8; bit += 1) { + value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1; + } + table[i] = value >>> 0; + } + return table; +})(); + +function crc32Update(crc, buffer) { + let next = crc >>> 0; + for (const byte of buffer) { + next = CRC32_TABLE[(next ^ byte) & 0xff] ^ (next >>> 8); + } + return next >>> 0; +} + +function dosDateTime(timestamp) { + const date = new Date(timestamp || Date.now()); + const year = Math.max(1980, date.getFullYear()); + return { + time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2), + date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(), + }; +} + function asciiFileName(name) { return safeDisplayName(name).replace(/[^\x20-\x7e]/g, "_").replace(/"/g, "'"); } diff --git a/test/dropServer.test.js b/test/dropServer.test.js index 8dfef23..86d1c6e 100644 --- a/test/dropServer.test.js +++ b/test/dropServer.test.js @@ -209,6 +209,312 @@ test("pending upload placeholder clears once the file lands", async () => { } }); +test("deleted uploads are not recreated by stale queued retries", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "waterdrop-deleted-retry-test-")); + const rendererDir = path.join(root, "renderer"); + const dataDir = path.join(root, "data"); + const downloads = path.join(root, "downloads"); + await fs.mkdir(rendererDir, { recursive: true }); + await fs.writeFile(path.join(rendererDir, "index.html"), "WaterDrop"); + + let server = await createDropServer({ dataDir, defaultDownloadDir: downloads, rendererDir, port: 48020 }); + try { + const id = "55555555-5555-4555-8555-555555555555"; + const upload = await fetch(new URL("api/files/raw", server.localUrl), { + method: "POST", + body: new Blob(["original"], { type: "text/plain" }), + headers: { + "Content-Type": "text/plain", + "X-WaterDrop-File-Name": encodeURIComponent("retry.txt"), + "X-WaterDrop-Mime-Type": "text/plain", + "X-WaterDrop-Upload-Id": id, + }, + }); + assert.equal(upload.status, 201); + + const deleted = await fetch(new URL(`api/files/${id}`, server.localUrl), { method: "DELETE" }); + assert.equal(deleted.status, 200); + + const staleRetry = await fetch(new URL("api/files/raw", server.localUrl), { + method: "POST", + body: new Blob(["stale retry"], { type: "text/plain" }), + headers: { + "Content-Type": "text/plain", + "X-WaterDrop-File-Name": encodeURIComponent("retry.txt"), + "X-WaterDrop-Mime-Type": "text/plain", + "X-WaterDrop-Upload-Id": id, + }, + }); + assert.equal(staleRetry.status, 200); + const staleRetryBody = await staleRetry.json(); + assert.equal(staleRetryBody.deleted, true); + assert.equal(staleRetryBody.files.length, 0); + + const afterRetry = await (await fetch(new URL("api/files", server.localUrl))).json(); + assert.equal(afterRetry.files.length, 0); + assert.equal(afterRetry.pending.length, 0); + assert.equal(afterRetry.deletedUploads.includes(id), true); + + await server.close(); + server = null; + server = await createDropServer({ dataDir, defaultDownloadDir: downloads, rendererDir, port: 48020 }); + + const staleRetryAfterRestart = await fetch(new URL("api/files/raw", server.localUrl), { + method: "POST", + body: new Blob(["stale retry after restart"], { type: "text/plain" }), + headers: { + "Content-Type": "text/plain", + "X-WaterDrop-File-Name": encodeURIComponent("retry.txt"), + "X-WaterDrop-Mime-Type": "text/plain", + "X-WaterDrop-Upload-Id": id, + }, + }); + assert.equal(staleRetryAfterRestart.status, 200); + const staleRetryAfterRestartBody = await staleRetryAfterRestart.json(); + assert.equal(staleRetryAfterRestartBody.deleted, true); + assert.equal(staleRetryAfterRestartBody.files.length, 0); + + const afterRestartRetry = await (await fetch(new URL("api/files", server.localUrl))).json(); + assert.equal(afterRestartRetry.files.length, 0); + assert.equal(afterRestartRetry.pending.length, 0); + assert.equal(afterRestartRetry.deletedUploads.includes(id), true); + } finally { + await server?.close(); + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("deleting a file clears matching in-flight upload placeholders", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "waterdrop-delete-pending-test-")); + const rendererDir = path.join(root, "renderer"); + const dataDir = path.join(root, "data"); + const downloads = path.join(root, "downloads"); + await fs.mkdir(rendererDir, { recursive: true }); + await fs.writeFile(path.join(rendererDir, "index.html"), "WaterDrop"); + + const server = await createDropServer({ dataDir, defaultDownloadDir: downloads, rendererDir, port: 48030 }); + try { + const id = "66666666-6666-4666-8666-666666666666"; + const upload = await fetch(new URL("api/files/raw", server.localUrl), { + method: "POST", + body: new Blob(["done"], { type: "text/plain" }), + headers: { + "Content-Type": "text/plain", + "X-WaterDrop-File-Name": encodeURIComponent("done.txt"), + "X-WaterDrop-Mime-Type": "text/plain", + "X-WaterDrop-Upload-Id": id, + }, + }); + assert.equal(upload.status, 201); + + server.store.addPendingUpload({ id, name: "done.txt", size: 4096 }); + const beforeDelete = await (await fetch(new URL("api/files", server.localUrl))).json(); + assert.equal(beforeDelete.files.length, 1); + assert.equal(beforeDelete.pending.length, 1); + + const deleted = await fetch(new URL(`api/files/${id}`, server.localUrl), { method: "DELETE" }); + assert.equal(deleted.status, 200); + + const afterDelete = await (await fetch(new URL("api/files", server.localUrl))).json(); + assert.equal(afterDelete.files.length, 0); + assert.equal(afterDelete.pending.length, 0); + assert.equal(afterDelete.deletedUploads.includes(id), true); + } finally { + await server.close(); + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("bulk upload folders group files, rename, download as zip, and delete children", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "waterdrop-folder-test-")); + const rendererDir = path.join(root, "renderer"); + const dataDir = path.join(root, "data"); + const downloads = path.join(root, "downloads"); + await fs.mkdir(rendererDir, { recursive: true }); + await fs.writeFile(path.join(rendererDir, "index.html"), "WaterDrop"); + + const server = await createDropServer({ dataDir, defaultDownloadDir: downloads, rendererDir, port: 48010 }); + try { + const base = server.localUrl; + const created = await fetch(new URL("api/folders", base), { + method: "POST", + body: JSON.stringify({}), + headers: { "Content-Type": "application/json" }, + }); + assert.equal(created.status, 201); + const createdBody = await created.json(); + assert.equal(createdBody.folder.kind, "folder"); + assert.equal(createdBody.folder.name, "Folder1"); + assert.equal(createdBody.folder.itemCount, 0); + const folderId = createdBody.folder.id; + + for (const [id, name, body] of [ + ["33333333-3333-4333-8333-333333333333", "one.txt", "one"], + ["44444444-4444-4444-8444-444444444444", "two.txt", "two"], + ]) { + const upload = await fetch(new URL("api/files/raw", base), { + method: "POST", + body: new Blob([body], { type: "text/plain" }), + headers: { + "Content-Type": "text/plain", + "X-WaterDrop-File-Name": encodeURIComponent(name), + "X-WaterDrop-Mime-Type": "text/plain", + "X-WaterDrop-Upload-Id": id, + "X-WaterDrop-Folder-Id": folderId, + }, + }); + assert.equal(upload.status, 201); + } + + const listed = await (await fetch(new URL("api/files", base))).json(); + assert.equal(listed.files.length, 1); + assert.equal(listed.files[0].kind, "folder"); + assert.equal(listed.files[0].itemCount, 2); + assert.equal(listed.files[0].size, 6); + assert.deepEqual(listed.files[0].files.map((file) => file.name), ["one.txt", "two.txt"]); + + const renamed = await fetch(new URL(`api/folders/${folderId}`, base), { + method: "PATCH", + body: JSON.stringify({ name: "Trip Photos" }), + headers: { "Content-Type": "application/json" }, + }); + assert.equal(renamed.status, 200); + const renamedBody = await renamed.json(); + assert.equal(renamedBody.folder.name, "Trip Photos"); + + const zip = await fetch(new URL(`api/files/${folderId}/download`, base)); + assert.equal(zip.status, 200); + assert.equal(zip.headers.get("content-type"), "application/zip"); + assert.match(zip.headers.get("content-disposition"), /^attachment; filename="Trip Photos\.zip"/); + const zipBytes = Buffer.from(await zip.arrayBuffer()); + assert.equal(zipBytes.readUInt32LE(0), 0x04034b50); + assert.equal(zipBytes.includes(Buffer.from("one.txt")), true); + assert.equal(zipBytes.includes(Buffer.from("two.txt")), true); + await waitForDownloads(base, folderId, 1); + + const savedZip = await server.store.copyToDownloadDir(folderId); + assert.equal(path.basename(savedZip), "Trip Photos.zip"); + const savedZipBytes = await fs.readFile(savedZip); + assert.equal(savedZipBytes.readUInt32LE(0), 0x04034b50); + + const deleted = await fetch(new URL(`api/files/${folderId}`, base), { method: "DELETE" }); + assert.equal(deleted.status, 200); + const afterDelete = await (await fetch(new URL("api/files", base))).json(); + assert.equal(afterDelete.files.length, 0); + assert.equal(server.store.files.length, 0); + } finally { + await server.close(); + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("assembles a parallel chunked upload into byte-identical bytes", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "waterdrop-chunk-test-")); + const rendererDir = path.join(root, "renderer"); + const dataDir = path.join(root, "data"); + const downloads = path.join(root, "downloads"); + await fs.mkdir(rendererDir, { recursive: true }); + await fs.writeFile(path.join(rendererDir, "index.html"), "WaterDrop"); + + const server = await createDropServer({ dataDir, defaultDownloadDir: downloads, rendererDir, port: 47960 }); + try { + const base = server.localUrl; + const payload = crypto.randomBytes(200000); + const expectedHash = sha256(payload); + const id = "33333333-3333-4333-8333-333333333333"; + const chunkSize = 16384; + + const offsets = []; + for (let start = 0; start < payload.length; start += chunkSize) offsets.push(start); + + const sendChunk = (start) => { + const end = Math.min(payload.length, start + chunkSize); + return fetch(new URL("api/files/raw", base), { + method: "POST", + body: payload.subarray(start, end), + headers: { + "Content-Type": "application/octet-stream", + "X-WaterDrop-File-Name": encodeURIComponent("blob.bin"), + "X-WaterDrop-Mime-Type": "application/octet-stream", + "X-WaterDrop-Upload-Id": id, + "X-WaterDrop-Chunk-Offset": String(start), + "X-WaterDrop-Total-Size": String(payload.length), + }, + }); + }; + + // Upload out of order and concurrently, exactly like the parallel client. + const responses = await Promise.all([...offsets].reverse().map(sendChunk)); + // A retried chunk after completion must resolve as a duplicate, not corrupt. + const retry = await (await sendChunk(offsets[0])).json(); + assert.equal(retry.duplicate, true); + + const finals = []; + for (const res of responses) { + const body = await res.json(); + if (res.status === 201 && body.files) finals.push(body.files[0]); + } + assert.equal(finals.length, 1); + assert.equal(finals[0].id, id); + assert.equal(finals[0].size, payload.length); + assert.equal(finals[0].sha256, expectedHash); + + const download = await fetch(new URL(`api/files/${id}/download`, base)); + const bytes = Buffer.from(await download.arrayBuffer()); + assert.equal(bytes.length, payload.length); + assert.equal(sha256(bytes), expectedHash); + assert.equal(bytes.equals(payload), true); + } finally { + await server.close(); + await fs.rm(root, { recursive: true, force: true }); + } +}); + +test("no-count range requests keep an accelerated download as one download", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "waterdrop-nocount-test-")); + const rendererDir = path.join(root, "renderer"); + const dataDir = path.join(root, "data"); + const downloads = path.join(root, "downloads"); + await fs.mkdir(rendererDir, { recursive: true }); + await fs.writeFile(path.join(rendererDir, "index.html"), "WaterDrop"); + + const server = await createDropServer({ dataDir, defaultDownloadDir: downloads, rendererDir, port: 47970 }); + try { + const base = server.localUrl; + const id = "44444444-4444-4444-8444-444444444444"; + const upload = await fetch(new URL("api/files/raw", base), { + method: "POST", + body: new Blob(["hello-range"]), + headers: { + "Content-Type": "text/plain", + "X-WaterDrop-File-Name": encodeURIComponent("r.txt"), + "X-WaterDrop-Mime-Type": "text/plain", + "X-WaterDrop-Upload-Id": id, + }, + }); + assert.equal(upload.status, 201); + + // Suppressed range parts (every part but the first of an accelerated download). + const suppressed = await fetch(new URL(`api/files/${id}/download`, base), { + headers: { Range: "bytes=0-1", "X-WaterDrop-No-Count": "1" }, + }); + assert.equal(suppressed.status, 206); + // The one counted part. + const counted = await fetch(new URL(`api/files/${id}/download`, base), { + headers: { Range: "bytes=2-3" }, + }); + assert.equal(counted.status, 206); + + await waitForDownloads(base, id, 1); + const listed = await (await fetch(new URL("api/files", base))).json(); + const file = listed.files.find((entry) => entry.id === id); + assert.equal(file.downloads, 1); + } finally { + await server.close(); + await fs.rm(root, { recursive: true, force: true }); + } +}); + function sha256(buffer) { return crypto.createHash("sha256").update(buffer).digest("hex"); }