jakob/PairDrop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

revert zipping and unzipping files on transfer to minimize needed browser memory. Use fileQueue instead.

Browse source
This commit is contained in:
schlagmichdoch 2023-01-27 01:27:22 +01:00
parent 1278009706
commit d35c27aa91
3 changed files with 169 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -28,8 +28,6 @@ Developed based on [Snapdrop](https://github.com/RobinLinus/snapdrop)
* Multiple files are downloaded as ZIP file * Multiple files are downloaded as ZIP file
* On iOS and Android the devices share menu is opened instead of downloading the files * On iOS and Android the devices share menu is opened instead of downloading the files
* Multiple files are transferred at once with an overall progress indicator * Multiple files are transferred at once with an overall progress indicator
* The integrity of the files is checked on receive
* All metadata is preserved
### Share Files Directly From Share / Context Menu ### Share Files Directly From Share / Context Menu
* [Share files directly from context menu on Windows](/docs/how-to.md#share-files-directly-from-context-menu-on-windows) * [Share files directly from context menu on Windows](/docs/how-to.md#share-files-directly-from-context-menu-on-windows)

228
public/scripts/network.js
View file

@ -197,27 +197,13 @@ class Peer {
} }
async createHeader(file) { async createHeader(file) {
let hashHex = await this.getHashHex(file);
return { return {
name: file.name, name: file.name,
mime: file.type, mime: file.type,
size: file.size, size: file.size,
hashHex: hashHex
}; };
} }
async getHashHex(file) {
if (!crypto.subtle) {
console.warn("PairDrops functionality to compare received with requested files works in secure contexts only (https or localhost).")
return;
}
const hashBuffer = await crypto.subtle.digest('SHA-256', await file.arrayBuffer());
// Convert hex to hash, see https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest#converting_a_digest_to_a_hex_string
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string
return(hashHex);
}
getResizedImageDataUrl(file, width = undefined, height = undefined, quality = 0.7) { getResizedImageDataUrl(file, width = undefined, height = undefined, quality = 0.7) {
return new Promise((resolve) => { return new Promise((resolve) => {
let image = new Image(); let image = new Image();
@ -254,58 +240,46 @@ class Peer {
} }
async requestFileTransfer(files) { async requestFileTransfer(files) {
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'prepare'})
let header = []; let header = [];
let combinedSize = 0; let totalSize = 0;
let imagesOnly = true
for (let i=0; i<files.length; i++) { for (let i=0; i<files.length; i++) {
Events.fire('set-progress', {peerId: this._peerId, progress: 0.8*i/files.length, status: 'prepare'})
header.push(await this.createHeader(files[i])); header.push(await this.createHeader(files[i]));
combinedSize += files[i].size; totalSize += files[i].size;
if (files[i].type.split('/')[0] !== 'image') imagesOnly = false;
} }
this._fileHeaderRequested = header;
let bytesCompleted = 0; Events.fire('set-progress', {peerId: this._peerId, progress: 0.8, status: 'prepare'})
zipper.createNewZipWriter();
for (let i=0; i<files.length; i++) {
await zipper.addFile(files[i], {
onprogress: (progress) => {
Events.fire('set-progress', {
peerId: this._peerId,
progress: (bytesCompleted + progress) / combinedSize,
status: 'prepare'
})
}
});
bytesCompleted += files[i].size;
}
this.zipFileRequested = await zipper.getZipFile();
let dataUrl = '';
if (files[0].type.split('/')[0] === 'image') { if (files[0].type.split('/')[0] === 'image') {
this.getResizedImageDataUrl(files[0], 400, null, 0.9).then(dataUrl => { dataUrl = await this.getResizedImageDataUrl(files[0], 400, null, 0.9);
this.sendJSON({type: 'request',
header: header,
size: combinedSize,
thumbnailDataUrl: dataUrl
});
})
} else {
this.sendJSON({type: 'request',
header: header,
size: combinedSize,
});
} }
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'prepare'})
this._filesRequested = files;
this.sendJSON({type: 'request',
header: header,
totalSize: totalSize,
imagesOnly: imagesOnly,
thumbnailDataUrl: dataUrl
});
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'wait'}) Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'wait'})
} }
async sendFiles() { async sendFiles() {
this._filesQueue.push({zipFile: this.zipFileRequested, fileHeader: this._fileHeaderRequested}); for (let i=0; i<this._filesRequested.length; i++) {
this._fileHeaderRequested = null this._filesQueue.push(this._filesRequested[i]);
}
this._filesRequested = null
if (this._busy) return; if (this._busy) return;
this._dequeueFile(); this._dequeueFile();
} }
_dequeueFile() { _dequeueFile() {
if (!this._filesQueue.length) return;
this._busy = true; this._busy = true;
const file = this._filesQueue.shift(); const file = this._filesQueue.shift();
this._sendFile(file); this._sendFile(file);
@ -314,10 +288,11 @@ class Peer {
async _sendFile(file) { async _sendFile(file) {
this.sendJSON({ this.sendJSON({
type: 'header', type: 'header',
size: file.zipFile.size, size: file.size,
fileHeader: file.fileHeader name: file.name,
mime: file.type
}); });
this._chunker = new FileChunker(file.zipFile, this._chunker = new FileChunker(file,
chunk => this._send(chunk), chunk => this._send(chunk),
offset => this._onPartitionEnd(offset)); offset => this._onPartitionEnd(offset));
this._chunker.nextPartition(); this._chunker.nextPartition();
@ -384,92 +359,116 @@ class Peer {
this.sendJSON({type: 'files-transfer-response', accepted: false}); this.sendJSON({type: 'files-transfer-response', accepted: false});
return; return;
} }
this._requestPending = true; if (window.iOS && request.totalSize >= 200*1024*1024) {
// iOS Safari can only put 400MB at once to memory.
// Request to send them in chunks of 200MB instead:
this.sendJSON({type: 'files-transfer-response', accepted: false, reason: 'ios-memory-limit'});
return;
}
this._requestPending = request;
Events.fire('files-transfer-request', { Events.fire('files-transfer-request', {
request: request, request: request,
peerId: this._peerId peerId: this._peerId
}); });
} }
_respondToFileTransferRequest(header, accepted) { _respondToFileTransferRequest(accepted) {
this._requestPending = false;
this._acceptedHeader = header;
this.sendJSON({type: 'files-transfer-response', accepted: accepted}); this.sendJSON({type: 'files-transfer-response', accepted: accepted});
if (accepted) this._busy = true; if (accepted) {
this._requestAccepted = this._requestPending;
this._totalBytesReceived = 0;
this._busy = true;
this._filesReceived = [];
}
this._requestPending = null;
} }
_onFilesHeader(header) {
_onFilesHeader(msg) { if (this._requestAccepted?.header.length) {
if (JSON.stringify(this._acceptedHeader) === JSON.stringify(msg.fileHeader)) {
this._lastProgress = 0; this._lastProgress = 0;
this._digester = new FileDigester(msg.size, blob => this._onFileReceived(blob, msg.fileHeader)); this._digester = new FileDigester({size: header.size, name: header.name, mime: header.mime},
this._acceptedHeader = null; this._requestAccepted.totalSize,
this._totalBytesReceived,
fileBlob => this._onFileReceived(fileBlob)
);
} }
} }
_abortTransfer() {
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'});
Events.fire('notify-user', 'Files are incorrect.');
this._filesReceived = [];
this._requestAccepted = null;
this._digester = null;
throw new Error("Received files differ from requested files. Abort!");
}
_onChunkReceived(chunk) { _onChunkReceived(chunk) {
if(!this._digester || !(chunk.byteLength || chunk.size)) return; if(!this._digester || !(chunk.byteLength || chunk.size)) return;
this._digester.unchunk(chunk); this._digester.unchunk(chunk);
const progress = this._digester.progress; const progress = this._digester.progress;
if (progress > 1) {
this._abortTransfer();
}
this._onDownloadProgress(progress); this._onDownloadProgress(progress);
// occasionally notify sender about our progress // occasionally notify sender about our progress
if (progress - this._lastProgress < 0.01) return; if (progress - this._lastProgress < 0.005 && progress !== 1) return;
this._lastProgress = progress; this._lastProgress = progress;
this._sendProgress(progress); this._sendProgress(progress);
} }
_onDownloadProgress(progress) { _onDownloadProgress(progress) {
if (this._busy) { Events.fire('set-progress', {peerId: this._peerId, progress: progress, status: 'transfer'});
Events.fire('set-progress', {peerId: this._peerId, progress: progress, status: 'transfer'});
}
} }
async _onFileReceived(zipBlob, fileHeader) { async _onFileReceived(fileBlob) {
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'process'}); const acceptedHeader = this._requestAccepted.header.shift();
this._busy = false; this._totalBytesReceived += fileBlob.size;
this.sendJSON({type: 'file-transfer-complete'}); this.sendJSON({type: 'file-transfer-complete'});
let zipEntries = await zipper.getEntries(zipBlob); const sameSize = fileBlob.size === acceptedHeader.size;
let files = []; const sameName = fileBlob.name === acceptedHeader.name
for (let i=0; i<zipEntries.length; i++) { if (!sameSize || !sameName) {
let fileBlob = await zipper.getData(zipEntries[i]); this._abortTransfer();
let hashHex = await this.getHashHex(fileBlob); }
let sameHex = hashHex === fileHeader[i].hashHex; this._filesReceived.push(fileBlob);
let sameSize = fileBlob.size === fileHeader[i].size; if (!this._requestAccepted.header.length) {
let sameName = zipEntries[i].filename === fileHeader[i].name this._busy = false;
if (!sameHex || !sameSize || !sameName) { Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'process'});
Events.fire('notify-user', 'Files are malformed.'); Events.fire('files-received', {sender: this._peerId, files: this._filesReceived, request: this._requestAccepted});
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'}); this._filesReceived = [];
throw new Error("Received files differ from requested files. Abort!"); this._requestAccepted = null;
}
files.push(new File([fileBlob], zipEntries[i].filename, {
type: fileHeader[i].mime,
lastModified: new Date().getTime()
}));
} }
Events.fire('files-received', {sender: this._peerId, files: files});
} }
_onFileTransferCompleted() { _onFileTransferCompleted() {
this._onDownloadProgress(1); this._chunker = null;
this._digester = null; if (!this._filesQueue.length) {
this._busy = false; this._busy = false;
this._dequeueFile(); Events.fire('notify-user', 'File transfer completed.');
Events.fire('notify-user', 'File transfer completed.'); } else {
this._dequeueFile();
}
} }
_onFileTransferRequestResponded(message) { _onFileTransferRequestResponded(message) {
if (!message.accepted) { if (!message.accepted) {
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'}); Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'});
this._filesRequested = null;
this.zipFile = null; if (message.reason === 'ios-memory-limit') {
Events.fire('notify-user', "Sending files to iOS is only possible up to 200MB at once");
}
return; return;
} }
Events.fire('file-transfer-accepted'); Events.fire('file-transfer-accepted');
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'transfer'});
this.sendFiles(); this.sendFiles();
} }
@ -690,16 +689,20 @@ class PeersManager {
} }
_onRespondToFileTransferRequest(detail) { _onRespondToFileTransferRequest(detail) {
this.peers[detail.to]._respondToFileTransferRequest(detail.header, detail.accepted); this.peers[detail.to]._respondToFileTransferRequest(detail.accepted);
} }
_onFilesSelected(message) { _onFilesSelected(message) {
let inputFiles = Array.from(message.files);
delete message.files;
let files = []; let files = [];
for (let i=0; i<message.files.length; i++) { const l = inputFiles.length;
// when filename is empty guess via suffix for (let i=0; i<l; i++) {
const file = message.files[i].type // when filetype is empty guess via suffix
? message.files[i] const inputFile = inputFiles.shift();
: new File([message.files[i]], message.files[i].name, {type: mime.getMimeByFilename(message.files[i].name)}); const file = inputFile.type
? inputFile
: new File([inputFile], inputFile.name, {type: mime.getMimeByFilename(inputFile.name)});
files.push(file) files.push(file)
} }
this.peers[message.to].requestFileTransfer(files); this.peers[message.to].requestFileTransfer(files);
@ -779,30 +782,35 @@ class FileChunker {
isFileEnd() { isFileEnd() {
return this._offset >= this._file.size; return this._offset >= this._file.size;
} }
get progress() {
return this._offset / this._file.size;
}
} }
class FileDigester { class FileDigester {
constructor(size, callback) { constructor(meta, totalSize, totalBytesReceived, callback) {
this._buffer = []; this._buffer = [];
this._bytesReceived = 0; this._bytesReceived = 0;
this._size = size; this._size = meta.size;
this._name = meta.name;
this._mime = meta.mime;
this._totalSize = totalSize;
this._totalBytesReceived = totalBytesReceived;
this._callback = callback; this._callback = callback;
} }
unchunk(chunk) { unchunk(chunk) {
this._buffer.push(chunk); this._buffer.push(chunk);
this._bytesReceived += chunk.byteLength || chunk.size; this._bytesReceived += chunk.byteLength || chunk.size;
this.progress = this._bytesReceived / this._size; this.progress = (this._totalBytesReceived + this._bytesReceived) / this._totalSize;
if (isNaN(this.progress)) this.progress = 1 if (isNaN(this.progress)) this.progress = 1
if (this._bytesReceived < this._size) return; if (this._bytesReceived < this._size) return;
// we are done // we are done
this._callback(new Blob(this._buffer)); const blob = new Blob(this._buffer)
this._buffer = null;
this._callback(new File([blob], this._name, {
type: this._mime,
lastModified: new Date().getTime()
}));
} }
} }

115
public/scripts/ui.js
View file

@ -471,21 +471,21 @@ class ReceiveFileDialog extends ReceiveDialog {
this.$shareOrDownloadBtn = this.$el.querySelector('#shareOrDownload'); this.$shareOrDownloadBtn = this.$el.querySelector('#shareOrDownload');
this.$receiveTitleNode = this.$el.querySelector('#receiveTitle') this.$receiveTitleNode = this.$el.querySelector('#receiveTitle')
Events.on('files-received', e => this._onFilesReceived(e.detail.sender, e.detail.files)); Events.on('files-received', e => this._onFilesReceived(e.detail.sender, e.detail.files, e.detail.request));
this._filesQueue = []; this._filesQueue = [];
} }
_onFilesReceived(sender, files) { _onFilesReceived(sender, files, request) {
this._nextFiles(sender, files); this._nextFiles(sender, files, request);
window.blop.play(); window.blop.play();
} }
_nextFiles(sender, nextFiles) { _nextFiles(sender, nextFiles, nextRequest) {
if (nextFiles) this._filesQueue.push({peerId: sender, files: nextFiles}); if (nextFiles) this._filesQueue.push({peerId: sender, files: nextFiles, request: nextRequest});
if (this._busy) return; if (this._busy) return;
this._busy = true; this._busy = true;
const {peerId, files} = this._filesQueue.shift(); const {peerId, files, request} = this._filesQueue.shift();
this._displayFiles(peerId, files); this._displayFiles(peerId, files, request);
} }
_dequeueFile() { _dequeueFile() {
@ -525,23 +525,20 @@ class ReceiveFileDialog extends ReceiveDialog {
}); });
} }
async _displayFiles(peerId, files) { async _displayFiles(peerId, files, request) {
if (this.continueCallback) this.$shareOrDownloadBtn.removeEventListener("click", this.continueCallback); if (this.continueCallback) this.$shareOrDownloadBtn.removeEventListener("click", this.continueCallback);
let url; let url;
let title; let title;
let filenameDownload; let filenameDownload;
let combinedSize = 0
let descriptor = "Image";
for (let i=0; i<files.length; i++) { let descriptor = request.imagesOnly ? "Image" : "File";
combinedSize += files[i].size;
if (files[i].type.split('/')[0] !== "image") descriptor = "File";
}
let size = this._formatFileSize(combinedSize); let size = this._formatFileSize(request.totalSize);
let description = files[0].name; let description = files[0].name;
let shareInsteadOfDownload = (window.iOS || window.android) && !!navigator.share && navigator.canShare({files});
if (files.length === 1) { if (files.length === 1) {
url = URL.createObjectURL(files[0]) url = URL.createObjectURL(files[0])
title = `PairDrop - ${descriptor} Received` title = `PairDrop - ${descriptor} Received`
@ -551,45 +548,47 @@ class ReceiveFileDialog extends ReceiveDialog {
description += ` and ${files.length-1} other ${descriptor.toLowerCase()}`; description += ` and ${files.length-1} other ${descriptor.toLowerCase()}`;
if(files.length>2) description += "s"; if(files.length>2) description += "s";
let bytesCompleted = 0; if(!shareInsteadOfDownload) {
zipper.createNewZipWriter(); let bytesCompleted = 0;
for (let i=0; i<files.length; i++) { zipper.createNewZipWriter();
await zipper.addFile(files[i], { for (let i=0; i<files.length; i++) {
onprogress: (progress) => { await zipper.addFile(files[i], {
Events.fire('set-progress', { onprogress: (progress) => {
peerId: peerId, Events.fire('set-progress', {
progress: (bytesCompleted + progress) / combinedSize, peerId: peerId,
status: 'process' progress: (bytesCompleted + progress) / request.totalSize,
}) status: 'process'
} })
}); }
bytesCompleted += files[i].size; });
} bytesCompleted += files[i].size;
url = await zipper.getBlobURL(); }
url = await zipper.getBlobURL();
let now = new Date(Date.now()); let now = new Date(Date.now());
let year = now.getFullYear().toString(); let year = now.getFullYear().toString();
let month = (now.getMonth()+1).toString(); let month = (now.getMonth()+1).toString();
month = month.length < 2 ? "0" + month : month; month = month.length < 2 ? "0" + month : month;
let date = now.getDate().toString(); let date = now.getDate().toString();
date = date.length < 2 ? "0" + date : date; date = date.length < 2 ? "0" + date : date;
let hours = now.getHours().toString(); let hours = now.getHours().toString();
hours = hours.length < 2 ? "0" + hours : hours; hours = hours.length < 2 ? "0" + hours : hours;
let minutes = now.getMinutes().toString(); let minutes = now.getMinutes().toString();
minutes = minutes.length < 2 ? "0" + minutes : minutes; minutes = minutes.length < 2 ? "0" + minutes : minutes;
filenameDownload = `PairDrop_files_${year+month+date}_${hours+minutes}.zip`; filenameDownload = `PairDrop_files_${year+month+date}_${hours+minutes}.zip`;
}
} }
this.$receiveTitleNode.textContent = title; this.$receiveTitleNode.textContent = title;
this.$fileDescriptionNode.textContent = description; this.$fileDescriptionNode.textContent = description;
this.$fileSizeNode.textContent = size; this.$fileSizeNode.textContent = size;
if ((window.iOS || window.android) && !!navigator.share && navigator.canShare({files})) { if (shareInsteadOfDownload) {
this.$shareOrDownloadBtn.innerText = "Share"; this.$shareOrDownloadBtn.innerText = "Share";
this.continueCallback = async _ => { this.continueCallback = async _ => {
navigator.share({ navigator.share({
files: files files: files
}).catch(err => console.error(err)); }).catch(err => console.error(err));
} }
this.$shareOrDownloadBtn.addEventListener("click", this.continueCallback); this.$shareOrDownloadBtn.addEventListener("click", this.continueCallback);
} else { } else {
@ -602,18 +601,14 @@ class ReceiveFileDialog extends ReceiveDialog {
document.title = `PairDrop - ${files.length} Files received`; document.title = `PairDrop - ${files.length} Files received`;
document.changeFavicon("images/favicon-96x96-notification.png"); document.changeFavicon("images/favicon-96x96-notification.png");
this.show(); this.show();
Events.fire('set-progress', { Events.fire('set-progress', {peerId: peerId, progress: 1, status: 'process'})
peerId: peerId,
progress: 1,
status: 'process'
})
this.$shareOrDownloadBtn.click(); this.$shareOrDownloadBtn.click();
}).catch(r => console.error(r)); }).catch(r => console.error(r));
} }
hide() { hide() {
this.$shareOrDownloadBtn.href = ''; this.$shareOrDownloadBtn.removeAttribute('href');
this.$shareOrDownloadBtn.download = ''; this.$shareOrDownloadBtn.removeAttribute('download');
this.$previewBox.innerHTML = ''; this.$previewBox.innerHTML = '';
super.hide(); super.hide();
this._dequeueFile(); this._dequeueFile();
@ -648,16 +643,9 @@ class ReceiveRequestDialog extends ReceiveDialog {
_onRequestFileTransfer(request, peerId) { _onRequestFileTransfer(request, peerId) {
this.correspondingPeerId = peerId; this.correspondingPeerId = peerId;
this.requestedHeader = request.header;
const peer = $(peerId); const peer = $(peerId);
let imagesOnly = true;
for(let i=0; i<request.header.length; i++) {
if (request.header[i].mime.split('/')[0] !== 'image') {
imagesOnly = false;
break;
}
}
this.$requestingPeerDisplayNameNode.innerText = peer.ui._displayName(); this.$requestingPeerDisplayNameNode.innerText = peer.ui._displayName();
const fileName = request.header[0].name; const fileName = request.header[0].name;
const fileNameSplit = fileName.split('.'); const fileNameSplit = fileName.split('.');
@ -667,14 +655,14 @@ class ReceiveRequestDialog extends ReceiveDialog {
if (request.header.length >= 2) { if (request.header.length >= 2) {
let fileOtherText = ` and ${request.header.length - 1} other `; let fileOtherText = ` and ${request.header.length - 1} other `;
fileOtherText += imagesOnly ? 'image' : 'file'; fileOtherText += request.imagesOnly ? 'image' : 'file';
if (request.header.length > 2) fileOtherText += "s"; if (request.header.length > 2) fileOtherText += "s";
this.$fileOtherNode.innerText = fileOtherText; this.$fileOtherNode.innerText = fileOtherText;
} }
this.$fileSizeNode.innerText = this._formatFileSize(request.size); this.$fileSizeNode.innerText = this._formatFileSize(request.totalSize);
if (request.thumbnailDataUrl) { if (request.thumbnailDataUrl?.substring(0, 22) === "data:image/jpeg;base64") {
let element = document.createElement('img'); let element = document.createElement('img');
element.src = request.thumbnailDataUrl; element.src = request.thumbnailDataUrl;
element.classList.add('element-preview'); element.classList.add('element-preview');
@ -690,7 +678,6 @@ class ReceiveRequestDialog extends ReceiveDialog {
_respondToFileTransferRequest(accepted) { _respondToFileTransferRequest(accepted) {
Events.fire('respond-to-files-transfer-request', { Events.fire('respond-to-files-transfer-request', {
to: this.correspondingPeerId, to: this.correspondingPeerId,
header: this.requestedHeader,
accepted: accepted accepted: accepted
}) })
if (accepted) { if (accepted) {
@ -1114,7 +1101,7 @@ class Toast extends Dialog {
if (this.hideTimeout) clearTimeout(this.hideTimeout); if (this.hideTimeout) clearTimeout(this.hideTimeout);
this.$el.textContent = message; this.$el.textContent = message;
this.show(); this.show();
this.hideTimeout = setTimeout(_ => this.hide(), 3000); this.hideTimeout = setTimeout(_ => this.hide(), 5000);
} }
} }