jakob/PairDrop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

revert zipping and unzipping files on transfer to minimize needed browser memory. Use fileQueue instead.

Browse source
This commit is contained in:
schlagmichdoch 2023-01-27 01:27:22 +01:00
parent 1278009706
commit d35c27aa91
3 changed files with 169 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -28,8 +28,6 @@ Developed based on [Snapdrop](https://github.com/RobinLinus/snapdrop)
* Multiple files are downloaded as ZIP file
* On iOS and Android the devices share menu is opened instead of downloading the files
* Multiple files are transferred at once with an overall progress indicator
* The integrity of the files is checked on receive
* All metadata is preserved
### Share Files Directly From Share / Context Menu
* [Share files directly from context menu on Windows](/docs/how-to.md#share-files-directly-from-context-menu-on-windows)

212
public/scripts/network.js
View file

@ -197,27 +197,13 @@ class Peer {
}
async createHeader(file) {
let hashHex = await this.getHashHex(file);
return {
name: file.name,
mime: file.type,
size: file.size,
hashHex: hashHex
};
}
async getHashHex(file) {
if (!crypto.subtle) {
console.warn("PairDrops functionality to compare received with requested files works in secure contexts only (https or localhost).")
return;
}
const hashBuffer = await crypto.subtle.digest('SHA-256', await file.arrayBuffer());
// Convert hex to hash, see https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest#converting_a_digest_to_a_hex_string
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string
return(hashHex);
}
getResizedImageDataUrl(file, width = undefined, height = undefined, quality = 0.7) {
return new Promise((resolve) => {
let image = new Image();
@ -254,58 +240,46 @@ class Peer {
}
async requestFileTransfer(files) {
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'prepare'})
let header = [];
let combinedSize = 0;
let totalSize = 0;
let imagesOnly = true
for (let i=0; i<files.length; i++) {
Events.fire('set-progress', {peerId: this._peerId, progress: 0.8*i/files.length, status: 'prepare'})
header.push(await this.createHeader(files[i]));
combinedSize += files[i].size;
totalSize += files[i].size;
if (files[i].type.split('/')[0] !== 'image') imagesOnly = false;
}
this._fileHeaderRequested = header;
let bytesCompleted = 0;
zipper.createNewZipWriter();
for (let i=0; i<files.length; i++) {
await zipper.addFile(files[i], {
onprogress: (progress) => {
Events.fire('set-progress', {
peerId: this._peerId,
progress: (bytesCompleted + progress) / combinedSize,
status: 'prepare'
})
}
});
bytesCompleted += files[i].size;
}
this.zipFileRequested = await zipper.getZipFile();
Events.fire('set-progress', {peerId: this._peerId, progress: 0.8, status: 'prepare'})
let dataUrl = '';
if (files[0].type.split('/')[0] === 'image') {
this.getResizedImageDataUrl(files[0], 400, null, 0.9).then(dataUrl => {
dataUrl = await this.getResizedImageDataUrl(files[0], 400, null, 0.9);
}
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'prepare'})
this._filesRequested = files;
this.sendJSON({type: 'request',
header: header,
size: combinedSize,
totalSize: totalSize,
imagesOnly: imagesOnly,
thumbnailDataUrl: dataUrl
});
})
} else {
this.sendJSON({type: 'request',
header: header,
size: combinedSize,
});
}
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'wait'})
}
async sendFiles() {
this._filesQueue.push({zipFile: this.zipFileRequested, fileHeader: this._fileHeaderRequested});
this._fileHeaderRequested = null
for (let i=0; i<this._filesRequested.length; i++) {
this._filesQueue.push(this._filesRequested[i]);
}
this._filesRequested = null
if (this._busy) return;
this._dequeueFile();
}
_dequeueFile() {
if (!this._filesQueue.length) return;
this._busy = true;
const file = this._filesQueue.shift();
this._sendFile(file);
@ -314,10 +288,11 @@ class Peer {
async _sendFile(file) {
this.sendJSON({
type: 'header',
size: file.zipFile.size,
fileHeader: file.fileHeader
size: file.size,
name: file.name,
mime: file.type
});
this._chunker = new FileChunker(file.zipFile,
this._chunker = new FileChunker(file,
chunk => this._send(chunk),
offset => this._onPartitionEnd(offset));
this._chunker.nextPartition();
@ -384,92 +359,116 @@ class Peer {
this.sendJSON({type: 'files-transfer-response', accepted: false});
return;
}
this._requestPending = true;
if (window.iOS && request.totalSize >= 200*1024*1024) {
// iOS Safari can only put 400MB at once to memory.
// Request to send them in chunks of 200MB instead:
this.sendJSON({type: 'files-transfer-response', accepted: false, reason: 'ios-memory-limit'});
return;
}
this._requestPending = request;
Events.fire('files-transfer-request', {
request: request,
peerId: this._peerId
});
}
_respondToFileTransferRequest(header, accepted) {
this._requestPending = false;
this._acceptedHeader = header;
_respondToFileTransferRequest(accepted) {
this.sendJSON({type: 'files-transfer-response', accepted: accepted});
if (accepted) this._busy = true;
if (accepted) {
this._requestAccepted = this._requestPending;
this._totalBytesReceived = 0;
this._busy = true;
this._filesReceived = [];
}
this._requestPending = null;
}
_onFilesHeader(msg) {
if (JSON.stringify(this._acceptedHeader) === JSON.stringify(msg.fileHeader)) {
_onFilesHeader(header) {
if (this._requestAccepted?.header.length) {
this._lastProgress = 0;
this._digester = new FileDigester(msg.size, blob => this._onFileReceived(blob, msg.fileHeader));
this._acceptedHeader = null;
this._digester = new FileDigester({size: header.size, name: header.name, mime: header.mime},
this._requestAccepted.totalSize,
this._totalBytesReceived,
fileBlob => this._onFileReceived(fileBlob)
);
}
}
_abortTransfer() {
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'});
Events.fire('notify-user', 'Files are incorrect.');
this._filesReceived = [];
this._requestAccepted = null;
this._digester = null;
throw new Error("Received files differ from requested files. Abort!");
}
_onChunkReceived(chunk) {
if(!this._digester || !(chunk.byteLength || chunk.size)) return;
this._digester.unchunk(chunk);
const progress = this._digester.progress;
if (progress > 1) {
this._abortTransfer();
}
this._onDownloadProgress(progress);
// occasionally notify sender about our progress
if (progress - this._lastProgress < 0.01) return;
if (progress - this._lastProgress < 0.005 && progress !== 1) return;
this._lastProgress = progress;
this._sendProgress(progress);
}
_onDownloadProgress(progress) {
if (this._busy) {
Events.fire('set-progress', {peerId: this._peerId, progress: progress, status: 'transfer'});
}
}
async _onFileReceived(zipBlob, fileHeader) {
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'process'});
this._busy = false;
async _onFileReceived(fileBlob) {
const acceptedHeader = this._requestAccepted.header.shift();
this._totalBytesReceived += fileBlob.size;
this.sendJSON({type: 'file-transfer-complete'});
let zipEntries = await zipper.getEntries(zipBlob);
let files = [];
for (let i=0; i<zipEntries.length; i++) {
let fileBlob = await zipper.getData(zipEntries[i]);
let hashHex = await this.getHashHex(fileBlob);
let sameHex = hashHex === fileHeader[i].hashHex;
let sameSize = fileBlob.size === fileHeader[i].size;
let sameName = zipEntries[i].filename === fileHeader[i].name
if (!sameHex || !sameSize || !sameName) {
Events.fire('notify-user', 'Files are malformed.');
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'});
throw new Error("Received files differ from requested files. Abort!");
const sameSize = fileBlob.size === acceptedHeader.size;
const sameName = fileBlob.name === acceptedHeader.name
if (!sameSize || !sameName) {
this._abortTransfer();
}
files.push(new File([fileBlob], zipEntries[i].filename, {
type: fileHeader[i].mime,
lastModified: new Date().getTime()
}));
this._filesReceived.push(fileBlob);
if (!this._requestAccepted.header.length) {
this._busy = false;
Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'process'});
Events.fire('files-received', {sender: this._peerId, files: this._filesReceived, request: this._requestAccepted});
this._filesReceived = [];
this._requestAccepted = null;
}
Events.fire('files-received', {sender: this._peerId, files: files});
}
_onFileTransferCompleted() {
this._onDownloadProgress(1);
this._digester = null;
this._chunker = null;
if (!this._filesQueue.length) {
this._busy = false;
this._dequeueFile();
Events.fire('notify-user', 'File transfer completed.');
} else {
this._dequeueFile();
}
}
_onFileTransferRequestResponded(message) {
if (!message.accepted) {
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'wait'});
this.zipFile = null;
this._filesRequested = null;
if (message.reason === 'ios-memory-limit') {
Events.fire('notify-user', "Sending files to iOS is only possible up to 200MB at once");
}
return;
}
Events.fire('file-transfer-accepted');
Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'transfer'});
this.sendFiles();
}
@ -690,16 +689,20 @@ class PeersManager {
}
_onRespondToFileTransferRequest(detail) {
this.peers[detail.to]._respondToFileTransferRequest(detail.header, detail.accepted);
this.peers[detail.to]._respondToFileTransferRequest(detail.accepted);
}
_onFilesSelected(message) {
let inputFiles = Array.from(message.files);
delete message.files;
let files = [];
for (let i=0; i<message.files.length; i++) {
// when filename is empty guess via suffix
const file = message.files[i].type
? message.files[i]
: new File([message.files[i]], message.files[i].name, {type: mime.getMimeByFilename(message.files[i].name)});
const l = inputFiles.length;
for (let i=0; i<l; i++) {
// when filetype is empty guess via suffix
const inputFile = inputFiles.shift();
const file = inputFile.type
? inputFile
: new File([inputFile], inputFile.name, {type: mime.getMimeByFilename(inputFile.name)});
files.push(file)
}
this.peers[message.to].requestFileTransfer(files);
@ -779,30 +782,35 @@ class FileChunker {
isFileEnd() {
return this._offset >= this._file.size;
}
get progress() {
return this._offset / this._file.size;
}
}
class FileDigester {
constructor(size, callback) {
constructor(meta, totalSize, totalBytesReceived, callback) {
this._buffer = [];
this._bytesReceived = 0;
this._size = size;
this._size = meta.size;
this._name = meta.name;
this._mime = meta.mime;
this._totalSize = totalSize;
this._totalBytesReceived = totalBytesReceived;
this._callback = callback;
}
unchunk(chunk) {
this._buffer.push(chunk);
this._bytesReceived += chunk.byteLength || chunk.size;
this.progress = this._bytesReceived / this._size;
this.progress = (this._totalBytesReceived + this._bytesReceived) / this._totalSize;
if (isNaN(this.progress)) this.progress = 1
if (this._bytesReceived < this._size) return;
// we are done
this._callback(new Blob(this._buffer));
const blob = new Blob(this._buffer)
this._buffer = null;
this._callback(new File([blob], this._name, {
type: this._mime,
lastModified: new Date().getTime()
}));
}
}

61
public/scripts/ui.js
View file

@ -471,21 +471,21 @@ class ReceiveFileDialog extends ReceiveDialog {
this.$shareOrDownloadBtn = this.$el.querySelector('#shareOrDownload');
this.$receiveTitleNode = this.$el.querySelector('#receiveTitle')
Events.on('files-received', e => this._onFilesReceived(e.detail.sender, e.detail.files));
Events.on('files-received', e => this._onFilesReceived(e.detail.sender, e.detail.files, e.detail.request));
this._filesQueue = [];
}
_onFilesReceived(sender, files) {
this._nextFiles(sender, files);
_onFilesReceived(sender, files, request) {
this._nextFiles(sender, files, request);
window.blop.play();
}
_nextFiles(sender, nextFiles) {
if (nextFiles) this._filesQueue.push({peerId: sender, files: nextFiles});
_nextFiles(sender, nextFiles, nextRequest) {
if (nextFiles) this._filesQueue.push({peerId: sender, files: nextFiles, request: nextRequest});
if (this._busy) return;
this._busy = true;
const {peerId, files} = this._filesQueue.shift();
this._displayFiles(peerId, files);
const {peerId, files, request} = this._filesQueue.shift();
this._displayFiles(peerId, files, request);
}
_dequeueFile() {
@ -525,23 +525,20 @@ class ReceiveFileDialog extends ReceiveDialog {
});
}
async _displayFiles(peerId, files) {
async _displayFiles(peerId, files, request) {
if (this.continueCallback) this.$shareOrDownloadBtn.removeEventListener("click", this.continueCallback);
let url;
let title;
let filenameDownload;
let combinedSize = 0
let descriptor = "Image";
for (let i=0; i<files.length; i++) {
combinedSize += files[i].size;
if (files[i].type.split('/')[0] !== "image") descriptor = "File";
}
let descriptor = request.imagesOnly ? "Image" : "File";
let size = this._formatFileSize(combinedSize);
let size = this._formatFileSize(request.totalSize);
let description = files[0].name;
let shareInsteadOfDownload = (window.iOS || window.android) && !!navigator.share && navigator.canShare({files});
if (files.length === 1) {
url = URL.createObjectURL(files[0])
title = `PairDrop - ${descriptor} Received`
@ -551,6 +548,7 @@ class ReceiveFileDialog extends ReceiveDialog {
description += ` and ${files.length-1} other ${descriptor.toLowerCase()}`;
if(files.length>2) description += "s";
if(!shareInsteadOfDownload) {
let bytesCompleted = 0;
zipper.createNewZipWriter();
for (let i=0; i<files.length; i++) {
@ -558,7 +556,7 @@ class ReceiveFileDialog extends ReceiveDialog {
onprogress: (progress) => {
Events.fire('set-progress', {
peerId: peerId,
progress: (bytesCompleted + progress) / combinedSize,
progress: (bytesCompleted + progress) / request.totalSize,
status: 'process'
})
}
@ -579,12 +577,13 @@ class ReceiveFileDialog extends ReceiveDialog {
minutes = minutes.length < 2 ? "0" + minutes : minutes;
filenameDownload = `PairDrop_files_${year+month+date}_${hours+minutes}.zip`;
}
}
this.$receiveTitleNode.textContent = title;
this.$fileDescriptionNode.textContent = description;
this.$fileSizeNode.textContent = size;
if ((window.iOS || window.android) && !!navigator.share && navigator.canShare({files})) {
if (shareInsteadOfDownload) {
this.$shareOrDownloadBtn.innerText = "Share";
this.continueCallback = async _ => {
navigator.share({
@ -602,18 +601,14 @@ class ReceiveFileDialog extends ReceiveDialog {
document.title = `PairDrop - ${files.length} Files received`;
document.changeFavicon("images/favicon-96x96-notification.png");
this.show();
Events.fire('set-progress', {
peerId: peerId,
progress: 1,
status: 'process'
})
Events.fire('set-progress', {peerId: peerId, progress: 1, status: 'process'})
this.$shareOrDownloadBtn.click();
}).catch(r => console.error(r));
}
hide() {
this.$shareOrDownloadBtn.href = '';
this.$shareOrDownloadBtn.download = '';
this.$shareOrDownloadBtn.removeAttribute('href');
this.$shareOrDownloadBtn.removeAttribute('download');
this.$previewBox.innerHTML = '';
super.hide();
this._dequeueFile();
@ -648,16 +643,9 @@ class ReceiveRequestDialog extends ReceiveDialog {
_onRequestFileTransfer(request, peerId) {
this.correspondingPeerId = peerId;
this.requestedHeader = request.header;
const peer = $(peerId);
let imagesOnly = true;
for(let i=0; i<request.header.length; i++) {
if (request.header[i].mime.split('/')[0] !== 'image') {
imagesOnly = false;
break;
}
}
this.$requestingPeerDisplayNameNode.innerText = peer.ui._displayName();
const fileName = request.header[0].name;
const fileNameSplit = fileName.split('.');
@ -667,14 +655,14 @@ class ReceiveRequestDialog extends ReceiveDialog {
if (request.header.length >= 2) {
let fileOtherText = ` and ${request.header.length - 1} other `;
fileOtherText += imagesOnly ? 'image' : 'file';
fileOtherText += request.imagesOnly ? 'image' : 'file';
if (request.header.length > 2) fileOtherText += "s";
this.$fileOtherNode.innerText = fileOtherText;
}
this.$fileSizeNode.innerText = this._formatFileSize(request.size);
this.$fileSizeNode.innerText = this._formatFileSize(request.totalSize);
if (request.thumbnailDataUrl) {
if (request.thumbnailDataUrl?.substring(0, 22) === "data:image/jpeg;base64") {
let element = document.createElement('img');
element.src = request.thumbnailDataUrl;
element.classList.add('element-preview');
@ -690,7 +678,6 @@ class ReceiveRequestDialog extends ReceiveDialog {
_respondToFileTransferRequest(accepted) {
Events.fire('respond-to-files-transfer-request', {
to: this.correspondingPeerId,
header: this.requestedHeader,
accepted: accepted
})
if (accepted) {
@ -1114,7 +1101,7 @@ class Toast extends Dialog {
if (this.hideTimeout) clearTimeout(this.hideTimeout);
this.$el.textContent = message;
this.show();
this.hideTimeout = setTimeout(_ => this.hide(), 3000);
this.hideTimeout = setTimeout(_ => this.hide(), 5000);
}
}