Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: allow files in directories to be downloaded onto local machine #2199

Merged
merged 15 commits into from
May 21, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 13 additions & 1 deletion src/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import * as resumableUpload from './resumable-upload';
import {Writable, Readable, pipeline, Transform, PassThrough} from 'stream';
import * as zlib from 'zlib';
import * as http from 'http';
import * as path from 'path';

import {
ExceptionMessages,
Expand Down Expand Up @@ -2097,7 +2098,18 @@ class File extends ServiceObject<File> {

const fileStream = this.createReadStream(options);
let receivedData = false;
if (destination) {

// Skip directory objects as they cannot be written to local filesystem
/**
* Temporary comment:
* Could consider returning an error, but TransferManager will have to handle rejected promises in Promises.all.
* Example: Promise.allSettled(promises) or Promise.all(promises.map(p => p.catch(e => e)))
hochoy marked this conversation as resolved.
Show resolved Hide resolved
*/
if (destination && destination.endsWith('/')) {
callback?.(null, Buffer.alloc(0));
} else if (destination) {
hochoy marked this conversation as resolved.
Show resolved Hide resolved
fs.mkdirSync(path.dirname(destination), {recursive: true});
hochoy marked this conversation as resolved.
Show resolved Hide resolved
hochoy marked this conversation as resolved.
Show resolved Hide resolved

fileStream
.on('error', callback)
.once('data', data => {
Expand Down
2 changes: 1 addition & 1 deletion src/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ export class TransferManager {
{},
options.passthroughOptions
);

if (options.prefix) {
passThroughOptionsCopy.destination = path.join(
options.prefix || '',
Expand All @@ -270,7 +271,6 @@ export class TransferManager {
}
promises.push(limit(() => file.download(passThroughOptionsCopy)));
}

return Promise.all(promises);
}

Expand Down
83 changes: 83 additions & 0 deletions test/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import * as resumableUpload from '../src/resumable-upload';
import * as sinon from 'sinon';
import * as tmp from 'tmp';
import * as zlib from 'zlib';
import * as path from 'path';

import {
Bucket,
Expand Down Expand Up @@ -2555,6 +2556,12 @@ describe('File', () => {
});

describe('with destination', () => {
const sandbox = sinon.createSandbox();

afterEach(() => {
sandbox.restore();
});

it('should write the file to a destination if provided', done => {
tmp.setGracefulCleanup();
tmp.file((err, tmpFilePath) => {
Expand Down Expand Up @@ -2678,6 +2685,82 @@ describe('File', () => {
});
});
});

it('should recursively create directory and write file contents if destination path is nested', done => {
tmp.setGracefulCleanup();
tmp.dir(async (err, tmpDirPath) => {
assert.ifError(err);

const fileContents = 'nested-abcdefghijklmnopqrstuvwxyz';

Object.assign(fileReadStream, {
_read(this: Readable) {
this.push(fileContents);
this.push(null);
},
});

const nestedPath = path.join(tmpDirPath, 'a', 'b', 'c', 'file.txt');

file.download({destination: nestedPath}, (err: Error) => {
assert.ifError(err);
assert.strictEqual(fs.existsSync(nestedPath), true);
fs.readFile(nestedPath, (err, tmpFileContents) => {
assert.ifError(err);
assert.strictEqual(fileContents, tmpFileContents.toString());
done();
});
});
});
});

it('should skip write if asked to write a directory object', done => {
// Temporary comment: Option 1: Spy on fs methods
const mkdirSync = sandbox.spy(fakeFs, 'mkdirSync');
const createWriteStream = sandbox.spy(fakeFs, 'createWriteStream');

tmp.setGracefulCleanup();
tmp.dir(async (err, tmpDirPath) => {
assert.ifError(err);

const fileContents = '';

Object.assign(fileReadStream, {
_read(this: Readable) {
this.push(fileContents);
this.push(null);
},
});

const nestedDir = path.join(tmpDirPath, 'a', 'b', 'c', '/');

file.download(
{destination: nestedDir},
(err: Error, buffer: Buffer) => {
try {
assert.ifError(err);

// Temporary comment: Option 1: Spy on fs methods
assert.strictEqual(createWriteStream.callCount, 0);
assert.strictEqual(mkdirSync.callCount, 0);

// Temporary comment: Option 2: Verify that no folder and/or file were created
assert.strictEqual(
fs.existsSync(path.dirname(nestedDir)),
false
);
assert.strictEqual(fs.existsSync(nestedDir), false);
hochoy marked this conversation as resolved.
Show resolved Hide resolved

// Temporary comment: Not very useful since no user would care about an empty buffer. Could remove.
assert.strictEqual(buffer.equals(Buffer.alloc(0)), true);
done();
} catch (e) {
done(e);
vishwarajanand marked this conversation as resolved.
Show resolved Hide resolved
}
}
);
});
});
});
});

Expand Down