diff --git a/handwritten/storage/src/bucket.ts b/handwritten/storage/src/bucket.ts index 23aefac9e3f..cddc5c24035 100644 --- a/handwritten/storage/src/bucket.ts +++ b/handwritten/storage/src/bucket.ts @@ -34,12 +34,15 @@ import {paginator} from '@google-cloud/paginator'; import {promisifyAll} from '@google-cloud/promisify'; import * as fs from 'fs'; import * as http from 'http'; -import mime from 'mime'; import * as path from 'path'; -import pLimit from 'p-limit'; import {promisify} from 'util'; import AsyncRetry from 'async-retry'; -import {convertObjKeysToSnakeCase, handleContextValidation} from './util.js'; +import { + convertObjKeysToSnakeCase, + handleContextValidation, + getMime, + getPLimit, +} from './util.js'; import {Acl, AclMetadata} from './acl.js'; import {Channel} from './channel.js'; @@ -1733,118 +1736,128 @@ class Bucket extends ServiceObject { const destinationFile = convertToFile(destination); callback = callback || util.noop; - if (!destinationFile.metadata.contentType) { - const destinationContentType = - mime.getType(destinationFile.name) || undefined; + void (async () => { + try { + if (!destinationFile.metadata.contentType) { + const mime = await getMime(); + const destinationContentType = + mime.getType(destinationFile.name) || undefined; - if (destinationContentType) { - destinationFile.metadata.contentType = destinationContentType; - } - } + if (destinationContentType) { + destinationFile.metadata.contentType = destinationContentType; + } + } - let maxRetries = this.storage.retryOptions.maxRetries; - if ( - (destinationFile?.instancePreconditionOpts?.ifGenerationMatch === - undefined && - options.ifGenerationMatch === undefined && - this.storage.retryOptions.idempotencyStrategy === - IdempotencyStrategy.RetryConditional) || - this.storage.retryOptions.idempotencyStrategy === - IdempotencyStrategy.RetryNever - ) { - maxRetries = 0; - } + let maxRetries = this.storage.retryOptions.maxRetries; + if ( + (destinationFile?.instancePreconditionOpts?.ifGenerationMatch === + undefined && + options.ifGenerationMatch === undefined && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + maxRetries = 0; + } - const deleteSourceObjects = options.deleteSourceObjects; + const deleteSourceObjects = options.deleteSourceObjects; - const requestQueryObject = Object.assign({}, options); - delete requestQueryObject.deleteSourceObjects; + const requestQueryObject = Object.assign({}, options); + delete requestQueryObject.deleteSourceObjects; - if (requestQueryObject.ifGenerationMatch === undefined) { - Object.assign( - requestQueryObject, - destinationFile.instancePreconditionOpts, - requestQueryObject - ); - } + if (requestQueryObject.ifGenerationMatch === undefined) { + Object.assign( + requestQueryObject, + destinationFile.instancePreconditionOpts, + requestQueryObject + ); + } - // Make the request from the destination File object. - destinationFile.request( - { - method: 'POST', - uri: '/compose', - maxRetries, - json: { - destination: { - contentType: destinationFile.metadata.contentType, - contentEncoding: destinationFile.metadata.contentEncoding, - contexts: - requestQueryObject.contexts || destinationFile.metadata.contexts, + // Make the request from the destination File object. + destinationFile.request( + { + method: 'POST', + uri: '/compose', + maxRetries, + json: { + destination: { + contentType: destinationFile.metadata.contentType, + contentEncoding: destinationFile.metadata.contentEncoding, + contexts: + requestQueryObject.contexts || + destinationFile.metadata.contexts, + }, + sourceObjects: (sources as File[]).map(source => { + const sourceObject = { + name: source.name, + } as SourceObject; + + const generation = + source.generation ?? source.metadata?.generation; + if (generation !== undefined) { + sourceObject.generation = parseInt(generation.toString()); + } + + return sourceObject; + }), + }, + qs: requestQueryObject, }, - sourceObjects: (sources as File[]).map(source => { - const sourceObject = { - name: source.name, - } as SourceObject; - - const generation = source.generation ?? source.metadata?.generation; - if (generation !== undefined) { - sourceObject.generation = parseInt(generation.toString()); + (err, resp) => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + if (err) { + callback!(err, null, resp); + return; } - return sourceObject; - }), - }, - qs: requestQueryObject, - }, - (err, resp) => { - this.storage.retryOptions.autoRetry = this.instanceRetryValue; - if (err) { - callback!(err, null, resp); - return; - } - - if (deleteSourceObjects) { - const deletePromises = (sources as File[]).map(source => { - const deleteOptions: DeleteOptions = { - ignoreNotFound: true, - userProject: options.userProject, - }; + if (deleteSourceObjects) { + const deletePromises = (sources as File[]).map(source => { + const deleteOptions: DeleteOptions = { + ignoreNotFound: true, + userProject: options.userProject, + }; + + const generation = + source.generation ?? source.metadata?.generation; + if (generation !== undefined) { + deleteOptions.ifGenerationMatch = generation; + } - const generation = source.generation ?? source.metadata?.generation; - if (generation !== undefined) { - deleteOptions.ifGenerationMatch = generation; - } + return source + .delete(deleteOptions) + .catch(deleteErr => deleteErr as Error); + }); - return source - .delete(deleteOptions) - .catch(deleteErr => deleteErr as Error); - }); + void (async () => { + // eslint-disable-next-line promise/no-promise-in-callback + const results = await Promise.all(deletePromises); + const errors = results.filter( + (res): res is Error => res instanceof Error + ); + + if (errors.length > 0) { + const cleanupErr = new ComposeCleanupError( + `Compose operation succeeded, but cleaning up source objects failed. Failed to delete ${errors.length} source object(s).`, + errors, + destinationFile, + resp + ); + callback!(cleanupErr, destinationFile, resp); + return; + } - void (async () => { - // eslint-disable-next-line promise/no-promise-in-callback - const results = await Promise.all(deletePromises); - const errors = results.filter( - (res): res is Error => res instanceof Error - ); - - if (errors.length > 0) { - const cleanupErr = new ComposeCleanupError( - `Compose operation succeeded, but cleaning up source objects failed. Failed to delete ${errors.length} source object(s).`, - errors, - destinationFile, - resp - ); - callback!(cleanupErr, destinationFile, resp); - return; + callback!(null, destinationFile, resp); + })(); + } else { + callback!(null, destinationFile, resp); } - - callback!(null, destinationFile, resp); - })(); - } else { - callback!(null, destinationFile, resp); - } + } + ); + } catch (err) { + callback!(err as Error, null, null); } - ); + })(); } createChannel( @@ -2288,6 +2301,7 @@ class Bucket extends ServiceObject { void (async () => { try { let promises = []; + const pLimit = await getPLimit(); const limit = pLimit(MAX_PARALLEL_LIMIT); const filesStream = this.getFilesStream(query); @@ -4721,6 +4735,7 @@ class Bucket extends ServiceObject { void (async () => { try { const [files] = await this.getFiles(options); + const pLimit = await getPLimit(); const limit = pLimit(MAX_PARALLEL_LIMIT); const promises = files.map(file => { return limit(() => processFile(file)); diff --git a/handwritten/storage/src/file.ts b/handwritten/storage/src/file.ts index c509ae3e60f..509c9dd3e8a 100644 --- a/handwritten/storage/src/file.ts +++ b/handwritten/storage/src/file.ts @@ -26,7 +26,6 @@ import {promisifyAll} from '@google-cloud/promisify'; import * as crypto from 'crypto'; import * as fs from 'fs'; -import mime from 'mime'; import * as resumableUpload from './resumable-upload.js'; import {Writable, Readable, pipeline, Transform, PipelineSource} from 'stream'; import * as zlib from 'zlib'; @@ -62,6 +61,7 @@ import { formatAsUTCISO, PassThroughShim, handleContextValidation, + getMime, } from './util.js'; import {CRC32C, CRC32CValidatorGenerator} from './crc32c.js'; import {HashStreamValidator} from './hash-stream-validator.js'; @@ -2158,23 +2158,7 @@ class File extends ServiceObject { options!.metadata!.contentType = options.contentType; } - if ( - !options!.metadata!.contentType || - options!.metadata!.contentType === 'auto' - ) { - const detectedContentType = mime.getType(this.name); - if (detectedContentType) { - options!.metadata!.contentType = detectedContentType; - } - } - - let gzip = options.gzip; - - if (gzip === 'auto') { - gzip = COMPRESSIBLE_MIME_REGEX.test(options!.metadata!.contentType || ''); - } - - if (gzip) { + if (options.gzip === true) { options!.metadata!.contentEncoding = 'gzip'; } @@ -2239,7 +2223,7 @@ class File extends ServiceObject { const transformStreams: Transform[] = []; - if (gzip) { + if (options.gzip === true) { transformStreams.push(zlib.createGzip()); } @@ -2280,96 +2264,127 @@ class File extends ServiceObject { fileWriteStreamMetadataReceived = true; }); - writeStream.once('writing', () => { - if (options.resumable === false) { - this.startSimpleUpload_(fileWriteStream, options); - } else { - this.startResumableUpload_(fileWriteStream, options); - } + writeStream.once('writing', async () => { + try { + if ( + !options!.metadata!.contentType || + options!.metadata!.contentType === 'auto' + ) { + const mime = await getMime(); + const detectedContentType = mime.getType(this.name); + if (detectedContentType) { + options!.metadata!.contentType = detectedContentType; + } + } - // remove temporary noop listener as we now create a pipeline that handles the errors - emitStream.removeListener('error', noop); + let gzip = options.gzip; - if (fileWriteStream.destroyed) { - let callbackCalled = false; - const onError = (err: Error) => { - if (!callbackCalled) { - callbackCalled = true; - pipelineCallback(err); + if (gzip === 'auto') { + gzip = COMPRESSIBLE_MIME_REGEX.test( + options!.metadata!.contentType || '' + ); + if (gzip) { + options!.metadata!.contentEncoding = 'gzip'; + transformStreams.unshift(zlib.createGzip()); } - }; - fileWriteStream.once('error', onError); - emitStream.destroy(); - - process.nextTick(() => { - fileWriteStream.removeListener('error', onError); - if (!callbackCalled) { - callbackCalled = true; - const err = - (fileWriteStream as Writable & {errored?: Error}).errored || - new Error('Write stream destroyed'); - pipelineCallback(err); - } - }); - return; - } + } - pipeline( - emitStream, - ...(transformStreams as [Transform]), - fileWriteStream, - async e => { - if (e) { - return pipelineCallback(e); - } + if (options.resumable === false) { + this.startSimpleUpload_(fileWriteStream, options); + } else { + this.startResumableUpload_(fileWriteStream, options); + } - // If this is a partial upload, we don't expect final metadata yet. - if (options.isPartialUpload) { - // Emit CRC32c for this completed chunk if hash validation is active. - if (hashCalculatingStream?.crc32c) { - writeStream.emit('crc32c', hashCalculatingStream.crc32c); + // remove temporary noop listener as we now create a pipeline that handles the errors + emitStream.removeListener('error', noop); + + if (fileWriteStream.destroyed) { + let callbackCalled = false; + const onError = (err: Error) => { + if (!callbackCalled) { + callbackCalled = true; + pipelineCallback(err); } - // Resolve the pipeline for this *partial chunk*. - return pipelineCallback(); - } + }; + fileWriteStream.once('error', onError); + emitStream.destroy(); + + process.nextTick(() => { + fileWriteStream.removeListener('error', onError); + if (!callbackCalled) { + callbackCalled = true; + const err = + (fileWriteStream as Writable & {errored?: Error}).errored || + new Error('Write stream destroyed'); + pipelineCallback(err); + } + }); + return; + } - // We want to make sure we've received the metadata from the server in order - // to properly validate the object's integrity. Depending on the type of upload, - // the stream could close before the response is returned. - if (!fileWriteStreamMetadataReceived) { - try { - await new Promise((resolve, reject) => { - fileWriteStream.once('metadata', resolve); - fileWriteStream.once('error', reject); - }); - } catch (e) { - return pipelineCallback(e as Error); + pipeline( + emitStream, + ...(transformStreams as [Transform]), + fileWriteStream, + async e => { + if (e) { + return pipelineCallback(e); } - } - // Emit the local CRC32C value for future validation, if validation is enabled. - if (hashCalculatingStream?.crc32c) { - writeStream.emit('crc32c', hashCalculatingStream.crc32c); - } + // If this is a partial upload, we don't expect final metadata yet. + if (options.isPartialUpload) { + // Emit CRC32c for this completed chunk if hash validation is active. + if (hashCalculatingStream?.crc32c) { + writeStream.emit('crc32c', hashCalculatingStream.crc32c); + } + // Resolve the pipeline for this *partial chunk*. + return pipelineCallback(); + } - try { - // Metadata may not be ready if the upload is a partial upload, - // nothing to validate yet. - const metadataNotReady = options.isPartialUpload && !this.metadata; - - if (hashCalculatingStream && !metadataNotReady) { - await this.#validateIntegrity(hashCalculatingStream, { - crc32c, - md5, - }); + // We want to make sure we've received the metadata from the server in order + // to properly validate the object's integrity. Depending on the type of upload, + // the stream could close before the response is returned. + if (!fileWriteStreamMetadataReceived) { + try { + await new Promise((resolve, reject) => { + fileWriteStream.once('metadata', resolve); + fileWriteStream.once('error', reject); + }); + } catch (e) { + return pipelineCallback(e as Error); + } } - pipelineCallback(); - } catch (e) { - pipelineCallback(e as Error); + // Emit the local CRC32C value for future validation, if validation is enabled. + if (hashCalculatingStream?.crc32c) { + writeStream.emit('crc32c', hashCalculatingStream.crc32c); + } + + try { + // Metadata may not be ready if the upload is a partial upload, + // nothing to validate yet. + const metadataNotReady = + options.isPartialUpload && !this.metadata; + + if (hashCalculatingStream && !metadataNotReady) { + await this.#validateIntegrity(hashCalculatingStream, { + crc32c, + md5, + }); + } + + pipelineCallback(); + } catch (e) { + pipelineCallback(e as Error); + } } - } - ); + ); + } catch (err) { + emitStream.removeListener('error', noop); + emitStream.destroy(err as Error); + fileWriteStream.destroy(err as Error); + pipelineCallback(err as Error); + } }); return writeStream; diff --git a/handwritten/storage/src/transfer-manager.ts b/handwritten/storage/src/transfer-manager.ts index 1e04aa08085..0e4882d1da1 100644 --- a/handwritten/storage/src/transfer-manager.ts +++ b/handwritten/storage/src/transfer-manager.ts @@ -24,7 +24,6 @@ import { RequestError, SkipReason, } from './file.js'; -import pLimit from 'p-limit'; import * as path from 'path'; import {createReadStream, existsSync, promises as fsp} from 'fs'; import {CRC32C} from './crc32c.js'; @@ -35,7 +34,11 @@ import {ApiError} from './nodejs-common/index.js'; import {GaxiosResponse, Headers} from 'gaxios'; import {createHash} from 'crypto'; import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; -import {getRuntimeTrackingString, getUserAgentString} from './util.js'; +import { + getRuntimeTrackingString, + getUserAgentString, + getPLimit, +} from './util.js'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore import {getPackageJSON} from './package-json-helper.cjs'; @@ -477,6 +480,7 @@ export class TransferManager { }; } + const pLimit = await getPLimit(); const limit = pLimit( options.concurrencyLimit || DEFAULT_PARALLEL_UPLOAD_LIMIT ); @@ -604,6 +608,7 @@ export class TransferManager { filesOrFolder: File[] | string[] | string, options: DownloadManyFilesOptions = {} ): Promise { + const pLimit = await getPLimit(); const limit = pLimit( options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT ); @@ -777,6 +782,7 @@ export class TransferManager { fileOrName: File | string, options: DownloadFileInChunksOptions = {} ): Promise { + const pLimit = await getPLimit(); let chunkSize = options.chunkSizeBytes || DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE; let limit = pLimit( @@ -904,6 +910,7 @@ export class TransferManager { options: UploadFileInChunksOptions = {}, generator: MultiPartHelperGenerator = defaultMultiPartGenerator ): Promise { + const pLimit = await getPLimit(); const chunkSize = options.chunkSizeBytes || UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE; const limit = pLimit( diff --git a/handwritten/storage/src/util.ts b/handwritten/storage/src/util.ts index 19d6b9efb27..feacafc76f9 100644 --- a/handwritten/storage/src/util.ts +++ b/handwritten/storage/src/util.ts @@ -318,3 +318,59 @@ export function handleContextValidation( return Promise.reject(err); } } + +let mimePromise: Promise | undefined; + +/** + * Lazily loads and returns the `mime` module. + * Caches the resolved module so dynamic import is evaluated only once. + * + * @internal + */ +export async function getMime(): Promise { + if (!mimePromise) { + mimePromise = import('mime') + .then(mod => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const modObj = mod as any; + const mime = + modObj && modObj.default && modObj.default.getType + ? modObj.default + : modObj && modObj.getType + ? modObj + : modObj.default || modObj; + return mime; + }) + .catch(err => { + mimePromise = undefined; + throw err; + }); + } + return mimePromise; +} + +let pLimitPromise: Promise | undefined; + +/** + * Lazily loads and returns the `p-limit` module. + * Caches the resolved module so dynamic import is evaluated only once. + * + * @internal + */ +export async function getPLimit(): Promise { + if (!pLimitPromise) { + pLimitPromise = import('p-limit') + .then(mod => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const modObj = mod as any; + const pLimit = + typeof mod === 'function' ? mod : modObj.default || modObj; + return pLimit; + }) + .catch(err => { + pLimitPromise = undefined; + throw err; + }); + } + return pLimitPromise; +} diff --git a/handwritten/storage/test/bucket.ts b/handwritten/storage/test/bucket.ts index 531db641588..d55434c9f9d 100644 --- a/handwritten/storage/test/bucket.ts +++ b/handwritten/storage/test/bucket.ts @@ -234,6 +234,11 @@ describe('Bucket', () => { ServiceObject: FakeServiceObject, util: fakeUtil, }, + './util.js': { + ...require('../src/util.js'), + getPLimit: async () => fakePLimit, + getMime: async () => mime, + }, './acl.js': {Acl: FakeAcl}, './file.js': {File: FakeFile}, './iam.js': {Iam: FakeIam},