Skip to content
Permalink
9bfb9ba527
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
1 lines (1 sloc) 15.1 KB
{"version":3,"file":"BufferScheduler.js","sourceRoot":"","sources":["../../../../storage-common/src/BufferScheduler.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAW9C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,eAAe;IAuF1B;;;;;;;;;;;OAWG;IACH,YACE,QAAkB,EAClB,UAAkB,EAClB,UAAkB,EAClB,eAAgC,EAChC,WAAmB,EACnB,QAAyB;QAlF3B;;WAEG;QACc,YAAO,GAAiB,IAAI,YAAY,EAAE,CAAC;QAO5D;;WAEG;QACK,WAAM,GAAW,CAAC,CAAC;QAE3B;;WAEG;QACK,gBAAW,GAAY,KAAK,CAAC;QAErC;;WAEG;QACK,YAAO,GAAY,KAAK,CAAC;QAEjC;;WAEG;QACK,8BAAyB,GAAW,CAAC,CAAC;QAO9C;;WAEG;QACK,eAAU,GAAW,CAAC,CAAC;QAE/B;;;;;;WAMG;QACK,wBAAmB,GAAa,EAAE,CAAC;QAE3C;;WAEG;QACK,qBAAgB,GAAW,CAAC,CAAC;QAErC;;WAEG;QACK,aAAQ,GAAmB,EAAE,CAAC;QAEtC;;WAEG;QACK,aAAQ,GAAmB,EAAE,CAAC;QAsBpC,IAAI,UAAU,IAAI,CAAC,EAAE;YACnB,MAAM,IAAI,UAAU,CAAC,gDAAgD,UAAU,EAAE,CAAC,CAAC;SACpF;QAED,IAAI,UAAU,IAAI,CAAC,EAAE;YACnB,MAAM,IAAI,UAAU,CAAC,gDAAgD,UAAU,EAAE,CAAC,CAAC;SACpF;QAED,IAAI,WAAW,IAAI,CAAC,EAAE;YACpB,MAAM,IAAI,UAAU,CAAC,iDAAiD,WAAW,EAAE,CAAC,CAAC;SACtF;QAED,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,EAAE;QACb,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAChC,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC1E,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;gBAEhC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE;oBACvB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;gBAC/B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;oBAC/B,OAAO;iBACR;gBAED,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,yBAAyB,KAAK,CAAC,EAAE;oBAC5D,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,UAAU,EAAE;wBACxE,MAAM,MAAM,GAAG,IAAI,CAAC,kCAAkC,EAAE,CAAC;wBACzD,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;6BAC7E,IAAI,CAAC,OAAO,CAAC;6BACb,KAAK,CAAC,MAAM,CAAC,CAAC;qBAClB;yBAAM,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE;wBACnD,OAAO;qBACR;yBAAM;wBACL,OAAO,EAAE,CAAC;qBACX;iBACF;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,oBAAoB,CAAC,IAAY;QACvC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,kCAAkC,CAAC,MAAqB;QAC9D,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7F;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC9D;QAED,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAC,IAAI,CAAC;QACrC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;OAQG;IACK,WAAW;QACjB,OAAO,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE;YAC/C,IAAI,MAAoB,CAAC;YAEzB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5B,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC,CAAC;aACjD;iBAAM;gBACL,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE;oBACrC,MAAM,GAAG,IAAI,CAAC,kCAAkC,EAAE,CAAC;oBACnD,IAAI,CAAC,UAAU,EAAE,CAAC;iBACnB;qBAAM;oBACL,gDAAgD;oBAChD,OAAO,KAAK,CAAC;iBACd;aACF;YAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB;QACnC,IAAI,MAAgC,CAAC;QACrC,GAAG;YACD,IAAI,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,WAAW,EAAE;gBACtD,OAAO;aACR;YAED,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,MAAM,EAAE;gBACV,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;aACrC;SACF,QAAQ,MAAM,EAAE;IACnB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,sBAAsB,CAAC,MAAoB;QACvD,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC;QAEjC,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC;QAE5B,IAAI;YACF,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE,EAChC,YAAY,EACZ,IAAI,CAAC,MAAM,GAAG,YAAY,CAC3B,CAAC;SACH;QAAC,OAAO,GAAQ,EAAE;YACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAChC,OAAO;SACR;QAED,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,MAAoB;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YAC5D,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACxB;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { EventEmitter } from \"events\";\nimport { Readable } from \"stream\";\nimport { PooledBuffer } from \"./PooledBuffer\";\n\n/**\n * OutgoingHandler is an async function triggered by BufferScheduler.\n */\nexport declare type OutgoingHandler = (\n body: () => NodeJS.ReadableStream,\n length: number,\n offset?: number\n) => Promise<any>;\n\n/**\n * This class accepts a Node.js Readable stream as input, and keeps reading data\n * from the stream into the internal buffer structure, until it reaches maxBuffers.\n * Every available buffer will try to trigger outgoingHandler.\n *\n * The internal buffer structure includes an incoming buffer array, and a outgoing\n * buffer array. The incoming buffer array includes the \"empty\" buffers can be filled\n * with new incoming data. The outgoing array includes the filled buffers to be\n * handled by outgoingHandler. Every above buffer size is defined by parameter bufferSize.\n *\n * NUM_OF_ALL_BUFFERS = BUFFERS_IN_INCOMING + BUFFERS_IN_OUTGOING + BUFFERS_UNDER_HANDLING\n *\n * NUM_OF_ALL_BUFFERS lesser than or equal to maxBuffers\n *\n * PERFORMANCE IMPROVEMENT TIPS:\n * 1. Input stream highWaterMark is better to set a same value with bufferSize\n * parameter, which will avoid Buffer.concat() operations.\n * 2. concurrency should set a smaller value than maxBuffers, which is helpful to\n * reduce the possibility when a outgoing handler waits for the stream data.\n * in this situation, outgoing handlers are blocked.\n * Outgoing queue shouldn't be empty.\n */\nexport class BufferScheduler {\n /**\n * Size of buffers in incoming and outgoing queues. This class will try to align\n * data read from Readable stream into buffer chunks with bufferSize defined.\n */\n private readonly bufferSize: number;\n\n /**\n * How many buffers can be created or maintained.\n */\n private readonly maxBuffers: number;\n\n /**\n * A Node.js Readable stream.\n */\n private readonly readable: Readable;\n\n /**\n * OutgoingHandler is an async function triggered by BufferScheduler when there\n * are available buffers in outgoing array.\n */\n private readonly outgoingHandler: OutgoingHandler;\n\n /**\n * An internal event emitter.\n */\n private readonly emitter: EventEmitter = new EventEmitter();\n\n /**\n * Concurrency of executing outgoingHandlers. (0 lesser than concurrency lesser than or equal to maxBuffers)\n */\n private readonly concurrency: number;\n\n /**\n * An internal offset marker to track data offset in bytes of next outgoingHandler.\n */\n private offset: number = 0;\n\n /**\n * An internal marker to track whether stream is end.\n */\n private isStreamEnd: boolean = false;\n\n /**\n * An internal marker to track whether stream or outgoingHandler returns error.\n */\n private isError: boolean = false;\n\n /**\n * How many handlers are executing.\n */\n private executingOutgoingHandlers: number = 0;\n\n /**\n * Encoding of the input Readable stream which has string data type instead of Buffer.\n */\n private encoding?: BufferEncoding;\n\n /**\n * How many buffers have been allocated.\n */\n private numBuffers: number = 0;\n\n /**\n * Because this class doesn't know how much data every time stream pops, which\n * is defined by highWaterMarker of the stream. So BufferScheduler will cache\n * data received from the stream, when data in unresolvedDataArray exceeds the\n * blockSize defined, it will try to concat a blockSize of buffer, fill into available\n * buffers from incoming and push to outgoing array.\n */\n private unresolvedDataArray: Buffer[] = [];\n\n /**\n * How much data consisted in unresolvedDataArray.\n */\n private unresolvedLength: number = 0;\n\n /**\n * The array includes all the available buffers can be used to fill data from stream.\n */\n private incoming: PooledBuffer[] = [];\n\n /**\n * The array (queue) includes all the buffers filled from stream data.\n */\n private outgoing: PooledBuffer[] = [];\n\n /**\n * Creates an instance of BufferScheduler.\n *\n * @param readable - A Node.js Readable stream\n * @param bufferSize - Buffer size of every maintained buffer\n * @param maxBuffers - How many buffers can be allocated\n * @param outgoingHandler - An async function scheduled to be\n * triggered when a buffer fully filled\n * with stream data\n * @param concurrency - Concurrency of executing outgoingHandlers (>0)\n * @param encoding - [Optional] Encoding of Readable stream when it's a string stream\n */\n constructor(\n readable: Readable,\n bufferSize: number,\n maxBuffers: number,\n outgoingHandler: OutgoingHandler,\n concurrency: number,\n encoding?: BufferEncoding\n ) {\n if (bufferSize <= 0) {\n throw new RangeError(`bufferSize must be larger than 0, current is ${bufferSize}`);\n }\n\n if (maxBuffers <= 0) {\n throw new RangeError(`maxBuffers must be larger than 0, current is ${maxBuffers}`);\n }\n\n if (concurrency <= 0) {\n throw new RangeError(`concurrency must be larger than 0, current is ${concurrency}`);\n }\n\n this.bufferSize = bufferSize;\n this.maxBuffers = maxBuffers;\n this.readable = readable;\n this.outgoingHandler = outgoingHandler;\n this.concurrency = concurrency;\n this.encoding = encoding;\n }\n\n /**\n * Start the scheduler, will return error when stream of any of the outgoingHandlers\n * returns error.\n *\n */\n public async do(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n this.readable.on(\"data\", (data) => {\n data = typeof data === \"string\" ? Buffer.from(data, this.encoding) : data;\n this.appendUnresolvedData(data);\n\n if (!this.resolveData()) {\n this.readable.pause();\n }\n });\n\n this.readable.on(\"error\", (err) => {\n this.emitter.emit(\"error\", err);\n });\n\n this.readable.on(\"end\", () => {\n this.isStreamEnd = true;\n this.emitter.emit(\"checkEnd\");\n });\n\n this.emitter.on(\"error\", (err) => {\n this.isError = true;\n this.readable.pause();\n reject(err);\n });\n\n this.emitter.on(\"checkEnd\", () => {\n if (this.outgoing.length > 0) {\n this.triggerOutgoingHandlers();\n return;\n }\n\n if (this.isStreamEnd && this.executingOutgoingHandlers === 0) {\n if (this.unresolvedLength > 0 && this.unresolvedLength < this.bufferSize) {\n const buffer = this.shiftBufferFromUnresolvedDataArray();\n this.outgoingHandler(() => buffer.getReadableStream(), buffer.size, this.offset)\n .then(resolve)\n .catch(reject);\n } else if (this.unresolvedLength >= this.bufferSize) {\n return;\n } else {\n resolve();\n }\n }\n });\n });\n }\n\n /**\n * Insert a new data into unresolved array.\n *\n * @param data -\n */\n private appendUnresolvedData(data: Buffer) {\n this.unresolvedDataArray.push(data);\n this.unresolvedLength += data.length;\n }\n\n /**\n * Try to shift a buffer with size in blockSize. The buffer returned may be less\n * than blockSize when data in unresolvedDataArray is less than bufferSize.\n *\n */\n private shiftBufferFromUnresolvedDataArray(buffer?: PooledBuffer): PooledBuffer {\n if (!buffer) {\n buffer = new PooledBuffer(this.bufferSize, this.unresolvedDataArray, this.unresolvedLength);\n } else {\n buffer.fill(this.unresolvedDataArray, this.unresolvedLength);\n }\n\n this.unresolvedLength -= buffer.size;\n return buffer;\n }\n\n /**\n * Resolve data in unresolvedDataArray. For every buffer with size in blockSize\n * shifted, it will try to get (or allocate a buffer) from incoming, and fill it,\n * then push it into outgoing to be handled by outgoing handler.\n *\n * Return false when available buffers in incoming are not enough, else true.\n *\n * @returns Return false when buffers in incoming are not enough, else true.\n */\n private resolveData(): boolean {\n while (this.unresolvedLength >= this.bufferSize) {\n let buffer: PooledBuffer;\n\n if (this.incoming.length > 0) {\n buffer = this.incoming.shift()!;\n this.shiftBufferFromUnresolvedDataArray(buffer);\n } else {\n if (this.numBuffers < this.maxBuffers) {\n buffer = this.shiftBufferFromUnresolvedDataArray();\n this.numBuffers++;\n } else {\n // No available buffer, wait for buffer returned\n return false;\n }\n }\n\n this.outgoing.push(buffer);\n this.triggerOutgoingHandlers();\n }\n return true;\n }\n\n /**\n * Try to trigger a outgoing handler for every buffer in outgoing. Stop when\n * concurrency reaches.\n */\n private async triggerOutgoingHandlers() {\n let buffer: PooledBuffer | undefined;\n do {\n if (this.executingOutgoingHandlers >= this.concurrency) {\n return;\n }\n\n buffer = this.outgoing.shift();\n if (buffer) {\n this.triggerOutgoingHandler(buffer);\n }\n } while (buffer);\n }\n\n /**\n * Trigger a outgoing handler for a buffer shifted from outgoing.\n *\n * @param buffer -\n */\n private async triggerOutgoingHandler(buffer: PooledBuffer): Promise<any> {\n const bufferLength = buffer.size;\n\n this.executingOutgoingHandlers++;\n this.offset += bufferLength;\n\n try {\n await this.outgoingHandler(\n () => buffer.getReadableStream(),\n bufferLength,\n this.offset - bufferLength\n );\n } catch (err: any) {\n this.emitter.emit(\"error\", err);\n return;\n }\n\n this.executingOutgoingHandlers--;\n this.reuseBuffer(buffer);\n this.emitter.emit(\"checkEnd\");\n }\n\n /**\n * Return buffer used by outgoing handler into incoming.\n *\n * @param buffer -\n */\n private reuseBuffer(buffer: PooledBuffer) {\n this.incoming.push(buffer);\n if (!this.isError && this.resolveData() && !this.isStreamEnd) {\n this.readable.resume();\n }\n }\n}\n"]}