BufferScheduler.js.map

{"version":3,"file":"BufferScheduler.js","sourceRoot":"","sources":["../../../../storage-common/src/BufferScheduler.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAW9C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,eAAe;IAuF1B;;;;;;;;;;;OAWG;IACH,YACE,QAAkB,EAClB,UAAkB,EAClB,UAAkB,EAClB,eAAgC,EAChC,WAAmB,EACnB,QAAyB;QAlF3B;;WAEG;QACc,YAAO,GAAiB,IAAI,YAAY,EAAE,CAAC;QAO5D;;WAEG;QACK,WAAM,GAAW,CAAC,CAAC;QAE3B;;WAEG;QACK,gBAAW,GAAY,KAAK,CAAC;QAErC;;WAEG;QACK,YAAO,GAAY,KAAK,CAAC;QAEjC;;WAEG;QACK,8BAAyB,GAAW,CAAC,CAAC;QAO9C;;WAEG;QACK,eAAU,GAAW,CAAC,CAAC;QAE/B;;;;;;WAMG;QACK,wBAAmB,GAAa,EAAE,CAAC;QAE3C;;WAEG;QACK,qBAAgB,GAAW,CAAC,CAAC;QAErC;;WAEG;QACK,aAAQ,GAAmB,EAAE,CAAC;QAEtC;;WAEG;QACK,aAAQ,GAAmB,EAAE,CAAC;QAsBpC,IAAI,UAAU,IAAI,CAAC,EAAE;YACnB,MAAM,IAAI,UAAU,CAAC,gDAAgD,UAAU,EAAE,CAAC,CAAC;SACpF;QAED,IAAI,UAAU,IAAI,CAAC,EAAE;YACnB,MAAM,IAAI,UAAU,CAAC,gDAAgD,UAAU,EAAE,CAAC,CAAC;SACpF;QAED,IAAI,WAAW,IAAI,CAAC,EAAE;YACpB,MAAM,IAAI,UAAU,CAAC,iDAAiD,WAAW,EAAE,CAAC,CAAC;SACtF;QAED,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,EAAE;QACb,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAChC,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC1E,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;gBAEhC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE;oBACvB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;gBAC/B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;oBAC/B,OAAO;iBACR;gBAED,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,yBAAyB,KAAK,CAAC,EAAE;oBAC5D,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,UAAU,EAAE;wBACxE,MAAM,MAAM,GAAG,IAAI,CAAC,kCAAkC,EAAE,CAAC;wBACzD,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;6BAC7E,IAAI,CAAC,OAAO,CAAC;6BACb,KAAK,CAAC,MAAM,CAAC,CAAC;qBAClB;yBAAM,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE;wBACnD,OAAO;qBACR;yBAAM;wBACL,OAAO,EAAE,CAAC;qBACX;iBACF;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,oBAAoB,CAAC,IAAY;QACvC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,kCAAkC,CAAC,MAAqB;QAC9D,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7F;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC9D;QAED,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAC,IAAI,CAAC;QACrC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;OAQG;IACK,WAAW;QACjB,OAAO,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE;YAC/C,IAAI,MAAoB,CAAC;YAEzB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5B,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC,CAAC;aACjD;iBAAM;gBACL,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE;oBACrC,MAAM,GAAG,IAAI,CAAC,kCAAkC,EAAE,CAAC;oBACnD,IAAI,CAAC,UAAU,EAAE,CAAC;iBACnB;qBAAM;oBACL,gDAAgD;oBAChD,OAAO,KAAK,CAAC;iBACd;aACF;YAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB;QACnC,IAAI,MAAgC,CAAC;QACrC,GAAG;YACD,IAAI,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,WAAW,EAAE;gBACtD,OAAO;aACR;YAED,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,MAAM,EAAE;gBACV,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;aACrC;SACF,QAAQ,MAAM,EAAE;IACnB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,sBAAsB,CAAC,MAAoB;QACvD,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC;QAEjC,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC;QAE5B,IAAI;YACF,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE,EAChC,YAAY,EACZ,IAAI,CAAC,MAAM,GAAG,YAAY,CAC3B,CAAC;SACH;QAAC,OAAO,GAAQ,EAAE;YACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAChC,OAAO;SACR;QAED,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,MAAoB;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YAC5D,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACxB;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { EventEmitter } from \"events\";\nimport { Readable } from \"stream\";\nimport { PooledBuffer } from \"./PooledBuffer\";\n\n/**\n * OutgoingHandler is an async function triggered by BufferScheduler.\n */\nexport declare type OutgoingHandler = (\n  body: () => NodeJS.ReadableStream,\n  length: number,\n  offset?: number\n) => Promise<any>;\n\n/**\n * This class accepts a Node.js Readable stream as input, and keeps reading data\n * from the stream into the internal buffer structure, until it reaches maxBuffers.\n * Every available buffer will try to trigger outgoingHandler.\n *\n * The internal buffer structure includes an incoming buffer array, and a outgoing\n * buffer array. The incoming buffer array includes the \"empty\" buffers can be filled\n * with new incoming data. The outgoing array includes the filled buffers to be\n * handled by outgoingHandler. Every above buffer size is defined by parameter bufferSize.\n *\n * NUM_OF_ALL_BUFFERS = BUFFERS_IN_INCOMING + BUFFERS_IN_OUTGOING + BUFFERS_UNDER_HANDLING\n *\n * NUM_OF_ALL_BUFFERS lesser than or equal to maxBuffers\n *\n * PERFORMANCE IMPROVEMENT TIPS:\n * 1. Input stream highWaterMark is better to set a same value with bufferSize\n *    parameter, which will avoid Buffer.concat() operations.\n * 2. concurrency should set a smaller value than maxBuffers, which is helpful to\n *    reduce the possibility when a outgoing handler waits for the stream data.\n *    in this situation, outgoing handlers are blocked.\n *    Outgoing queue shouldn't be empty.\n */\nexport class BufferScheduler {\n  /**\n   * Size of buffers in incoming and outgoing queues. This class will try to align\n   * data read from Readable stream into buffer chunks with bufferSize defined.\n   */\n  private readonly bufferSize: number;\n\n  /**\n   * How many buffers can be created or maintained.\n   */\n  private readonly maxBuffers: number;\n\n  /**\n   * A Node.js Readable stream.\n   */\n  private readonly readable: Readable;\n\n  /**\n   * OutgoingHandler is an async function triggered by BufferScheduler when there\n   * are available buffers in outgoing array.\n   */\n  private readonly outgoingHandler: OutgoingHandler;\n\n  /**\n   * An internal event emitter.\n   */\n  private readonly emitter: EventEmitter = new EventEmitter();\n\n  /**\n   * Concurrency of executing outgoingHandlers. (0 lesser than concurrency lesser than or equal to maxBuffers)\n   */\n  private readonly concurrency: number;\n\n  /**\n   * An internal offset marker to track data offset in bytes of next outgoingHandler.\n   */\n  private offset: number = 0;\n\n  /**\n   * An internal marker to track whether stream is end.\n   */\n  private isStreamEnd: boolean = false;\n\n  /**\n   * An internal marker to track whether stream or outgoingHandler returns error.\n   */\n  private isError: boolean = false;\n\n  /**\n   * How many handlers are executing.\n   */\n  private executingOutgoingHandlers: number = 0;\n\n  /**\n   * Encoding of the input Readable stream which has string data type instead of Buffer.\n   */\n  private encoding?: BufferEncoding;\n\n  /**\n   * How many buffers have been allocated.\n   */\n  private numBuffers: number = 0;\n\n  /**\n   * Because this class doesn't know how much data every time stream pops, which\n   * is defined by highWaterMarker of the stream. So BufferScheduler will cache\n   * data received from the stream, when data in unresolvedDataArray exceeds the\n   * blockSize defined, it will try to concat a blockSize of buffer, fill into available\n   * buffers from incoming and push to outgoing array.\n   */\n  private unresolvedDataArray: Buffer[] = [];\n\n  /**\n   * How much data consisted in unresolvedDataArray.\n   */\n  private unresolvedLength: number = 0;\n\n  /**\n   * The array includes all the available buffers can be used to fill data from stream.\n   */\n  private incoming: PooledBuffer[] = [];\n\n  /**\n   * The array (queue) includes all the buffers filled from stream data.\n   */\n  private outgoing: PooledBuffer[] = [];\n\n  /**\n   * Creates an instance of BufferScheduler.\n   *\n   * @param readable - A Node.js Readable stream\n   * @param bufferSize - Buffer size of every maintained buffer\n   * @param maxBuffers - How many buffers can be allocated\n   * @param outgoingHandler - An async function scheduled to be\n   *                                          triggered when a buffer fully filled\n   *                                          with stream data\n   * @param concurrency - Concurrency of executing outgoingHandlers (>0)\n   * @param encoding - [Optional] Encoding of Readable stream when it's a string stream\n   */\n  constructor(\n    readable: Readable,\n    bufferSize: number,\n    maxBuffers: number,\n    outgoingHandler: OutgoingHandler,\n    concurrency: number,\n    encoding?: BufferEncoding\n  ) {\n    if (bufferSize <= 0) {\n      throw new RangeError(`bufferSize must be larger than 0, current is ${bufferSize}`);\n    }\n\n    if (maxBuffers <= 0) {\n      throw new RangeError(`maxBuffers must be larger than 0, current is ${maxBuffers}`);\n    }\n\n    if (concurrency <= 0) {\n      throw new RangeError(`concurrency must be larger than 0, current is ${concurrency}`);\n    }\n\n    this.bufferSize = bufferSize;\n    this.maxBuffers = maxBuffers;\n    this.readable = readable;\n    this.outgoingHandler = outgoingHandler;\n    this.concurrency = concurrency;\n    this.encoding = encoding;\n  }\n\n  /**\n   * Start the scheduler, will return error when stream of any of the outgoingHandlers\n   * returns error.\n   *\n   */\n  public async do(): Promise<void> {\n    return new Promise<void>((resolve, reject) => {\n      this.readable.on(\"data\", (data) => {\n        data = typeof data === \"string\" ? Buffer.from(data, this.encoding) : data;\n        this.appendUnresolvedData(data);\n\n        if (!this.resolveData()) {\n          this.readable.pause();\n        }\n      });\n\n      this.readable.on(\"error\", (err) => {\n        this.emitter.emit(\"error\", err);\n      });\n\n      this.readable.on(\"end\", () => {\n        this.isStreamEnd = true;\n        this.emitter.emit(\"checkEnd\");\n      });\n\n      this.emitter.on(\"error\", (err) => {\n        this.isError = true;\n        this.readable.pause();\n        reject(err);\n      });\n\n      this.emitter.on(\"checkEnd\", () => {\n        if (this.outgoing.length > 0) {\n          this.triggerOutgoingHandlers();\n          return;\n        }\n\n        if (this.isStreamEnd && this.executingOutgoingHandlers === 0) {\n          if (this.unresolvedLength > 0 && this.unresolvedLength < this.bufferSize) {\n            const buffer = this.shiftBufferFromUnresolvedDataArray();\n            this.outgoingHandler(() => buffer.getReadableStream(), buffer.size, this.offset)\n              .then(resolve)\n              .catch(reject);\n          } else if (this.unresolvedLength >= this.bufferSize) {\n            return;\n          } else {\n            resolve();\n          }\n        }\n      });\n    });\n  }\n\n  /**\n   * Insert a new data into unresolved array.\n   *\n   * @param data -\n   */\n  private appendUnresolvedData(data: Buffer) {\n    this.unresolvedDataArray.push(data);\n    this.unresolvedLength += data.length;\n  }\n\n  /**\n   * Try to shift a buffer with size in blockSize. The buffer returned may be less\n   * than blockSize when data in unresolvedDataArray is less than bufferSize.\n   *\n   */\n  private shiftBufferFromUnresolvedDataArray(buffer?: PooledBuffer): PooledBuffer {\n    if (!buffer) {\n      buffer = new PooledBuffer(this.bufferSize, this.unresolvedDataArray, this.unresolvedLength);\n    } else {\n      buffer.fill(this.unresolvedDataArray, this.unresolvedLength);\n    }\n\n    this.unresolvedLength -= buffer.size;\n    return buffer;\n  }\n\n  /**\n   * Resolve data in unresolvedDataArray. For every buffer with size in blockSize\n   * shifted, it will try to get (or allocate a buffer) from incoming, and fill it,\n   * then push it into outgoing to be handled by outgoing handler.\n   *\n   * Return false when available buffers in incoming are not enough, else true.\n   *\n   * @returns Return false when buffers in incoming are not enough, else true.\n   */\n  private resolveData(): boolean {\n    while (this.unresolvedLength >= this.bufferSize) {\n      let buffer: PooledBuffer;\n\n      if (this.incoming.length > 0) {\n        buffer = this.incoming.shift()!;\n        this.shiftBufferFromUnresolvedDataArray(buffer);\n      } else {\n        if (this.numBuffers < this.maxBuffers) {\n          buffer = this.shiftBufferFromUnresolvedDataArray();\n          this.numBuffers++;\n        } else {\n          // No available buffer, wait for buffer returned\n          return false;\n        }\n      }\n\n      this.outgoing.push(buffer);\n      this.triggerOutgoingHandlers();\n    }\n    return true;\n  }\n\n  /**\n   * Try to trigger a outgoing handler for every buffer in outgoing. Stop when\n   * concurrency reaches.\n   */\n  private async triggerOutgoingHandlers() {\n    let buffer: PooledBuffer | undefined;\n    do {\n      if (this.executingOutgoingHandlers >= this.concurrency) {\n        return;\n      }\n\n      buffer = this.outgoing.shift();\n      if (buffer) {\n        this.triggerOutgoingHandler(buffer);\n      }\n    } while (buffer);\n  }\n\n  /**\n   * Trigger a outgoing handler for a buffer shifted from outgoing.\n   *\n   * @param buffer -\n   */\n  private async triggerOutgoingHandler(buffer: PooledBuffer): Promise<any> {\n    const bufferLength = buffer.size;\n\n    this.executingOutgoingHandlers++;\n    this.offset += bufferLength;\n\n    try {\n      await this.outgoingHandler(\n        () => buffer.getReadableStream(),\n        bufferLength,\n        this.offset - bufferLength\n      );\n    } catch (err: any) {\n      this.emitter.emit(\"error\", err);\n      return;\n    }\n\n    this.executingOutgoingHandlers--;\n    this.reuseBuffer(buffer);\n    this.emitter.emit(\"checkEnd\");\n  }\n\n  /**\n   * Return buffer used by outgoing handler into incoming.\n   *\n   * @param buffer -\n   */\n  private reuseBuffer(buffer: PooledBuffer) {\n    this.incoming.push(buffer);\n    if (!this.isError && this.resolveData() && !this.isStreamEnd) {\n      this.readable.resume();\n    }\n  }\n}\n"]}