Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
codeql-action/node_modules/@azure/storage-blob/dist-esm/storage-internal-avro/src/AvroReader.js.map
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"version":3,"file":"AvroReader.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/AvroReader.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AAElC,sCAAsC;AACtC,iEAAiE;AAEjE,OAAO,oBAAoB,CAAC;AAC5B,OAAO,EACL,cAAc,EACd,eAAe,EACf,eAAe,EACf,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAanD,MAAM,OAAO,UAAU;IAuCrB,YACE,UAAwB,EACxB,YAA2B,EAC3B,kBAA2B,EAC3B,uBAAgC;QAEhC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,YAAY,IAAI,UAAU,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,kBAAkB,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,uBAAuB,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,CAAC,CAAC;IACrD,CAAC;IAhCD,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAGD,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IA2BO,KAAK,CAAC,UAAU,CAAC,UAA4B,EAAE;QACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,MAAM,EAAE;YACzF,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE;YACzC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;QAED,sEAAsE;QACtE,sCAAsC;QACtC,IAAI,CAAC,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,UAAU,EAAE;YACnF,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QAEH,iBAAiB;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAU,CAAC,cAAc,CAAC,CAAC;QAC9C,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,EAAE;YAChE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC7C;QAED,6DAA6D;QAC7D,IAAI,CAAC,WAAW,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,qBAAqB,EAAE;YAC5F,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QAEH,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAU,CAAC,eAAe,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;SAC1E;QAED,IAAI,CAAC,sBAAsB,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE;YACxE,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QACH,oBAAoB;QACpB,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAElF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE;YAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE;gBAC1C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;gBAClF,IAAI,CAAC,sBAAuB,EAAE,CAAC;aAChC;SACF;IACH,CAAC;IAEM,OAAO;QACZ,OAAO,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,sBAAuB,GAAG,CAAC,CAAC;IAChE,CAAC;IAEa,YAAY,CACxB,UAA4B,EAAE;;YAE9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBACtB,cAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA,CAAC;aAChC;YAED,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE;gBACrB,MAAM,MAAM,GAAG,cAAM,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;oBAC1D,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC,CAAA,CAAC;gBAEH,IAAI,CAAC,sBAAuB,EAAE,CAAC;gBAC/B,IAAI,CAAC,YAAa,EAAE,CAAC;gBAErB,IAAI,IAAI,CAAC,sBAAsB,KAAK,CAAC,EAAE;oBACrC,MAAM,MAAM,GAAG,cAAM,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,qBAAqB,EAAE;wBACtF,WAAW,EAAE,OAAO,CAAC,WAAW;qBACjC,CAAC,CAAA,CAAC;oBAEH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBACzE,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;oBAEtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAY,EAAE,MAAM,CAAC,EAAE;wBAC3C,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;qBACrD;oBAED,IAAI;wBACF,IAAI,CAAC,sBAAsB,GAAG,cAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE;4BACxE,WAAW,EAAE,OAAO,CAAC,WAAW;yBACjC,CAAC,CAAA,CAAC;qBACJ;oBAAC,OAAO,GAAQ,EAAE;wBACjB,gCAAgC;wBAChC,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;qBACjC;oBAED,IAAI,IAAI,CAAC,sBAAuB,GAAG,CAAC,EAAE;wBACpC,oBAAoB;wBACpB,cAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA,CAAC;qBACnF;iBACF;gBACD,oBAAM,MAAM,CAAA,CAAC;aACd;QACH,CAAC;KAAA;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\n// TODO: Do a review of non-interfaces\n/* eslint-disable @azure/azure-sdk/ts-use-interface-parameters */\n\nimport \"@azure/core-paging\";\nimport {\n AVRO_CODEC_KEY,\n AVRO_INIT_BYTES,\n AVRO_SCHEMA_KEY,\n AVRO_SYNC_MARKER_SIZE,\n} from \"./AvroConstants\";\nimport { AvroParser, AvroType } from \"./AvroParser\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { AvroReadable } from \"./AvroReadable\";\nimport { arraysEqual } from \"./utils/utils.common\";\n\n/**\n * Options to configure the {@link AvroReader.parseObjects} operation.\n */\nexport interface AvroParseOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class AvroReader {\n private readonly _dataStream: AvroReadable;\n\n private readonly _headerStream: AvroReadable;\n\n private _syncMarker?: Uint8Array;\n\n private _metadata?: Record<string, string>;\n\n private _itemType?: AvroType;\n\n private _itemsRemainingInBlock?: number;\n\n // Remembers where we started if partial data stream was provided.\n private readonly _initialBlockOffset: number;\n\n /// The byte offset within the Avro file (both header and data)\n /// of the start of the current block.\n private _blockOffset: number;\n public get blockOffset(): number {\n return this._blockOffset;\n }\n\n private _objectIndex: number;\n public get objectIndex(): number {\n return this._objectIndex;\n }\n\n private _initialized: boolean;\n\n constructor(dataStream: AvroReadable);\n\n constructor(\n dataStream: AvroReadable,\n headerStream: AvroReadable,\n currentBlockOffset: number,\n indexWithinCurrentBlock: number\n );\n\n constructor(\n dataStream: AvroReadable,\n headerStream?: AvroReadable,\n currentBlockOffset?: number,\n indexWithinCurrentBlock?: number\n ) {\n this._dataStream = dataStream;\n this._headerStream = headerStream || dataStream;\n this._initialized = false;\n this._blockOffset = currentBlockOffset || 0;\n this._objectIndex = indexWithinCurrentBlock || 0;\n this._initialBlockOffset = currentBlockOffset || 0;\n }\n\n private async initialize(options: AvroParseOptions = {}): Promise<void> {\n const header = await AvroParser.readFixedBytes(this._headerStream, AVRO_INIT_BYTES.length, {\n abortSignal: options.abortSignal,\n });\n if (!arraysEqual(header, AVRO_INIT_BYTES)) {\n throw new Error(\"Stream is not an Avro file.\");\n }\n\n // File metadata is written as if defined by the following map schema:\n // { \"type\": \"map\", \"values\": \"bytes\"}\n this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString, {\n abortSignal: options.abortSignal,\n });\n\n // Validate codec\n const codec = this._metadata![AVRO_CODEC_KEY];\n if (!(codec === undefined || codec === null || codec === \"null\")) {\n throw new Error(\"Codecs are not supported\");\n }\n\n // The 16-byte, randomly-generated sync marker for this file.\n this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AVRO_SYNC_MARKER_SIZE, {\n abortSignal: options.abortSignal,\n });\n\n // Parse the schema\n const schema = JSON.parse(this._metadata![AVRO_SCHEMA_KEY]);\n this._itemType = AvroType.fromSchema(schema);\n\n if (this._blockOffset === 0) {\n this._blockOffset = this._initialBlockOffset + this._dataStream.position;\n }\n\n this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream, {\n abortSignal: options.abortSignal,\n });\n // skip block length\n await AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });\n\n this._initialized = true;\n if (this._objectIndex && this._objectIndex > 0) {\n for (let i = 0; i < this._objectIndex; i++) {\n await this._itemType.read(this._dataStream, { abortSignal: options.abortSignal });\n this._itemsRemainingInBlock!--;\n }\n }\n }\n\n public hasNext(): boolean {\n return !this._initialized || this._itemsRemainingInBlock! > 0;\n }\n\n public async *parseObjects(\n options: AvroParseOptions = {}\n ): AsyncIterableIterator<Record<string, any> | null> {\n if (!this._initialized) {\n await this.initialize(options);\n }\n\n while (this.hasNext()) {\n const result = await this._itemType!.read(this._dataStream, {\n abortSignal: options.abortSignal,\n });\n\n this._itemsRemainingInBlock!--;\n this._objectIndex!++;\n\n if (this._itemsRemainingInBlock === 0) {\n const marker = await AvroParser.readFixedBytes(this._dataStream, AVRO_SYNC_MARKER_SIZE, {\n abortSignal: options.abortSignal,\n });\n\n this._blockOffset = this._initialBlockOffset + this._dataStream.position;\n this._objectIndex = 0;\n\n if (!arraysEqual(this._syncMarker!, marker)) {\n throw new Error(\"Stream is not a valid Avro file.\");\n }\n\n try {\n this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream, {\n abortSignal: options.abortSignal,\n });\n } catch (err: any) {\n // We hit the end of the stream.\n this._itemsRemainingInBlock = 0;\n }\n\n if (this._itemsRemainingInBlock! > 0) {\n // Ignore block size\n await AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });\n }\n }\n yield result;\n }\n }\n}\n"]} |