From fc49c5b4d532ce871d6646ececc25b2f4482eaf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sat, 1 Feb 2025 10:13:27 +0100 Subject: [PATCH 1/3] refactor: move endianness check to separate file --- src/IOBuffer.ts | 11 +++-------- src/env.ts | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 src/env.ts diff --git a/src/IOBuffer.ts b/src/IOBuffer.ts index c939557..f4df760 100644 --- a/src/IOBuffer.ts +++ b/src/IOBuffer.ts @@ -1,13 +1,8 @@ +import { isHostBigEndian } from './env'; import { decode, encode } from './text'; const defaultByteLength = 1024 * 8; -const hostBigEndian = (() => { - const array = new Uint8Array(4); - const view = new Uint32Array(array.buffer); - return !((view[0] = 1) & array[0]); -})(); - type InputData = number | ArrayBufferLike | ArrayBufferView | IOBuffer | Buffer; const typedArrays = { @@ -21,7 +16,7 @@ const typedArrays = { int64: globalThis.BigInt64Array, float32: globalThis.Float32Array, float64: globalThis.Float64Array, -}; +} as const; type TypedArrays = typeof typedArrays; @@ -319,7 +314,7 @@ export class IOBuffer { const offset = this.byteOffset + this.offset; const slice = this.buffer.slice(offset, offset + bytes); if ( - this.littleEndian === hostBigEndian && + this.littleEndian === isHostBigEndian() && type !== 'uint8' && type !== 'int8' ) { diff --git a/src/env.ts b/src/env.ts new file mode 100644 index 0000000..f1d860a --- /dev/null +++ b/src/env.ts @@ -0,0 +1,15 @@ +let hostIsBigEndian: boolean | undefined; + +/** + * Checks if the host system is big-endian. + * @returns `true` if the host system is big-endian, `false` if it is little-endian. + */ +export function isHostBigEndian() { + return hostIsBigEndian ?? (hostIsBigEndian = detectEndianness()); +} + +function detectEndianness() { + const array = new Uint8Array(4); + const view = new Uint32Array(array.buffer); + return !((view[0] = 1) & array[0]); +} From 86e612fd0a609c26e1684ba33f8cd570c272a178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sat, 1 Feb 2025 12:21:58 +0100 Subject: [PATCH 2/3] fix: don't try to grow the buffer more than allowed by the system Closes: https://github.com/image-js/fast-png/issues/42 --- src/IOBuffer.ts | 14 ++++----- src/__tests__/write.test.ts | 3 ++ src/env.ts | 58 +++++++++++++++++++++++++++++++++++++ src/text.ts | 11 +++++++ tsconfig.json | 1 + 5 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/IOBuffer.ts b/src/IOBuffer.ts index f4df760..b0e9afb 100644 --- a/src/IOBuffer.ts +++ b/src/IOBuffer.ts @@ -1,4 +1,4 @@ -import { isHostBigEndian } from './env'; +import { increaseBufferSize, isHostBigEndian } from './env'; import { decode, encode } from './text'; const defaultByteLength = 1024 * 8; @@ -246,13 +246,11 @@ export class IOBuffer { public ensureAvailable(byteLength = 1): this { if (!this.available(byteLength)) { const lengthNeeded = this.offset + byteLength; - const newLength = lengthNeeded * 2; - const newArray = new Uint8Array(newLength); - newArray.set(new Uint8Array(this.buffer)); - this.buffer = newArray.buffer; - this.length = newLength; - this.byteLength = newLength; - this._data = new DataView(this.buffer); + const newBuffer = increaseBufferSize(this.buffer, lengthNeeded); + this.buffer = newBuffer; + this.length = newBuffer.byteLength; + this.byteLength = newBuffer.byteLength; + this._data = new DataView(newBuffer); } return this; } diff --git a/src/__tests__/write.test.ts b/src/__tests__/write.test.ts index 15333a8..5e62a60 100644 --- a/src/__tests__/write.test.ts +++ b/src/__tests__/write.test.ts @@ -122,6 +122,9 @@ describe('write data', () => { theBuffer.seek(20); theBuffer.ensureAvailable(30); expect(theBuffer.byteLength).toBeGreaterThanOrEqual(50); + expect(() => theBuffer.ensureAvailable(Number.MAX_SAFE_INTEGER)).toThrow( + RangeError, + ); }); it('writeUtf8', () => { diff --git a/src/env.ts b/src/env.ts index f1d860a..d9b4e6f 100644 --- a/src/env.ts +++ b/src/env.ts @@ -13,3 +13,61 @@ function detectEndianness() { const view = new Uint32Array(array.buffer); return !((view[0] = 1) & array[0]); } + +/** + * Returns a new buffer with the same contents as the input buffer, but with a larger size. + * Try to at least double the size of the buffer to avoid frequent reallocations. + * @param buffer - The buffer to increase the size of. + * @param minimumSize - The minimum size of the new buffer. + * @returns The new buffer. + */ +export function increaseBufferSize( + buffer: ArrayBuffer, + minimumSize: number, +): ArrayBuffer { + let maxAllowedLength = 2 ** 29; // 512MB + if (minimumSize > maxAllowedLength) { + // The check is expensive, don't do it until a large buffer is requested. + maxAllowedLength = getMaxUint8ArrayLength(); + if (minimumSize > maxAllowedLength) { + throw new RangeError( + `Cannot create a buffer with more than ${maxAllowedLength} bytes`, + ); + } + } + // Don't try to allocate more than the maximum allowed length. + const doubleOrMax = Math.min(buffer.byteLength * 2, maxAllowedLength); + const newLength = Math.max(doubleOrMax, minimumSize); + const newBuffer = new ArrayBuffer(newLength); + new Uint8Array(newBuffer).set(new Uint8Array(buffer)); + return newBuffer; +} + +let maxUint8ArrayLength: number | undefined; + +/** + * Detects and returns the maximum length that of an `ArrayBuffer`. + * @returns The maximum length that an `ArrayBuffer` can have. + */ +export function getMaxUint8ArrayLength() { + return ( + maxUint8ArrayLength ?? (maxUint8ArrayLength = detectMaxUint8ArrayLength()) + ); +} + +function detectMaxUint8ArrayLength() { + // Use a binary search to find the maximum length of an Uint8Array. + let low = 1; + let high = Number.MAX_SAFE_INTEGER - 1; + while (low < high) { + const mid = Math.trunc((low + high) / 2); + try { + // eslint-disable-next-line no-new + new Uint8Array(mid); + low = mid + 1; + } catch { + high = mid; + } + } + return low - 1; +} diff --git a/src/text.ts b/src/text.ts index 3ff8fee..6588b8f 100644 --- a/src/text.ts +++ b/src/text.ts @@ -1,3 +1,9 @@ +/** + * Decode a Uint8Array to a string. + * @param bytes - The Uint8Array to decode. + * @param encoding - The encoding to use. Defaults to 'utf8'. + * @returns The decoded string. + */ export function decode(bytes: Uint8Array, encoding = 'utf8'): string { const decoder = new TextDecoder(encoding); return decoder.decode(bytes); @@ -5,6 +11,11 @@ export function decode(bytes: Uint8Array, encoding = 'utf8'): string { const encoder = new TextEncoder(); +/** + * Encode a string as UTF-8 to a Uint8Array. + * @param str - The string to encode. + * @returns The encoded Uint8Array. + */ export function encode(str: string): Uint8Array { return encoder.encode(str); } diff --git a/tsconfig.json b/tsconfig.json index 63c0de5..e297dbc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,7 @@ "sourceMap": true, "strict": true, "target": "es2022", + "lib": ["es2022"], "skipLibCheck": true, "allowJs": false, "noEmit": true From 0d1486a83ba72f2c6b44388188cb91cee96e31c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sat, 1 Feb 2025 12:39:04 +0100 Subject: [PATCH 3/3] chore: remove old eslintrc --- .eslintrc.yml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .eslintrc.yml diff --git a/.eslintrc.yml b/.eslintrc.yml deleted file mode 100644 index b16b5f8..0000000 --- a/.eslintrc.yml +++ /dev/null @@ -1,3 +0,0 @@ -extends: cheminfo-typescript -rules: - '@typescript-eslint/prefer-for-of': off