From f95e7ea2204c023c93fe73894d2c6e8c632f6beb Mon Sep 17 00:00:00 2001 From: Daan Vanden Bosch Date: Sat, 27 Jul 2019 21:47:49 +0200 Subject: [PATCH] All unit tests pass again. Removed dependency on text-encoding. Refactored cursors to reuse more code. --- package.json | 1 - src/data_formats/compression/prs/compress.ts | 7 +- .../compression/prs/decompress.ts | 4 +- src/data_formats/cursor/AbstractCursor.ts | 253 +++++++++++++++ ...ferCursor.ts => AbstractWritableCursor.ts} | 66 ++-- src/data_formats/cursor/ArrayBufferCursor.ts | 264 +--------------- src/data_formats/cursor/BufferCursor.ts | 30 +- src/data_formats/cursor/Cursor.test.ts | 12 - ....test.ts => ResizableBufferCursor.test.ts} | 8 +- .../cursor/ResizableBufferCursor.ts | 298 ++---------------- .../cursor/WritableArrayBufferCursor.ts | 122 ------- .../cursor/WritableCursor.test.ts | 12 +- src/data_formats/cursor/index.ts | 14 - src/data_formats/encryption/prc.ts | 9 +- src/data_formats/parsing/ninja/index.ts | 11 +- src/data_formats/parsing/quest/bin.ts | 4 +- src/data_formats/parsing/quest/dat.ts | 4 +- src/data_formats/parsing/quest/qst.ts | 13 +- tsconfig.json | 2 +- yarn.lock | 5 - 20 files changed, 378 insertions(+), 761 deletions(-) create mode 100644 src/data_formats/cursor/AbstractCursor.ts rename src/data_formats/cursor/{WritableResizableBufferCursor.ts => AbstractWritableCursor.ts} (64%) rename src/data_formats/cursor/{WritableResizableBufferCursor.test.ts => ResizableBufferCursor.test.ts} (84%) delete mode 100644 src/data_formats/cursor/WritableArrayBufferCursor.ts delete mode 100644 src/data_formats/cursor/index.ts diff --git a/package.json b/package.json index fc459c39..bfd11cd3 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "react-dom": "^16.8.6", "react-virtualized": "^9.21.1", "react-virtualized-select": "^3.1.3", - "text-encoding": "^0.7.0", "three": "^0.106.2" }, "scripts": { diff --git a/src/data_formats/compression/prs/compress.ts b/src/data_formats/compression/prs/compress.ts index 1354741a..0fc34801 100644 --- a/src/data_formats/compression/prs/compress.ts +++ b/src/data_formats/compression/prs/compress.ts @@ -1,6 +1,6 @@ import { Cursor } from "../../cursor/Cursor"; +import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; import { WritableCursor } from "../../cursor/WritableCursor"; -import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor"; import { ResizableBuffer } from "../../ResizableBuffer"; export function prs_compress(src: Cursor): Cursor { @@ -117,10 +117,7 @@ class Context { constructor(cursor: Cursor) { this.src = cursor; - this.dst = new WritableResizableBufferCursor( - new ResizableBuffer(cursor.size), - cursor.endianness - ); + this.dst = new ResizableBufferCursor(new ResizableBuffer(cursor.size), cursor.endianness); } set_bit(bit: number): void { diff --git a/src/data_formats/compression/prs/decompress.ts b/src/data_formats/compression/prs/decompress.ts index 22543123..c4f4ced4 100644 --- a/src/data_formats/compression/prs/decompress.ts +++ b/src/data_formats/compression/prs/decompress.ts @@ -1,7 +1,7 @@ import Logger from "js-logger"; import { Cursor } from "../../cursor/Cursor"; +import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; import { WritableCursor } from "../../cursor/WritableCursor"; -import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor"; import { ResizableBuffer } from "../../ResizableBuffer"; const logger = Logger.get("data_formats/compression/prs/decompress"); @@ -63,7 +63,7 @@ class Context { constructor(cursor: Cursor) { this.src = cursor; - this.dst = new WritableResizableBufferCursor( + this.dst = new ResizableBufferCursor( new ResizableBuffer(Math.floor(1.5 * cursor.size)), cursor.endianness ); diff --git a/src/data_formats/cursor/AbstractCursor.ts b/src/data_formats/cursor/AbstractCursor.ts new file mode 100644 index 00000000..084d61d2 --- /dev/null +++ b/src/data_formats/cursor/AbstractCursor.ts @@ -0,0 +1,253 @@ +import { Endianness } from ".."; +import { Vec2, Vec3 } from "../vector"; +import { Cursor } from "./Cursor"; + +export abstract class AbstractCursor implements Cursor { + abstract readonly size: number; + + protected _position: number = 0; + + get position(): number { + return this._position; + } + + protected little_endian!: boolean; + + get endianness(): Endianness { + return this.little_endian ? Endianness.Little : Endianness.Big; + } + + set endianness(endianness: Endianness) { + this.little_endian = endianness === Endianness.Little; + } + + get bytes_left(): number { + return this.size - this.position; + } + + protected readonly offset: number; + + protected abstract readonly backing_buffer: ArrayBuffer; + protected abstract readonly dv: DataView; + + constructor(endianness: Endianness, offset: number) { + this.endianness = endianness; + this.offset = offset; + } + + seek(offset: number): this { + return this.seek_start(this.position + offset); + } + + seek_start(offset: number): this { + if (offset < 0 || offset > this.size) { + throw new Error(`Offset ${offset} is out of bounds.`); + } + + this._position = offset; + return this; + } + + seek_end(offset: number): this { + if (offset < 0 || offset > this.size) { + throw new Error(`Offset ${offset} is out of bounds.`); + } + + this._position = this.size - offset; + return this; + } + + u8(): number { + return this.u8_at(this._position++); + } + + u8_at(offset: number): number { + this.check_offset(offset, 1); + return this.dv.getUint8(this.offset + offset); + } + + u16(): number { + const r = this.u16_at(this.position); + this._position += 2; + return r; + } + + u16_at(offset: number): number { + this.check_offset(offset, 2); + return this.dv.getUint16(this.offset + offset, this.little_endian); + } + + u32(): number { + const r = this.u32_at(this.position); + this._position += 4; + return r; + } + + u32_at(offset: number): number { + this.check_offset(offset, 4); + return this.dv.getUint32(this.offset + offset, this.little_endian); + } + + i8(): number { + return this.i8_at(this._position++); + } + + i8_at(offset: number): number { + this.check_offset(offset, 1); + return this.dv.getInt8(this.offset + offset); + } + + i16(): number { + const r = this.i16_at(this.position); + this._position += 2; + return r; + } + + i16_at(offset: number): number { + this.check_offset(offset, 2); + return this.dv.getInt16(this.offset + offset, this.little_endian); + } + + i32(): number { + const r = this.i32_at(this.position); + this._position += 4; + return r; + } + + i32_at(offset: number): number { + this.check_offset(offset, 4); + return this.dv.getInt32(this.offset + offset, this.little_endian); + } + + f32(): number { + const r = this.f32_at(this.position); + this._position += 4; + return r; + } + + f32_at(offset: number): number { + this.check_offset(offset, 4); + return this.dv.getFloat32(this.offset + offset, this.little_endian); + } + + u8_array(n: number): number[] { + this.check_size("n", n, n); + + const array = []; + + for (let i = 0; i < n; ++i) { + array.push(this.dv.getUint8(this.offset + this._position++)); + } + + return array; + } + + u16_array(n: number): number[] { + this.check_size("n", n, 2 * n); + + const array = []; + + for (let i = 0; i < n; ++i) { + array.push(this.dv.getUint16(this.offset + this.position, this.little_endian)); + this._position += 2; + } + + return array; + } + + u32_array(n: number): number[] { + this.check_size("n", n, 4 * n); + + const array = []; + + for (let i = 0; i < n; ++i) { + array.push(this.dv.getUint32(this.offset + this.position, this.little_endian)); + this._position += 4; + } + + return array; + } + + vec2_f32(): Vec2 { + return new Vec2(this.f32(), this.f32()); + } + + vec3_f32(): Vec3 { + return new Vec3(this.f32(), this.f32(), this.f32()); + } + + string_ascii(max_byte_length: number, null_terminated: boolean, drop_remaining: boolean) { + let code_points: number[] = []; + + for (let i = 0; i < max_byte_length; i++) { + const code_point = this.u8(); + + if (null_terminated && code_point === 0) { + if (drop_remaining) { + this.seek(max_byte_length - i - 1); + } + + break; + } + + code_points.push(code_point); + } + + return String.fromCodePoint(...code_points); + } + + string_utf16(max_byte_length: number, null_terminated: boolean, drop_remaining: boolean) { + let code_points: number[] = []; + let len = Math.floor(max_byte_length / 2); + + for (let i = 0; i < len; i++) { + const code_point = this.u16(); + + if (null_terminated && code_point === 0) { + if (drop_remaining) { + this.seek(2 * (len - i - 1)); + } + + break; + } + + code_points.push(code_point); + } + + return String.fromCodePoint(...code_points); + } + + array_buffer(size: number = this.size - this.position): ArrayBuffer { + this.check_size("size", size, size); + const r = this.backing_buffer.slice( + this.offset + this.position, + this.offset + this.position + size + ); + this._position += size; + return r; + } + + copy_to_uint8_array(array: Uint8Array, size: number = this.size - this.position): this { + this.check_size("size", size, size); + array.set(new Uint8Array(this.backing_buffer, this.offset + this.position, size)); + this._position += size; + return this; + } + + abstract take(size: number): Cursor; + + protected check_size(name: string, value: number, byte_size: number): void { + if (byte_size < 0 || byte_size > this.size - this.position) { + throw new Error(`${name} ${value} is out of bounds.`); + } + } + + /** + * Checks whether we can read size bytes at offset. + */ + private check_offset(offset: number, size: number): void { + if (offset < 0 || offset + size > this.size) { + throw new Error(`Offset ${offset} is out of bounds.`); + } + } +} diff --git a/src/data_formats/cursor/WritableResizableBufferCursor.ts b/src/data_formats/cursor/AbstractWritableCursor.ts similarity index 64% rename from src/data_formats/cursor/WritableResizableBufferCursor.ts rename to src/data_formats/cursor/AbstractWritableCursor.ts index 2ab8b906..da50e2ae 100644 --- a/src/data_formats/cursor/WritableResizableBufferCursor.ts +++ b/src/data_formats/cursor/AbstractWritableCursor.ts @@ -1,21 +1,10 @@ -import { WritableCursor } from "./WritableCursor"; -import { ResizableBufferCursor } from "./ResizableBufferCursor"; +import { Vec2, Vec3 } from "../vector"; +import { AbstractCursor } from "./AbstractCursor"; import { Cursor } from "./Cursor"; -import { ASCII_ENCODER } from "."; -import { Vec3, Vec2 } from "../vector"; +import { WritableCursor } from "./WritableCursor"; -export class WritableResizableBufferCursor extends ResizableBufferCursor implements WritableCursor { - get size(): number { - return this._size; - } - - set size(size: number) { - if (size > this._size) { - this.ensure_size(size - this._size); - } else { - this._size = size; - } - } +export abstract class AbstractWritableCursor extends AbstractCursor implements WritableCursor { + abstract size: number; write_u8(value: number): this { this.ensure_size(1); @@ -53,9 +42,7 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme write_u8_array(array: number[]): this { this.ensure_size(array.length); - new Uint8Array(this.buffer.backing_buffer, this.offset + this.position).set( - new Uint8Array(array) - ); + new Uint8Array(this.backing_buffer, this.offset + this.position).set(new Uint8Array(array)); this._position += array.length; return this; } @@ -104,7 +91,7 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme this.ensure_size(size); other.copy_to_uint8_array( - new Uint8Array(this.buffer.backing_buffer, this.offset + this.position, size), + new Uint8Array(this.backing_buffer, this.offset + this.position, size), size ); @@ -115,15 +102,15 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme write_string_ascii(str: string, byte_length: number): this { this.ensure_size(byte_length); - const encoded = ASCII_ENCODER.encode(str); - const encoded_length = Math.min(encoded.byteLength, byte_length); - let i = 0; + const len = Math.min(byte_length, str.length); - while (i < encoded_length) { - this.write_u8(encoded[i++]); + for (let i = 0; i < len; i++) { + this.write_u8(str.codePointAt(i)!); } - while (i++ < byte_length) { + let pad_len = byte_length - len; + + for (let i = 0; i < pad_len; i++) { this.write_u8(0); } @@ -133,30 +120,25 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme write_string_utf16(str: string, byte_length: number): this { this.ensure_size(byte_length); - const encoded = this.utf16_encoder.encode(str); - const encoded_length = Math.min(encoded.byteLength, byte_length); - let i = 0; + const max_len = Math.floor(byte_length / 2); + const len = Math.min(max_len, str.length); - while (i < encoded_length) { - this.write_u8(encoded[i++]); + for (let i = 0; i < len; i++) { + this.write_u16(str.codePointAt(i)!); } - while (i++ < byte_length) { - this.write_u8(0); + let pad_len = max_len - len; + + for (let i = 0; i < pad_len; i++) { + this.write_u16(0); } return this; } - private ensure_size(size: number): void { - const needed = this.position + size - this._size; - - if (needed > 0) { - this._size += needed; - - if (this.buffer.size < this.offset + this._size) { - this.buffer.size = this.offset + this._size; - } + protected ensure_size(size: number): void { + if (size > this.bytes_left) { + throw new Error(`${size} Bytes required but only ${this.bytes_left} available.`); } } } diff --git a/src/data_formats/cursor/ArrayBufferCursor.ts b/src/data_formats/cursor/ArrayBufferCursor.ts index 792bb2e8..ae67e84b 100644 --- a/src/data_formats/cursor/ArrayBufferCursor.ts +++ b/src/data_formats/cursor/ArrayBufferCursor.ts @@ -1,56 +1,27 @@ -import { - ASCII_DECODER, - UTF_16BE_DECODER, - UTF_16BE_ENCODER, - UTF_16LE_DECODER, - UTF_16LE_ENCODER, -} from "."; import { Endianness } from ".."; -import { Vec2, Vec3 } from "../vector"; -import { Cursor } from "./Cursor"; +import { AbstractWritableCursor } from "./AbstractWritableCursor"; +import { WritableCursor } from "./WritableCursor"; /** * A cursor for reading from an array buffer or part of an array buffer. */ -export class ArrayBufferCursor implements Cursor { - get offset(): number { - return this.dv.byteOffset; - } +export class ArrayBufferCursor extends AbstractWritableCursor implements WritableCursor { + private _size: number; get size(): number { - return this.dv.byteLength; + return this._size; } set size(size: number) { - this.dv = new DataView(this.buffer, this.offset, size); + if (size > this.backing_buffer.byteLength - this.offset) { + throw new Error(`Size ${size} is out of bounds.`); + } + + this._size = size; } - protected _position: number; - - get position(): number { - return this._position; - } - - protected little_endian!: boolean; - - get endianness(): Endianness { - return this.little_endian ? Endianness.Little : Endianness.Big; - } - - set endianness(endianness: Endianness) { - this.little_endian = endianness === Endianness.Little; - this.utf16_decoder = this.little_endian ? UTF_16LE_DECODER : UTF_16BE_DECODER; - this.utf16_encoder = this.little_endian ? UTF_16LE_ENCODER : UTF_16BE_ENCODER; - } - - get bytes_left(): number { - return this.size - this.position; - } - - protected buffer: ArrayBuffer; + protected backing_buffer: ArrayBuffer; protected dv: DataView; - protected utf16_decoder!: TextDecoder; - protected utf16_encoder!: TextEncoder; /** * @param buffer The buffer to read from. @@ -62,219 +33,18 @@ export class ArrayBufferCursor implements Cursor { buffer: ArrayBuffer, endianness: Endianness, offset: number = 0, - size: number = buffer.byteLength + size: number = buffer.byteLength - offset ) { - this.buffer = buffer; - this.dv = new DataView(buffer, offset, size); - this.endianness = endianness; - this._position = 0; - } - - seek(offset: number): this { - return this.seek_start(this.position + offset); - } - - seek_start(offset: number): this { - if (offset < 0 || offset > this.size) { - throw new Error(`Offset ${offset} is out of bounds.`); - } - - this._position = offset; - return this; - } - - seek_end(offset: number): this { - if (offset < 0 || offset > this.size) { - throw new Error(`Offset ${offset} is out of bounds.`); - } - - this._position = this.size - offset; - return this; - } - - u8(): number { - return this.u8_at(this._position++); - } - - u8_at(offset: number): number { - return this.dv.getUint8(offset); - } - - u16(): number { - const r = this.u16_at(this.position); - this._position += 2; - return r; - } - - u16_at(offset: number): number { - return this.dv.getUint16(offset, this.little_endian); - } - - u32(): number { - const r = this.u32_at(this.position); - this._position += 4; - return r; - } - - u32_at(offset: number): number { - return this.dv.getUint32(offset, this.little_endian); - } - - i8(): number { - return this.i8_at(this._position++); - } - - i8_at(offset: number): number { - return this.dv.getInt8(offset); - } - - i16(): number { - const r = this.i16_at(this.position); - this._position += 2; - return r; - } - - i16_at(offset: number): number { - return this.dv.getInt16(offset, this.little_endian); - } - - i32(): number { - const r = this.i32_at(this.position); - this._position += 4; - return r; - } - - i32_at(offset: number): number { - return this.dv.getInt32(offset, this.little_endian); - } - - f32(): number { - const r = this.f32_at(this.position); - this._position += 4; - return r; - } - - f32_at(offset: number): number { - return this.dv.getFloat32(offset, this.little_endian); - } - - u8_array(n: number): number[] { - const array = []; - for (let i = 0; i < n; ++i) array.push(this.dv.getUint8(this._position++)); - return array; - } - - u16_array(n: number): number[] { - const array = []; - - for (let i = 0; i < n; ++i) { - array.push(this.dv.getUint16(this.position, this.little_endian)); - this._position += 2; - } - - return array; - } - - u32_array(n: number): number[] { - const array = []; - - for (let i = 0; i < n; ++i) { - array.push(this.dv.getUint32(this.position, this.little_endian)); - this._position += 4; - } - - return array; - } - - vec2_f32(): Vec2 { - return new Vec2(this.f32(), this.f32()); - } - - vec3_f32(): Vec3 { - return new Vec3(this.f32(), this.f32(), this.f32()); + super(endianness, offset); + this._size = size; + this.backing_buffer = buffer; + this.dv = new DataView(buffer, 0, buffer.byteLength); } take(size: number): ArrayBufferCursor { const offset = this.offset + this.position; - const wrapper = new ArrayBufferCursor(this.buffer, this.endianness, offset, size); + const wrapper = new ArrayBufferCursor(this.backing_buffer, this.endianness, offset, size); this._position += size; return wrapper; } - - string_ascii( - max_byte_length: number, - null_terminated: boolean, - drop_remaining: boolean - ): string { - const string_length = null_terminated - ? this.index_of_u8(0, max_byte_length) - this.position - : max_byte_length; - - const view = new DataView(this.buffer, this.offset + this.position, string_length); - const r = ASCII_DECODER.decode(view); - - this._position += drop_remaining - ? max_byte_length - : Math.min(string_length + 1, max_byte_length); - - return r; - } - - string_utf16( - max_byte_length: number, - null_terminated: boolean, - drop_remaining: boolean - ): string { - const string_length = null_terminated - ? this.index_of_u16(0, max_byte_length) - this.position - : Math.floor(max_byte_length / 2) * 2; - - const view = new DataView(this.buffer, this.offset + this.position, string_length); - const r = this.utf16_decoder.decode(view); - - this._position += drop_remaining - ? max_byte_length - : Math.min(string_length + 2, max_byte_length); - - return r; - } - - array_buffer(size: number = this.size - this.position): ArrayBuffer { - const r = this.buffer.slice( - this.offset + this.position, - this.offset + this.position + size - ); - this._position += size; - return r; - } - - copy_to_uint8_array(array: Uint8Array, size: number = this.size - this.position): this { - array.set(new Uint8Array(this.buffer, this.offset + this.position, size)); - this._position += size; - return this; - } - - private index_of_u8(value: number, max_byte_length: number): number { - const max_pos = Math.min(this.position + max_byte_length, this.size); - - for (let i = this.position; i < max_pos; ++i) { - if (this.dv.getUint8(i) === value) { - return i; - } - } - - return this.position + max_byte_length; - } - - private index_of_u16(value: number, max_byte_length: number): number { - const max_pos = Math.min(this.position + max_byte_length, this.size); - - for (let i = this.position; i < max_pos; i += 2) { - if (this.dv.getUint16(i, this.little_endian) === value) { - return i; - } - } - - return this.position + max_byte_length; - } } diff --git a/src/data_formats/cursor/BufferCursor.ts b/src/data_formats/cursor/BufferCursor.ts index fb125eb3..a6f65016 100644 --- a/src/data_formats/cursor/BufferCursor.ts +++ b/src/data_formats/cursor/BufferCursor.ts @@ -1,7 +1,18 @@ -import { ArrayBufferCursor } from "./ArrayBufferCursor"; import { Endianness } from ".."; +import { AbstractCursor } from "./AbstractCursor"; +import { Cursor } from "./Cursor"; + +export class BufferCursor extends AbstractCursor implements Cursor { + readonly size: number; + + protected buffer: Buffer; + + protected get backing_buffer(): ArrayBuffer { + return this.buffer.buffer; + } + + protected dv: DataView; -export class BufferCursor extends ArrayBufferCursor { /** * @param buffer The buffer to read from. * @param endianness Decides in which byte order multi-byte integers and floats will be interpreted. @@ -12,7 +23,7 @@ export class BufferCursor extends ArrayBufferCursor { buffer: Buffer, endianness: Endianness, offset: number = 0, - size: number = buffer.byteLength + size: number = buffer.byteLength - offset ) { if (offset < 0 || offset > buffer.byteLength) { throw new Error(`Offset ${offset} is out of bounds.`); @@ -22,6 +33,17 @@ export class BufferCursor extends ArrayBufferCursor { throw new Error(`Size ${size} is out of bounds.`); } - super(buffer.buffer, endianness, buffer.byteOffset + offset, size); + super(endianness, buffer.byteOffset + offset); + + this.buffer = buffer; + this.size = size; + this.dv = new DataView(buffer.buffer, 0, buffer.buffer.byteLength); + } + + take(size: number): BufferCursor { + const offset = this.offset + this.position; + const wrapper = new BufferCursor(this.buffer, this.endianness, offset, size); + this._position += size; + return wrapper; } } diff --git a/src/data_formats/cursor/Cursor.test.ts b/src/data_formats/cursor/Cursor.test.ts index 237cf683..a6674284 100644 --- a/src/data_formats/cursor/Cursor.test.ts +++ b/src/data_formats/cursor/Cursor.test.ts @@ -5,8 +5,6 @@ import { ArrayBufferCursor } from "./ArrayBufferCursor"; import { BufferCursor } from "./BufferCursor"; import { Cursor } from "./Cursor"; import { ResizableBufferCursor } from "./ResizableBufferCursor"; -import { WritableArrayBufferCursor } from "./WritableArrayBufferCursor"; -import { WritableResizableBufferCursor } from "./WritableResizableBufferCursor"; /** * Run a test on every cursor implementation with every endianness. @@ -50,16 +48,6 @@ function test_all( endianness, new ResizableBufferCursor(rbuf(endianness), endianness), ]), - ...endiannesses.map(endianness => [ - WritableArrayBufferCursor.name, - endianness, - new WritableArrayBufferCursor(new Uint8Array(bytes(endianness)).buffer, endianness), - ]), - ...endiannesses.map(endianness => [ - WritableResizableBufferCursor.name, - endianness, - new WritableResizableBufferCursor(rbuf(endianness), endianness), - ]), ] as any; for (const [cursor_name, endianness, cursor] of cursors) { diff --git a/src/data_formats/cursor/WritableResizableBufferCursor.test.ts b/src/data_formats/cursor/ResizableBufferCursor.test.ts similarity index 84% rename from src/data_formats/cursor/WritableResizableBufferCursor.test.ts rename to src/data_formats/cursor/ResizableBufferCursor.test.ts index e268309d..d56e6c28 100644 --- a/src/data_formats/cursor/WritableResizableBufferCursor.test.ts +++ b/src/data_formats/cursor/ResizableBufferCursor.test.ts @@ -1,6 +1,6 @@ -import { WritableResizableBufferCursor } from "./WritableResizableBufferCursor"; -import { ResizableBuffer } from "../ResizableBuffer"; import { Endianness } from ".."; +import { ResizableBuffer } from "../ResizableBuffer"; +import { ResizableBufferCursor } from "./ResizableBufferCursor"; /** * Writes two integers to a cursor backed with a buffer of size 0. @@ -13,7 +13,7 @@ function test_integer_write(method_name: string): void { let expected_number_2 = 7348942; const buf = new ResizableBuffer(8); - const cursor = new WritableResizableBufferCursor(buf, Endianness.Little); + const cursor = new ResizableBufferCursor(buf, Endianness.Little); expect(buf.size).toBe(0); expect(cursor.size).toBe(0); @@ -33,7 +33,7 @@ test_integer_write("write_u32"); test_integer_write("write_i32"); test("write, seek backwards then take", () => { - const cursor = new WritableResizableBufferCursor(new ResizableBuffer(0), Endianness.Little); + const cursor = new ResizableBufferCursor(new ResizableBuffer(0), Endianness.Little); cursor .write_u32(1) .write_u32(2) diff --git a/src/data_formats/cursor/ResizableBufferCursor.ts b/src/data_formats/cursor/ResizableBufferCursor.ts index 0a1095bb..6252b4fd 100644 --- a/src/data_formats/cursor/ResizableBufferCursor.ts +++ b/src/data_formats/cursor/ResizableBufferCursor.ts @@ -1,59 +1,33 @@ -import { - ASCII_DECODER, - UTF_16BE_DECODER, - UTF_16BE_ENCODER, - UTF_16LE_DECODER, - UTF_16LE_ENCODER, -} from "."; import { Endianness } from ".."; import { ResizableBuffer } from "../ResizableBuffer"; -import { Vec2, Vec3 } from "../vector"; -import { Cursor } from "./Cursor"; - -export class ResizableBufferCursor implements Cursor { - private _offset: number; - - get offset(): number { - return this._offset; - } +import { AbstractWritableCursor } from "./AbstractWritableCursor"; +import { WritableCursor } from "./WritableCursor"; +export class ResizableBufferCursor extends AbstractWritableCursor implements WritableCursor { protected _size: number; get size(): number { return this._size; } - protected _position: number; - - get position(): number { - return this._position; - } - - protected little_endian!: boolean; - - get endianness(): Endianness { - return this.little_endian ? Endianness.Little : Endianness.Big; - } - - set endianness(endianness: Endianness) { - this.little_endian = endianness === Endianness.Little; - this.utf16_decoder = this.little_endian ? UTF_16LE_DECODER : UTF_16BE_DECODER; - this.utf16_encoder = this.little_endian ? UTF_16LE_ENCODER : UTF_16BE_ENCODER; - } - - get bytes_left(): number { - return this.size - this.position; + set size(size: number) { + if (size > this._size) { + this.ensure_size(size - this._size); + } else { + this._size = size; + } } protected buffer: ResizableBuffer; + protected get backing_buffer(): ArrayBuffer { + return this.buffer.backing_buffer; + } + protected get dv(): DataView { return this.buffer.view; } - protected utf16_decoder: TextDecoder = UTF_16BE_DECODER; - protected utf16_encoder: TextEncoder = UTF_16BE_ENCODER; - /** * @param buffer The buffer to read from. * @param endianness Decides in which byte order multi-byte integers and floats will be interpreted. @@ -64,7 +38,7 @@ export class ResizableBufferCursor implements Cursor { buffer: ResizableBuffer, endianness: Endianness, offset: number = 0, - size: number = buffer.size + size: number = buffer.size - offset ) { if (offset < 0 || offset > buffer.size) { throw new Error(`Offset ${offset} is out of bounds.`); @@ -74,152 +48,10 @@ export class ResizableBufferCursor implements Cursor { throw new Error(`Size ${size} is out of bounds.`); } + super(endianness, offset); + this.buffer = buffer; - this.endianness = endianness; - this._offset = offset; this._size = size; - this._position = 0; - } - - seek(offset: number): this { - return this.seek_start(this.position + offset); - } - - seek_start(offset: number): this { - if (offset < 0 || offset > this.size) { - throw new Error(`Offset ${offset} is out of bounds.`); - } - - this._position = offset; - return this; - } - - seek_end(offset: number): this { - if (offset < 0 || offset > this.size) { - throw new Error(`Offset ${offset} is out of bounds.`); - } - - this._position = this.size - offset; - return this; - } - - u8(): number { - return this.u8_at(this._position++); - } - - u8_at(offset: number): number { - this.check_offset(offset, 1); - return this.dv.getUint8(this.offset + offset); - } - - u16(): number { - const r = this.u16_at(this.position); - this._position += 2; - return r; - } - - u16_at(offset: number): number { - this.check_offset(offset, 2); - return this.dv.getUint16(this.offset + offset, this.little_endian); - } - - u32(): number { - const r = this.u32_at(this.position); - this._position += 4; - return r; - } - - u32_at(offset: number): number { - this.check_offset(offset, 4); - return this.dv.getUint32(this.offset + offset, this.little_endian); - } - - i8(): number { - return this.i8_at(this._position++); - } - - i8_at(offset: number): number { - this.check_offset(offset, 1); - return this.dv.getInt8(this.offset + offset); - } - - i16(): number { - const r = this.i16_at(this.position); - this._position += 2; - return r; - } - - i16_at(offset: number): number { - this.check_offset(offset, 2); - return this.dv.getInt16(this.offset + offset, this.little_endian); - } - - i32(): number { - const r = this.i32_at(this.position); - this._position += 4; - return r; - } - - i32_at(offset: number): number { - this.check_offset(offset, 4); - return this.dv.getInt32(this.offset + offset, this.little_endian); - } - - f32(): number { - const r = this.f32_at(this.position); - this._position += 4; - return r; - } - - f32_at(offset: number): number { - this.check_offset(offset, 4); - return this.dv.getFloat32(this.offset + offset, this.little_endian); - } - - u8_array(n: number): number[] { - this.check_size("n", n, n); - - const array = []; - - for (let i = 0; i < n; ++i) { - array.push(this.dv.getUint8(this.offset + this._position++)); - } - - return array; - } - - u16_array(n: number): number[] { - this.check_size("n", n, 2 * n); - - const array = []; - - for (let i = 0; i < n; ++i) { - array.push(this.dv.getUint16(this.offset + this.position, this.little_endian)); - this._position += 2; - } - - return array; - } - - u32_array(n: number): number[] { - this.check_size("n", n, 4 * n); - - const array = []; - - for (let i = 0; i < n; ++i) { - array.push(this.dv.getUint32(this.offset + this.position, this.little_endian)); - this._position += 4; - } - - return array; - } - - vec2_f32(): Vec2 { - return new Vec2(this.f32(), this.f32()); - } - - vec3_f32(): Vec3 { - return new Vec3(this.f32(), this.f32(), this.f32()); } take(size: number): ResizableBufferCursor { @@ -231,101 +63,15 @@ export class ResizableBufferCursor implements Cursor { return wrapper; } - string_ascii( - max_byte_length: number, - null_terminated: boolean, - drop_remaining: boolean - ): string { - this.check_size("max_byte_length", max_byte_length, max_byte_length); + protected ensure_size(size: number): void { + const needed = this.position + size - this._size; - const string_length = null_terminated - ? this.index_of_u8(0, max_byte_length) - this.position - : max_byte_length; + if (needed > 0) { + this._size += needed; - const view = this.buffer.sub_view(this.offset + this.position, string_length); - const r = ASCII_DECODER.decode(view); - - this._position += drop_remaining - ? max_byte_length - : Math.min(string_length + 1, max_byte_length); - - return r; - } - - string_utf16( - max_byte_length: number, - null_terminated: boolean, - drop_remaining: boolean - ): string { - this.check_size("max_byte_length", max_byte_length, max_byte_length); - - const string_length = null_terminated - ? this.index_of_u16(0, max_byte_length) - this.position - : Math.floor(max_byte_length / 2) * 2; - - const view = this.buffer.sub_view(this.offset + this.position, string_length); - const r = this.utf16_decoder.decode(view); - - this._position += drop_remaining - ? max_byte_length - : Math.min(string_length + 2, max_byte_length); - - return r; - } - - array_buffer(size: number = this.size - this.position): ArrayBuffer { - this.check_size("size", size, size); - const r = this.buffer.backing_buffer.slice( - this.offset + this.position, - this.offset + this.position + size - ); - this._position += size; - return r; - } - - copy_to_uint8_array(array: Uint8Array, size: number = this.size - this.position): this { - this.check_size("size", size, size); - array.set(new Uint8Array(this.buffer.backing_buffer, this.offset + this.position, size)); - this._position += size; - return this; - } - - private check_size(name: string, value: number, byte_size: number): void { - if (byte_size < 0 || byte_size > this.size - this.position) { - throw new Error(`${name} ${value} is out of bounds.`); - } - } - - /** - * Checks whether we can read size bytes at offset. - */ - protected check_offset(offset: number, size: number): void { - if (offset < 0 || offset + size > this.size) { - throw new Error(`Offset ${offset} is out of bounds.`); - } - } - - private index_of_u8(value: number, max_byte_length: number): number { - const max_pos = Math.min(this.position + max_byte_length, this.size); - - for (let i = this.position; i < max_pos; ++i) { - if (this.dv.getUint8(this.offset + i) === value) { - return i; + if (this.buffer.size < this.offset + this._size) { + this.buffer.size = this.offset + this._size; } } - - return this.position + max_byte_length; - } - - private index_of_u16(value: number, max_byte_length: number): number { - const max_pos = Math.min(this.position + max_byte_length, this.size); - - for (let i = this.position; i < max_pos; i += 2) { - if (this.dv.getUint16(this.offset + i, this.little_endian) === value) { - return i; - } - } - - return this.position + max_byte_length; } } diff --git a/src/data_formats/cursor/WritableArrayBufferCursor.ts b/src/data_formats/cursor/WritableArrayBufferCursor.ts deleted file mode 100644 index bf668f5f..00000000 --- a/src/data_formats/cursor/WritableArrayBufferCursor.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { ASCII_ENCODER } from "."; -import { Vec2, Vec3 } from "../vector"; -import { ArrayBufferCursor } from "./ArrayBufferCursor"; -import { Cursor } from "./Cursor"; -import { WritableCursor } from "./WritableCursor"; - -/** - * A cursor for reading and writing from an array buffer or part of an array buffer. - */ -export class WritableArrayBufferCursor extends ArrayBufferCursor implements WritableCursor { - write_u8(value: number): this { - this.dv.setUint8(this._position++, value); - return this; - } - - write_u16(value: number): this { - this.dv.setUint16(this.position, value, this.little_endian); - this._position += 2; - return this; - } - - write_u32(value: number): this { - this.dv.setUint32(this.position, value, this.little_endian); - this._position += 4; - return this; - } - - write_i32(value: number): this { - this.dv.setInt32(this.position, value, this.little_endian); - this._position += 4; - return this; - } - - write_f32(value: number): this { - this.dv.setFloat32(this.position, value, this.little_endian); - this._position += 4; - return this; - } - - write_u8_array(array: number[]): this { - new Uint8Array(this.buffer, this.offset + this.position).set(new Uint8Array(array)); - this._position += array.length; - return this; - } - - write_u16_array(array: number[]): this { - const len = array.length; - - for (let i = 0; i < len; i++) { - this.write_u16(array[i]); - } - - return this; - } - - write_u32_array(array: number[]): this { - const len = array.length; - - for (let i = 0; i < len; i++) { - this.write_u32(array[i]); - } - - return this; - } - - write_vec2_f32(value: Vec2): this { - this.dv.setFloat32(this.position, value.x, this.little_endian); - this.dv.setFloat32(this.position + 4, value.y, this.little_endian); - this._position += 8; - return this; - } - - write_vec3_f32(value: Vec3): this { - this.dv.setFloat32(this.position, value.x, this.little_endian); - this.dv.setFloat32(this.position + 4, value.y, this.little_endian); - this.dv.setFloat32(this.position + 8, value.z, this.little_endian); - this._position += 12; - return this; - } - - write_cursor(other: Cursor): this { - const size = other.size - other.position; - other.copy_to_uint8_array( - new Uint8Array(this.buffer, this.offset + this.position, size), - size - ); - this._position += size; - return this; - } - - write_string_ascii(str: string, byte_length: number): this { - const encoded = ASCII_ENCODER.encode(str); - const encoded_length = Math.min(encoded.byteLength, byte_length); - let i = 0; - - while (i < encoded_length) { - this.write_u8(encoded[i++]); - } - - while (i++ < byte_length) { - this.write_u8(0); - } - - return this; - } - - write_string_utf16(str: string, byte_length: number): this { - const encoded = this.utf16_encoder.encode(str); - const encoded_length = Math.min(encoded.byteLength, byte_length); - let i = 0; - - while (i < encoded_length) { - this.write_u8(encoded[i++]); - } - - while (i++ < byte_length) { - this.write_u8(0); - } - - return this; - } -} diff --git a/src/data_formats/cursor/WritableCursor.test.ts b/src/data_formats/cursor/WritableCursor.test.ts index 6de2131b..ddac836b 100644 --- a/src/data_formats/cursor/WritableCursor.test.ts +++ b/src/data_formats/cursor/WritableCursor.test.ts @@ -1,9 +1,9 @@ import { Endianness } from ".."; import { enum_values } from "../../enums"; import { ResizableBuffer } from "../ResizableBuffer"; -import { WritableArrayBufferCursor } from "./WritableArrayBufferCursor"; +import { ArrayBufferCursor } from "./ArrayBufferCursor"; +import { ResizableBufferCursor } from "./ResizableBufferCursor"; import { WritableCursor } from "./WritableCursor"; -import { WritableResizableBufferCursor } from "./WritableResizableBufferCursor"; /** * Run a test on every writable cursor implementation with every endianness. @@ -33,14 +33,14 @@ function test_all( const cursors: [string, Endianness, WritableCursor][] = [ ...endiannesses.map(endianness => [ - WritableArrayBufferCursor.name, + ArrayBufferCursor.name, endianness, - new WritableArrayBufferCursor(new Uint8Array(bytes(endianness)).buffer, endianness), + new ArrayBufferCursor(new Uint8Array(bytes(endianness)).buffer, endianness), ]), ...endiannesses.map(endianness => [ - WritableResizableBufferCursor.name, + ResizableBufferCursor.name, endianness, - new WritableResizableBufferCursor(rbuf(endianness), endianness), + new ResizableBufferCursor(rbuf(endianness), endianness), ]), ] as any; diff --git a/src/data_formats/cursor/index.ts b/src/data_formats/cursor/index.ts deleted file mode 100644 index 0b2a682d..00000000 --- a/src/data_formats/cursor/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -// TODO: remove dependency on text-encoding because it is no longer maintained. -import { TextDecoder, TextEncoder } from "text-encoding"; - -export const ASCII_DECODER = new TextDecoder("ascii"); -export const UTF_16BE_DECODER = new TextDecoder("utf-16be"); -export const UTF_16LE_DECODER = new TextDecoder("utf-16le"); - -export const ASCII_ENCODER = new TextEncoder("ascii"); -export const UTF_16BE_ENCODER = new TextEncoder("utf-16be", { - NONSTANDARD_allowLegacyEncoding: true, -}); -export const UTF_16LE_ENCODER = new TextEncoder("utf-16le", { - NONSTANDARD_allowLegacyEncoding: true, -}); diff --git a/src/data_formats/encryption/prc.ts b/src/data_formats/encryption/prc.ts index a634051e..e0cc3a9f 100644 --- a/src/data_formats/encryption/prc.ts +++ b/src/data_formats/encryption/prc.ts @@ -1,6 +1,6 @@ -import { Cursor } from "../cursor/Cursor"; -import { WritableArrayBufferCursor } from "../cursor/WritableArrayBufferCursor"; import { Endianness } from ".."; +import { ArrayBufferCursor } from "../cursor/ArrayBufferCursor"; +import { Cursor } from "../cursor/Cursor"; /** * Decrypts the bytes left in cursor. @@ -21,10 +21,7 @@ class PrcDecryptor { // Size should be divisible by 4. const actual_size = cursor.bytes_left; const size = Math.ceil(actual_size / 4) * 4; - const out_cursor = new WritableArrayBufferCursor( - new ArrayBuffer(actual_size), - cursor.endianness - ); + const out_cursor = new ArrayBufferCursor(new ArrayBuffer(actual_size), cursor.endianness); for (let pos = 0; pos < size; pos += 4) { let u32; diff --git a/src/data_formats/parsing/ninja/index.ts b/src/data_formats/parsing/ninja/index.ts index 144b608e..3092c960 100644 --- a/src/data_formats/parsing/ninja/index.ts +++ b/src/data_formats/parsing/ninja/index.ts @@ -128,9 +128,14 @@ function parse_ninja( context: any ): NjObject[] { // POF0 and other chunks types are ignored. - return parse_iff(cursor) - .filter(chunk => chunk.type === NJCM) - .flatMap(chunk => parse_sibling_objects(chunk.data, parse_model, context)); + const njcm_chunks = parse_iff(cursor).filter(chunk => chunk.type === NJCM); + const objects: NjObject[] = []; + + for (const chunk of njcm_chunks) { + objects.push(...parse_sibling_objects(chunk.data, parse_model, context)); + } + + return objects; } // TODO: cache model and object offsets so we don't reparse the same data. diff --git a/src/data_formats/parsing/quest/bin.ts b/src/data_formats/parsing/quest/bin.ts index 33860e2c..b4c00504 100644 --- a/src/data_formats/parsing/quest/bin.ts +++ b/src/data_formats/parsing/quest/bin.ts @@ -1,8 +1,8 @@ import Logger from "js-logger"; import { Endianness } from "../.."; import { Cursor } from "../../cursor/Cursor"; +import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; import { WritableCursor } from "../../cursor/WritableCursor"; -import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor"; import { ResizableBuffer } from "../../ResizableBuffer"; import { Opcode, OPCODES, Type } from "./opcodes"; @@ -195,7 +195,7 @@ export function write_bin(bin: BinFile): ArrayBuffer { const buffer = new ResizableBuffer( object_code_offset + 10 * bin.instructions.length + 4 * labels.length ); - const cursor = new WritableResizableBufferCursor(buffer, Endianness.Little); + const cursor = new ResizableBufferCursor(buffer, Endianness.Little); cursor.write_u32(object_code_offset); cursor.write_u32(0); // Placeholder for the labels offset. diff --git a/src/data_formats/parsing/quest/dat.ts b/src/data_formats/parsing/quest/dat.ts index 76f91ba1..4ec4403d 100644 --- a/src/data_formats/parsing/quest/dat.ts +++ b/src/data_formats/parsing/quest/dat.ts @@ -2,7 +2,7 @@ import Logger from "js-logger"; import { groupBy } from "lodash"; import { Endianness } from "../.."; import { Cursor } from "../../cursor/Cursor"; -import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor"; +import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; import { ResizableBuffer } from "../../ResizableBuffer"; import { Vec3 } from "../../vector"; @@ -159,7 +159,7 @@ export function write_dat({ objs, npcs, unknowns }: DatFile): ResizableBuffer { npcs.length * (16 + NPC_SIZE) + unknowns.reduce((a, b) => a + b.total_size, 0) ); - const cursor = new WritableResizableBufferCursor(buffer, Endianness.Little); + const cursor = new ResizableBufferCursor(buffer, Endianness.Little); const grouped_objs = groupBy(objs, obj => obj.area_id); const obj_area_ids = Object.keys(grouped_objs) diff --git a/src/data_formats/parsing/quest/qst.ts b/src/data_formats/parsing/quest/qst.ts index 937cb39f..f5029995 100644 --- a/src/data_formats/parsing/quest/qst.ts +++ b/src/data_formats/parsing/quest/qst.ts @@ -1,11 +1,10 @@ import Logger from "js-logger"; -import { Cursor } from "../../cursor/Cursor"; -import { WritableArrayBufferCursor } from "../../cursor/WritableArrayBufferCursor"; import { Endianness } from "../.."; -import { WritableCursor } from "../../cursor/WritableCursor"; -import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor"; -import { ResizableBuffer } from "../../ResizableBuffer"; import { ArrayBufferCursor } from "../../cursor/ArrayBufferCursor"; +import { Cursor } from "../../cursor/Cursor"; +import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; +import { WritableCursor } from "../../cursor/WritableCursor"; +import { ResizableBuffer } from "../../ResizableBuffer"; const logger = Logger.get("data_formats/parsing/quest/qst"); @@ -92,7 +91,7 @@ export function write_qst(params: WriteQstParams): ArrayBuffer { .map(f => 88 + Math.ceil(f.data.byteLength / 1024) * 1056) .reduce((a, b) => a + b); const buffer = new ArrayBuffer(total_size); - const cursor = new WritableArrayBufferCursor(buffer, Endianness.Little); + const cursor = new ArrayBufferCursor(buffer, Endianness.Little); write_file_headers(cursor, files); write_file_chunks(cursor, files); @@ -166,7 +165,7 @@ function parse_files(cursor: Cursor, expected_sizes: Map): QstCo (file = { name: file_name, expected_size, - cursor: new WritableResizableBufferCursor( + cursor: new ResizableBufferCursor( new ResizableBuffer(expected_size || 10 * 1024), Endianness.Little ), diff --git a/tsconfig.json b/tsconfig.json index 75c52afb..546e127e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "sourceMap": true, "module": "es6", "target": "es6", - "lib": ["dom", "dom.iterable", "esnext"], + "lib": ["es6", "dom", "dom.iterable"], "allowJs": true, "skipLibCheck": true, "allowSyntheticDefaultImports": true, diff --git a/yarn.lock b/yarn.lock index 049684c7..e6d30f54 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8133,11 +8133,6 @@ test-exclude@^5.2.3: read-pkg-up "^4.0.0" require-main-filename "^2.0.0" -text-encoding@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.7.0.tgz#f895e836e45990624086601798ea98e8f36ee643" - integrity sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA== - text-table@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"