Sancaros/phantasmal-world
1
0
Fork 0
mirror of https://github.com/DaanVandenBosch/phantasmal-world.git synced 2025-04-04 22:58:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

All unit tests pass again. Removed dependency on text-encoding. Refactored cursors to reuse more code.

Browse Source
This commit is contained in:
Daan Vanden Bosch 2019-07-27 21:47:49 +02:00
parent 4eeda19837
commit f95e7ea220
20 changed files with 378 additions and 761 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -24,7 +24,6 @@
"react-dom": "^16.8.6",
"react-virtualized": "^9.21.1",
"react-virtualized-select": "^3.1.3",
"text-encoding": "^0.7.0",
"three": "^0.106.2"
},
"scripts": {

7
src/data_formats/compression/prs/compress.ts
View File

@ -1,6 +1,6 @@
import { Cursor } from "../../cursor/Cursor";
import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { WritableCursor } from "../../cursor/WritableCursor";
import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
export function prs_compress(src: Cursor): Cursor {
@ -117,10 +117,7 @@ class Context {
constructor(cursor: Cursor) {
this.src = cursor;
this.dst = new WritableResizableBufferCursor(
new ResizableBuffer(cursor.size),
cursor.endianness
);
this.dst = new ResizableBufferCursor(new ResizableBuffer(cursor.size), cursor.endianness);
}
set_bit(bit: number): void {

4
src/data_formats/compression/prs/decompress.ts
View File

@ -1,7 +1,7 @@
import Logger from "js-logger";
import { Cursor } from "../../cursor/Cursor";
import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { WritableCursor } from "../../cursor/WritableCursor";
import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
const logger = Logger.get("data_formats/compression/prs/decompress");
@ -63,7 +63,7 @@ class Context {
constructor(cursor: Cursor) {
this.src = cursor;
this.dst = new WritableResizableBufferCursor(
this.dst = new ResizableBufferCursor(
new ResizableBuffer(Math.floor(1.5 * cursor.size)),
cursor.endianness
);

253
src/data_formats/cursor/AbstractCursor.ts Normal file
View File

@ -0,0 +1,253 @@
import { Endianness } from "..";
import { Vec2, Vec3 } from "../vector";
import { Cursor } from "./Cursor";
export abstract class AbstractCursor implements Cursor {
abstract readonly size: number;
protected _position: number = 0;
get position(): number {
return this._position;
}
protected little_endian!: boolean;
get endianness(): Endianness {
return this.little_endian ? Endianness.Little : Endianness.Big;
}
set endianness(endianness: Endianness) {
this.little_endian = endianness === Endianness.Little;
}
get bytes_left(): number {
return this.size - this.position;
}
protected readonly offset: number;
protected abstract readonly backing_buffer: ArrayBuffer;
protected abstract readonly dv: DataView;
constructor(endianness: Endianness, offset: number) {
this.endianness = endianness;
this.offset = offset;
}
seek(offset: number): this {
return this.seek_start(this.position + offset);
}
seek_start(offset: number): this {
if (offset < 0 || offset > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
this._position = offset;
return this;
}
seek_end(offset: number): this {
if (offset < 0 || offset > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
this._position = this.size - offset;
return this;
}
u8(): number {
return this.u8_at(this._position++);
}
u8_at(offset: number): number {
this.check_offset(offset, 1);
return this.dv.getUint8(this.offset + offset);
}
u16(): number {
const r = this.u16_at(this.position);
this._position += 2;
return r;
}
u16_at(offset: number): number {
this.check_offset(offset, 2);
return this.dv.getUint16(this.offset + offset, this.little_endian);
}
u32(): number {
const r = this.u32_at(this.position);
this._position += 4;
return r;
}
u32_at(offset: number): number {
this.check_offset(offset, 4);
return this.dv.getUint32(this.offset + offset, this.little_endian);
}
i8(): number {
return this.i8_at(this._position++);
}
i8_at(offset: number): number {
this.check_offset(offset, 1);
return this.dv.getInt8(this.offset + offset);
}
i16(): number {
const r = this.i16_at(this.position);
this._position += 2;
return r;
}
i16_at(offset: number): number {
this.check_offset(offset, 2);
return this.dv.getInt16(this.offset + offset, this.little_endian);
}
i32(): number {
const r = this.i32_at(this.position);
this._position += 4;
return r;
}
i32_at(offset: number): number {
this.check_offset(offset, 4);
return this.dv.getInt32(this.offset + offset, this.little_endian);
}
f32(): number {
const r = this.f32_at(this.position);
this._position += 4;
return r;
}
f32_at(offset: number): number {
this.check_offset(offset, 4);
return this.dv.getFloat32(this.offset + offset, this.little_endian);
}
u8_array(n: number): number[] {
this.check_size("n", n, n);
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint8(this.offset + this._position++));
}
return array;
}
u16_array(n: number): number[] {
this.check_size("n", n, 2 * n);
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint16(this.offset + this.position, this.little_endian));
this._position += 2;
}
return array;
}
u32_array(n: number): number[] {
this.check_size("n", n, 4 * n);
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint32(this.offset + this.position, this.little_endian));
this._position += 4;
}
return array;
}
vec2_f32(): Vec2 {
return new Vec2(this.f32(), this.f32());
}
vec3_f32(): Vec3 {
return new Vec3(this.f32(), this.f32(), this.f32());
}
string_ascii(max_byte_length: number, null_terminated: boolean, drop_remaining: boolean) {
let code_points: number[] = [];
for (let i = 0; i < max_byte_length; i++) {
const code_point = this.u8();
if (null_terminated && code_point === 0) {
if (drop_remaining) {
this.seek(max_byte_length - i - 1);
}
break;
}
code_points.push(code_point);
}
return String.fromCodePoint(...code_points);
}
string_utf16(max_byte_length: number, null_terminated: boolean, drop_remaining: boolean) {
let code_points: number[] = [];
let len = Math.floor(max_byte_length / 2);
for (let i = 0; i < len; i++) {
const code_point = this.u16();
if (null_terminated && code_point === 0) {
if (drop_remaining) {
this.seek(2 * (len - i - 1));
}
break;
}
code_points.push(code_point);
}
return String.fromCodePoint(...code_points);
}
array_buffer(size: number = this.size - this.position): ArrayBuffer {
this.check_size("size", size, size);
const r = this.backing_buffer.slice(
this.offset + this.position,
this.offset + this.position + size
);
this._position += size;
return r;
}
copy_to_uint8_array(array: Uint8Array, size: number = this.size - this.position): this {
this.check_size("size", size, size);
array.set(new Uint8Array(this.backing_buffer, this.offset + this.position, size));
this._position += size;
return this;
}
abstract take(size: number): Cursor;
protected check_size(name: string, value: number, byte_size: number): void {
if (byte_size < 0 || byte_size > this.size - this.position) {
throw new Error(`${name} ${value} is out of bounds.`);
}
}
/**
* Checks whether we can read size bytes at offset.
*/
private check_offset(offset: number, size: number): void {
if (offset < 0 || offset + size > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
}
}

66
src/data_formats/cursor/WritableResizableBufferCursor.ts → src/data_formats/cursor/AbstractWritableCursor.ts
View File

@ -1,21 +1,10 @@
import { WritableCursor } from "./WritableCursor";
import { ResizableBufferCursor } from "./ResizableBufferCursor";
import { Vec2, Vec3 } from "../vector";
import { AbstractCursor } from "./AbstractCursor";
import { Cursor } from "./Cursor";
import { ASCII_ENCODER } from ".";
import { Vec3, Vec2 } from "../vector";
import { WritableCursor } from "./WritableCursor";
export class WritableResizableBufferCursor extends ResizableBufferCursor implements WritableCursor {
get size(): number {
return this._size;
}
set size(size: number) {
if (size > this._size) {
this.ensure_size(size - this._size);
} else {
this._size = size;
}
}
export abstract class AbstractWritableCursor extends AbstractCursor implements WritableCursor {
abstract size: number;
write_u8(value: number): this {
this.ensure_size(1);
@ -53,9 +42,7 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme
write_u8_array(array: number[]): this {
this.ensure_size(array.length);
new Uint8Array(this.buffer.backing_buffer, this.offset + this.position).set(
new Uint8Array(array)
);
new Uint8Array(this.backing_buffer, this.offset + this.position).set(new Uint8Array(array));
this._position += array.length;
return this;
}
@ -104,7 +91,7 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme
this.ensure_size(size);
other.copy_to_uint8_array(
new Uint8Array(this.buffer.backing_buffer, this.offset + this.position, size),
new Uint8Array(this.backing_buffer, this.offset + this.position, size),
size
);
@ -115,15 +102,15 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme
write_string_ascii(str: string, byte_length: number): this {
this.ensure_size(byte_length);
const encoded = ASCII_ENCODER.encode(str);
const encoded_length = Math.min(encoded.byteLength, byte_length);
let i = 0;
const len = Math.min(byte_length, str.length);
while (i < encoded_length) {
this.write_u8(encoded[i++]);
for (let i = 0; i < len; i++) {
this.write_u8(str.codePointAt(i)!);
}
while (i++ < byte_length) {
let pad_len = byte_length - len;
for (let i = 0; i < pad_len; i++) {
this.write_u8(0);
}
@ -133,30 +120,25 @@ export class WritableResizableBufferCursor extends ResizableBufferCursor impleme
write_string_utf16(str: string, byte_length: number): this {
this.ensure_size(byte_length);
const encoded = this.utf16_encoder.encode(str);
const encoded_length = Math.min(encoded.byteLength, byte_length);
let i = 0;
const max_len = Math.floor(byte_length / 2);
const len = Math.min(max_len, str.length);
while (i < encoded_length) {
this.write_u8(encoded[i++]);
for (let i = 0; i < len; i++) {
this.write_u16(str.codePointAt(i)!);
}
while (i++ < byte_length) {
this.write_u8(0);
let pad_len = max_len - len;
for (let i = 0; i < pad_len; i++) {
this.write_u16(0);
}
return this;
}
private ensure_size(size: number): void {
const needed = this.position + size - this._size;
if (needed > 0) {
this._size += needed;
if (this.buffer.size < this.offset + this._size) {
this.buffer.size = this.offset + this._size;
}
protected ensure_size(size: number): void {
if (size > this.bytes_left) {
throw new Error(`${size} Bytes required but only ${this.bytes_left} available.`);
}
}
}

264
src/data_formats/cursor/ArrayBufferCursor.ts
View File

@ -1,56 +1,27 @@
import {
ASCII_DECODER,
UTF_16BE_DECODER,
UTF_16BE_ENCODER,
UTF_16LE_DECODER,
UTF_16LE_ENCODER,
} from ".";
import { Endianness } from "..";
import { Vec2, Vec3 } from "../vector";
import { Cursor } from "./Cursor";
import { AbstractWritableCursor } from "./AbstractWritableCursor";
import { WritableCursor } from "./WritableCursor";
/**
* A cursor for reading from an array buffer or part of an array buffer.
*/
export class ArrayBufferCursor implements Cursor {
get offset(): number {
return this.dv.byteOffset;
}
export class ArrayBufferCursor extends AbstractWritableCursor implements WritableCursor {
private _size: number;
get size(): number {
return this.dv.byteLength;
return this._size;
}
set size(size: number) {
this.dv = new DataView(this.buffer, this.offset, size);
if (size > this.backing_buffer.byteLength - this.offset) {
throw new Error(`Size ${size} is out of bounds.`);
}
this._size = size;
}
protected _position: number;
get position(): number {
return this._position;
}
protected little_endian!: boolean;
get endianness(): Endianness {
return this.little_endian ? Endianness.Little : Endianness.Big;
}
set endianness(endianness: Endianness) {
this.little_endian = endianness === Endianness.Little;
this.utf16_decoder = this.little_endian ? UTF_16LE_DECODER : UTF_16BE_DECODER;
this.utf16_encoder = this.little_endian ? UTF_16LE_ENCODER : UTF_16BE_ENCODER;
}
get bytes_left(): number {
return this.size - this.position;
}
protected buffer: ArrayBuffer;
protected backing_buffer: ArrayBuffer;
protected dv: DataView;
protected utf16_decoder!: TextDecoder;
protected utf16_encoder!: TextEncoder;
/**
* @param buffer The buffer to read from.
@ -62,219 +33,18 @@ export class ArrayBufferCursor implements Cursor {
buffer: ArrayBuffer,
endianness: Endianness,
offset: number = 0,
size: number = buffer.byteLength
size: number = buffer.byteLength - offset
) {
this.buffer = buffer;
this.dv = new DataView(buffer, offset, size);
this.endianness = endianness;
this._position = 0;
}
seek(offset: number): this {
return this.seek_start(this.position + offset);
}
seek_start(offset: number): this {
if (offset < 0 || offset > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
this._position = offset;
return this;
}
seek_end(offset: number): this {
if (offset < 0 || offset > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
this._position = this.size - offset;
return this;
}
u8(): number {
return this.u8_at(this._position++);
}
u8_at(offset: number): number {
return this.dv.getUint8(offset);
}
u16(): number {
const r = this.u16_at(this.position);
this._position += 2;
return r;
}
u16_at(offset: number): number {
return this.dv.getUint16(offset, this.little_endian);
}
u32(): number {
const r = this.u32_at(this.position);
this._position += 4;
return r;
}
u32_at(offset: number): number {
return this.dv.getUint32(offset, this.little_endian);
}
i8(): number {
return this.i8_at(this._position++);
}
i8_at(offset: number): number {
return this.dv.getInt8(offset);
}
i16(): number {
const r = this.i16_at(this.position);
this._position += 2;
return r;
}
i16_at(offset: number): number {
return this.dv.getInt16(offset, this.little_endian);
}
i32(): number {
const r = this.i32_at(this.position);
this._position += 4;
return r;
}
i32_at(offset: number): number {
return this.dv.getInt32(offset, this.little_endian);
}
f32(): number {
const r = this.f32_at(this.position);
this._position += 4;
return r;
}
f32_at(offset: number): number {
return this.dv.getFloat32(offset, this.little_endian);
}
u8_array(n: number): number[] {
const array = [];
for (let i = 0; i < n; ++i) array.push(this.dv.getUint8(this._position++));
return array;
}
u16_array(n: number): number[] {
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint16(this.position, this.little_endian));
this._position += 2;
}
return array;
}
u32_array(n: number): number[] {
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint32(this.position, this.little_endian));
this._position += 4;
}
return array;
}
vec2_f32(): Vec2 {
return new Vec2(this.f32(), this.f32());
}
vec3_f32(): Vec3 {
return new Vec3(this.f32(), this.f32(), this.f32());
super(endianness, offset);
this._size = size;
this.backing_buffer = buffer;
this.dv = new DataView(buffer, 0, buffer.byteLength);
}
take(size: number): ArrayBufferCursor {
const offset = this.offset + this.position;
const wrapper = new ArrayBufferCursor(this.buffer, this.endianness, offset, size);
const wrapper = new ArrayBufferCursor(this.backing_buffer, this.endianness, offset, size);
this._position += size;
return wrapper;
}
string_ascii(
max_byte_length: number,
null_terminated: boolean,
drop_remaining: boolean
): string {
const string_length = null_terminated
? this.index_of_u8(0, max_byte_length) - this.position
: max_byte_length;
const view = new DataView(this.buffer, this.offset + this.position, string_length);
const r = ASCII_DECODER.decode(view);
this._position += drop_remaining
? max_byte_length
: Math.min(string_length + 1, max_byte_length);
return r;
}
string_utf16(
max_byte_length: number,
null_terminated: boolean,
drop_remaining: boolean
): string {
const string_length = null_terminated
? this.index_of_u16(0, max_byte_length) - this.position
: Math.floor(max_byte_length / 2) * 2;
const view = new DataView(this.buffer, this.offset + this.position, string_length);
const r = this.utf16_decoder.decode(view);
this._position += drop_remaining
? max_byte_length
: Math.min(string_length + 2, max_byte_length);
return r;
}
array_buffer(size: number = this.size - this.position): ArrayBuffer {
const r = this.buffer.slice(
this.offset + this.position,
this.offset + this.position + size
);
this._position += size;
return r;
}
copy_to_uint8_array(array: Uint8Array, size: number = this.size - this.position): this {
array.set(new Uint8Array(this.buffer, this.offset + this.position, size));
this._position += size;
return this;
}
private index_of_u8(value: number, max_byte_length: number): number {
const max_pos = Math.min(this.position + max_byte_length, this.size);
for (let i = this.position; i < max_pos; ++i) {
if (this.dv.getUint8(i) === value) {
return i;
}
}
return this.position + max_byte_length;
}
private index_of_u16(value: number, max_byte_length: number): number {
const max_pos = Math.min(this.position + max_byte_length, this.size);
for (let i = this.position; i < max_pos; i += 2) {
if (this.dv.getUint16(i, this.little_endian) === value) {
return i;
}
}
return this.position + max_byte_length;
}
}

30
src/data_formats/cursor/BufferCursor.ts
View File

@ -1,7 +1,18 @@
import { ArrayBufferCursor } from "./ArrayBufferCursor";
import { Endianness } from "..";
import { AbstractCursor } from "./AbstractCursor";
import { Cursor } from "./Cursor";
export class BufferCursor extends AbstractCursor implements Cursor {
readonly size: number;
protected buffer: Buffer;
protected get backing_buffer(): ArrayBuffer {
return this.buffer.buffer;
}
protected dv: DataView;
export class BufferCursor extends ArrayBufferCursor {
/**
* @param buffer The buffer to read from.
* @param endianness Decides in which byte order multi-byte integers and floats will be interpreted.
@ -12,7 +23,7 @@ export class BufferCursor extends ArrayBufferCursor {
buffer: Buffer,
endianness: Endianness,
offset: number = 0,
size: number = buffer.byteLength
size: number = buffer.byteLength - offset
) {
if (offset < 0 || offset > buffer.byteLength) {
throw new Error(`Offset ${offset} is out of bounds.`);
@ -22,6 +33,17 @@ export class BufferCursor extends ArrayBufferCursor {
throw new Error(`Size ${size} is out of bounds.`);
}
super(buffer.buffer, endianness, buffer.byteOffset + offset, size);
super(endianness, buffer.byteOffset + offset);
this.buffer = buffer;
this.size = size;
this.dv = new DataView(buffer.buffer, 0, buffer.buffer.byteLength);
}
take(size: number): BufferCursor {
const offset = this.offset + this.position;
const wrapper = new BufferCursor(this.buffer, this.endianness, offset, size);
this._position += size;
return wrapper;
}
}

12
src/data_formats/cursor/Cursor.test.ts
View File

@ -5,8 +5,6 @@ import { ArrayBufferCursor } from "./ArrayBufferCursor";
import { BufferCursor } from "./BufferCursor";
import { Cursor } from "./Cursor";
import { ResizableBufferCursor } from "./ResizableBufferCursor";
import { WritableArrayBufferCursor } from "./WritableArrayBufferCursor";
import { WritableResizableBufferCursor } from "./WritableResizableBufferCursor";
/**
* Run a test on every cursor implementation with every endianness.
@ -50,16 +48,6 @@ function test_all(
endianness,
new ResizableBufferCursor(rbuf(endianness), endianness),
]),
...endiannesses.map(endianness => [
WritableArrayBufferCursor.name,
endianness,
new WritableArrayBufferCursor(new Uint8Array(bytes(endianness)).buffer, endianness),
]),
...endiannesses.map(endianness => [
WritableResizableBufferCursor.name,
endianness,
new WritableResizableBufferCursor(rbuf(endianness), endianness),
]),
] as any;
for (const [cursor_name, endianness, cursor] of cursors) {

8
src/data_formats/cursor/WritableResizableBufferCursor.test.ts → src/data_formats/cursor/ResizableBufferCursor.test.ts
View File

@ -1,6 +1,6 @@
import { WritableResizableBufferCursor } from "./WritableResizableBufferCursor";
import { ResizableBuffer } from "../ResizableBuffer";
import { Endianness } from "..";
import { ResizableBuffer } from "../ResizableBuffer";
import { ResizableBufferCursor } from "./ResizableBufferCursor";
/**
* Writes two integers to a cursor backed with a buffer of size 0.
@ -13,7 +13,7 @@ function test_integer_write(method_name: string): void {
let expected_number_2 = 7348942;
const buf = new ResizableBuffer(8);
const cursor = new WritableResizableBufferCursor(buf, Endianness.Little);
const cursor = new ResizableBufferCursor(buf, Endianness.Little);
expect(buf.size).toBe(0);
expect(cursor.size).toBe(0);
@ -33,7 +33,7 @@ test_integer_write("write_u32");
test_integer_write("write_i32");
test("write, seek backwards then take", () => {
const cursor = new WritableResizableBufferCursor(new ResizableBuffer(0), Endianness.Little);
const cursor = new ResizableBufferCursor(new ResizableBuffer(0), Endianness.Little);
cursor
.write_u32(1)
.write_u32(2)

298
src/data_formats/cursor/ResizableBufferCursor.ts
View File

@ -1,59 +1,33 @@
import {
ASCII_DECODER,
UTF_16BE_DECODER,
UTF_16BE_ENCODER,
UTF_16LE_DECODER,
UTF_16LE_ENCODER,
} from ".";
import { Endianness } from "..";
import { ResizableBuffer } from "../ResizableBuffer";
import { Vec2, Vec3 } from "../vector";
import { Cursor } from "./Cursor";
export class ResizableBufferCursor implements Cursor {
private _offset: number;
get offset(): number {
return this._offset;
}
import { AbstractWritableCursor } from "./AbstractWritableCursor";
import { WritableCursor } from "./WritableCursor";
export class ResizableBufferCursor extends AbstractWritableCursor implements WritableCursor {
protected _size: number;
get size(): number {
return this._size;
}
protected _position: number;
get position(): number {
return this._position;
}
protected little_endian!: boolean;
get endianness(): Endianness {
return this.little_endian ? Endianness.Little : Endianness.Big;
}
set endianness(endianness: Endianness) {
this.little_endian = endianness === Endianness.Little;
this.utf16_decoder = this.little_endian ? UTF_16LE_DECODER : UTF_16BE_DECODER;
this.utf16_encoder = this.little_endian ? UTF_16LE_ENCODER : UTF_16BE_ENCODER;
}
get bytes_left(): number {
return this.size - this.position;
set size(size: number) {
if (size > this._size) {
this.ensure_size(size - this._size);
} else {
this._size = size;
}
}
protected buffer: ResizableBuffer;
protected get backing_buffer(): ArrayBuffer {
return this.buffer.backing_buffer;
}
protected get dv(): DataView {
return this.buffer.view;
}
protected utf16_decoder: TextDecoder = UTF_16BE_DECODER;
protected utf16_encoder: TextEncoder = UTF_16BE_ENCODER;
/**
* @param buffer The buffer to read from.
* @param endianness Decides in which byte order multi-byte integers and floats will be interpreted.
@ -64,7 +38,7 @@ export class ResizableBufferCursor implements Cursor {
buffer: ResizableBuffer,
endianness: Endianness,
offset: number = 0,
size: number = buffer.size
size: number = buffer.size - offset
) {
if (offset < 0 || offset > buffer.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
@ -74,152 +48,10 @@ export class ResizableBufferCursor implements Cursor {
throw new Error(`Size ${size} is out of bounds.`);
}
super(endianness, offset);
this.buffer = buffer;
this.endianness = endianness;
this._offset = offset;
this._size = size;
this._position = 0;
}
seek(offset: number): this {
return this.seek_start(this.position + offset);
}
seek_start(offset: number): this {
if (offset < 0 || offset > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
this._position = offset;
return this;
}
seek_end(offset: number): this {
if (offset < 0 || offset > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
this._position = this.size - offset;
return this;
}
u8(): number {
return this.u8_at(this._position++);
}
u8_at(offset: number): number {
this.check_offset(offset, 1);
return this.dv.getUint8(this.offset + offset);
}
u16(): number {
const r = this.u16_at(this.position);
this._position += 2;
return r;
}
u16_at(offset: number): number {
this.check_offset(offset, 2);
return this.dv.getUint16(this.offset + offset, this.little_endian);
}
u32(): number {
const r = this.u32_at(this.position);
this._position += 4;
return r;
}
u32_at(offset: number): number {
this.check_offset(offset, 4);
return this.dv.getUint32(this.offset + offset, this.little_endian);
}
i8(): number {
return this.i8_at(this._position++);
}
i8_at(offset: number): number {
this.check_offset(offset, 1);
return this.dv.getInt8(this.offset + offset);
}
i16(): number {
const r = this.i16_at(this.position);
this._position += 2;
return r;
}
i16_at(offset: number): number {
this.check_offset(offset, 2);
return this.dv.getInt16(this.offset + offset, this.little_endian);
}
i32(): number {
const r = this.i32_at(this.position);
this._position += 4;
return r;
}
i32_at(offset: number): number {
this.check_offset(offset, 4);
return this.dv.getInt32(this.offset + offset, this.little_endian);
}
f32(): number {
const r = this.f32_at(this.position);
this._position += 4;
return r;
}
f32_at(offset: number): number {
this.check_offset(offset, 4);
return this.dv.getFloat32(this.offset + offset, this.little_endian);
}
u8_array(n: number): number[] {
this.check_size("n", n, n);
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint8(this.offset + this._position++));
}
return array;
}
u16_array(n: number): number[] {
this.check_size("n", n, 2 * n);
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint16(this.offset + this.position, this.little_endian));
this._position += 2;
}
return array;
}
u32_array(n: number): number[] {
this.check_size("n", n, 4 * n);
const array = [];
for (let i = 0; i < n; ++i) {
array.push(this.dv.getUint32(this.offset + this.position, this.little_endian));
this._position += 4;
}
return array;
}
vec2_f32(): Vec2 {
return new Vec2(this.f32(), this.f32());
}
vec3_f32(): Vec3 {
return new Vec3(this.f32(), this.f32(), this.f32());
}
take(size: number): ResizableBufferCursor {
@ -231,101 +63,15 @@ export class ResizableBufferCursor implements Cursor {
return wrapper;
}
string_ascii(
max_byte_length: number,
null_terminated: boolean,
drop_remaining: boolean
): string {
this.check_size("max_byte_length", max_byte_length, max_byte_length);
protected ensure_size(size: number): void {
const needed = this.position + size - this._size;
const string_length = null_terminated
? this.index_of_u8(0, max_byte_length) - this.position
: max_byte_length;
if (needed > 0) {
this._size += needed;
const view = this.buffer.sub_view(this.offset + this.position, string_length);
const r = ASCII_DECODER.decode(view);
this._position += drop_remaining
? max_byte_length
: Math.min(string_length + 1, max_byte_length);
return r;
}
string_utf16(
max_byte_length: number,
null_terminated: boolean,
drop_remaining: boolean
): string {
this.check_size("max_byte_length", max_byte_length, max_byte_length);
const string_length = null_terminated
? this.index_of_u16(0, max_byte_length) - this.position
: Math.floor(max_byte_length / 2) * 2;
const view = this.buffer.sub_view(this.offset + this.position, string_length);
const r = this.utf16_decoder.decode(view);
this._position += drop_remaining
? max_byte_length
: Math.min(string_length + 2, max_byte_length);
return r;
}
array_buffer(size: number = this.size - this.position): ArrayBuffer {
this.check_size("size", size, size);
const r = this.buffer.backing_buffer.slice(
this.offset + this.position,
this.offset + this.position + size
);
this._position += size;
return r;
}
copy_to_uint8_array(array: Uint8Array, size: number = this.size - this.position): this {
this.check_size("size", size, size);
array.set(new Uint8Array(this.buffer.backing_buffer, this.offset + this.position, size));
this._position += size;
return this;
}
private check_size(name: string, value: number, byte_size: number): void {
if (byte_size < 0 || byte_size > this.size - this.position) {
throw new Error(`${name} ${value} is out of bounds.`);
}
}
/**
* Checks whether we can read size bytes at offset.
*/
protected check_offset(offset: number, size: number): void {
if (offset < 0 || offset + size > this.size) {
throw new Error(`Offset ${offset} is out of bounds.`);
}
}
private index_of_u8(value: number, max_byte_length: number): number {
const max_pos = Math.min(this.position + max_byte_length, this.size);
for (let i = this.position; i < max_pos; ++i) {
if (this.dv.getUint8(this.offset + i) === value) {
return i;
if (this.buffer.size < this.offset + this._size) {
this.buffer.size = this.offset + this._size;
}
}
return this.position + max_byte_length;
}
private index_of_u16(value: number, max_byte_length: number): number {
const max_pos = Math.min(this.position + max_byte_length, this.size);
for (let i = this.position; i < max_pos; i += 2) {
if (this.dv.getUint16(this.offset + i, this.little_endian) === value) {
return i;
}
}
return this.position + max_byte_length;
}
}

122
src/data_formats/cursor/WritableArrayBufferCursor.ts
View File

@ -1,122 +0,0 @@
import { ASCII_ENCODER } from ".";
import { Vec2, Vec3 } from "../vector";
import { ArrayBufferCursor } from "./ArrayBufferCursor";
import { Cursor } from "./Cursor";
import { WritableCursor } from "./WritableCursor";
/**
* A cursor for reading and writing from an array buffer or part of an array buffer.
*/
export class WritableArrayBufferCursor extends ArrayBufferCursor implements WritableCursor {
write_u8(value: number): this {
this.dv.setUint8(this._position++, value);
return this;
}
write_u16(value: number): this {
this.dv.setUint16(this.position, value, this.little_endian);
this._position += 2;
return this;
}
write_u32(value: number): this {
this.dv.setUint32(this.position, value, this.little_endian);
this._position += 4;
return this;
}
write_i32(value: number): this {
this.dv.setInt32(this.position, value, this.little_endian);
this._position += 4;
return this;
}
write_f32(value: number): this {
this.dv.setFloat32(this.position, value, this.little_endian);
this._position += 4;
return this;
}
write_u8_array(array: number[]): this {
new Uint8Array(this.buffer, this.offset + this.position).set(new Uint8Array(array));
this._position += array.length;
return this;
}
write_u16_array(array: number[]): this {
const len = array.length;
for (let i = 0; i < len; i++) {
this.write_u16(array[i]);
}
return this;
}
write_u32_array(array: number[]): this {
const len = array.length;
for (let i = 0; i < len; i++) {
this.write_u32(array[i]);
}
return this;
}
write_vec2_f32(value: Vec2): this {
this.dv.setFloat32(this.position, value.x, this.little_endian);
this.dv.setFloat32(this.position + 4, value.y, this.little_endian);
this._position += 8;
return this;
}
write_vec3_f32(value: Vec3): this {
this.dv.setFloat32(this.position, value.x, this.little_endian);
this.dv.setFloat32(this.position + 4, value.y, this.little_endian);
this.dv.setFloat32(this.position + 8, value.z, this.little_endian);
this._position += 12;
return this;
}
write_cursor(other: Cursor): this {
const size = other.size - other.position;
other.copy_to_uint8_array(
new Uint8Array(this.buffer, this.offset + this.position, size),
size
);
this._position += size;
return this;
}
write_string_ascii(str: string, byte_length: number): this {
const encoded = ASCII_ENCODER.encode(str);
const encoded_length = Math.min(encoded.byteLength, byte_length);
let i = 0;
while (i < encoded_length) {
this.write_u8(encoded[i++]);
}
while (i++ < byte_length) {
this.write_u8(0);
}
return this;
}
write_string_utf16(str: string, byte_length: number): this {
const encoded = this.utf16_encoder.encode(str);
const encoded_length = Math.min(encoded.byteLength, byte_length);
let i = 0;
while (i < encoded_length) {
this.write_u8(encoded[i++]);
}
while (i++ < byte_length) {
this.write_u8(0);
}
return this;
}
}

12
src/data_formats/cursor/WritableCursor.test.ts
View File

@ -1,9 +1,9 @@
import { Endianness } from "..";
import { enum_values } from "../../enums";
import { ResizableBuffer } from "../ResizableBuffer";
import { WritableArrayBufferCursor } from "./WritableArrayBufferCursor";
import { ArrayBufferCursor } from "./ArrayBufferCursor";
import { ResizableBufferCursor } from "./ResizableBufferCursor";
import { WritableCursor } from "./WritableCursor";
import { WritableResizableBufferCursor } from "./WritableResizableBufferCursor";
/**
* Run a test on every writable cursor implementation with every endianness.
@ -33,14 +33,14 @@ function test_all(
const cursors: [string, Endianness, WritableCursor][] = [
...endiannesses.map(endianness => [
WritableArrayBufferCursor.name,
ArrayBufferCursor.name,
endianness,
new WritableArrayBufferCursor(new Uint8Array(bytes(endianness)).buffer, endianness),
new ArrayBufferCursor(new Uint8Array(bytes(endianness)).buffer, endianness),
]),
...endiannesses.map(endianness => [
WritableResizableBufferCursor.name,
ResizableBufferCursor.name,
endianness,
new WritableResizableBufferCursor(rbuf(endianness), endianness),
new ResizableBufferCursor(rbuf(endianness), endianness),
]),
] as any;

14
src/data_formats/cursor/index.ts
View File

@ -1,14 +0,0 @@
// TODO: remove dependency on text-encoding because it is no longer maintained.
import { TextDecoder, TextEncoder } from "text-encoding";
export const ASCII_DECODER = new TextDecoder("ascii");
export const UTF_16BE_DECODER = new TextDecoder("utf-16be");
export const UTF_16LE_DECODER = new TextDecoder("utf-16le");
export const ASCII_ENCODER = new TextEncoder("ascii");
export const UTF_16BE_ENCODER = new TextEncoder("utf-16be", {
NONSTANDARD_allowLegacyEncoding: true,
});
export const UTF_16LE_ENCODER = new TextEncoder("utf-16le", {
NONSTANDARD_allowLegacyEncoding: true,
});

9
src/data_formats/encryption/prc.ts
View File

@ -1,6 +1,6 @@
import { Cursor } from "../cursor/Cursor";
import { WritableArrayBufferCursor } from "../cursor/WritableArrayBufferCursor";
import { Endianness } from "..";
import { ArrayBufferCursor } from "../cursor/ArrayBufferCursor";
import { Cursor } from "../cursor/Cursor";
/**
* Decrypts the bytes left in cursor.
@ -21,10 +21,7 @@ class PrcDecryptor {
// Size should be divisible by 4.
const actual_size = cursor.bytes_left;
const size = Math.ceil(actual_size / 4) * 4;
const out_cursor = new WritableArrayBufferCursor(
new ArrayBuffer(actual_size),
cursor.endianness
);
const out_cursor = new ArrayBufferCursor(new ArrayBuffer(actual_size), cursor.endianness);
for (let pos = 0; pos < size; pos += 4) {
let u32;

11
src/data_formats/parsing/ninja/index.ts
View File

@ -128,9 +128,14 @@ function parse_ninja<M extends NjModel>(
context: any
): NjObject<M>[] {
// POF0 and other chunks types are ignored.
return parse_iff(cursor)
.filter(chunk => chunk.type === NJCM)
.flatMap(chunk => parse_sibling_objects(chunk.data, parse_model, context));
const njcm_chunks = parse_iff(cursor).filter(chunk => chunk.type === NJCM);
const objects: NjObject<M>[] = [];
for (const chunk of njcm_chunks) {
objects.push(...parse_sibling_objects(chunk.data, parse_model, context));
}
return objects;
}
// TODO: cache model and object offsets so we don't reparse the same data.

4
src/data_formats/parsing/quest/bin.ts
View File

@ -1,8 +1,8 @@
import Logger from "js-logger";
import { Endianness } from "../..";
import { Cursor } from "../../cursor/Cursor";
import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { WritableCursor } from "../../cursor/WritableCursor";
import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
import { Opcode, OPCODES, Type } from "./opcodes";
@ -195,7 +195,7 @@ export function write_bin(bin: BinFile): ArrayBuffer {
const buffer = new ResizableBuffer(
object_code_offset + 10 * bin.instructions.length + 4 * labels.length
);
const cursor = new WritableResizableBufferCursor(buffer, Endianness.Little);
const cursor = new ResizableBufferCursor(buffer, Endianness.Little);
cursor.write_u32(object_code_offset);
cursor.write_u32(0); // Placeholder for the labels offset.

4
src/data_formats/parsing/quest/dat.ts
View File

@ -2,7 +2,7 @@ import Logger from "js-logger";
import { groupBy } from "lodash";
import { Endianness } from "../..";
import { Cursor } from "../../cursor/Cursor";
import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor";
import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
import { Vec3 } from "../../vector";
@ -159,7 +159,7 @@ export function write_dat({ objs, npcs, unknowns }: DatFile): ResizableBuffer {
npcs.length * (16 + NPC_SIZE) +
unknowns.reduce((a, b) => a + b.total_size, 0)
);
const cursor = new WritableResizableBufferCursor(buffer, Endianness.Little);
const cursor = new ResizableBufferCursor(buffer, Endianness.Little);
const grouped_objs = groupBy(objs, obj => obj.area_id);
const obj_area_ids = Object.keys(grouped_objs)

13
src/data_formats/parsing/quest/qst.ts
View File

@ -1,11 +1,10 @@
import Logger from "js-logger";
import { Cursor } from "../../cursor/Cursor";
import { WritableArrayBufferCursor } from "../../cursor/WritableArrayBufferCursor";
import { Endianness } from "../..";
import { WritableCursor } from "../../cursor/WritableCursor";
import { WritableResizableBufferCursor } from "../../cursor/WritableResizableBufferCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
import { ArrayBufferCursor } from "../../cursor/ArrayBufferCursor";
import { Cursor } from "../../cursor/Cursor";
import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { WritableCursor } from "../../cursor/WritableCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
const logger = Logger.get("data_formats/parsing/quest/qst");
@ -92,7 +91,7 @@ export function write_qst(params: WriteQstParams): ArrayBuffer {
.map(f => 88 + Math.ceil(f.data.byteLength / 1024) * 1056)
.reduce((a, b) => a + b);
const buffer = new ArrayBuffer(total_size);
const cursor = new WritableArrayBufferCursor(buffer, Endianness.Little);
const cursor = new ArrayBufferCursor(buffer, Endianness.Little);
write_file_headers(cursor, files);
write_file_chunks(cursor, files);
@ -166,7 +165,7 @@ function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstCo
(file = {
name: file_name,
expected_size,
cursor: new WritableResizableBufferCursor(
cursor: new ResizableBufferCursor(
new ResizableBuffer(expected_size || 10 * 1024),
Endianness.Little
),

2
tsconfig.json
View File

@ -4,7 +4,7 @@
"sourceMap": true,
"module": "es6",
"target": "es6",
"lib": ["dom", "dom.iterable", "esnext"],
"lib": ["es6", "dom", "dom.iterable"],
"allowJs": true,
"skipLibCheck": true,
"allowSyntheticDefaultImports": true,

5
yarn.lock
View File

@ -8133,11 +8133,6 @@ test-exclude@^5.2.3:
read-pkg-up "^4.0.0"
require-main-filename "^2.0.0"
text-encoding@^0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.7.0.tgz#f895e836e45990624086601798ea98e8f36ee643"
integrity sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA==
text-table@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"