diff --git a/src/core/data_formats/parsing/quest/index.ts b/src/core/data_formats/parsing/quest/index.ts index 2ea76db9..8ef27a27 100644 --- a/src/core/data_formats/parsing/quest/index.ts +++ b/src/core/data_formats/parsing/quest/index.ts @@ -13,7 +13,7 @@ import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; import { Endianness } from "../../Endianness"; import { parse_bin, write_bin } from "./bin"; import { DatFile, DatNpc, DatObject, DatUnknown, parse_dat, write_dat } from "./dat"; -import { QuestNpc, QuestObject, QuestEvent } from "./entities"; +import { QuestEvent, QuestNpc, QuestObject } from "./entities"; import { Episode } from "./Episode"; import { object_data, ObjectType, pso_id_to_object_type } from "./object_types"; import { parse_qst, QstContainedFile, write_qst } from "./qst"; @@ -59,7 +59,7 @@ export function parse_quest(cursor: Cursor, lenient: boolean = false): Quest | u let bin_file: QstContainedFile | undefined; for (const file of qst.files) { - const file_name = file.name.trim().toLowerCase(); + const file_name = file.filename.trim().toLowerCase(); if (file_name.endsWith(".dat")) { dat_file = file; @@ -158,14 +158,14 @@ export function write_quest_qst(quest: Quest, file_name: string): ArrayBuffer { return write_qst({ files: [ { - name: base_file_name + ".dat", + filename: base_file_name + ".dat", id: quest.id, data: prs_compress( new ResizableBufferCursor(dat, Endianness.Little), ).array_buffer(), }, { - name: base_file_name + ".bin", + filename: base_file_name + ".bin", id: quest.id, data: prs_compress(new ArrayBufferCursor(bin, Endianness.Little)).array_buffer(), }, diff --git a/src/core/data_formats/parsing/quest/qst.test.ts b/src/core/data_formats/parsing/quest/qst.test.ts index 54accd0e..eb4d615c 100644 --- a/src/core/data_formats/parsing/quest/qst.test.ts +++ b/src/core/data_formats/parsing/quest/qst.test.ts @@ -1,8 +1,25 @@ import { walk_qst_files } from "../../../../../test/src/utils"; -import { parse_qst, write_qst } from "./qst"; +import { parse_qst, Version, write_qst } from "./qst"; import { Endianness } from "../../Endianness"; import { BufferCursor } from "../../cursor/BufferCursor"; import { ArrayBufferCursor } from "../../cursor/ArrayBufferCursor"; +import * as fs from "fs"; + +test("Parse a GC quest.", () => { + const buf = fs.readFileSync("test/resources/lost_heat_sword_gc.qst"); + const qst = parse_qst(new BufferCursor(buf, Endianness.Little)); + + expect(qst).toBeDefined(); + expect(qst!.version).toBe(Version.GC); + expect(qst!.online).toBe(true); + expect(qst!.files.length).toBe(2); + expect(qst!.files[0].id).toBe(58); + expect(qst!.files[0].filename).toBe("quest58.bin"); + expect(qst!.files[0].quest_name).toBe("PSO/Lost HEAT SWORD"); + expect(qst!.files[1].id).toBe(58); + expect(qst!.files[1].filename).toBe("quest58.dat"); + expect(qst!.files[1].quest_name).toBe("PSO/Lost HEAT SWORD"); +}); /** * Parse a file, convert the resulting structure to QST again and check whether the end result is equal to the original. diff --git a/src/core/data_formats/parsing/quest/qst.ts b/src/core/data_formats/parsing/quest/qst.ts index 441acaf1..93354e10 100644 --- a/src/core/data_formats/parsing/quest/qst.ts +++ b/src/core/data_formats/parsing/quest/qst.ts @@ -4,37 +4,44 @@ import { Cursor } from "../../cursor/Cursor"; import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor"; import { WritableCursor } from "../../cursor/WritableCursor"; import { ResizableBuffer } from "../../ResizableBuffer"; -import { basename } from "../../../util"; +import { basename, defined } from "../../../util"; import { LogManager } from "../../../Logger"; const logger = LogManager.get("core/data_formats/parsing/quest/qst"); +const BB_HEADER_SIZE = 88; +const PC_GC_HEADER_SIZE = 60; +const ONLINE_QUEST = 0x44; +const DOWNLOAD_QUEST = 0xa6; +const CHUNK_BODY_SIZE = 1024; + export enum Version { - PC, /** - * Dreamcast/GameCube + * Dreamcast */ - DC_GC, + DC, + /** + * GameCube + */ + GC, + PC, /** * BlueBurst */ BB, - /** - * Dreamcast Download - */ - DC_DOWNLOAD, } export type QstContainedFile = { - id?: number; - name: string; - name_2?: string; // Unsure what this is - data: ArrayBuffer; + readonly id?: number; + readonly filename: string; + readonly quest_name?: string; + readonly data: ArrayBuffer; }; export type ParseQstResult = { - version: Version; - files: QstContainedFile[]; + readonly version: Version; + readonly online: boolean; + readonly files: readonly QstContainedFile[]; }; /** @@ -43,60 +50,66 @@ export type ParseQstResult = { */ export function parse_qst(cursor: Cursor): ParseQstResult | undefined { // A .qst file contains two 88-byte headers that describe the embedded .dat and .bin files. - let version = Version.PC; + // Read headers and contained files. + const headers = parse_headers(cursor); - // Detect version. - const version_a = cursor.u8(); - cursor.seek(1); - const version_b = cursor.u8(); - - if (version_a === 0x44) { - version = Version.DC_GC; - } else if (version_a === 0x58) { - if (version_b === 0x44) { - version = Version.BB; - } - } else if (version_a === 0xa6) { - version = Version.DC_DOWNLOAD; - } - - if (version === Version.BB) { - // Read headers and contained files. - cursor.seek_start(0); - - const headers = parse_headers(cursor); - - const files = parse_files(cursor, new Map(headers.map(h => [h.file_name, h.size]))); - - for (const file of files) { - const header = headers.find(h => h.file_name === file.name); - - if (header) { - file.id = header.quest_id; - file.name_2 = header.file_name_2; - } - } - - return { - version, - files, - }; - } else { - logger.error(`Can't parse ${Version[version]} QST files.`); + if (headers.length < 2) { + logger.error( + `Corrupt .qst file, expected at least 2 headers but only found ${headers.length}.`, + ); return undefined; } + + let version: Version | undefined = undefined; + let online: boolean | undefined = undefined; + + for (const header of headers) { + if (version != undefined && header.version !== version) { + logger.error( + `Corrupt .qst file, header version ${Version[header.version]} for file ${ + header.file_name + } doesn't match the previous header's version ${Version[version]}.`, + ); + return undefined; + } + + if (online != undefined && header.online !== online) { + logger.error( + `Corrupt .qst file, header type ${ + header.online ? '"online"' : '"download"' + } for file ${header.file_name} doesn't match the previous header's type ${ + online ? '"online"' : '"download"' + }.`, + ); + return undefined; + } + + version = header.version; + online = header.online; + } + + defined(version, "version"); + defined(online, "online"); + + const files = parse_files(cursor, version, new Map(headers.map(h => [h.file_name, h]))); + + return { + version, + online, + files, + }; } export type QstContainedFileParam = { - id?: number; - name: string; - name_2?: string; - data: ArrayBuffer; + readonly id?: number; + readonly filename: string; + readonly quest_name?: string; + readonly data: ArrayBuffer; }; export type WriteQstParams = { - version?: Version; - files: QstContainedFileParam[]; + readonly version?: Version; + readonly files: readonly QstContainedFileParam[]; }; /** @@ -121,10 +134,12 @@ export function write_qst(params: WriteQstParams): ArrayBuffer { } type QstHeader = { - quest_id: number; - file_name: string; - file_name_2: string; - size: number; + readonly version: Version; + readonly online: boolean; + readonly quest_id: number; + readonly name: string; + readonly file_name: string; + readonly size: number; }; function parse_headers(cursor: Cursor): QstHeader[] { @@ -133,21 +148,103 @@ function parse_headers(cursor: Cursor): QstHeader[] { let prev_quest_id: number | undefined = undefined; let prev_file_name: string | undefined = undefined; + // .qst files should have two headers, some malformed files have more. for (let i = 0; i < 4; ++i) { - cursor.seek(4); - const quest_id = cursor.u16(); - cursor.seek(38); - const file_name = cursor.string_ascii(16, true, true); - const size = cursor.u32(); - // Not sure what this is: - const file_name_2 = cursor.string_ascii(24, true, true); + // Detect version and whether it's an online or download quest. + let version; + let online; + const version_a = cursor.u8(); + cursor.seek(1); + const version_b = cursor.u8(); + cursor.seek(-3); + + if (version_a === BB_HEADER_SIZE && version_b === ONLINE_QUEST) { + version = Version.BB; + online = true; + } else if (version_a === PC_GC_HEADER_SIZE && version_b === ONLINE_QUEST) { + version = Version.PC; + online = true; + } else if (version_b === PC_GC_HEADER_SIZE) { + const pos = cursor.position; + cursor.seek(35); + + if (cursor.u8() === 0) { + version = Version.GC; + } else { + version = Version.DC; + } + + cursor.seek_start(pos); + + if (version_a === ONLINE_QUEST) { + online = true; + } else if (version_a === DOWNLOAD_QUEST) { + online = false; + } else { + break; + } + } else { + break; + } + + // Read header. + let header_size; + let quest_id: number; + let name: string; + let file_name: string; + let size: number; + + switch (version) { + case Version.DC: + cursor.seek(1); // Skip online/download. + quest_id = cursor.u8(); + header_size = cursor.u16(); + name = cursor.string_ascii(32, true, true); + cursor.seek(3); + file_name = cursor.string_ascii(16, true, true); + size = cursor.u32(); + break; + + case Version.GC: + cursor.seek(1); // Skip online/download. + quest_id = cursor.u8(); + header_size = cursor.u16(); + name = cursor.string_ascii(32, true, true); + cursor.seek(4); + file_name = cursor.string_ascii(16, true, true); + size = cursor.u32(); + break; + + case Version.PC: + header_size = cursor.u16(); + cursor.seek(1); // Skip online/download. + quest_id = cursor.u8(); + name = cursor.string_ascii(32, true, true); + cursor.seek(4); + file_name = cursor.string_ascii(16, true, true); + size = cursor.u32(); + break; + + case Version.BB: + header_size = cursor.u16(); + cursor.seek(2); // Skip online/download. + quest_id = cursor.u16(); + cursor.seek(38); + file_name = cursor.string_ascii(16, true, true); + size = cursor.u32(); + name = cursor.string_ascii(24, true, true); + break; + } + + // Use some simple heuristics to figure out whether the file contains more than two headers. + // Some malformed .qst files have extra headers. if ( prev_quest_id != undefined && prev_file_name != undefined && (quest_id !== prev_quest_id || basename(file_name) !== basename(prev_file_name)) ) { - cursor.seek(-88); + cursor.seek(-header_size); break; } @@ -155,9 +252,11 @@ function parse_headers(cursor: Cursor): QstHeader[] { prev_file_name = file_name; headers.push({ + version, + online, quest_id, + name, file_name, - file_name_2, size, }); } @@ -165,9 +264,13 @@ function parse_headers(cursor: Cursor): QstHeader[] { return headers; } -function parse_files(cursor: Cursor, expected_sizes: Map): QstContainedFile[] { +function parse_files( + cursor: Cursor, + version: Version, + headers: Map, +): QstContainedFile[] { // Files are interleaved in 1056 byte chunks. - // Each chunk has a 24 byte header, 1024 byte data segment and an 8 byte trailer. + // Each chunk has a 20 or 24 byte header, 1024 byte data segment and an 4 or 8 byte trailer. const files = new Map< string, { @@ -178,29 +281,63 @@ function parse_files(cursor: Cursor, expected_sizes: Map): QstCo } >(); - while (cursor.bytes_left >= 1056) { + let chunk_size: number; // Size including padding, header and trailer. + let trailer_size: number; + + switch (version) { + case Version.DC: + case Version.GC: + case Version.PC: + chunk_size = CHUNK_BODY_SIZE + 24; + trailer_size = 4; + break; + + case Version.BB: + chunk_size = CHUNK_BODY_SIZE + 32; + trailer_size = 8; + break; + } + + while (cursor.bytes_left >= chunk_size) { const start_position = cursor.position; - // Read meta data. - const chunk_no = cursor.seek(4).u8(); - const file_name = cursor.seek(3).string_ascii(16, true, true); + // Read chunk header. + let chunk_no: number; + switch (version) { + case Version.DC: + case Version.GC: + cursor.seek(1); + chunk_no = cursor.u8(); + cursor.seek(2); + break; + + case Version.PC: + cursor.seek(3); + chunk_no = cursor.u8(); + break; + + case Version.BB: + cursor.seek(4); + chunk_no = cursor.u32(); + break; + } + + const file_name = cursor.string_ascii(16, true, true); let file = files.get(file_name); if (!file) { - const expected_size = expected_sizes.get(file_name); - files.set( - file_name, - (file = { - name: file_name, - expected_size, - cursor: new ResizableBufferCursor( - new ResizableBuffer(expected_size || 10 * 1024), - Endianness.Little, - ), - chunk_nos: new Set(), - }), - ); + const header = headers.get(file_name); + file = { + name: file_name, + expected_size: header?.size, + cursor: new ResizableBufferCursor( + new ResizableBuffer(header?.size ?? 10 * 1024), + Endianness.Little, + ), + chunk_nos: new Set(), + }; + files.set(file_name, file); } if (file.chunk_nos.has(chunk_no)) { @@ -212,28 +349,28 @@ function parse_files(cursor: Cursor, expected_sizes: Map): QstCo } // Read file data. - let size = cursor.seek(1024).u32(); - cursor.seek(-1028); + let size = cursor.seek(CHUNK_BODY_SIZE).u32(); + cursor.seek(-CHUNK_BODY_SIZE - 4); - if (size > 1024) { + if (size > CHUNK_BODY_SIZE) { logger.warn( - `Data segment size of ${size} is larger than expected maximum size, reading just 1024 bytes.`, + `Data segment size of ${size} is larger than expected maximum size, reading just ${CHUNK_BODY_SIZE} bytes.`, ); - size = 1024; + size = CHUNK_BODY_SIZE; } const data = cursor.take(size); - const chunk_position = chunk_no * 1024; + const chunk_position = chunk_no * CHUNK_BODY_SIZE; file.cursor.size = Math.max(chunk_position + size, file.cursor.size); file.cursor.seek_start(chunk_position).write_cursor(data); // Skip the padding and the trailer. - cursor.seek(1032 - data.size); + cursor.seek(CHUNK_BODY_SIZE + trailer_size - data.size); - if (cursor.position !== start_position + 1056) { + if (cursor.position !== start_position + chunk_size) { throw new Error( `Read ${cursor.position - - start_position} file chunk message bytes instead of expected 1056.`, + start_position} file chunk message bytes instead of expected ${chunk_size}.`, ); } } @@ -255,9 +392,10 @@ function parse_files(cursor: Cursor, expected_sizes: Map): QstCo } // Detect missing file chunks. - const actual_size = Math.max(file.cursor.size, file.expected_size || 0); + const actual_size = Math.max(file.cursor.size, file.expected_size ?? 0); + const expected_chunk_count = Math.ceil(actual_size / CHUNK_BODY_SIZE); - for (let chunk_no = 0; chunk_no < Math.ceil(actual_size / 1024); ++chunk_no) { + for (let chunk_no = 0; chunk_no < expected_chunk_count; ++chunk_no) { if (!file.chunk_nos.has(chunk_no)) { logger.warn(`File ${file.name} is missing chunk ${chunk_no}.`); } @@ -267,8 +405,11 @@ function parse_files(cursor: Cursor, expected_sizes: Map): QstCo const contained_files: QstContainedFile[] = []; for (const file of files.values()) { + const header = headers.get(file.name); contained_files.push({ - name: file.name, + id: header?.quest_id, + filename: file.name, + quest_name: header?.name, data: file.cursor.seek_start(0).array_buffer(), }); } @@ -276,10 +417,10 @@ function parse_files(cursor: Cursor, expected_sizes: Map): QstCo return contained_files; } -function write_file_headers(cursor: WritableCursor, files: QstContainedFileParam[]): void { +function write_file_headers(cursor: WritableCursor, files: readonly QstContainedFileParam[]): void { for (const file of files) { - if (file.name.length > 15) { - throw new Error(`File ${file.name} has a name longer than 15 characters.`); + if (file.filename.length > 15) { + throw new Error(`File ${file.filename} has a name longer than 15 characters.`); } cursor.write_u16(88); // Header size. @@ -290,25 +431,25 @@ function write_file_headers(cursor: WritableCursor, files: QstContainedFileParam cursor.write_u8(0); } - cursor.write_string_ascii(file.name, 16); + cursor.write_string_ascii(file.filename, 16); cursor.write_u32(file.data.byteLength); let file_name_2: string; - if (file.name_2 == null) { + if (file.quest_name == null) { // Not sure this makes sense. - const dot_pos = file.name.lastIndexOf("."); + const dot_pos = file.filename.lastIndexOf("."); file_name_2 = dot_pos === -1 - ? file.name + "_j" - : file.name.slice(0, dot_pos) + "_j" + file.name.slice(dot_pos); + ? file.filename + "_j" + : file.filename.slice(0, dot_pos) + "_j" + file.filename.slice(dot_pos); } else { - file_name_2 = file.name_2; + file_name_2 = file.quest_name; } if (file_name_2.length > 24) { throw Error( - `File ${file.name} has a file_name_2 length (${file_name_2}) longer than 24 characters.`, + `File ${file.filename} has a file_name_2 length (${file_name_2}) longer than 24 characters.`, ); } @@ -316,13 +457,13 @@ function write_file_headers(cursor: WritableCursor, files: QstContainedFileParam } } -function write_file_chunks(cursor: WritableCursor, files: QstContainedFileParam[]): void { +function write_file_chunks(cursor: WritableCursor, files: readonly QstContainedFileParam[]): void { // Files are interleaved in 1056 byte chunks. // Each chunk has a 24 byte header, 1024 byte data segment and an 8 byte trailer. const files_to_chunk = files.map(file => ({ no: 0, data: new ArrayBufferCursor(file.data, Endianness.Little), - name: file.name, + name: file.filename, })); let done = 0; diff --git a/src/viewer/stores/TextureStore.ts b/src/viewer/stores/TextureStore.ts index aaa2e164..f108665a 100644 --- a/src/viewer/stores/TextureStore.ts +++ b/src/viewer/stores/TextureStore.ts @@ -26,7 +26,7 @@ export class TextureStore extends Store { load_file = async (file: File): Promise => { try { - const ext = filename_extension(file.name); + const ext = filename_extension(file.name).toLowerCase(); const buffer = await read_file(file); if (ext === "xvm") { diff --git a/test/resources/lost_heat_sword_gc.qst b/test/resources/lost_heat_sword_gc.qst new file mode 100644 index 00000000..6d338ea7 Binary files /dev/null and b/test/resources/lost_heat_sword_gc.qst differ