Sancaros/phantasmal-world
1
0
Fork 0
mirror of https://github.com/DaanVandenBosch/phantasmal-world.git synced 2025-04-04 22:58:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

The different .qst formats are now unpacked correctly.

Browse Source
This commit is contained in:
Daan Vanden Bosch 2020-01-02 13:44:34 +01:00
parent 2c12f47c4d
commit 0035588e43
5 changed files with 279 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/core/data_formats/parsing/quest/index.ts
View File

@ -13,7 +13,7 @@ import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { Endianness } from "../../Endianness";
import { parse_bin, write_bin } from "./bin";
import { DatFile, DatNpc, DatObject, DatUnknown, parse_dat, write_dat } from "./dat";
import { QuestNpc, QuestObject, QuestEvent } from "./entities";
import { QuestEvent, QuestNpc, QuestObject } from "./entities";
import { Episode } from "./Episode";
import { object_data, ObjectType, pso_id_to_object_type } from "./object_types";
import { parse_qst, QstContainedFile, write_qst } from "./qst";
@ -59,7 +59,7 @@ export function parse_quest(cursor: Cursor, lenient: boolean = false): Quest | u
let bin_file: QstContainedFile | undefined;
for (const file of qst.files) {
const file_name = file.name.trim().toLowerCase();
const file_name = file.filename.trim().toLowerCase();
if (file_name.endsWith(".dat")) {
dat_file = file;
@ -158,14 +158,14 @@ export function write_quest_qst(quest: Quest, file_name: string): ArrayBuffer {
return write_qst({
files: [
{
name: base_file_name + ".dat",
filename: base_file_name + ".dat",
id: quest.id,
data: prs_compress(
new ResizableBufferCursor(dat, Endianness.Little),
).array_buffer(),
},
{
name: base_file_name + ".bin",
filename: base_file_name + ".bin",
id: quest.id,
data: prs_compress(new ArrayBufferCursor(bin, Endianness.Little)).array_buffer(),
},

19
src/core/data_formats/parsing/quest/qst.test.ts
View File

@ -1,8 +1,25 @@
import { walk_qst_files } from "../../../../../test/src/utils";
import { parse_qst, write_qst } from "./qst";
import { parse_qst, Version, write_qst } from "./qst";
import { Endianness } from "../../Endianness";
import { BufferCursor } from "../../cursor/BufferCursor";
import { ArrayBufferCursor } from "../../cursor/ArrayBufferCursor";
import * as fs from "fs";
test("Parse a GC quest.", () => {
const buf = fs.readFileSync("test/resources/lost_heat_sword_gc.qst");
const qst = parse_qst(new BufferCursor(buf, Endianness.Little));
expect(qst).toBeDefined();
expect(qst!.version).toBe(Version.GC);
expect(qst!.online).toBe(true);
expect(qst!.files.length).toBe(2);
expect(qst!.files[0].id).toBe(58);
expect(qst!.files[0].filename).toBe("quest58.bin");
expect(qst!.files[0].quest_name).toBe("PSO/Lost HEAT SWORD");
expect(qst!.files[1].id).toBe(58);
expect(qst!.files[1].filename).toBe("quest58.dat");
expect(qst!.files[1].quest_name).toBe("PSO/Lost HEAT SWORD");
});
/**
* Parse a file, convert the resulting structure to QST again and check whether the end result is equal to the original.

371
src/core/data_formats/parsing/quest/qst.ts
View File

@ -4,37 +4,44 @@ import { Cursor } from "../../cursor/Cursor";
import { ResizableBufferCursor } from "../../cursor/ResizableBufferCursor";
import { WritableCursor } from "../../cursor/WritableCursor";
import { ResizableBuffer } from "../../ResizableBuffer";
import { basename } from "../../../util";
import { basename, defined } from "../../../util";
import { LogManager } from "../../../Logger";
const logger = LogManager.get("core/data_formats/parsing/quest/qst");
const BB_HEADER_SIZE = 88;
const PC_GC_HEADER_SIZE = 60;
const ONLINE_QUEST = 0x44;
const DOWNLOAD_QUEST = 0xa6;
const CHUNK_BODY_SIZE = 1024;
export enum Version {
PC,
/**
* Dreamcast/GameCube
* Dreamcast
*/
DC_GC,
DC,
/**
* GameCube
*/
GC,
PC,
/**
* BlueBurst
*/
BB,
/**
* Dreamcast Download
*/
DC_DOWNLOAD,
}
export type QstContainedFile = {
id?: number;
name: string;
name_2?: string; // Unsure what this is
data: ArrayBuffer;
readonly id?: number;
readonly filename: string;
readonly quest_name?: string;
readonly data: ArrayBuffer;
};
export type ParseQstResult = {
version: Version;
files: QstContainedFile[];
readonly version: Version;
readonly online: boolean;
readonly files: readonly QstContainedFile[];
};
/**
@ -43,60 +50,66 @@ export type ParseQstResult = {
*/
export function parse_qst(cursor: Cursor): ParseQstResult | undefined {
// A .qst file contains two 88-byte headers that describe the embedded .dat and .bin files.
let version = Version.PC;
// Read headers and contained files.
const headers = parse_headers(cursor);
// Detect version.
const version_a = cursor.u8();
cursor.seek(1);
const version_b = cursor.u8();
if (version_a === 0x44) {
version = Version.DC_GC;
} else if (version_a === 0x58) {
if (version_b === 0x44) {
version = Version.BB;
}
} else if (version_a === 0xa6) {
version = Version.DC_DOWNLOAD;
}
if (version === Version.BB) {
// Read headers and contained files.
cursor.seek_start(0);
const headers = parse_headers(cursor);
const files = parse_files(cursor, new Map(headers.map(h => [h.file_name, h.size])));
for (const file of files) {
const header = headers.find(h => h.file_name === file.name);
if (header) {
file.id = header.quest_id;
file.name_2 = header.file_name_2;
}
}
return {
version,
files,
};
} else {
logger.error(`Can't parse ${Version[version]} QST files.`);
if (headers.length < 2) {
logger.error(
`Corrupt .qst file, expected at least 2 headers but only found ${headers.length}.`,
);
return undefined;
}
let version: Version | undefined = undefined;
let online: boolean | undefined = undefined;
for (const header of headers) {
if (version != undefined && header.version !== version) {
logger.error(
`Corrupt .qst file, header version ${Version[header.version]} for file ${
header.file_name
} doesn't match the previous header's version ${Version[version]}.`,
);
return undefined;
}
if (online != undefined && header.online !== online) {
logger.error(
`Corrupt .qst file, header type ${
header.online ? '"online"' : '"download"'
} for file ${header.file_name} doesn't match the previous header's type ${
online ? '"online"' : '"download"'
}.`,
);
return undefined;
}
version = header.version;
online = header.online;
}
defined(version, "version");
defined(online, "online");
const files = parse_files(cursor, version, new Map(headers.map(h => [h.file_name, h])));
return {
version,
online,
files,
};
}
export type QstContainedFileParam = {
id?: number;
name: string;
name_2?: string;
data: ArrayBuffer;
readonly id?: number;
readonly filename: string;
readonly quest_name?: string;
readonly data: ArrayBuffer;
};
export type WriteQstParams = {
version?: Version;
files: QstContainedFileParam[];
readonly version?: Version;
readonly files: readonly QstContainedFileParam[];
};
/**
@ -121,10 +134,12 @@ export function write_qst(params: WriteQstParams): ArrayBuffer {
}
type QstHeader = {
quest_id: number;
file_name: string;
file_name_2: string;
size: number;
readonly version: Version;
readonly online: boolean;
readonly quest_id: number;
readonly name: string;
readonly file_name: string;
readonly size: number;
};
function parse_headers(cursor: Cursor): QstHeader[] {
@ -133,21 +148,103 @@ function parse_headers(cursor: Cursor): QstHeader[] {
let prev_quest_id: number | undefined = undefined;
let prev_file_name: string | undefined = undefined;
// .qst files should have two headers, some malformed files have more.
for (let i = 0; i < 4; ++i) {
cursor.seek(4);
const quest_id = cursor.u16();
cursor.seek(38);
const file_name = cursor.string_ascii(16, true, true);
const size = cursor.u32();
// Not sure what this is:
const file_name_2 = cursor.string_ascii(24, true, true);
// Detect version and whether it's an online or download quest.
let version;
let online;
const version_a = cursor.u8();
cursor.seek(1);
const version_b = cursor.u8();
cursor.seek(-3);
if (version_a === BB_HEADER_SIZE && version_b === ONLINE_QUEST) {
version = Version.BB;
online = true;
} else if (version_a === PC_GC_HEADER_SIZE && version_b === ONLINE_QUEST) {
version = Version.PC;
online = true;
} else if (version_b === PC_GC_HEADER_SIZE) {
const pos = cursor.position;
cursor.seek(35);
if (cursor.u8() === 0) {
version = Version.GC;
} else {
version = Version.DC;
}
cursor.seek_start(pos);
if (version_a === ONLINE_QUEST) {
online = true;
} else if (version_a === DOWNLOAD_QUEST) {
online = false;
} else {
break;
}
} else {
break;
}
// Read header.
let header_size;
let quest_id: number;
let name: string;
let file_name: string;
let size: number;
switch (version) {
case Version.DC:
cursor.seek(1); // Skip online/download.
quest_id = cursor.u8();
header_size = cursor.u16();
name = cursor.string_ascii(32, true, true);
cursor.seek(3);
file_name = cursor.string_ascii(16, true, true);
size = cursor.u32();
break;
case Version.GC:
cursor.seek(1); // Skip online/download.
quest_id = cursor.u8();
header_size = cursor.u16();
name = cursor.string_ascii(32, true, true);
cursor.seek(4);
file_name = cursor.string_ascii(16, true, true);
size = cursor.u32();
break;
case Version.PC:
header_size = cursor.u16();
cursor.seek(1); // Skip online/download.
quest_id = cursor.u8();
name = cursor.string_ascii(32, true, true);
cursor.seek(4);
file_name = cursor.string_ascii(16, true, true);
size = cursor.u32();
break;
case Version.BB:
header_size = cursor.u16();
cursor.seek(2); // Skip online/download.
quest_id = cursor.u16();
cursor.seek(38);
file_name = cursor.string_ascii(16, true, true);
size = cursor.u32();
name = cursor.string_ascii(24, true, true);
break;
}
// Use some simple heuristics to figure out whether the file contains more than two headers.
// Some malformed .qst files have extra headers.
if (
prev_quest_id != undefined &&
prev_file_name != undefined &&
(quest_id !== prev_quest_id || basename(file_name) !== basename(prev_file_name))
) {
cursor.seek(-88);
cursor.seek(-header_size);
break;
}
@ -155,9 +252,11 @@ function parse_headers(cursor: Cursor): QstHeader[] {
prev_file_name = file_name;
headers.push({
version,
online,
quest_id,
name,
file_name,
file_name_2,
size,
});
}
@ -165,9 +264,13 @@ function parse_headers(cursor: Cursor): QstHeader[] {
return headers;
}
function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstContainedFile[] {
function parse_files(
cursor: Cursor,
version: Version,
headers: Map<string, QstHeader>,
): QstContainedFile[] {
// Files are interleaved in 1056 byte chunks.
// Each chunk has a 24 byte header, 1024 byte data segment and an 8 byte trailer.
// Each chunk has a 20 or 24 byte header, 1024 byte data segment and an 4 or 8 byte trailer.
const files = new Map<
string,
{
@ -178,29 +281,63 @@ function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstCo
}
>();
while (cursor.bytes_left >= 1056) {
let chunk_size: number; // Size including padding, header and trailer.
let trailer_size: number;
switch (version) {
case Version.DC:
case Version.GC:
case Version.PC:
chunk_size = CHUNK_BODY_SIZE + 24;
trailer_size = 4;
break;
case Version.BB:
chunk_size = CHUNK_BODY_SIZE + 32;
trailer_size = 8;
break;
}
while (cursor.bytes_left >= chunk_size) {
const start_position = cursor.position;
// Read meta data.
const chunk_no = cursor.seek(4).u8();
const file_name = cursor.seek(3).string_ascii(16, true, true);
// Read chunk header.
let chunk_no: number;
switch (version) {
case Version.DC:
case Version.GC:
cursor.seek(1);
chunk_no = cursor.u8();
cursor.seek(2);
break;
case Version.PC:
cursor.seek(3);
chunk_no = cursor.u8();
break;
case Version.BB:
cursor.seek(4);
chunk_no = cursor.u32();
break;
}
const file_name = cursor.string_ascii(16, true, true);
let file = files.get(file_name);
if (!file) {
const expected_size = expected_sizes.get(file_name);
files.set(
file_name,
(file = {
name: file_name,
expected_size,
cursor: new ResizableBufferCursor(
new ResizableBuffer(expected_size || 10 * 1024),
Endianness.Little,
),
chunk_nos: new Set(),
}),
);
const header = headers.get(file_name);
file = {
name: file_name,
expected_size: header?.size,
cursor: new ResizableBufferCursor(
new ResizableBuffer(header?.size ?? 10 * 1024),
Endianness.Little,
),
chunk_nos: new Set(),
};
files.set(file_name, file);
}
if (file.chunk_nos.has(chunk_no)) {
@ -212,28 +349,28 @@ function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstCo
}
// Read file data.
let size = cursor.seek(1024).u32();
cursor.seek(-1028);
let size = cursor.seek(CHUNK_BODY_SIZE).u32();
cursor.seek(-CHUNK_BODY_SIZE - 4);
if (size > 1024) {
if (size > CHUNK_BODY_SIZE) {
logger.warn(
`Data segment size of ${size} is larger than expected maximum size, reading just 1024 bytes.`,
`Data segment size of ${size} is larger than expected maximum size, reading just ${CHUNK_BODY_SIZE} bytes.`,
);
size = 1024;
size = CHUNK_BODY_SIZE;
}
const data = cursor.take(size);
const chunk_position = chunk_no * 1024;
const chunk_position = chunk_no * CHUNK_BODY_SIZE;
file.cursor.size = Math.max(chunk_position + size, file.cursor.size);
file.cursor.seek_start(chunk_position).write_cursor(data);
// Skip the padding and the trailer.
cursor.seek(1032 - data.size);
cursor.seek(CHUNK_BODY_SIZE + trailer_size - data.size);
if (cursor.position !== start_position + 1056) {
if (cursor.position !== start_position + chunk_size) {
throw new Error(
`Read ${cursor.position -
start_position} file chunk message bytes instead of expected 1056.`,
start_position} file chunk message bytes instead of expected ${chunk_size}.`,
);
}
}
@ -255,9 +392,10 @@ function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstCo
}
// Detect missing file chunks.
const actual_size = Math.max(file.cursor.size, file.expected_size || 0);
const actual_size = Math.max(file.cursor.size, file.expected_size ?? 0);
const expected_chunk_count = Math.ceil(actual_size / CHUNK_BODY_SIZE);
for (let chunk_no = 0; chunk_no < Math.ceil(actual_size / 1024); ++chunk_no) {
for (let chunk_no = 0; chunk_no < expected_chunk_count; ++chunk_no) {
if (!file.chunk_nos.has(chunk_no)) {
logger.warn(`File ${file.name} is missing chunk ${chunk_no}.`);
}
@ -267,8 +405,11 @@ function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstCo
const contained_files: QstContainedFile[] = [];
for (const file of files.values()) {
const header = headers.get(file.name);
contained_files.push({
name: file.name,
id: header?.quest_id,
filename: file.name,
quest_name: header?.name,
data: file.cursor.seek_start(0).array_buffer(),
});
}
@ -276,10 +417,10 @@ function parse_files(cursor: Cursor, expected_sizes: Map<string, number>): QstCo
return contained_files;
}
function write_file_headers(cursor: WritableCursor, files: QstContainedFileParam[]): void {
function write_file_headers(cursor: WritableCursor, files: readonly QstContainedFileParam[]): void {
for (const file of files) {
if (file.name.length > 15) {
throw new Error(`File ${file.name} has a name longer than 15 characters.`);
if (file.filename.length > 15) {
throw new Error(`File ${file.filename} has a name longer than 15 characters.`);
}
cursor.write_u16(88); // Header size.
@ -290,25 +431,25 @@ function write_file_headers(cursor: WritableCursor, files: QstContainedFileParam
cursor.write_u8(0);
}
cursor.write_string_ascii(file.name, 16);
cursor.write_string_ascii(file.filename, 16);
cursor.write_u32(file.data.byteLength);
let file_name_2: string;
if (file.name_2 == null) {
if (file.quest_name == null) {
// Not sure this makes sense.
const dot_pos = file.name.lastIndexOf(".");
const dot_pos = file.filename.lastIndexOf(".");
file_name_2 =
dot_pos === -1
? file.name + "_j"
: file.name.slice(0, dot_pos) + "_j" + file.name.slice(dot_pos);
? file.filename + "_j"
: file.filename.slice(0, dot_pos) + "_j" + file.filename.slice(dot_pos);
} else {
file_name_2 = file.name_2;
file_name_2 = file.quest_name;
}
if (file_name_2.length > 24) {
throw Error(
`File ${file.name} has a file_name_2 length (${file_name_2}) longer than 24 characters.`,
`File ${file.filename} has a file_name_2 length (${file_name_2}) longer than 24 characters.`,
);
}
@ -316,13 +457,13 @@ function write_file_headers(cursor: WritableCursor, files: QstContainedFileParam
}
}
function write_file_chunks(cursor: WritableCursor, files: QstContainedFileParam[]): void {
function write_file_chunks(cursor: WritableCursor, files: readonly QstContainedFileParam[]): void {
// Files are interleaved in 1056 byte chunks.
// Each chunk has a 24 byte header, 1024 byte data segment and an 8 byte trailer.
const files_to_chunk = files.map(file => ({
no: 0,
data: new ArrayBufferCursor(file.data, Endianness.Little),
name: file.name,
name: file.filename,
}));
let done = 0;

2
src/viewer/stores/TextureStore.ts
View File

@ -26,7 +26,7 @@ export class TextureStore extends Store {
load_file = async (file: File): Promise<void> => {
try {
const ext = filename_extension(file.name);
const ext = filename_extension(file.name).toLowerCase();
const buffer = await read_file(file);
if (ext === "xvm") {

BIN
test/resources/lost_heat_sword_gc.qst Normal file
View File

Binary file not shown.