From 40b8464eb082da6b9c6844a9e93c083b86d08526 Mon Sep 17 00:00:00 2001 From: Daan Vanden Bosch Date: Tue, 20 Oct 2020 20:45:33 +0200 Subject: [PATCH] Added PRS decompression code and object code parser. --- .../assembly}/opcodes.schema.json | 0 .../assetsGeneration/assembly}/opcodes.yml | 7 - lib/build.gradle.kts | 130 ++++ .../phantasmal/lib/assembly/Instructions.kt | 152 +++++ .../world/phantasmal/lib/assembly/Opcode.kt | 176 +++++ .../dataFlowAnalysis/ControlFlowGraph.kt | 10 + .../dataFlowAnalysis/GetMapDesignations.kt | 55 ++ .../dataFlowAnalysis/GetRegisterValue.kt | 9 + .../dataFlowAnalysis/GetStackValue.kt | 6 + .../lib/assembly/dataFlowAnalysis/ValueSet.kt | 198 ++++++ .../lib/compression/prs/PrsDecompress.kt | 123 ++++ .../phantasmal/lib/fileFormats/quest/Bin.kt | 23 +- .../phantasmal/lib/fileFormats/quest/Dat.kt | 83 +-- .../lib/fileFormats/quest/ObjectCode.kt | 617 ++++++++++++++++++ .../lib/fileFormats/quest/ObjectType.kt | 3 + .../phantasmal/lib/fileFormats/quest/Quest.kt | 169 +++++ .../lib/fileFormats/quest/QuestNpc.kt | 27 + .../lib/fileFormats/quest/QuestObject.kt | 15 + .../dataFlowAnalysis/ValueSetTests.kt | 129 ++++ 19 files changed, 1873 insertions(+), 59 deletions(-) rename {web/src/assets_generation/resources/asm => lib/assetsGeneration/assembly}/opcodes.schema.json (100%) rename {web/src/assets_generation/resources/asm => lib/assetsGeneration/assembly}/opcodes.yml (99%) create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Instructions.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Opcode.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ControlFlowGraph.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetMapDesignations.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetRegisterValue.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetStackValue.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSet.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/compression/prs/PrsDecompress.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectCode.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectType.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Quest.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestNpc.kt create mode 100644 lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestObject.kt create mode 100644 lib/src/commonTest/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSetTests.kt diff --git a/web/src/assets_generation/resources/asm/opcodes.schema.json b/lib/assetsGeneration/assembly/opcodes.schema.json similarity index 100% rename from web/src/assets_generation/resources/asm/opcodes.schema.json rename to lib/assetsGeneration/assembly/opcodes.schema.json diff --git a/web/src/assets_generation/resources/asm/opcodes.yml b/lib/assetsGeneration/assembly/opcodes.yml similarity index 99% rename from web/src/assets_generation/resources/asm/opcodes.yml rename to lib/assetsGeneration/assembly/opcodes.yml index 3ff07ac8..f305b57d 100644 --- a/web/src/assets_generation/resources/asm/opcodes.yml +++ b/lib/assetsGeneration/assembly/opcodes.yml @@ -1751,13 +1751,6 @@ opcodes: mnemonic: default_camera_pos1 params: [] - - code: 0xf8 - params: - - type: reg_tup_ref - reg_tup: # TODO: determine type and access - - type: any - access: read - - code: 0xfa mnemonic: get_gc_number params: diff --git a/lib/build.gradle.kts b/lib/build.gradle.kts index b77b9b23..42c71c85 100644 --- a/lib/build.gradle.kts +++ b/lib/build.gradle.kts @@ -1,7 +1,17 @@ +import org.snakeyaml.engine.v2.api.Load +import org.snakeyaml.engine.v2.api.LoadSettings +import java.io.PrintWriter + plugins { kotlin("multiplatform") } +buildscript { + dependencies { + classpath("org.snakeyaml:snakeyaml-engine:2.1") + } +} + val kotlinLoggingVersion: String by project.extra kotlin { @@ -15,6 +25,7 @@ kotlin { } commonMain { + kotlin.setSrcDirs(kotlin.srcDirs + file("build/generated-src/commonMain/kotlin")) dependencies { api(project(":core")) api("io.github.microutils:kotlin-logging:$kotlinLoggingVersion") @@ -39,3 +50,122 @@ kotlin { tasks.register("test") { dependsOn("allTests") } + +val generateOpcodes = tasks.register("generateOpcodes") { + group = "code generation" + + val packageName = "world.phantasmal.lib.assembly" + val opcodesFile = file("assetsGeneration/assembly/opcodes.yml") + val outputFile = file( + "build/generated-src/commonMain/kotlin/${packageName.replace('.', '/')}/Opcodes.kt" + ) + + inputs.file(opcodesFile) + outputs.file(outputFile) + + @Suppress("UNCHECKED_CAST") + doLast { + val root = Load(LoadSettings.builder().build()) + .loadFromInputStream(opcodesFile.inputStream()) as Map + + outputFile.printWriter() + .use { writer -> + writer.println("package $packageName") + writer.println() + writer.println("val OPCODES: Array = Array(256) { null }") + writer.println("val OPCODES_F8: Array = Array(256) { null }") + writer.println("val OPCODES_F9: Array = Array(256) { null }") + + (root["opcodes"] as List>).forEach { opcode -> + opcodeToCode(writer, opcode) + } + } + } +} + +fun opcodeToCode(writer: PrintWriter, opcode: Map) { + val code = (opcode["code"] as String).drop(2).toInt(16) + val codeStr = code.toString(16).padStart(2, '0') + val mnemonic = opcode["mnemonic"] as String? ?: "unknown_$codeStr" + val description = opcode["description"] as String? + val stack = opcode["stack"] as String? + + val valName = "OP_" + mnemonic + .replace("!=", "ne") + .replace("=", "e") + .replace("<", "l") + .replace(">", "g") + .toUpperCase() + + val stackInteraction = when (stack) { + "push" -> "StackInteraction.Push" + "pop" -> "StackInteraction.Pop" + else -> "null" + } + + @Suppress("UNCHECKED_CAST") + val params = paramsToCode(opcode["params"] as List>, 4) + + val array = when (code) { + in 0..0xFF -> "OPCODES" + in 0xF800..0xF8FF -> "OPCODES_F8" + in 0xF900..0xF9FF -> "OPCODES_F9" + else -> error("Invalid opcode $codeStr ($mnemonic).") + } + + writer.println( + """ + |val $valName = Opcode( + | 0x$codeStr, + | "$mnemonic", + | ${description?.let { "\"$it\"" }}, + | $params, + | $stackInteraction, + |).also { ${array}[0x$codeStr] = it }""".trimMargin() + ) +} + +fun paramsToCode(params: List>, indent: Int): String { + val i = " ".repeat(indent) + + if (params.isEmpty()) return "emptyList()" + + return params.joinToString(",\n", "listOf(\n", ",\n${i})") { param -> + @Suppress("UNCHECKED_CAST") + val type = when (param["type"]) { + "any" -> "AnyType()" + "byte" -> "ByteType" + "word" -> "WordType" + "dword" -> "DWordType" + "float" -> "FloatType" + "label" -> "LabelType()" + "instruction_label" -> "ILabelType" + "data_label" -> "DLabelType" + "string_label" -> "SLabelType" + "string" -> "StringType" + "instruction_label_var" -> "ILabelVarType" + "reg_ref" -> "RegRefType" + "reg_tup_ref" -> """RegTupRefType(${ + paramsToCode(param["reg_tup"] as List>, indent + 4) + })""" + "reg_ref_var" -> "RegRefVarType" + "pointer" -> "PointerType" + else -> error("Type ${param["type"]} not implemented.") + } + + val doc = (param["doc"] as String?)?.let { "\"$it\"" } ?: "null" + + val access = when (param["access"]) { + "read" -> "ParamAccess.Read" + "write" -> "ParamAccess.Write" + "read_write" -> "ParamAccess.ReadWrite" + else -> "null" + } + + "$i Param(${type}, ${doc}, ${access})" + } +} + +val build by tasks.build + +build.dependsOn(generateOpcodes) diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Instructions.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Instructions.kt new file mode 100644 index 00000000..d74be29b --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Instructions.kt @@ -0,0 +1,152 @@ +package world.phantasmal.lib.assembly + +import world.phantasmal.lib.buffer.Buffer +import kotlin.math.min + +/** + * Opcode invocation. + */ +class Instruction( + val opcode: Opcode, + val args: List, + val srcLoc: InstructionSrcLoc?, +) { + /** + * Maps each parameter by index to its arguments. + */ + val paramToArgs: List> + + init { + val len = min(opcode.params.size, args.size) + val paramToArgs: MutableList> = mutableListOf() + + for (i in 0 until len) { + val type = opcode.params[i].type + val arg = args[i] + paramToArgs[i] = mutableListOf() + + if (type is ILabelVarType || type is RegRefVarType) { + for (j in i until args.size) { + paramToArgs[i].add(args[j]) + } + } else { + paramToArgs[i].add(arg) + } + } + + this.paramToArgs = paramToArgs + } +} + +/** + * Returns the byte size of the entire instruction, i.e. the sum of the opcode size and all + * argument sizes. + */ +fun instructionSize(instruction: Instruction, dcGcFormat: Boolean): Int { + val opcode = instruction.opcode + val pLen = min(opcode.params.size, instruction.paramToArgs.size) + var argSize = 0 + + for (i in 0 until pLen) { + val type = opcode.params[i].type + val args = instruction.paramToArgs[i] + + argSize += when (type) { + is ByteType, + is RegRefType, + is RegTupRefType, + -> 1 + + is WordType, + is LabelType, + is ILabelType, + is DLabelType, + is SLabelType, + -> 2 + + is DWordType, + is FloatType, + -> 4 + + is StringType -> { + if (dcGcFormat) { + (args[0].value as String).length + 1 + } else { + 2 * (args[0].value as String).length + 2 + } + } + + is ILabelVarType -> 1 + 2 * args.size + + is RegRefVarType -> 1 + args.size + + else -> error("Parameter type ${type::class} not implemented.") + } + } + + return opcode.size + argSize +} + +/** + * Instruction argument. + */ +data class Arg(val value: Any) + +enum class SegmentType { + Instructions, + Data, + String, +} + +/** + * Segment of object code. A segment starts with an instruction, byte or string character that is + * referenced by one or more labels. The segment ends right before the next instruction, byte or + * string character that is referenced by a label. + */ +sealed class Segment(val type: SegmentType, val labels: MutableList) + +class InstructionSegment( + labels: MutableList, + val instructions: List, + val srcLoc: SegmentSrcLoc, +) : Segment(SegmentType.Instructions, labels) + +class DataSegment( + labels: MutableList, + val data: Buffer, + val srcLoc: SegmentSrcLoc, +) : Segment(SegmentType.Data, labels) + +class StringSegment( + labels: MutableList, + val value: String, + val srcLoc: SegmentSrcLoc, +) : Segment(SegmentType.String, labels) + +/** + * Position and length of related source assembly code. + */ +open class SrcLoc( + val lineNo: Int, + val col: Int, + val len: Int, +) + +/** + * Locations of the instruction parts in the source assembly code. + */ +class InstructionSrcLoc( + val mnemonic: SrcLoc?, + val args: List, + val stackArgs: List, +) + +/** + * Locations of an instruction's stack arguments in the source assembly code. + */ +class StackArgSrcLoc(lineNo: Int, col: Int, len: Int, val value: Int) : SrcLoc(lineNo, col, len) + +/** + * Locations of a segment's labels in the source assembly code. + */ +class SegmentSrcLoc(val labels: List) diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Opcode.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Opcode.kt new file mode 100644 index 00000000..815b6f3a --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/Opcode.kt @@ -0,0 +1,176 @@ +package world.phantasmal.lib.assembly + +/** + * Abstract super type of all types. + */ +open class AnyType + +/** + * Purely abstract super type of all value types. + */ +sealed class ValueType : AnyType() + +/** + * 8-Bit integer. + */ +object ByteType : ValueType() + +/** + * 16-Bit integer. + */ +object WordType : ValueType() + +/** + * 32-Bit integer. + */ +object DWordType : ValueType() + +/** + * 32-Bit floating point number. + */ +object FloatType : ValueType() + +/** + * Abstract super type of all label types. + */ +open class LabelType : ValueType() + +/** + * Named reference to an instruction. + */ +object ILabelType : LabelType() + +/** + * Named reference to a data segment. + */ +object DLabelType : LabelType() + +/** + * Named reference to a string segment. + */ +object SLabelType : LabelType() + +/** + * String of arbitrary size. + */ +object StringType : LabelType() + +/** + * Arbitrary amount of instruction labels. + */ +object ILabelVarType : LabelType() + +/** + * Purely abstract super type of all reference types. + */ +sealed class RefType : AnyType() + +/** + * Reference to one or more registers. + */ +object RegRefType : RefType() + +/** + * Reference to a fixed amount of consecutive registers of specific types. + * The only parameterized type. + */ +class RegTupRefType(val registerTuples: List) : RefType() + +/** + * Arbitrary amount of register references. + */ +object RegRefVarType : RefType() + +/** + * Raw memory pointer. + */ +object PointerType : AnyType() + +const val MIN_SIGNED_DWORD_VALUE = Int.MIN_VALUE +const val MAX_SIGNED_DWORD_VALUE = Int.MAX_VALUE +const val MIN_UNSIGNED_DWORD_VALUE = UInt.MIN_VALUE +const val MAX_UNSIGNED_DWORD_VALUE = UInt.MAX_VALUE +const val MIN_DWORD_VALUE = MIN_SIGNED_DWORD_VALUE +const val MAX_DWORD_VALUE = MAX_UNSIGNED_DWORD_VALUE + +enum class ParamAccess { + Read, + Write, + ReadWrite, +} + +class Param( + val type: AnyType, + /** + * Documentation string. + */ + val doc: String?, + /** + * The way referenced registers are accessed by the instruction. Only set when type is a + * register reference. + */ + val access: ParamAccess?, +) + +enum class StackInteraction { + Push, + Pop, +} + +/** + * Opcode for script object code. Invoked by instructions. + */ +class Opcode( + /** + * 1- Or 2-byte big-endian representation of this opcode as used in object code. + */ + val code: Int, + /** + * String representation of this opcode as used in assembly. + */ + val mnemonic: String, + /** + * Documentation string. + */ + val doc: String?, + /** + * Parameters passed in directly or via the stack, depending on the value of [stack]. + */ + val params: List, + /** + * Stack interaction. + */ + val stack: StackInteraction?, +) { + /** + * Byte size of the opcode, either 1 or 2. + */ + val size: Int = if (code < 0xFF) 1 else 2 + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + other as Opcode + return code == other.code + } + + override fun hashCode(): Int = code +} + +fun codeToOpcode(code: Int): Opcode = + when { + code <= 0xFF -> getOpcode(code, code, OPCODES) + code <= 0xF8FF -> getOpcode(code, code and 0xFF, OPCODES_F8) + else -> getOpcode(code, code and 0xFF, OPCODES_F9) + } + +private fun getOpcode(code: Int, index: Int, opcodes: Array): Opcode { + var opcode = opcodes[index] + + if (opcode == null) { + opcode = Opcode(code, "unknown_${code.toString(16)}", null, emptyList(), null) + opcodes[index] = opcode + } + + return opcode +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ControlFlowGraph.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ControlFlowGraph.kt new file mode 100644 index 00000000..4b496290 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ControlFlowGraph.kt @@ -0,0 +1,10 @@ +package world.phantasmal.lib.assembly.dataFlowAnalysis + +import world.phantasmal.lib.assembly.InstructionSegment + +class ControlFlowGraph { + companion object { + fun create(segments: List): ControlFlowGraph = + TODO() + } +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetMapDesignations.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetMapDesignations.kt new file mode 100644 index 00000000..4e11bd98 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetMapDesignations.kt @@ -0,0 +1,55 @@ +package world.phantasmal.lib.assembly.dataFlowAnalysis + +import mu.KotlinLogging +import world.phantasmal.lib.assembly.InstructionSegment +import world.phantasmal.lib.assembly.OP_BB_MAP_DESIGNATE +import world.phantasmal.lib.assembly.OP_MAP_DESIGNATE +import world.phantasmal.lib.assembly.OP_MAP_DESIGNATE_EX + +private val logger = KotlinLogging.logger {} + +fun getMapDesignations( + instructionSegments: List, + func0Segment: InstructionSegment, +): Map { + val mapDesignations = mutableMapOf() + var cfg: ControlFlowGraph? = null + + for (inst in func0Segment.instructions) { + when (inst.opcode) { + OP_MAP_DESIGNATE, + OP_MAP_DESIGNATE_EX, + -> { + if (cfg == null) { + cfg = ControlFlowGraph.create(instructionSegments) + } + + val areaId = getRegisterValue(cfg, inst, inst.args[0].value as Int) + + if (areaId.size != 1) { + logger.warn { "Couldn't determine area ID for mapDesignate instruction." } + continue + } + + val variantIdRegister = + inst.args[0].value as Int + (if (inst.opcode == OP_MAP_DESIGNATE) 2 else 3) + val variantId = getRegisterValue(cfg, inst, variantIdRegister) + + if (variantId.size != 1) { + logger.warn { + "Couldn't determine area variant ID for mapDesignate instruction." + } + continue + } + + mapDesignations[areaId[0]!!] = variantId[0]!! + } + + OP_BB_MAP_DESIGNATE -> { + mapDesignations[inst.args[0].value as Int] = inst.args[2].value as Int + } + } + } + + return mapDesignations +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetRegisterValue.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetRegisterValue.kt new file mode 100644 index 00000000..e0dc32aa --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetRegisterValue.kt @@ -0,0 +1,9 @@ +package world.phantasmal.lib.assembly.dataFlowAnalysis + +import world.phantasmal.lib.assembly.Instruction + +/** + * Computes the possible values of a register right before a specific instruction. + */ +fun getRegisterValue(cfg: ControlFlowGraph, instruction: Instruction, register: Int): ValueSet = + TODO() diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetStackValue.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetStackValue.kt new file mode 100644 index 00000000..a9a459d3 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/GetStackValue.kt @@ -0,0 +1,6 @@ +package world.phantasmal.lib.assembly.dataFlowAnalysis + +import world.phantasmal.lib.assembly.Instruction + +fun getStackValue(cfg: ControlFlowGraph, instruction: Instruction, position: Int): ValueSet = + TODO() diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSet.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSet.kt new file mode 100644 index 00000000..8cad104a --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSet.kt @@ -0,0 +1,198 @@ +package world.phantasmal.lib.assembly.dataFlowAnalysis + +import kotlin.math.max +import kotlin.math.min + +/** + * Represents a sorted set of integers. + */ +class ValueSet : Iterable { + private val intervals: MutableList = mutableListOf() + + val size: Int + get() = + intervals.fold(0) { acc, i -> acc + i.end - i.start + 1 } + + operator fun get(i: Int): Int? { + var idx = i + + for ((start, end) in intervals) { + val size = end - start + 1 + + if (idx < size) { + return start + idx + } else { + idx -= size + } + } + + return null + } + + fun isEmpty(): Boolean = + intervals.isEmpty() + + fun minOrNull(): Int? = + intervals.firstOrNull()?.start + + fun maxOrNull(): Int? = + intervals.lastOrNull()?.end + + operator fun contains(value: Int): Boolean { + for (int in intervals) { + if (value in int) { + return true + } + } + + return false + } + + /** + * Sets this ValueSet to the given integer. + */ + fun setValue(value: Int): ValueSet { + intervals.clear() + intervals.add(Interval(value, value)) + return this + } + + /** + * Sets this ValueSet to the values in the given interval. + * + * @param start lower bound, inclusive + * @param end upper bound, inclusive + */ + fun setInterval(start: Int, end: Int): ValueSet { + require(end >= start) { + "Interval upper bound should be greater than or equal to lower bound, got [${start}, ${end}]." + } + + intervals.clear() + intervals.add(Interval(start, end)) + return this + } + + /** + * Doesn't take into account integer overflow. + */ + fun scalarAdd(s: Int): ValueSet { + for (int in intervals) { + int.start += s + int.end += s + } + + return this + } + + /** + * Doesn't take into account integer overflow. + */ + fun scalarSub(s: Int): ValueSet { + return scalarAdd(-s) + } + + /** + * Doesn't take into account integer overflow. + */ + fun scalarMul(s: Int): ValueSet { + for (int in intervals) { + int.start *= s + int.end *= s + } + + return this + } + + /** + * Integer division. + */ + fun scalarDiv(s: Int): ValueSet { + for (int in intervals) { + int.start = int.start / s + int.end = int.end / s + } + + return this + } + + fun union(other: ValueSet): ValueSet { + var i = 0 + + outer@ for (b in other.intervals) { + while (i < intervals.size) { + val a = intervals[i] + + if (b.end < a.start - 1) { + // b lies entirely before a, insert it right before a. + intervals.add(i, b.copy()) + i++ + continue@outer + } else if (b.start <= a.end + 1) { + // a and b overlap or form a continuous interval (e.g. [1, 2] and [3, 4]). + a.start = min(a.start, b.start) + + // Merge all intervals that overlap with b. + val j = i + 1 + + while (j < intervals.size) { + if (b.end >= intervals[j].start - 1) { + a.end = intervals[j].end + intervals.removeAt(j) + } else { + break + } + } + + a.end = max(a.end, b.end) + i++ + continue@outer + } else { + // b lies entirely after a, check next a. + i++ + } + } + + // b lies after every a, add it to the end of our intervals. + intervals.add(b.copy()) + } + + return this + } + + override fun iterator(): Iterator = + object : Iterator { + private var intIdx = 0 + private var nextValue: Int? = minOrNull() + + override fun hasNext(): Boolean = + nextValue != null + + override fun next(): Int { + val v = nextValue ?: throw NoSuchElementException() + + nextValue = + if (v < intervals[intIdx].end) { + v + 1 + } else { + intIdx++ + + if (intIdx < intervals.size) { + intervals[intIdx].start + } else { + null + } + } + + return v + } + } +} + +/** + * Closed interval [start, end]. + */ +private data class Interval(var start: Int, var end: Int) { + operator fun contains(value: Int): Boolean = + value in start..end +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/compression/prs/PrsDecompress.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/compression/prs/PrsDecompress.kt new file mode 100644 index 00000000..1539c7ba --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/compression/prs/PrsDecompress.kt @@ -0,0 +1,123 @@ +package world.phantasmal.lib.compression.prs + +import mu.KotlinLogging +import world.phantasmal.core.PwResult +import world.phantasmal.core.PwResultBuilder +import world.phantasmal.core.Severity +import world.phantasmal.core.Success +import world.phantasmal.lib.buffer.Buffer +import world.phantasmal.lib.cursor.BufferCursor +import world.phantasmal.lib.cursor.Cursor +import world.phantasmal.lib.cursor.WritableCursor +import kotlin.math.floor +import kotlin.math.min + +private val logger = KotlinLogging.logger {} + +fun prsDecompress(cursor: Cursor): PwResult { + try { + val ctx = Context(cursor) + + while (true) { + if (ctx.readFlagBit() == 1u) { + // Single byte copy. + ctx.copyU8() + } else { + // Multi byte copy. + var length: UInt + var offset: Int + + if (ctx.readFlagBit() == 0u) { + // Short copy. + length = (ctx.readFlagBit() shl 1) or ctx.readFlagBit() + length += 2u + + offset = ctx.readU8().toInt() - 256 + } else { + // Long copy or end of file. + offset = ctx.readU16().toInt() + + // Two zero bytes implies that this is the end of the file. + if (offset == 0) { + break + } + + // Do we need to read a length byte, or is it encoded in what we already have? + length = (offset and 0b111).toUInt() + offset = offset shr 3 + + if (length == 0u) { + length = ctx.readU8().toUInt() + length += 1u + } else { + length += 2u + } + + offset -= 8192 + } + + ctx.offsetCopy(offset, length) + } + } + + return Success(ctx.dst.seekStart(0u)) + } catch (e: Throwable) { + return PwResultBuilder(logger) + .addProblem(Severity.Error, "PRS-compressed stream is corrupt.", cause = e) + .failure() + } +} + +class Context(cursor: Cursor) { + private val src: Cursor = cursor + val dst: WritableCursor = BufferCursor( + Buffer.withCapacity(floor(1.5 * cursor.size.toDouble()).toUInt(), cursor.endianness), + ) + private var flags = 0u + private var flagBitsLeft = 0 + + fun readFlagBit(): UInt { + // Fetch a new flag byte when the previous byte has been processed. + if (flagBitsLeft == 0) { + flags = readU8().toUInt() + flagBitsLeft = 8 + } + + val bit = flags and 1u + flags = flags shr 1 + flagBitsLeft -= 1 + return bit + } + + fun copyU8() { + dst.writeU8(readU8()) + } + + fun readU8(): UByte = src.u8() + + fun readU16(): UShort = src.u16() + + fun offsetCopy(offset: Int, length: UInt) { + require(offset in -8192..0) { + "offset was ${offset}, should be between -8192 and 0." + } + + require(length in 1u..256u) { + "length was ${length}, should be between 1 and 256." + } + + // The length can be larger than -offset, in that case we copy -offset bytes size/-offset times. + val bufSize = min((-offset).toUInt(), length) + + dst.seek(offset) + val buf = dst.take(bufSize) + dst.seek(-offset - bufSize.toInt()) + + repeat((length / bufSize).toInt()) { + dst.writeCursor(buf) + buf.seekStart(0u) + } + + dst.writeCursor(buf.take(length % bufSize)) + } +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Bin.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Bin.kt index 4dab1e2e..68a4ec4c 100644 --- a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Bin.kt +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Bin.kt @@ -1,6 +1,7 @@ package world.phantasmal.lib.fileFormats.quest import mu.KotlinLogging +import world.phantasmal.lib.buffer.Buffer import world.phantasmal.lib.cursor.Cursor private val logger = KotlinLogging.logger {} @@ -16,7 +17,7 @@ class BinFile( val questName: String, val shortDescription: String, val longDescription: String, -// val objectCode: ArrayBuffer, + val objectCode: Buffer, val labelOffsets: IntArray, val shopItems: UIntArray, ) @@ -64,15 +65,15 @@ fun parseBin(cursor: Cursor): BinFile { cursor.seek(1) language = cursor.u8().toUInt() questId = cursor.u16().toUInt() - questName = cursor.stringAscii(32u, true, true) - shortDescription = cursor.stringAscii(128u, true, true) - longDescription = cursor.stringAscii(288u, true, true) + questName = cursor.stringAscii(32u, nullTerminated = true, dropRemaining = true) + shortDescription = cursor.stringAscii(128u, nullTerminated = true, dropRemaining = true) + longDescription = cursor.stringAscii(288u, nullTerminated = true, dropRemaining = true) } else { questId = cursor.u32() language = cursor.u32() - questName = cursor.stringUtf16(64u, true, true) - shortDescription = cursor.stringUtf16(256u, true, true) - longDescription = cursor.stringUtf16(576u, true, true) + questName = cursor.stringUtf16(64u, nullTerminated = true, dropRemaining = true) + shortDescription = cursor.stringUtf16(256u, nullTerminated = true, dropRemaining = true) + longDescription = cursor.stringUtf16(576u, nullTerminated = true, dropRemaining = true) } if (size != cursor.size) { @@ -91,9 +92,9 @@ fun parseBin(cursor: Cursor): BinFile { .seekStart(labelOffsetTableOffset) .i32Array(labelOffsetCount) -// val objectCode = cursor -// .seekStart(objectCodeOffset) -// .arrayBuffer(labelOffsetTableOffset - objectCodeOffset); + val objectCode = cursor + .seekStart(objectCodeOffset) + .buffer(labelOffsetTableOffset - objectCodeOffset) return BinFile( format, @@ -102,7 +103,7 @@ fun parseBin(cursor: Cursor): BinFile { questName, shortDescription, longDescription, -// objectCode, + objectCode, labelOffsets, shopItems, ) diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Dat.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Dat.kt index 7ae3e287..a804c17f 100644 --- a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Dat.kt +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Dat.kt @@ -6,12 +6,13 @@ import world.phantasmal.lib.cursor.Cursor private val logger = KotlinLogging.logger {} -private const val OBJECT_BYTE_SIZE = 68u; -private const val NPC_BYTE_SIZE = 72u; -private const val EVENT_ACTION_SPAWN_NPCS: UByte = 0x8u; -private const val EVENT_ACTION_UNLOCK: UByte = 0xAu; -private const val EVENT_ACTION_LOCK: UByte = 0xBu; -private const val EVENT_ACTION_TRIGGER_EVENT: UByte = 0xCu; +private const val EVENT_ACTION_SPAWN_NPCS: UByte = 0x8u +private const val EVENT_ACTION_UNLOCK: UByte = 0xAu +private const val EVENT_ACTION_LOCK: UByte = 0xBu +private const val EVENT_ACTION_TRIGGER_EVENT: UByte = 0xCu + +const val OBJECT_BYTE_SIZE = 68u +const val NPC_BYTE_SIZE = 72u class DatFile( val objs: List, @@ -69,24 +70,24 @@ fun parseDat(cursor: Cursor): DatFile { val unknowns = mutableListOf() while (cursor.hasBytesLeft()) { - val entityType = cursor.u32(); - val totalSize = cursor.u32(); - val areaId = cursor.u32(); - val entitiesSize = cursor.u32(); + val entityType = cursor.u32() + val totalSize = cursor.u32() + val areaId = cursor.u32() + val entitiesSize = cursor.u32() if (entityType == 0u) { - break; + break } else { require(entitiesSize == totalSize - 16u) { "Malformed DAT file. Expected an entities size of ${totalSize - 16u}, got ${entitiesSize}." } - val entitiesCursor = cursor.take(entitiesSize); + val entitiesCursor = cursor.take(entitiesSize) when (entityType) { - 1u -> parseEntities(entitiesCursor, areaId, objs, OBJECT_BYTE_SIZE); - 2u -> parseEntities(entitiesCursor, areaId, npcs, NPC_BYTE_SIZE); - 3u -> parseEvents(entitiesCursor, areaId, events); + 1u -> parseEntities(entitiesCursor, areaId, objs, OBJECT_BYTE_SIZE) + 2u -> parseEntities(entitiesCursor, areaId, npcs, NPC_BYTE_SIZE) + 3u -> parseEvents(entitiesCursor, areaId, events) else -> { // Unknown entity types 4 and 5 (challenge mode). unknowns.add(DatUnknown( @@ -127,38 +128,38 @@ private fun parseEntities( entities.add(DatEntity( areaId, data = cursor.buffer(entitySize), - )); + )) } } private fun parseEvents(cursor: Cursor, areaId: UInt, events: MutableList) { - val actionsOffset = cursor.u32(); - cursor.seek(4); // Always 0x10 - val eventCount = cursor.u32(); - cursor.seek(3); // Always 0 - val eventType = cursor.u8(); + val actionsOffset = cursor.u32() + cursor.seek(4) // Always 0x10 + val eventCount = cursor.u32() + cursor.seek(3) // Always 0 + val eventType = cursor.u8() require(eventType == (0x32u).toUByte()) { "Can't parse challenge mode quests yet." } - cursor.seekStart(actionsOffset); - val actionsCursor = cursor.take(cursor.bytesLeft); - cursor.seekStart(16u); + cursor.seekStart(actionsOffset) + val actionsCursor = cursor.take(cursor.bytesLeft) + cursor.seekStart(16u) repeat(eventCount.toInt()) { - val id = cursor.u32(); - cursor.seek(4); // Always 0x100 - val sectionId = cursor.u16(); - val wave = cursor.u16(); - val delay = cursor.u16(); - val unknown = cursor.u16(); // "wavesetting"? - val eventActionsOffset = cursor.u32(); + val id = cursor.u32() + cursor.seek(4) // Always 0x100 + val sectionId = cursor.u16() + val wave = cursor.u16() + val delay = cursor.u16() + val unknown = cursor.u16() // "wavesetting"? + val eventActionsOffset = cursor.u32() val actions: MutableList = if (eventActionsOffset < actionsCursor.size) { - actionsCursor.seekStart(eventActionsOffset); - parseEventActions(actionsCursor); + actionsCursor.seekStart(eventActionsOffset) + parseEventActions(actionsCursor) } else { logger.warn { "Invalid event actions offset $eventActionsOffset for event ${id}." } mutableListOf() @@ -181,22 +182,22 @@ private fun parseEvents(cursor: Cursor, areaId: UInt, events: MutableList { @@ -204,7 +205,7 @@ private fun parseEventActions(cursor: Cursor): MutableList { outer@ while (cursor.hasBytesLeft()) { when (val type = cursor.u8()) { - (1u).toUByte() -> break@outer; + (1u).toUByte() -> break@outer EVENT_ACTION_SPAWN_NPCS -> actions.add(DatEventAction.SpawnNpcs( @@ -229,10 +230,10 @@ private fun parseEventActions(cursor: Cursor): MutableList { else -> { logger.warn { "Unexpected event action type ${type}." } - break@outer; + break@outer } } } - return actions; + return actions } diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectCode.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectCode.kt new file mode 100644 index 00000000..f0650f3e --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectCode.kt @@ -0,0 +1,617 @@ +package world.phantasmal.lib.fileFormats.quest + +import mu.KotlinLogging +import world.phantasmal.core.PwResult +import world.phantasmal.core.PwResultBuilder +import world.phantasmal.core.Severity +import world.phantasmal.lib.assembly.* +import world.phantasmal.lib.assembly.dataFlowAnalysis.ControlFlowGraph +import world.phantasmal.lib.assembly.dataFlowAnalysis.getRegisterValue +import world.phantasmal.lib.assembly.dataFlowAnalysis.getStackValue +import world.phantasmal.lib.buffer.Buffer +import world.phantasmal.lib.cursor.BufferCursor +import world.phantasmal.lib.cursor.Cursor +import kotlin.math.ceil +import kotlin.math.min + +private val logger = KotlinLogging.logger {} + +val SEGMENT_PRIORITY = mapOf( + SegmentType.Instructions to 2, + SegmentType.String to 1, + SegmentType.Data to 0, +) + +val BUILTIN_FUNCTIONS = setOf( + 60, + 70, + 80, + 90, + 100, + 110, + 120, + 130, + 140, + 800, + 810, + 820, + 830, + 840, + 850, + 860, +) + +fun parseObjectCode( + objectCode: Buffer, + labelOffsets: IntArray, + entryLabels: Set, + lenient: Boolean, + dcGcFormat: Boolean, +): PwResult> { + val cursor = BufferCursor(objectCode) + val labelHolder = LabelHolder(labelOffsets) + val result = PwResultBuilder>(logger) + val offsetToSegment = mutableMapOf() + + findAndParseSegments( + cursor, + labelHolder, + entryLabels.map { it to SegmentType.Instructions }.toMap(), + offsetToSegment, + lenient, + dcGcFormat, + ) + + val segments: MutableList = mutableListOf() + + // Put segments in an array and parse left-over segments as data. + var offset = 0 + + while (offset < cursor.size.toInt()) { + var segment: Segment? = offsetToSegment[offset] + + // If we have a segment, add it. Otherwise create a new data segment. + if (segment == null) { + val labels = labelHolder.getLabels(offset) + var endOffset: Int + + if (labels == null) { + endOffset = cursor.size.toInt() + + for (label in labelHolder.labels) { + if (label.offset > offset) { + endOffset = label.offset + break + } + } + } else { + val info = labelHolder.getInfo(labels[0])!! + endOffset = info.next?.offset ?: cursor.size.toInt() + } + + cursor.seekStart(offset.toUInt()) + parseDataSegment( + offsetToSegment, + cursor, + endOffset, + labels?.toMutableList() ?: mutableListOf() + ) + + segment = offsetToSegment[offset] + + check(endOffset > offset) { + "Next offset $endOffset was smaller than or equal to current offset ${offset}." + } + checkNotNull(segment) { "Couldn't create segment for offset ${offset}." } + } + + segments.add(segment) + + offset += when (segment) { + is InstructionSegment -> segment.instructions.sumBy { instructionSize(it, dcGcFormat) } + + is DataSegment -> segment.data.size.toInt() + + // String segments should be multiples of 4 bytes. + is StringSegment -> 4 * ceil((segment.value.length + 1) / 2.0).toInt() + } + } + + // Add unreferenced labels to their segment. + for ((label, labelOffset) in labelHolder.labels) { + val segment = offsetToSegment[labelOffset] + + if (segment == null) { + result.addProblem( + Severity.Warning, + "Label $label doesn't point to anything.", + "Label $label with offset $labelOffset doesn't point to anything.", + ) + } else { + if (label !in segment.labels) { + segment.labels.add(label) + segment.labels.sort() + } + } + } + + // Sanity check parsed object code. + if (cursor.size != offset.toUInt()) { + result.addProblem( + Severity.Error, + "The script code is corrupt.", + "Expected to parse ${cursor.size} bytes but parsed $offset instead.", + ) + + if (!lenient) { + return result.failure() + } + } + + return result.success(segments) +} + +private fun findAndParseSegments( + cursor: Cursor, + labelHolder: LabelHolder, + labels: Map, + offsetToSegment: MutableMap, + lenient: Boolean, + dcGcFormat: Boolean, +) { + var newLabels = labels + var startSegmentCount: Int + + // Iteratively parse segments from label references. + do { + startSegmentCount = offsetToSegment.size + + for ((label, type) in newLabels) { + parseSegment(offsetToSegment, labelHolder, cursor, label, type, lenient, dcGcFormat) + } + + // Find label references. + val sortedSegments = offsetToSegment.entries + .filter { (_, s) -> s is InstructionSegment } + .sortedBy { it.key } + .map { (_, s) -> s as InstructionSegment } + + val cfg = ControlFlowGraph.create(sortedSegments) + + newLabels = mutableMapOf() + + for (segment in sortedSegments) { + for (instruction in segment.instructions) { + var i = 0 + + while (i < instruction.opcode.params.size) { + val param = instruction.opcode.params[i] + + when (param.type) { + is ILabelType -> + getArgLabelValues( + cfg, + newLabels, + instruction, + i, + SegmentType.Instructions, + ) + + is ILabelVarType -> { + // Never on the stack. + // Eat all remaining arguments. + while (i < instruction.args.size) { + newLabels[instruction.args[i].value as Int] = SegmentType.Instructions + i++ + } + } + + is DLabelType -> + getArgLabelValues(cfg, newLabels, instruction, i, SegmentType.Data) + + is SLabelType -> + getArgLabelValues(cfg, newLabels, instruction, i, SegmentType.String) + + is RegTupRefType -> { + // Never on the stack. + val arg = instruction.args[i] + + for (j in param.type.registerTuples.indices) { + val regTup = param.type.registerTuples[j] + + if (regTup.type is ILabelType) { + val labelValues = getRegisterValue( + cfg, + instruction, + arg.value as Int + j, + ) + + if (labelValues.size <= 10) { + for (label in labelValues) { + newLabels[label] = SegmentType.Instructions + } + } + } + } + } + } + } + } + } + } while (offsetToSegment.size > startSegmentCount) +} + +/** + * Returns immediate arguments or stack arguments. + */ +private fun getArgLabelValues( + cfg: ControlFlowGraph, + labels: MutableMap, + instruction: Instruction, + paramIdx: Int, + segmentType: SegmentType, +) { + if (instruction.opcode.stack === StackInteraction.Pop) { + val stackValues = getStackValue( + cfg, + instruction, + instruction.opcode.params.size - paramIdx - 1, + ) + + if (stackValues.size <= 10) { + for (value in stackValues) { + val oldType = labels[value] + + if ( + oldType == null || + SEGMENT_PRIORITY.getValue(segmentType) > SEGMENT_PRIORITY.getValue(oldType) + ) { + labels[value] = segmentType + } + } + } + } else { + val value = instruction.args[paramIdx].value as Int + val oldType = labels[value] + + if ( + oldType == null || + SEGMENT_PRIORITY.getValue(segmentType) > SEGMENT_PRIORITY.getValue(oldType) + ) { + labels[value] = segmentType + } + } +} + +private fun parseSegment( + offsetToSegment: MutableMap, + labelHolder: LabelHolder, + cursor: Cursor, + label: Int, + type: SegmentType, + lenient: Boolean, + dcGcFormat: Boolean, +) { + try { + val info = labelHolder.getInfo(label) + + if (info == null) { + if (label !in BUILTIN_FUNCTIONS) { + logger.warn { "Label $label is not registered in the label table." } + } + + return + } + + // Check whether we've already parsed this segment and reparse it if necessary. + val segment = offsetToSegment[info.offset] + + val labels: MutableList = + if (segment == null) { + mutableListOf(label) + } else { + if (label !in segment.labels) { + segment.labels.add(label) + segment.labels.sort() + } + + if (SEGMENT_PRIORITY.getValue(type) > SEGMENT_PRIORITY.getValue(segment.type)) { + segment.labels + } else { + return + } + } + + val endOffset = info.next?.offset ?: cursor.size.toInt() + cursor.seekStart(info.offset.toUInt()) + + return when (type) { + SegmentType.Instructions -> + parseInstructionsSegment( + offsetToSegment, + labelHolder, + cursor, + endOffset, + labels, + info.next?.label, + lenient, + dcGcFormat, + ) + + SegmentType.Data -> + parseDataSegment(offsetToSegment, cursor, endOffset, labels) + + SegmentType.String -> + parseStringSegment(offsetToSegment, cursor, endOffset, labels, dcGcFormat) + } + } catch (e: Throwable) { + if (lenient) { + logger.error(e) { "Couldn't fully parse object code segment." } + } else { + throw e + } + } +} + +private fun parseInstructionsSegment( + offsetToSegment: MutableMap, + labelHolder: LabelHolder, + cursor: Cursor, + endOffset: Int, + labels: MutableList, + nextLabel: Int?, + lenient: Boolean, + dcGcFormat: Boolean, +) { + val instructions = mutableListOf() + + val segment = InstructionSegment( + labels, + instructions, + SegmentSrcLoc(emptyList()) + ) + offsetToSegment[cursor.position.toInt()] = segment + + while (cursor.position < endOffset.toUInt()) { + // Parse the opcode. + val mainOpcode = cursor.u8() + + val fullOpcode = when (mainOpcode.toInt()) { + 0xF8, 0xF9 -> ((mainOpcode.toUInt() shl 8) or cursor.u8().toUInt()).toInt() + else -> mainOpcode.toInt() + } + + val opcode = codeToOpcode(fullOpcode) + + // Parse the arguments. + try { + val args = parseInstructionArguments(cursor, opcode, dcGcFormat) + instructions.add(Instruction(opcode, args, null)) + } catch (e: Throwable) { + if (lenient) { + logger.error(e) { + "Exception occurred while parsing arguments for instruction ${opcode.mnemonic}." + } + instructions.add(Instruction(opcode, emptyList(), null)) + } else { + throw e + } + } + } + + // Recurse on label drop-through. + if (nextLabel != null) { + // Find the first ret or jmp. + var dropThrough = true + + for (i in instructions.size - 1 downTo 0) { + val opcode = instructions[i].opcode + + if (opcode == OP_RET || opcode == OP_JMP) { + dropThrough = false + break + } + } + + if (dropThrough) { + parseSegment( + offsetToSegment, + labelHolder, + cursor, + nextLabel, + SegmentType.Instructions, + lenient, + dcGcFormat, + ) + } + } +} + +private fun parseDataSegment( + offsetToSegment: MutableMap, + cursor: Cursor, + endOffset: Int, + labels: MutableList, +) { + val startOffset = cursor.position + val segment = DataSegment( + labels, + cursor.buffer(endOffset.toUInt() - startOffset), + SegmentSrcLoc(listOf()), + ) + offsetToSegment[startOffset.toInt()] = segment +} + +private fun parseStringSegment( + offsetToSegment: MutableMap, + cursor: Cursor, + endOffset: Int, + labels: MutableList, + dcGcFormat: Boolean, +) { + val startOffset = cursor.position + val segment = StringSegment( + labels, + if (dcGcFormat) { + cursor.stringAscii( + endOffset.toUInt() - startOffset, + nullTerminated = true, + dropRemaining = true + ) + } else { + cursor.stringUtf16( + endOffset.toUInt() - startOffset, + nullTerminated = true, + dropRemaining = true + ) + }, + SegmentSrcLoc(listOf()) + ) + offsetToSegment[startOffset.toInt()] = segment +} + +private fun parseInstructionArguments( + cursor: Cursor, + opcode: Opcode, + dcGcFormat: Boolean, +): List { + val args = mutableListOf() + + if (opcode.stack != StackInteraction.Pop) { + for (param in opcode.params) { + when (param.type) { + is ByteType -> + args.add(Arg(cursor.u8().toInt())) + + is WordType -> + args.add(Arg(cursor.u16().toInt())) + + is DWordType -> + args.add(Arg(cursor.i32())) + + is FloatType -> + args.add(Arg(cursor.f32())) + + is LabelType, + is ILabelType, + is DLabelType, + is SLabelType, + -> { + args.add(Arg(cursor.u16().toInt())) + } + + is StringType -> { + val maxBytes = min(4096u, cursor.bytesLeft) + args.add(Arg( + if (dcGcFormat) { + cursor.stringAscii( + maxBytes, + nullTerminated = true, + dropRemaining = false + ) + } else { + cursor.stringUtf16( + maxBytes, + nullTerminated = true, + dropRemaining = false + ) + }, + )) + } + + is ILabelVarType -> { + val argSize = cursor.u8() + args.addAll(cursor.u16Array(argSize.toUInt()).map { Arg(it.toInt()) }) + } + + is RegRefType, + is RegTupRefType, + -> { + args.add(Arg(cursor.u8().toInt())) + } + + is RegRefVarType -> { + val argSize = cursor.u8() + args.addAll(cursor.u8Array(argSize.toUInt()).map { Arg(it.toInt()) }) + } + + else -> error("Parameter type ${param.type} not implemented.") + } + } + } + + return args +} + +private data class LabelAndOffset(val label: Int, val offset: Int) +private data class OffsetAndIndex(val offset: Int, val index: Int) +private class LabelInfo(val offset: Int, val next: LabelAndOffset?) + +private class LabelHolder(labelOffsets: IntArray) { + /** + * Mapping of labels to their offset and index into [labels]. + */ + private val labelMap: MutableMap = mutableMapOf() + + /** + * Mapping of offsets to lists of labels. + */ + private val offsetMap: MutableMap> = mutableMapOf() + + /** + * Labels and their offset sorted by offset and then label. + */ + val labels: List + + init { + val labels = mutableListOf() + + // Populate the main label list. + for (label in labelOffsets.indices) { + val offset = labelOffsets[label] + + if (offset != -1) { + labels.add(LabelAndOffset(label, offset)) + } + } + + // Sort by offset, then label. + labels.sortWith { a, b -> + if (a.offset - b.offset != 0) a.offset - b.offset + else a.label - b.label + } + + this.labels = labels + + // Populate the label and offset maps. + for (index in 0 until labels.size) { + val (label, offset) = labels[index] + + labelMap[label] = OffsetAndIndex(offset, index) + + offsetMap.getOrPut(offset) { mutableListOf() }.add(label) + } + } + + fun getLabels(offset: Int): List? = offsetMap[offset] + + fun getInfo(label: Int): LabelInfo? { + val offsetAndIndex = labelMap[label] ?: return null + + // Find the next label with a different offset. + var next: LabelAndOffset? = null + + for (i in offsetAndIndex.index + 1 until labels.size) { + next = labels[i] + + // Skip the label if it points to the same offset. + if (next.offset > offsetAndIndex.offset) { + break + } else { + next = null + } + } + + return LabelInfo(offsetAndIndex.offset, next) + } +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectType.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectType.kt new file mode 100644 index 00000000..2770ae57 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/ObjectType.kt @@ -0,0 +1,3 @@ +package world.phantasmal.lib.fileFormats.quest + +enum class ObjectType : EntityType diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Quest.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Quest.kt new file mode 100644 index 00000000..6850c566 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/Quest.kt @@ -0,0 +1,169 @@ +package world.phantasmal.lib.fileFormats.quest + +import mu.KotlinLogging +import world.phantasmal.core.PwResult +import world.phantasmal.core.PwResultBuilder +import world.phantasmal.core.Severity +import world.phantasmal.core.Success +import world.phantasmal.lib.assembly.InstructionSegment +import world.phantasmal.lib.assembly.OP_SET_EPISODE +import world.phantasmal.lib.assembly.Segment +import world.phantasmal.lib.assembly.dataFlowAnalysis.getMapDesignations +import world.phantasmal.lib.compression.prs.prsDecompress +import world.phantasmal.lib.cursor.Cursor + +private val logger = KotlinLogging.logger {} + +class Quest( + var id: Int, + var language: Int, + var name: String, + var shortDescription: String, + var longDescription: String, + var episode: Episode, + val objects: List, + val npcs: List, + val events: List, + val datUnknowns: List, + val objectCode: List, + val shopItems: UIntArray, + val mapDesignations: Map, +) + +fun parseBinDatToQuest( + binCursor: Cursor, + datCursor: Cursor, + lenient: Boolean = false, +): PwResult { + val rb = PwResultBuilder(logger) + + // Decompress and parse files. + val binDecompressed = prsDecompress(binCursor) + rb.addResult(binDecompressed) + + if (binDecompressed !is Success) { + return rb.failure() + } + + val bin = parseBin(binDecompressed.value) + + val datDecompressed = prsDecompress(datCursor) + rb.addResult(datDecompressed) + + if (datDecompressed !is Success) { + return rb.failure() + } + + val dat = parseDat(datDecompressed.value) + val objects = dat.objs.map { QuestObject(it.areaId.toInt(), it.data) } + // Initialize NPCs with random episode and correct it later. + val npcs = dat.npcs.map { QuestNpc(Episode.I, it.areaId.toInt(), it.data) } + + // Extract episode and map designations from object code. + var episode = Episode.I + var mapDesignations = emptyMap() + + val objectCodeResult = parseObjectCode( + bin.objectCode, + bin.labelOffsets, + extractScriptEntryPoints(objects, npcs), + lenient, + bin.format == BinFormat.DC_GC, + ) + + rb.addResult(objectCodeResult) + + if (objectCodeResult !is Success) { + return rb.failure() + } + + val objectCode = objectCodeResult.value + + if (objectCode.isEmpty()) { + rb.addProblem(Severity.Warning, "File contains no instruction labels.") + } else { + val instructionSegments = objectCode.filterIsInstance() + + var label0Segment: InstructionSegment? = null + + for (segment in instructionSegments) { + if (0 in segment.labels) { + label0Segment = segment + break + } + } + + if (label0Segment != null) { + episode = getEpisode(rb, label0Segment) + + for (npc in npcs) { + npc.episode = episode + } + + mapDesignations = getMapDesignations(instructionSegments, label0Segment) + } else { + rb.addProblem(Severity.Warning, "No instruction segment for label 0 found.") + } + } + + return rb.success(Quest( + id = bin.questId.toInt(), + language = bin.language.toInt(), + name = bin.questName, + shortDescription = bin.shortDescription, + longDescription = bin.longDescription, + episode, + objects, + npcs, + events = dat.events, + datUnknowns = dat.unknowns, + objectCode, + shopItems = bin.shopItems, + mapDesignations, + )) +} + +/** + * Defaults to episode I. + */ +private fun getEpisode(rb: PwResultBuilder<*>, func0Segment: InstructionSegment): Episode { + val setEpisode = func0Segment.instructions.find { + it.opcode == OP_SET_EPISODE + } + + if (setEpisode == null) { + logger.debug { "Function 0 has no set_episode instruction." } + return Episode.I + } + + return when (val episode = setEpisode.args[0].value) { + 0 -> Episode.I + 1 -> Episode.II + 2 -> Episode.IV + else -> { + rb.addProblem( + Severity.Warning, + "Unknown episode $episode in function 0 set_episode instruction." + ) + Episode.I + } + } +} + +private fun extractScriptEntryPoints( + objects: List, + npcs: List, +): Set { + val entryPoints = mutableSetOf(0) + + objects.forEach { obj -> + obj.scriptLabel?.let(entryPoints::add) + obj.scriptLabel2?.let(entryPoints::add) + } + + npcs.forEach { npc -> + entryPoints.add(npc.scriptLabel) + } + + return entryPoints +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestNpc.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestNpc.kt new file mode 100644 index 00000000..93f44792 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestNpc.kt @@ -0,0 +1,27 @@ +package world.phantasmal.lib.fileFormats.quest + +import world.phantasmal.lib.buffer.Buffer +import kotlin.math.roundToInt + +class QuestNpc(var episode: Episode, var areaId: Int, val data: Buffer) { + /** + * Only seems to be valid for non-enemies. + */ + var scriptLabel: Int + get() = data.getF32(60u).roundToInt() + set(value) { + data.setF32(60u, value.toFloat()) + } + + var skin: Int + get() = data.getI32(64u) + set(value) { + data.setI32(64u, value) + } + + init { + require(data.size == NPC_BYTE_SIZE) { + "Data size should be $NPC_BYTE_SIZE but was ${data.size}." + } + } +} diff --git a/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestObject.kt b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestObject.kt new file mode 100644 index 00000000..83181503 --- /dev/null +++ b/lib/src/commonMain/kotlin/world/phantasmal/lib/fileFormats/quest/QuestObject.kt @@ -0,0 +1,15 @@ +package world.phantasmal.lib.fileFormats.quest + +import world.phantasmal.lib.buffer.Buffer + +class QuestObject(var areaId: Int, val data: Buffer) { + var type: ObjectType = TODO() + val scriptLabel: Int? = TODO() + val scriptLabel2: Int? = TODO() + + init { + require(data.size == OBJECT_BYTE_SIZE) { + "Data size should be $OBJECT_BYTE_SIZE but was ${data.size}." + } + } +} diff --git a/lib/src/commonTest/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSetTests.kt b/lib/src/commonTest/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSetTests.kt new file mode 100644 index 00000000..66c0a561 --- /dev/null +++ b/lib/src/commonTest/kotlin/world/phantasmal/lib/assembly/dataFlowAnalysis/ValueSetTests.kt @@ -0,0 +1,129 @@ +package world.phantasmal.lib.assembly.dataFlowAnalysis + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ValueSetTests { + @Test + fun empty_set_has_size_0() { + val vs = ValueSet() + + assertEquals(0, vs.size) + } + + @Test + fun get() { + val vs = ValueSet().setInterval(10, 13) + .union(ValueSet().setInterval(20, 22)) + + assertEquals(7, vs.size) + assertEquals(10, vs[0]) + assertEquals(11, vs[1]) + assertEquals(12, vs[2]) + assertEquals(13, vs[3]) + assertEquals(20, vs[4]) + assertEquals(21, vs[5]) + assertEquals(22, vs[6]) + } + + @Test + fun contains() { + val vs = ValueSet().setInterval(-20, 13) + .union(ValueSet().setInterval(20, 22)) + + assertEquals(37, vs.size) + assertFalse(-9001 in vs) + assertFalse(-21 in vs) + assertTrue(-20 in vs) + assertTrue(13 in vs) + assertFalse(14 in vs) + assertFalse(19 in vs) + assertTrue(20 in vs) + assertTrue(22 in vs) + assertFalse(23 in vs) + assertFalse(9001 in vs) + } + + @Test + fun setValue() { + val vs = ValueSet() + vs.setValue(100) + vs.setValue(4) + vs.setValue(24324) + + assertEquals(1, vs.size) + assertEquals(24324, vs[0]) + } + + @Test + fun union() { + val vs = ValueSet() + .union(ValueSet().setValue(21)) + .union(ValueSet().setValue(4968)) + + assertEquals(2, vs.size) + assertEquals(21, vs[0]) + assertEquals(4968, vs[1]) + } + + @Test + fun union_of_intervals() { + val vs = ValueSet() + .union(ValueSet().setInterval(10, 12)) + .union(ValueSet().setInterval(14, 16)) + + assertEquals(6, vs.size) + assertTrue(arrayOf(10, 11, 12, 14, 15, 16).all { it in vs }) + + vs.union(ValueSet().setInterval(13, 13)) + + assertEquals(7, vs.size) + assertEquals(10, vs[0]) + assertEquals(11, vs[1]) + assertEquals(12, vs[2]) + assertEquals(13, vs[3]) + assertEquals(14, vs[4]) + assertEquals(15, vs[5]) + assertEquals(16, vs[6]) + + vs.union(ValueSet().setInterval(1, 2)) + + assertEquals(9, vs.size) + assertTrue(arrayOf(1, 2, 10, 11, 12, 13, 14, 15, 16).all { it in vs }) + + vs.union(ValueSet().setInterval(30, 32)) + + assertEquals(12, vs.size) + assertTrue(arrayOf(1, 2, 10, 11, 12, 13, 14, 15, 16, 30, 31, 32).all { it in vs }) + + vs.union(ValueSet().setInterval(20, 21)) + + assertEquals(14, vs.size) + assertTrue(arrayOf(1, 2, 10, 11, 12, 13, 14, 15, 16, 20, 21, 30, 31, 32).all { it in vs }) + } + + @Test + fun iterator() { + val vs = ValueSet() + .union(ValueSet().setInterval(5, 7)) + .union(ValueSet().setInterval(14, 16)) + + val iter = vs.iterator() + + assertTrue(iter.hasNext()) + assertEquals(5, iter.next()) + assertTrue(iter.hasNext()) + assertEquals(6, iter.next()) + assertTrue(iter.hasNext()) + assertEquals(7, iter.next()) + assertTrue(iter.hasNext()) + assertEquals(14, iter.next()) + assertTrue(iter.hasNext()) + assertEquals(15, iter.next()) + assertTrue(iter.hasNext()) + assertEquals(16, iter.next()) + assertFalse(iter.hasNext()) + } +}