From 29b2e754dd3026bfdfa660dfcbb404b58731757a Mon Sep 17 00:00:00 2001 From: Daan Vanden Bosch Date: Mon, 12 Aug 2019 17:18:42 +0200 Subject: [PATCH] Fixed float parsing bug in assembly lexer. --- .../scripting/AssemblyLexer.test.ts | 42 ++++++ src/quest_editor/scripting/AssemblyLexer.ts | 127 +++++++++++------- .../ui/AssemblyEditorComponent.tsx | 4 +- 3 files changed, 119 insertions(+), 54 deletions(-) create mode 100644 src/quest_editor/scripting/AssemblyLexer.test.ts diff --git a/src/quest_editor/scripting/AssemblyLexer.test.ts b/src/quest_editor/scripting/AssemblyLexer.test.ts new file mode 100644 index 00000000..0d7e363f --- /dev/null +++ b/src/quest_editor/scripting/AssemblyLexer.test.ts @@ -0,0 +1,42 @@ +import { AssemblyLexer, FloatToken, TokenType } from "./AssemblyLexer"; + +test("valid floats", () => { + const lexer = new AssemblyLexer(); + + expect((lexer.tokenize_line("808.9")[0] as FloatToken).value).toBeCloseTo(808.9, 4); + expect((lexer.tokenize_line("-0.9")[0] as FloatToken).value).toBeCloseTo(-0.9, 2); + expect((lexer.tokenize_line("1e-3")[0] as FloatToken).value).toBeCloseTo(0.001, 4); + expect((lexer.tokenize_line("-6e2")[0] as FloatToken).value).toBeCloseTo(-600, 3); +}); + +test("invalid floats", () => { + const lexer = new AssemblyLexer(); + + const tokens1 = lexer.tokenize_line(" 808.9a "); + + expect(tokens1.length).toBe(1); + expect(tokens1[0].type).toBe(TokenType.InvalidNumber); + expect(tokens1[0].col).toBe(2); + expect(tokens1[0].len).toBe(6); + + const tokens2 = lexer.tokenize_line(" -55e "); + + expect(tokens2.length).toBe(1); + expect(tokens2[0].type).toBe(TokenType.InvalidNumber); + expect(tokens2[0].col).toBe(3); + expect(tokens2[0].len).toBe(4); + + const tokens3 = lexer.tokenize_line(".7429"); + + expect(tokens3.length).toBe(1); + expect(tokens3[0].type).toBe(TokenType.InvalidSection); + expect(tokens3[0].col).toBe(1); + expect(tokens3[0].len).toBe(5); + + const tokens4 = lexer.tokenize_line("\t\t\t4. test"); + + expect(tokens4.length).toBe(2); + expect(tokens4[0].type).toBe(TokenType.InvalidNumber); + expect(tokens4[0].col).toBe(4); + expect(tokens4[0].len).toBe(2); +}); diff --git a/src/quest_editor/scripting/AssemblyLexer.ts b/src/quest_editor/scripting/AssemblyLexer.ts index f9e2a551..1d85fd49 100644 --- a/src/quest_editor/scripting/AssemblyLexer.ts +++ b/src/quest_editor/scripting/AssemblyLexer.ts @@ -189,10 +189,6 @@ export class AssemblyLexer { return this.line.charAt(this.index); } - private peek_prev(): string { - return this.line.charAt(this.index - 1); - } - private skip(): void { this.index++; } @@ -213,76 +209,103 @@ export class AssemblyLexer { return this.line.slice(this._mark, this.index); } + private eat_rest_of_token(): void { + while (this.has_next()) { + const char = this.next(); + + if (/[\s,]/.test(char)) { + this.back(); + break; + } + } + } + private tokenize_number_or_label(): IntToken | FloatToken | InvalidNumberToken | LabelToken { this.mark(); const col = this.col; this.skip(); let is_label = false; - let is_float = false; - let is_hex = false; while (this.has_next()) { const char = this.peek(); - if (/\d/.test(char)) { + if ("." === char || "e" === char) { + return this.tokenize_float(col); + } else if ("x" === char) { + return this.tokenize_hex_number(col); + } else if (":" === char) { + is_label = true; this.skip(); - } else if ("." === char) { - if (is_float || is_hex) { - break; - } else { - is_float = true; - this.skip(); - } - } else if ("x" === char && this.marked_len() === 1 && this.peek_prev() === "0") { - if (is_float || is_hex) { - break; - } else { - is_hex = true; - this.skip(); - } - } else if (/[a-fA-F]/.test(char)) { - if (is_hex) { - this.skip(); - } else { - break; - } - } else { - if (char === ":" && !is_float && !is_hex) { - is_label = true; - } - break; + } else if (/[\s,]/.test(char)) { + break; + } else { + this.skip(); } } - let value: number; - - if (is_float) { - value = parseFloat(this.slice()); - } else if (is_hex) { - value = parseInt(this.slice(), 16); - } else { - value = parseInt(this.slice(), 10); - } - - if (is_label) { - this.skip(); - } + const value = parseInt(this.slice(), 10); return { - type: isNaN(value) - ? TokenType.InvalidNumber - : is_label - ? TokenType.Label - : is_float - ? TokenType.Float - : TokenType.Int, + type: Number.isInteger(value) + ? is_label + ? TokenType.Label + : TokenType.Int + : TokenType.InvalidNumber, col, len: this.marked_len(), value, }; } + private tokenize_hex_number(col: number): IntToken | InvalidNumberToken { + this.eat_rest_of_token(); + const hex_str = this.slice(); + + if (/^0x[\da-fA-F]+$/.test(hex_str)) { + const value = parseInt(hex_str, 16); + + if (Number.isInteger(value)) { + return { + type: TokenType.Int, + col, + len: this.marked_len(), + value, + }; + } + } + + return { + type: TokenType.InvalidNumber, + col, + len: this.marked_len(), + }; + } + + private tokenize_float(col: number): FloatToken | InvalidNumberToken { + this.eat_rest_of_token(); + const float_str = this.slice(); + + if (/^-?\d+(\.\d+)?(e-?\d+)?$/.test(float_str)) { + const value = parseFloat(float_str); + + if (Number.isFinite(value)) { + return { + type: TokenType.Float, + col, + len: this.marked_len(), + value, + }; + } + } + + return { + type: TokenType.InvalidNumber, + col, + len: this.marked_len(), + }; + } + private tokenize_register_or_ident(): RegisterToken | IdentToken | InvalidIdentToken { const col = this.col; this.skip(); diff --git a/src/quest_editor/ui/AssemblyEditorComponent.tsx b/src/quest_editor/ui/AssemblyEditorComponent.tsx index d3203974..c841ca28 100644 --- a/src/quest_editor/ui/AssemblyEditorComponent.tsx +++ b/src/quest_editor/ui/AssemblyEditorComponent.tsx @@ -26,9 +26,9 @@ const ASM_SYNTAX: languages.IMonarchLanguage = { [/[^\s]+:/, "tag"], // Numbers. - [/-?\d+\.\d+/, "number.float"], + [/-?\d+(\.\d+)?(e-?\d+)?/, "number.float"], [/0x[0-9a-fA-F]+/, "number.hex"], - [/-?[0-9]+?/, "number"], + [/-?[0-9]+/, "number"], // Identifiers. [/[a-z][a-z0-9_=<>!]*/, "identifier"],