From 147aeb65e6067b6784abd14ecb97fe4f28bcb084 Mon Sep 17 00:00:00 2001 From: Jeremy Faivre Date: Sat, 25 Jan 2025 16:47:58 +0100 Subject: [PATCH] First iteration on language server protocol, and many position fixes --- build.hxml | 3 +- src/loreline/Interpreter.hx | 47 +- src/loreline/Lens.hx | 194 +++++ src/loreline/Lexer.hx | 273 +++--- src/loreline/Node.hx | 75 +- src/loreline/Parser.hx | 274 ++++-- src/loreline/Position.hx | 90 +- src/loreline/Printer.hx | 39 +- src/loreline/Quotes.hx | 7 + src/loreline/Utf8.hx | 161 ++++ {cli/loreline => src/loreline/cli}/Cli.hx | 11 +- .../loreline/cli/CliColors.hx | 4 +- .../loreline/cli}/CliMacros.hx | 2 +- src/loreline/lsp/Protocol.hx | 476 ++++++++++ src/loreline/lsp/Server.hx | 824 ++++++++++++++++++ src/loreline/lsp/SymbolPrinter.hx | 328 +++++++ 16 files changed, 2544 insertions(+), 264 deletions(-) create mode 100644 src/loreline/Lens.hx create mode 100644 src/loreline/Utf8.hx rename {cli/loreline => src/loreline/cli}/Cli.hx (98%) rename cli/loreline/Colors.hx => src/loreline/cli/CliColors.hx (99%) rename {cli/loreline => src/loreline/cli}/CliMacros.hx (98%) create mode 100644 src/loreline/lsp/Protocol.hx create mode 100644 src/loreline/lsp/Server.hx create mode 100644 src/loreline/lsp/SymbolPrinter.hx diff --git a/build.hxml b/build.hxml index 4857949..0db90f1 100644 --- a/build.hxml +++ b/build.hxml @@ -1,3 +1,2 @@ --class-path src ---class-path cli ---main loreline.Cli \ No newline at end of file +--main loreline.cli.Cli \ No newline at end of file diff --git a/src/loreline/Interpreter.hx b/src/loreline/Interpreter.hx index c6064cf..74169a9 100644 --- a/src/loreline/Interpreter.hx +++ b/src/loreline/Interpreter.hx @@ -5,6 +5,10 @@ import haxe.ds.StringMap; import loreline.Lexer; import loreline.Node; +using StringTools; +using loreline.Utf8; + + /** * A state during the runtime execution of a loreline script */ @@ -56,7 +60,7 @@ enum RuntimeAccess { * When exiting that scope, the related temporary states associated to it are destroyed */ @:structInit -class Scope { +class RuntimeScope { /** * The scope id, a unique integer value in the stack @@ -198,13 +202,13 @@ class Interpreter { * The current execution stack, which consists of scopes added on top of one another. * Each scope can have its own local beats and temporary states. */ - final stack:Array = []; + final stack:Array = []; /** * Current scope associated with current execution state */ - var currentScope(get,never):Scope; - function get_currentScope():Scope { + var currentScope(get,never):RuntimeScope; + function get_currentScope():RuntimeScope { return stack.length > 0 ? stack[stack.length - 1] : null; } @@ -479,7 +483,7 @@ class Interpreter { } - function push(scope:Scope):Void { + function push(scope:RuntimeScope):Void { scope.id = nextScopeId++; stack.push(scope); @@ -531,7 +535,7 @@ class Interpreter { } - function initializeState(state:NStateDecl, scope:Scope) { + function initializeState(state:NStateDecl, scope:RuntimeScope) { var runtimeState:RuntimeState = null; if (state.temporary) { @@ -918,18 +922,27 @@ class Interpreter { } function evaluateString(str:NStringLiteral):{text:String, tags:Array} { - final buf = new StringBuf(); + final buf = new loreline.Utf8.Utf8Buf(); final tags:Array = []; var offset = 0; - for (part in str.parts) { + var keepWhitespace = (str.quotes != Unquoted); + + for (i in 0...str.parts.length) { + final part = str.parts[i]; + switch (part.type) { case Raw(text): - offset += text.length; + if (!keepWhitespace) { + text = text.ltrim(); + } + final len = text.uLength(); + if (len > 0) keepWhitespace = false; + offset += len; buf.add(text); case Expr(expr): - + keepWhitespace = false; if (expr is NAccess) { // When providing a character object, // implicitly read the character's `name` field @@ -940,20 +953,20 @@ class Interpreter { final characterFields = evaluateExpression(expr); final value = getField(characterFields, 'name') ?? name; final text = valueToString(value); - offset += text.length; + offset += text.uLength(); buf.add(text); case _: final value = evaluateExpression(expr); final text = valueToString(value); - offset += text.length; + offset += text.uLength(); buf.add(text); } } else { final value = evaluateExpression(expr); final text = valueToString(value); - offset += text.length; + offset += text.uLength(); buf.add(text); } @@ -1066,7 +1079,7 @@ class Interpreter { case Number, Boolean, Null: lit.value; case Array: [for (elem in (lit.value:Array)) evaluateExpression(elem)]; - case Object: + case Object(_): final obj = new Map(); for (field in (lit.value:Array)) { obj.set(field.name, evaluateExpression(field.value)); @@ -1354,12 +1367,12 @@ class Interpreter { throw new RuntimeError('Cannot compare ${getTypeName(leftType)} and ${getTypeName(rightType)}', pos ?? currentScope?.node?.pos ?? script.pos); } - case OpAnd | OpOr: + case OpAnd(_) | OpOr(_): switch [leftType, rightType] { case [TBool, TBool]: switch op { - case OpAnd: left && right; - case OpOr: left || right; + case OpAnd(_): left && right; + case OpOr(_): left || right; case _: throw "Unreachable"; } case _: diff --git a/src/loreline/Lens.hx b/src/loreline/Lens.hx new file mode 100644 index 0000000..491aa10 --- /dev/null +++ b/src/loreline/Lens.hx @@ -0,0 +1,194 @@ +package loreline; + +import loreline.Node; +import loreline.Position; + +/** + * Definition reference found in a script + */ +@:structInit class Definition { + /** The node where this definition appears */ + public var node:Node; + /** The exact position of the definition */ + public var pos:Position; + /** Any references to this definition */ + public var references:Array = []; +} + +/** + * Reference to a definition found in a script + */ +@:structInit class Reference { + /** The node containing this reference */ + public var node:Node; + /** The exact position of the reference */ + public var pos:Position; +} + +/** + * Utility class for analyzing Loreline scripts without executing them. + * Provides methods for finding nodes, variables, references, etc. + */ +class Lens { + /** The script being analyzed */ + final script:Script; + + /** Map of all nodes by their unique ID */ + final nodesById:Map = []; + + /** Map of node IDs to their parent nodes */ + final parentNodes:Map = []; + + /** Map of node IDs to their child nodes */ + final childNodes:Map> = []; + + public function new(script:Script) { + this.script = script; + initialize(); + } + + /** + * Initialize all the lookups and analysis data + */ + function initialize() { + // First pass: Build node maps and collect definitions + script.each((node, parent) -> { + // Track nodes by ID + nodesById.set(node.id, node); + + // Track parent relationships + if (parent != null) { + parentNodes.set(node.id, parent); + + // And track the other way around + var children = childNodes.get(parent.id); + if (children == null) { + children = []; + childNodes.set(parent.id, children); + } + children.push(node); + } + }); + } + + /** + * Gets the nodes at the given position + * @param pos Position to check + * @return Most specific node at that position, or null if none found + */ + public function getNodeAtPosition(pos:Position):Null { + var bestMatch:Null = null; + + script.each((node, parent) -> { + final nodePos = node.pos; + if (nodePos.length > 0 && + nodePos.offset <= pos.offset && + nodePos.offset + nodePos.length >= pos.offset) { + + bestMatch = node; + } + }); + + return bestMatch; + } + + /** + * Gets all nodes of a specific type + * @param nodeType Class type to find + * @return Array of matching nodes + */ + public function getNodesOfType(nodeType:Class):Array { + final matches:Array = []; + script.each((node, _) -> { + if (Std.isOfType(node, nodeType)) { + matches.push(cast node); + } + }); + return matches; + } + + /** + * Gets the parent node of a given node + * @param node Child node + * @return Parent node or null if none found + */ + public function getParentNode(node:Node):Null { + return parentNodes.get(node.id); + } + + /** + * Gets the parent node of a given node + * @param node Child node + * @return Parent node or null if none found + */ + public function getParentOfType(node:Node, type:Class):Null { + var current:Any = node; + while (current != null) { + current = getParentNode(current); + if (current != null && Type.getClass(current) == type) { + return current; + } + } + return null; + } + + /** + * Gets all ancestor nodes of a given node + * @param node Starting node + * @return Array of ancestor nodes from immediate parent to root + */ + public function getAncestors(node:Node):Array { + final ancestors:Array = []; + var current = node; + while (current != null) { + current = parentNodes.get(current.id); + if (current != null) { + ancestors.push(current); + } + } + return ancestors; + } + + /** + * Finds all nodes that match a predicate function + * @param predicate Function that returns true for matching nodes + * @return Array of matching nodes + */ + public function findNodes(predicate:(node:Node) -> Bool):Array { + final matches:Array = []; + script.each((node, _) -> { + if (predicate(node)) { + matches.push(node); + } + }); + return matches; + } + + /** + * Finds and returns the beat declaration referenced by the given transition. + * This method searches through the beat declarations to find a match based on the transition's properties. + * @param transition The transition object containing the reference to search for + * @return The referenced beat declaration if found, null otherwise + */ + public function findReferencedBeat(transition:NTransition):Null { + + var result:Null = null; + + var parent = parentNodes.get(transition.id); + while (result == null && parent != null) { + parent.each((node, _) -> { + if (Type.getClass(node) == NBeatDecl) { + final beatDecl:NBeatDecl = cast node; + if (beatDecl.name == transition.target) { + result = beatDecl; + } + } + }); + parent = parentNodes.get(parent.id); + } + + return result; + + } + +} \ No newline at end of file diff --git a/src/loreline/Lexer.hx b/src/loreline/Lexer.hx index ee7175c..6afecc0 100644 --- a/src/loreline/Lexer.hx +++ b/src/loreline/Lexer.hx @@ -1,6 +1,7 @@ package loreline; using StringTools; +using loreline.Utf8; /** * Represents an error that occurred during lexical analysis. @@ -127,10 +128,10 @@ enum TokenType { OpGreaterEq; /** Less than or equal operator (<=) */ OpLessEq; - /** Logical AND operator (&&) */ - OpAnd; - /** Logical OR operator (||) */ - OpOr; + /** Logical AND operator (&& / and) */ + OpAnd(word:Bool); + /** Logical OR operator (|| / or) */ + OpOr(word:Bool); /** Logical NOT operator (!) */ OpNot; @@ -221,8 +222,8 @@ class TokenTypeHelpers { case [OpLess, OpLess]: true; case [OpGreaterEq, OpGreaterEq]: true; case [OpLessEq, OpLessEq]: true; - case [OpAnd, OpAnd]: true; - case [OpOr, OpOr]: true; + case [OpAnd(_), OpAnd(_)]: true; + case [OpOr(_), OpOr(_)]: true; case [OpNot, OpNot]: true; case [LNull, LNull]: true; case [Identifier(n1), Identifier(n2)]: n1 == n2; @@ -329,8 +330,8 @@ class Lexer { "true" => TokenType.LBoolean(true), "false" => TokenType.LBoolean(false), "null" => TokenType.LNull, - "and" => TokenType.OpAnd, - "or" => TokenType.OpOr + "and" => TokenType.OpAnd(true), + "or" => TokenType.OpOr(true) ]; /** @@ -419,7 +420,7 @@ class Lexer { */ public function new(input:String) { this.input = input; - this.length = input.length; + this.length = input.uLength(); reset(); } @@ -524,7 +525,7 @@ class Lexer { startLine = line; startColumn = column; - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); if (c == "\n".code || c == "\r".code) { final lineBreakToken = readLineBreak(); @@ -621,7 +622,7 @@ class Lexer { case "&".code: if (peek() == "&".code) { advance(2); - makeToken(OpAnd, startPos); + makeToken(OpAnd(false), startPos); } else { advance(); @@ -631,7 +632,7 @@ class Lexer { case "|".code: if (peek() == "|".code) { advance(2); - makeToken(OpOr, startPos); + makeToken(OpOr(false), startPos); } else { advance(); @@ -690,7 +691,7 @@ class Lexer { // Count spaces/tabs while (pos < length) { - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); if (c == " ".code) { spaces++; } else if (c == "\t".code) { @@ -702,7 +703,7 @@ class Lexer { } // Check if line is empty or only whitespace - if (pos >= length || input.charCodeAt(pos) == "\n".code || input.charCodeAt(pos) == "\r".code) { + if (pos >= length || input.uCharCodeAt(pos) == "\n".code || input.uCharCodeAt(pos) == "\r".code) { // Return previous indentation level for empty lines return indentStack[indentStack.length - 1]; } @@ -760,9 +761,9 @@ class Lexer { */ function readLineBreak():Token { final start = makePosition(); - if (input.charCodeAt(pos) == "\r".code) { + if (input.uCharCodeAt(pos) == "\r".code) { advance(); - if (pos < length && input.charCodeAt(pos) == "\n".code) { + if (pos < length && input.uCharCodeAt(pos) == "\n".code) { advance(); } } @@ -788,12 +789,12 @@ class Lexer { */ function matchIdentifier(pos:Int):Null { // Handle empty strings first - if (input.length == 0) { + if (input.uLength() == 0) { return null; } // Check if the first character is a valid identifier start - var firstChar = input.charCodeAt(pos + 0); + var firstChar = input.uCharCodeAt(pos + 0); if (!isIdentifierStart(firstChar)) { return null; } @@ -803,15 +804,15 @@ class Lexer { // Check subsequent characters until we find an invalid one // or reach the end of the string - while (identifierLength < input.length) { - if (!isIdentifierPart(input.charCodeAt(pos + identifierLength))) { + while (identifierLength < input.uLength()) { + if (!isIdentifierPart(input.uCharCodeAt(pos + identifierLength))) { break; } identifierLength++; } // Return the substring that contains our identifier - return input.substr(pos, identifierLength); + return input.uSubstr(pos, identifierLength); } /** @@ -820,28 +821,28 @@ class Lexer { function skipWhitespaceAndComments(pos:Int):Int { final startPos = pos; var foundContent = false; - while (pos < input.length) { + while (pos < input.uLength()) { // Skip whitespace - while (pos < input.length && (input.charCodeAt(pos) == " ".code || input.charCodeAt(pos) == "\t".code)) { + while (pos < input.uLength() && (input.uCharCodeAt(pos) == " ".code || input.uCharCodeAt(pos) == "\t".code)) { pos++; foundContent = true; } // Check for comments - if (pos < input.length - 1) { - if (input.charCodeAt(pos) == "/".code) { - if (input.charCodeAt(pos + 1) == "/".code) { + if (pos < input.uLength() - 1) { + if (input.uCharCodeAt(pos) == "/".code) { + if (input.uCharCodeAt(pos + 1) == "/".code) { // Single line comment - invalid in single line pos = startPos; return pos; } - else if (input.charCodeAt(pos + 1) == "*".code) { + else if (input.uCharCodeAt(pos + 1) == "*".code) { // Multi-line comment pos += 2; foundContent = true; var commentClosed = false; - while (pos < input.length - 1) { - if (input.charCodeAt(pos) == "*".code && input.charCodeAt(pos + 1) == "/".code) { + while (pos < input.uLength() - 1) { + if (input.uCharCodeAt(pos) == "*".code && input.uCharCodeAt(pos + 1) == "/".code) { pos += 2; commentClosed = true; break; @@ -867,11 +868,13 @@ class Lexer { * @return True if an if condition starts at the position, false otherwise */ function isIfStart(pos:Int):Bool { + pos = skipWhitespaceAndComments(pos); + // Check "if" literal first - if (input.charCodeAt(pos) != "i".code) return false; + if (input.uCharCodeAt(pos) != "i".code) return false; pos++; - if (input.charCodeAt(pos) != "f".code) return false; + if (input.uCharCodeAt(pos) != "f".code) return false; pos++; // Save initial position to restore it later @@ -881,11 +884,11 @@ class Lexer { inline function readIdent():Bool { var result = true; - if (pos >= input.length) { + if (pos >= input.uLength()) { result = false; } else { - var c = input.charCodeAt(pos); + var c = input.uCharCodeAt(pos); // First char must be letter or underscore if (!isIdentifierStart(c)) { @@ -895,8 +898,8 @@ class Lexer { pos++; // Continue reading identifier chars - while (pos < input.length) { - c = input.charCodeAt(pos); + while (pos < input.uLength()) { + c = input.uCharCodeAt(pos); if (!isIdentifierPart(c)) break; pos++; } @@ -909,28 +912,28 @@ class Lexer { pos = skipWhitespaceAndComments(pos); // Handle optional ! for negation - if (pos < input.length && input.charCodeAt(pos) == "!".code) { + if (pos < input.uLength() && input.uCharCodeAt(pos) == "!".code) { pos++; pos = skipWhitespaceAndComments(pos); } // If directly followed with (, that's a valid if - if (input.charCodeAt(pos) == "(".code) { + if (input.uCharCodeAt(pos) == "(".code) { return true; } - // If "if" is directly followed by an identifier start, that's not a if - if (pos == startPos && isIdentifierStart(input.charCodeAt(startPos))) { + // If "if" is directly followed by an identifier start (without space), that's not a if + if (pos == startPos && isIdentifierStart(input.uCharCodeAt(startPos))) { return false; } // Must start with identifier or opening parenthesis - if (pos >= input.length || !isIdentifierStart(input.charCodeAt(pos))) { + if (pos >= input.uLength() || !isIdentifierStart(input.uCharCodeAt(pos))) { return false; } - while (pos < input.length) { - if (input.charCodeAt(pos) == "(".code) { + while (pos < input.uLength()) { + if (input.uCharCodeAt(pos) == "(".code) { // Function call return true; } else { @@ -940,11 +943,11 @@ class Lexer { } pos = skipWhitespaceAndComments(pos); - if (pos >= input.length) { + if (pos >= input.uLength()) { return true; } - var c = input.charCodeAt(pos); + var c = input.uCharCodeAt(pos); // Handle dot access if (c == ".".code) { @@ -954,46 +957,46 @@ class Lexer { return false; } pos = skipWhitespaceAndComments(pos); - if (pos >= input.length) { + if (pos >= input.uLength()) { return true; } - c = input.charCodeAt(pos); + c = input.uCharCodeAt(pos); } // Handle bracket access if (c == "[".code) { pos++; var bracketLevel = 1; - while (pos < input.length && bracketLevel > 0) { - c = input.charCodeAt(pos); + while (pos < input.uLength() && bracketLevel > 0) { + c = input.uCharCodeAt(pos); if (c == "[".code) bracketLevel++; if (c == "]".code) bracketLevel--; pos++; } pos = skipWhitespaceAndComments(pos); - if (pos >= input.length) { + if (pos >= input.uLength()) { return true; } - c = input.charCodeAt(pos); + c = input.uCharCodeAt(pos); } // Check for and delimiter - if (c == "a".code && input.charCodeAt(pos + 1) == "n".code && input.charCodeAt(pos + 2) == "d".code && !isIdentifierStart(input.charCodeAt(pos + 3))) { + if (c == "a".code && input.uCharCodeAt(pos + 1) == "n".code && input.uCharCodeAt(pos + 2) == "d".code && !isIdentifierStart(input.uCharCodeAt(pos + 3))) { return true; } // Check for or delimiter - if (c == "o".code && input.charCodeAt(pos + 1) == "r".code && !isIdentifierStart(input.charCodeAt(pos + 2))) { + if (c == "o".code && input.uCharCodeAt(pos + 1) == "r".code && !isIdentifierStart(input.uCharCodeAt(pos + 2))) { return true; } // Check for various delimiters typical from if condition - if (c == "(".code || c == "&".code || c == "|".code || ((input.charCodeAt(pos + 1) == "=".code) && c == "=".code) || c == ">".code || c == "<".code || (input.charCodeAt(pos + 1) != "=".code && (c == "+".code || c == "-".code || c == "*".code || c == "/".code))) { + if (c == "(".code || c == "&".code || c == "|".code || ((input.uCharCodeAt(pos + 1) == "=".code) && c == "=".code) || c == ">".code || c == "<".code || (input.uCharCodeAt(pos + 1) != "=".code && (c == "+".code || c == "-".code || c == "*".code || c == "/".code || c == "{".code))) { return true; } // If we're at end or newline, it's valid - if (c == "\n".code || c == "\r".code || pos >= input.length) { + if (c == "\n".code || c == "\r".code || pos >= input.uLength()) { pos = startPos; return true; } @@ -1019,7 +1022,7 @@ class Lexer { var startPos = pos; // Check for -> - if (input.charCodeAt(pos) != "-".code || pos >= input.length - 1 || input.charCodeAt(pos + 1) != ">".code) { + if (input.uCharCodeAt(pos) != "-".code || pos >= input.uLength() - 1 || input.uCharCodeAt(pos + 1) != ">".code) { return false; } pos += 2; @@ -1028,14 +1031,14 @@ class Lexer { pos = skipWhitespaceAndComments(pos); // Read identifier - if (pos >= input.length || !isIdentifierStart(input.charCodeAt(pos))) { + if (pos >= input.uLength() || !isIdentifierStart(input.uCharCodeAt(pos))) { pos = startPos; return false; } // Move past identifier pos++; - while (pos < input.length && isIdentifierPart(input.charCodeAt(pos))) { + while (pos < input.uLength() && isIdentifierPart(input.uCharCodeAt(pos))) { pos++; } @@ -1043,8 +1046,8 @@ class Lexer { pos = skipWhitespaceAndComments(pos); // Check that we're at end of line, end of input, or only have whitespace/comments left - if (pos < input.length) { - var c = input.charCodeAt(pos); + if (pos < input.uLength()) { + var c = input.uCharCodeAt(pos); if (c != "\n".code && c != "\r".code && c != " ".code && c != "\t".code && c != "/".code) { pos = startPos; return false; @@ -1070,11 +1073,11 @@ class Lexer { inline function readIdent():Bool { var result = true; - if (pos >= input.length) { + if (pos >= input.uLength()) { result = false; } else { - var c = input.charCodeAt(pos); + var c = input.uCharCodeAt(pos); // First char must be letter or underscore if (!isIdentifierStart(c)) { @@ -1084,8 +1087,8 @@ class Lexer { pos++; // Continue reading identifier chars - while (pos < input.length) { - c = input.charCodeAt(pos); + while (pos < input.uLength()) { + c = input.uCharCodeAt(pos); if (!isIdentifierPart(c)) break; pos++; } @@ -1102,15 +1105,15 @@ class Lexer { } // Keep reading segments until we find opening parenthesis - while (pos < input.length) { + while (pos < input.uLength()) { pos = skipWhitespaceAndComments(pos); - if (pos >= input.length) { + if (pos >= input.uLength()) { pos = startPos; return false; } - var c = input.charCodeAt(pos); + var c = input.uCharCodeAt(pos); // Found opening parenthesis - success! if (c == "(".code) { @@ -1133,8 +1136,8 @@ class Lexer { if (c == "[".code) { // Skip everything until closing bracket pos++; - while (pos < input.length) { - if (input.charCodeAt(pos) == "]".code) { + while (pos < input.uLength()) { + if (input.uCharCodeAt(pos) == "]".code) { pos++; break; } @@ -1167,11 +1170,11 @@ class Lexer { inline function readIdent():Bool { var result = true; - if (pos >= input.length) { + if (pos >= input.uLength()) { result = false; } else { - var c = input.charCodeAt(pos); + var c = input.uCharCodeAt(pos); // First char must be letter or underscore if (!isIdentifierStart(c)) { @@ -1181,8 +1184,8 @@ class Lexer { pos++; // Continue reading identifier chars - while (pos < input.length) { - c = input.charCodeAt(pos); + while (pos < input.uLength()) { + c = input.uCharCodeAt(pos); if (!isIdentifierPart(c)) break; pos++; } @@ -1199,19 +1202,19 @@ class Lexer { } // Keep reading segments until we find opening parenthesis - while (pos < input.length) { + while (pos < input.uLength()) { pos = skipWhitespaceAndComments(pos); - if (pos >= input.length) { + if (pos >= input.uLength()) { pos = startPos; return false; } - var c = input.charCodeAt(pos); + var c = input.uCharCodeAt(pos); // Found assign operator if (c == "=".code || - (input.charCodeAt(pos + 1) == "=".code && (c == "+".code || c == "-".code || c == "*".code || c == "/".code))) { + (input.uCharCodeAt(pos + 1) == "=".code && (c == "+".code || c == "-".code || c == "*".code || c == "/".code))) { pos = startPos; return true; } @@ -1231,8 +1234,8 @@ class Lexer { if (c == "[".code) { // Skip everything until closing bracket pos++; - while (pos < input.length) { - if (input.charCodeAt(pos) == "]".code) { + while (pos < input.uLength()) { + if (input.uCharCodeAt(pos) == "]".code) { pos++; break; } @@ -1260,12 +1263,12 @@ class Lexer { function isColon(pos:Int, skipWhitespaces:Bool = true):Bool { if (skipWhitespaces) { - while (pos < input.length && (input.charCodeAt(pos) == " ".code || input.charCodeAt(pos) == "\t".code)) { + while (pos < input.uLength() && (input.uCharCodeAt(pos) == " ".code || input.uCharCodeAt(pos) == "\t".code)) { pos++; } } - return pos < input.length && input.charCodeAt(pos) == ":".code; + return pos < input.uLength() && input.uCharCodeAt(pos) == ":".code; } @@ -1379,9 +1382,9 @@ class Lexer { for (i in 0...str.length) { var found = false; - var code = str.charCodeAt(i); + var code = str.uCharCodeAt(i); for (j in 0...specialChars.length) { - if (code == specialChars.charCodeAt(j)) { + if (code == specialChars.uCharCodeAt(j)) { found = true; break; } @@ -1400,15 +1403,15 @@ class Lexer { */ function isNumber(input:String):Bool { var pos:Int = 0; - var length = input.length; + var length = input.uLength(); - while (pos < length && isDigit(input.charCodeAt(pos))) { + while (pos < length && isDigit(input.uCharCodeAt(pos))) { pos++; } - if (pos < length && input.charCodeAt(pos) == ".".code && pos + 1 < length && isDigit(input.charCodeAt(pos + 1))) { + if (pos < length && input.uCharCodeAt(pos) == ".".code && pos + 1 < length && isDigit(input.uCharCodeAt(pos + 1))) { pos++; - while (pos < length && isDigit(input.charCodeAt(pos))) pos++; + while (pos < length && isDigit(input.uCharCodeAt(pos))) pos++; } return pos == length; @@ -1425,12 +1428,12 @@ class Lexer { if (strictExprs > 0) return null; // Look ahead to validate if this could be an unquoted string start - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); final cc = peek(); // Skip if it's a comment if (c == "/".code && pos < length - 1) { - final next = input.charCodeAt(pos + 1); + final next = input.uCharCodeAt(pos + 1); if (next == "/".code || next == "*".code) return null; } @@ -1524,7 +1527,7 @@ class Lexer { // If we get here, we can start reading the unquoted string final start = makePosition(); - final buf = new StringBuf(); + final buf = new loreline.Utf8.Utf8Buf(); final attachments = new Array(); final startLine = line; @@ -1540,7 +1543,7 @@ class Lexer { var hasContent = false; while (pos < length) { - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); final isSpace = isWhitespace(c); if (!hasContent) { @@ -1560,7 +1563,7 @@ class Lexer { escaped = true; advance(); } - else if (tagStart == -1 && isSpace && !hasContent) { + else if (tagStart == -1 && isSpace && !hasContent && attachments.length == 0) { // Skip leading white spaces advance(); } @@ -1569,26 +1572,26 @@ class Lexer { break; } // Check for trailing if - else if (tagStart == -1 && parent == KwBeat && c == "i".code && input.charCodeAt(pos+1) == "f".code && isIfStart(pos)) { + else if (tagStart == -1 && parent == KwBeat && isIfStart(pos)) { break; } // Check for comment start - else if (tagStart == -1 && (c == "/".code && pos < length - 1 && (input.charCodeAt(pos+1) == "/".code || input.charCodeAt(pos+1) == "*".code))) { + else if (tagStart == -1 && (c == "/".code && pos < length - 1 && (input.uCharCodeAt(pos+1) == "/".code || input.uCharCodeAt(pos+1) == "*".code))) { break; } // Check for arrow start - else if (tagStart == -1 && c == "-".code && pos < length - 1 && input.charCodeAt(pos+1) == ">".code && isTransitionStart(pos)) { + else if (tagStart == -1 && c == "-".code && pos < length - 1 && input.uCharCodeAt(pos+1) == ">".code && isTransitionStart(pos)) { break; } else if (allowTags && c == "<".code) { if (tagStart != -1) { error("Unexpected < inside tag"); } - final nextChar = pos + 1 < length ? input.charCodeAt(pos + 1) : 0; + final nextChar = pos + 1 < length ? input.uCharCodeAt(pos + 1) : 0; tagIsClosing = nextChar == "/".code; final checkPos = pos + (tagIsClosing ? 2 : 1); if (checkPos < length) { - final nameStart = input.charCodeAt(checkPos); + final nameStart = input.uCharCodeAt(checkPos); if (isIdentifierStart(nameStart) || nameStart == "_".code || nameStart == "$".code || (tagIsClosing && nameStart == ">".code)) { tagStart = buf.length; } @@ -1616,7 +1619,7 @@ class Lexer { advance(); currentColumn++; - if (input.charCodeAt(pos) == "{".code) { + if (input.uCharCodeAt(pos) == "{".code) { advance(); currentColumn++; @@ -1625,15 +1628,15 @@ class Lexer { final interpLength = pos - tokenStartPos; attachments.push(Interpolation(true, tagStart != -1, tokens, interpStart, interpLength)); - buf.add(input.substr(tokenStartPos, interpLength)); + buf.add(input.uSubstr(tokenStartPos, interpLength)); } - else if (isIdentifierStart(input.charCodeAt(pos))) { + else if (isIdentifierStart(input.uCharCodeAt(pos))) { final interpPos = new Position(interpLine, interpColumn + 1, pos); final tokens = readFieldAccessInterpolation(interpPos); final interpLength = pos - tokenStartPos; attachments.push(Interpolation(false, tagStart != -1, tokens, interpStart, interpLength)); - buf.add(input.substr(tokenStartPos, interpLength)); + buf.add(input.uSubstr(tokenStartPos, interpLength)); } else { error("Expected identifier or { after $"); @@ -1661,7 +1664,7 @@ class Lexer { // If we found valid content, return the string token if (valid) { final content = buf.toString().rtrim(); - if (content.length > 0 && hasNonSpecialChar(content) && !isNumber(content)) { + if (content.uLength() > 0 && hasNonSpecialChar(content) && !isNumber(content)) { attachments.sort(compareAttachments); return makeToken(LString( @@ -1688,7 +1691,7 @@ class Lexer { */ function readString(stringStart:Position):Token { advance(); // Skip opening quote - final buf = new StringBuf(); + final buf = new loreline.Utf8.Utf8Buf(); final attachments = new Array(); var escaped = false; var tagStart = -1; @@ -1700,7 +1703,7 @@ class Lexer { var allowTags = (parentBlockType() == KwBeat); while (pos < length) { - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); if (escaped) { buf.addChar("\\".code); @@ -1726,11 +1729,11 @@ class Lexer { if (tagStart != -1) { error("Unexpected < inside tag"); } - final nextChar = pos + 1 < length ? input.charCodeAt(pos + 1) : 0; + final nextChar = pos + 1 < length ? input.uCharCodeAt(pos + 1) : 0; tagIsClosing = nextChar == "/".code; final checkPos = pos + (tagIsClosing ? 2 : 1); if (checkPos < length) { - final nameStart = input.charCodeAt(checkPos); + final nameStart = input.uCharCodeAt(checkPos); if (isIdentifierStart(nameStart) || nameStart == "_".code || nameStart == "$".code || (tagIsClosing && nameStart == ">".code)) { tagStart = buf.length; } @@ -1758,7 +1761,7 @@ class Lexer { advance(); currentColumn++; - if (input.charCodeAt(pos) == "{".code) { + if (input.uCharCodeAt(pos) == "{".code) { advance(); currentColumn++; @@ -1767,15 +1770,15 @@ class Lexer { final interpLength = pos - tokenStartPos; attachments.push(Interpolation(true, tagStart != -1, tokens, interpStart, interpLength)); - buf.add(input.substr(tokenStartPos, interpLength)); + buf.add(input.uSubstr(tokenStartPos, interpLength)); } - else if (isIdentifierStart(input.charCodeAt(pos))) { + else if (isIdentifierStart(input.uCharCodeAt(pos))) { final interpPos = new Position(interpLine, interpColumn + 1, pos); final tokens = readFieldAccessInterpolation(interpPos); final interpLength = pos - tokenStartPos; attachments.push(Interpolation(false, tagStart != -1, tokens, interpStart, interpLength)); - buf.add(input.substr(tokenStartPos, interpLength)); + buf.add(input.uSubstr(tokenStartPos, interpLength)); } else { error("Expected identifier or { after $"); @@ -1837,21 +1840,19 @@ class Lexer { var currentLine = interpStart.line; while (pos < length && braceLevel > 0) { - if (input.charCodeAt(pos) == "}".code && --braceLevel == 0) { + if (input.uCharCodeAt(pos) == "}".code && --braceLevel == 0) { advance(); break; } - if (input.charCodeAt(pos) == '"'.code) { + if (input.uCharCodeAt(pos) == '"'.code) { final stringPos = new Position(currentLine, currentColumn, pos); tokens.push(readString(stringPos)); currentColumn += (pos - stringPos.offset); continue; } - final tokenStart = new Position(currentLine, currentColumn, pos); final token = nextToken(); - token.pos = tokenStart; tokens.push(token); if (token.type == LineBreak) { @@ -1863,9 +1864,9 @@ class Lexer { } else { var tokenLength = switch (token.type) { - case Identifier(name): name.length; - case LString(q, s, _): s.length + (q != Unquoted ? 2 : 0); - case LNumber(n): Std.string(n).length; + case Identifier(name): name.uLength(); + case LString(q, s, _): s.uLength() + (q != Unquoted ? 2 : 0); + case LNumber(n): Std.string(n).uLength(); case _: 1; } currentColumn += tokenLength; @@ -1921,16 +1922,16 @@ class Lexer { function readFieldAccessInterpolation(stringStart:Position):Array { final tokens = new Array(); - if (!isIdentifierStart(input.charCodeAt(pos))) { + if (!isIdentifierStart(input.uCharCodeAt(pos))) { error("Expected identifier in field access"); } tokens.push(readIdentifierTokenInInterpolation(stringStart)); - while (pos < length - 1 && input.charCodeAt(pos) == ".".code && isIdentifierStart(input.charCodeAt(pos + 1))) { + while (pos < length - 1 && input.uCharCodeAt(pos) == ".".code && isIdentifierStart(input.uCharCodeAt(pos + 1))) { final dotPos = makePositionRelativeTo(stringStart); advance(); - if (pos >= length || !isIdentifierStart(input.charCodeAt(pos))) { + if (pos >= length || !isIdentifierStart(input.uCharCodeAt(pos))) { break; } tokens.push(new Token(Dot, dotPos)); @@ -1950,12 +1951,12 @@ class Lexer { final startOffset = pos; while (pos < length) { - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); if (!isIdentifierPart(c)) break; advance(); } - final name = input.substr(startOffset, pos - startOffset); + final name = input.uSubstr(startOffset, pos - startOffset); final tokenType = KEYWORDS.exists(name) ? KEYWORDS.get(name) : Identifier(name); return new Token( tokenType, @@ -1974,7 +1975,7 @@ class Lexer { var i = stringStart.offset; while (i < pos) { - if (input.charCodeAt(i) == "\n".code) { + if (input.uCharCodeAt(i) == "\n".code) { line++; column = 1; } @@ -1997,12 +1998,12 @@ class Lexer { final contentStart = pos; while (pos < length) { - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); if (c == "\n".code || c == "\r".code) break; advance(); } - return makeToken(CommentLine(input.substr(contentStart, pos - contentStart)), start); + return makeToken(CommentLine(input.uSubstr(contentStart, pos - contentStart)), start); } /** @@ -2017,16 +2018,16 @@ class Lexer { var nestLevel = 1; while (pos < length && nestLevel > 0) { - if (input.charCodeAt(pos) == "*".code && peek() == "/".code) { + if (input.uCharCodeAt(pos) == "*".code && peek() == "/".code) { nestLevel--; if (nestLevel == 0) { - final content = input.substr(contentStart, pos - contentStart); + final content = input.uSubstr(contentStart, pos - contentStart); advance(2); return makeToken(CommentMultiLine(content), start); } advance(2); } - else if (input.charCodeAt(pos) == "/".code && peek() == "*".code) { + else if (input.uCharCodeAt(pos) == "/".code && peek() == "*".code) { nestLevel++; advance(2); } @@ -2047,16 +2048,16 @@ class Lexer { final start = makePosition(); final startPos = pos; - while (pos < length && isDigit(input.charCodeAt(pos))) { + while (pos < length && isDigit(input.uCharCodeAt(pos))) { advance(); } - if (pos < length && input.charCodeAt(pos) == ".".code && pos + 1 < length && isDigit(input.charCodeAt(pos + 1))) { + if (pos < length && input.uCharCodeAt(pos) == ".".code && pos + 1 < length && isDigit(input.uCharCodeAt(pos + 1))) { advance(); - while (pos < length && isDigit(input.charCodeAt(pos))) advance(); + while (pos < length && isDigit(input.uCharCodeAt(pos))) advance(); } - return makeToken(LNumber(Std.parseFloat(input.substr(startPos, pos - startPos))), start); + return makeToken(LNumber(Std.parseFloat(input.uSubstr(startPos, pos - startPos))), start); } /** @@ -2068,12 +2069,12 @@ class Lexer { final startPos = pos; while (pos < length) { - final c = input.charCodeAt(pos); + final c = input.uCharCodeAt(pos); if (!isIdentifierPart(c)) break; advance(); } - final word = input.substr(startPos, pos - startPos); + final word = input.uSubstr(startPos, pos - startPos); final tokenType = KEYWORDS.exists(word) ? KEYWORDS.get(word) : Identifier(word); return makeToken( tokenType, @@ -2109,7 +2110,7 @@ class Lexer { */ inline function advance(count:Int = 1) { while (count-- > 0 && pos < length) { - if (input.charCodeAt(pos) == "\n".code) { + if (input.uCharCodeAt(pos) == "\n".code) { line++; column = 1; } @@ -2126,7 +2127,7 @@ class Lexer { * @return Character code at the offset position, or 0 if beyond input length */ inline function peek(offset:Int = 1):Int { - return pos + offset < length ? input.charCodeAt(pos + offset) : 0; + return pos + offset < length ? input.uCharCodeAt(pos + offset) : 0; } /** @@ -2143,7 +2144,7 @@ class Lexer { */ function skipWhitespace() { while (pos < length) { - switch (input.charCodeAt(pos)) { + switch (input.uCharCodeAt(pos)) { case " ".code | "\t".code: advance(); case _: diff --git a/src/loreline/Node.hx b/src/loreline/Node.hx index da6ac97..2edf0cb 100644 --- a/src/loreline/Node.hx +++ b/src/loreline/Node.hx @@ -291,6 +291,11 @@ class NCharacterDecl extends AstNode { */ public var properties:Array; + /** + * Block style of this character + */ + public var style:BlockStyle; + /** * Creates a new character declaration node. * @param pos Position in source where this character appears @@ -303,6 +308,7 @@ class NCharacterDecl extends AstNode { super(id, pos, leadingComments, trailingComments); this.name = name; this.properties = properties; + this.style = Plain; } public override function each(handleNode:(node:Node, parent:Node)->Void):Void { @@ -325,6 +331,7 @@ class NCharacterDecl extends AstNode { final json = super.toJson(); json.name = name; json.properties = [for (prop in properties) prop.toJson()]; + json.style = style.toString(); return json; } @@ -345,6 +352,11 @@ class NBeatDecl extends AstNode { */ public var body:Array; + /** + * Block style of this beat + */ + public var style:BlockStyle; + /** * Creates a new beat declaration node. * @param pos Position in source where this beat appears @@ -357,6 +369,7 @@ class NBeatDecl extends AstNode { super(id, pos, leadingComments, trailingComments); this.name = name; this.body = body; + this.style = Plain; } public override function each(handleNode:(node:Node, parent:Node)->Void):Void { @@ -379,6 +392,7 @@ class NBeatDecl extends AstNode { final json = super.toJson(); json.name = name; json.body = [for (node in body) node.toJson()]; + json.style = style.toString(); return json; } @@ -406,6 +420,21 @@ enum StringPartType { } +enum abstract BlockStyle(Int) { + + var Plain = 0; + + var Braces = 1; + + public function toString() { + return switch abstract { + case Plain: "Plain"; + case Braces: "Braces"; + } + } + +} + /** * Represents a string part that can appear in string literals. */ @@ -487,6 +516,7 @@ class NStringLiteral extends NExpr { public function new(id:Int, pos:Position, quotes:Quotes, parts:Array, ?leadingComments:Array, ?trailingComments:Array) { super(id, pos, leadingComments, trailingComments); this.parts = parts; + this.quotes = quotes; } public override function each(handleNode:(node:Node, parent:Node)->Void):Void { @@ -511,10 +541,7 @@ class NStringLiteral extends NExpr { for (part in parts) part.toJson() ]; json.parts = parts; - json.quotes = switch quotes { - case Unquoted: "Unquoted"; - case DoubleQuotes: "DoubleQuotes"; - } + json.quotes = quotes.toString(); return json; } @@ -619,6 +646,11 @@ class NChoiceStatement extends AstNode { */ public var options:Array; + /** + * The block style of the choice statement + */ + public var style:BlockStyle; + /** * Creates a new choice statement node. * @param pos Position in source where this choice appears @@ -629,6 +661,7 @@ class NChoiceStatement extends AstNode { public function new(id:Int, pos:Position, options:Array, ?leadingComments:Array, ?trailingComments:Array) { super(id, pos, leadingComments, trailingComments); this.options = options; + this.style = Plain; } public override function each(handleNode:(node:Node, parent:Node)->Void):Void { @@ -650,6 +683,7 @@ class NChoiceStatement extends AstNode { public override function toJson():Dynamic { final json = super.toJson(); json.options = [for (option in options) option.toJson()]; + json.style = style.toString(); return json; } } @@ -673,6 +707,11 @@ class NChoiceOption extends AstNode { */ public var body:Array; + /** + * The block style of the body + */ + public var style:BlockStyle; + /** * Creates a new choice option node. * @param pos Position in source where this option appears @@ -687,15 +726,12 @@ class NChoiceOption extends AstNode { this.text = text; this.condition = condition; this.body = body; + this.style = Plain; } public override function each(handleNode:(node:Node, parent:Node)->Void):Void { super.each(handleNode); - if (condition != null) { - handleNode(condition, this); - condition.each(handleNode); - } if (body != null) { for (i in 0...body.length) { final child = body[i]; @@ -703,6 +739,14 @@ class NChoiceOption extends AstNode { child.each(handleNode); } } + if (text != null) { + handleNode(text, this); + text.each(handleNode); + } + if (condition != null) { + handleNode(condition, this); + condition.each(handleNode); + } } /** @@ -714,6 +758,7 @@ class NChoiceOption extends AstNode { json.text = text.toJson(); if (condition != null) json.condition = condition.toJson(); json.body = [for (node in body) node.toJson()]; + json.style = style.toString(); return json; } } @@ -728,6 +773,11 @@ class NBlock extends AstNode { */ public var body:Array; + /** + * The block style of the body. + */ + public var style:BlockStyle; + /** * Creates a new block node. * @param pos Position in source where this beat appears @@ -738,6 +788,7 @@ class NBlock extends AstNode { public function new(id:Int, pos:Position, body:Array, ?leadingComments:Array, ?trailingComments:Array) { super(id, pos, leadingComments, trailingComments); this.body = body; + this.style = Plain; } public override function each(handleNode:(node:Node, parent:Node)->Void):Void { @@ -759,6 +810,7 @@ class NBlock extends AstNode { public override function toJson():Dynamic { final json = super.toJson(); json.body = [for (node in body) node.toJson()]; + json.style = style.toString(); return json; } @@ -850,8 +902,10 @@ class NIfStatement extends AstNode { final json = super.toJson(); json.condition = condition.toJson(); json.thenBranch = [for (node in thenBranch.body) node.toJson()]; + json.thenStyle = thenBranch.style.toString(); if (elseBranch != null) { json.elseBranch = [for (node in elseBranch.body) node.toJson()]; + json.elseStyle = elseBranch.style.toString(); if ((elseLeadingComments != null && elseLeadingComments.length > 0) || (elseTrailingComments != null && elseTrailingComments.length > 0)) { final comments:Dynamic = json.comments ?? {}; if (elseLeadingComments != null && elseLeadingComments.length > 0) { @@ -1000,13 +1054,14 @@ class NLiteral extends NExpr { elem; } }]; - case Object: + case Object(style): if (value != null) { json.value = [for (field in (value:Array)) field.toJson()]; } else { json.value = []; } + json.style = style.toString(); case _: json.value = value; } @@ -1027,7 +1082,7 @@ enum LiteralType { /** Array literal */ Array; /** Object literal */ - Object; + Object(style:BlockStyle); } /** diff --git a/src/loreline/Parser.hx b/src/loreline/Parser.hx index 0797ec3..c1b222d 100644 --- a/src/loreline/Parser.hx +++ b/src/loreline/Parser.hx @@ -3,6 +3,8 @@ package loreline; import loreline.Lexer; import loreline.Node; +using loreline.Utf8; + /** * Represents a parsing error with position information. */ @@ -57,6 +59,12 @@ class Parser { this.lineBreakAfterToken = false; this.nextNodeId = 0; this.rootBeat = null; + + trace('--'); + for (token in tokens) { + trace(token.toString()); + } + trace('--'); } /** @@ -111,12 +119,12 @@ class Parser { switch (tokens[current].type) { case CommentLine(content): if (pendingComments == null) pendingComments = []; - pendingComments.push(new Comment(nextNodeId++, tokens[current].pos, content, false)); + pendingComments.push(new Comment(nextNodeId++, currentPos(), content, false)); case CommentMultiLine(content): if (pendingComments == null) pendingComments = []; - pendingComments.push(new Comment(nextNodeId++, tokens[current].pos, content, true)); + pendingComments.push(new Comment(nextNodeId++, currentPos(), content, true)); case LineBreak: - lastLineBreak = tokens[current].pos; + lastLineBreak = currentPos(); lineBreakAfterToken = true; case _: } @@ -134,6 +142,45 @@ class Parser { return tokens[current - 1]; } + /** + * Gets the previously consumed token that was an identifier. + * @return The previous token that was an identifier + */ + function prevIdentifier():Token { + var n = current - 1; + while (n >= 0) { + switch tokens[n].type { + case Identifier(_): + return tokens[n]; + case _: + } + n--; + } + return null; + } + + /** + * Gets the previously consumed token that was not a white space or comment. + * @return The previous token that was an identifier + */ + function prevNonWhitespaceOrComment():Token { + var n = current - 1; + while (n >= 0) { + switch tokens[n].type { + case CommentLine(_) | CommentMultiLine(_) | Indent | Unindent | LineBreak: + // Skip + case _: + return tokens[n]; + } + n--; + } + return null; + } + + function currentPos():Position { + return tokens[current]?.pos ?? new Position(1, 1, 0, 0); + } + /** * Checks if the current token matches the expected type. * @param type TokenType to check against @@ -193,7 +240,7 @@ class Parser { nextNodeId = 1; } - final startPos = tokens[current].pos; + final startPos = currentPos(); final nodes = []; final script = new Script(nextNodeId++, startPos, nodes); @@ -228,7 +275,7 @@ class Parser { if (isComment(tokens[current].type)) { pendingComments.push(new Comment( nextNodeId++, - tokens[current].pos, + currentPos(), switch(tokens[current].type) { case CommentLine(content): content; case CommentMultiLine(content): content; @@ -238,10 +285,13 @@ class Parser { )); } advance(); + if (isAtEnd()) { + throw new ParseError("Unexpected end of file", currentPos()); + } } if (isAtEnd()) { - throw new ParseError("Unexpected end of file", tokens[current].pos); + throw new ParseError("Unexpected end of file", currentPos()); } inline function ensureInBeat(node:AstNode):AstNode { @@ -254,7 +304,7 @@ class Parser { case KwNew: advance(); if (!check(KwState)) { - throw new ParseError("Expected 'state' after 'new'", tokens[current].pos); + throw new ParseError("Expected 'state' after 'new'", currentPos()); } ensureInBeat(parseStateDecl(true)); case KwBeat: parseBeatDecl(); @@ -267,7 +317,7 @@ class Parser { case KwIf: ensureInBeat(parseIfStatement()); case Arrow: ensureInBeat(parseTransition()); case _: - throw new ParseError('Unexpected token: ${tokens[current].type}', tokens[current].pos); + throw new ParseError('Unexpected token: ${tokens[current].type}', currentPos()); } } @@ -276,7 +326,7 @@ class Parser { var body:Array; var result = null; if (rootBeat == null) { - final startPos = tokens[current].pos; + final startPos = currentPos(); body = []; rootBeat = new NBeatDecl(nextNodeId++, startPos, "_", body); result = rootBeat; @@ -296,14 +346,14 @@ class Parser { * @return Import statement node */ function parseImport():NImport { - final startPos = tokens[current].pos; + final startPos = currentPos(); final imp = new NImport(nextNodeId++, startPos, null); expect(KwImport); final path = switch tokens[current].type { case LString(s, _): s; - case _: throw new ParseError("Expected string literal for import path", tokens[current].pos); + case _: throw new ParseError("Expected string literal for import path", currentPos()); } advance(); @@ -318,13 +368,13 @@ class Parser { * @return Dialogue statement node */ function parseDialogueStatement():NDialogueStatement { - final startPos = tokens[current].pos; + final startPos = currentPos(); final dialogue = new NDialogueStatement(nextNodeId++, startPos, null, null); // Parse character name dialogue.character = switch (tokens[current].type) { case Identifier(name): name; - case _: throw new ParseError("Expected character name", tokens[current].pos); + case _: throw new ParseError("Expected character name", currentPos()); }; advance(); // Move past identifier @@ -335,6 +385,9 @@ class Parser { // Parse dialogue content dialogue.content = parseStringLiteral(); + // Update position + dialogue.pos = dialogue.pos.extendedTo(dialogue.content.pos); + return dialogue; } @@ -342,10 +395,9 @@ class Parser { * Parses a block of statements enclosed in braces. * @return Array of parsed statement nodes */ - function parseStatementBlock():Array { + function parseStatementBlock(statements:Array):BlockStyle { final blockEnd:TokenType = parseBlockStart().type == Indent ? Unindent : RBrace; - final statements:Array = []; while (!check(blockEnd) && !isAtEnd()) { // Handle line breaks and comments @@ -367,7 +419,9 @@ class Parser { } expect(blockEnd); - return statements; + + return (blockEnd == RBrace) ? Braces : Plain; + } /** @@ -376,7 +430,7 @@ class Parser { * @return State declaration node */ function parseStateDecl(temporary:Bool):NStateDecl { - final startPos = tokens[current].pos; + final startPos = currentPos(); final stateNode = new NStateDecl(nextNodeId++, startPos, temporary, null); expect(KwState); @@ -387,6 +441,8 @@ class Parser { // Parse state fields stateNode.fields = cast(parseObjectLiteral(), NLiteral); + stateNode.pos = stateNode.pos.extendedTo(prevNonWhitespaceOrComment().pos); + return stateNode; } @@ -395,7 +451,7 @@ class Parser { * @return Object field node */ function parseObjectField():NObjectField { - final startPos = tokens[current].pos; + final startPos = currentPos(); final name = expectIdentifier(); final objectField = new NObjectField(nextNodeId++, startPos, name, null); @@ -412,13 +468,17 @@ class Parser { * @return Beat declaration node */ function parseBeatDecl():NBeatDecl { - final startPos = tokens[current].pos; + final startPos = currentPos(); final beatNode = new NBeatDecl(nextNodeId++, startPos, null, [], []); expect(KwBeat); + beatNode.pos = startPos.extendedTo(currentPos()); + // final namePos = currentPos(); + // beatNode.pos.length += namePos.offset + namePos.length - (beatNode.pos.offset + beatNode.pos.length); beatNode.name = expectIdentifier(); final blockEnd:TokenType = parseBlockStart().type == Indent ? Unindent : RBrace; + beatNode.style = (blockEnd == RBrace) ? Braces : Plain; attachComments(beatNode); @@ -432,6 +492,8 @@ class Parser { while (match(LineBreak) || (blockEnd != Unindent && match(Unindent))) {} expect(blockEnd); + beatNode.pos = beatNode.pos.extendedTo(prevNonWhitespaceOrComment().pos); + return beatNode; } @@ -506,7 +568,7 @@ class Parser { return indentToken; } else { - throw new ParseError('Expected ${TokenType.LBrace} or ${TokenType.Indent}, got ${tokens[current].type}', tokens[current].pos); + throw new ParseError('Expected ${TokenType.LBrace} or ${TokenType.Indent}, got ${tokens[current].type}', currentPos()); } } @@ -516,13 +578,14 @@ class Parser { * @return Character declaration node */ function parseCharacterDecl():NCharacterDecl { - final startPos = tokens[current].pos; + final startPos = currentPos(); final characterNode = new NCharacterDecl(nextNodeId++, startPos, null, []); expect(KwCharacter); characterNode.name = expectIdentifier(); final blockEnd:TokenType = parseBlockStart().type == Indent ? Unindent : RBrace; + characterNode.style = (blockEnd == RBrace) ? Braces : Plain; attachComments(characterNode); @@ -536,6 +599,8 @@ class Parser { while (match(LineBreak) || (blockEnd != Unindent && match(Unindent))) {} expect(blockEnd); + characterNode.pos = characterNode.pos.extendedTo(prevNonWhitespaceOrComment().pos); + return characterNode; } @@ -544,7 +609,7 @@ class Parser { * @return Text statement node */ function parseTextStatement():NTextStatement { - final startPos = tokens[current].pos; + final startPos = currentPos(); final statement = attachComments(new NTextStatement(nextNodeId++, startPos, null)); statement.content = parseStringLiteral(); return statement; @@ -555,12 +620,13 @@ class Parser { * @return Choice statement node */ function parseChoiceStatement():NChoiceStatement { - final startPos = tokens[current].pos; + final startPos = currentPos(); final choiceNode = new NChoiceStatement(nextNodeId++, startPos, []); expect(KwChoice); final blockEnd:TokenType = parseBlockStart().type == Indent ? Unindent : RBrace; + choiceNode.style = (blockEnd == RBrace) ? Braces : Plain; attachComments(choiceNode); @@ -573,6 +639,9 @@ class Parser { expect(blockEnd); + // Update statement position to wrap all its sub expressions + choiceNode.pos = choiceNode.pos.extendedTo(prevNonWhitespaceOrComment().pos); + return choiceNode; } @@ -581,7 +650,7 @@ class Parser { * @return Choice option node */ function parseChoiceOption(blockEnd:TokenType):NChoiceOption { - final startPos = tokens[current].pos; + final startPos = currentPos(); final choiceOption = attachComments(new NChoiceOption(nextNodeId++, startPos, null, null, [])); choiceOption.text = parseStringLiteral(); @@ -592,12 +661,16 @@ class Parser { // Parse option body if (checkBlockStart()) { - choiceOption.body = parseStatementBlock(); + choiceOption.body = []; + choiceOption.style = parseStatementBlock(choiceOption.body); } else if (!check(blockEnd)) { // If not end of choice choiceOption.body = [parseNode()]; + choiceOption.style = Plain; } + choiceOption.pos = choiceOption.pos.extendedTo(prevNonWhitespaceOrComment().pos); + return choiceOption; } @@ -626,6 +699,7 @@ class Parser { final op = previous().type; final assignment = attachComments(new NAssign(nextNodeId++, expr.pos, expr, op, null)); assignment.value = parseExpression(); + assignment.pos = assignment.pos.extendedTo(assignment.value.pos); return assignment; } @@ -650,7 +724,7 @@ class Parser { * @return If statement node */ function parseIfStatement():NIfStatement { - final startPos = tokens[current].pos; + final startPos = currentPos(); final ifNode = new NIfStatement(nextNodeId++, startPos, null, null, null); expect(KwIf); @@ -659,25 +733,31 @@ class Parser { while (match(LineBreak)) {} attachComments(ifNode); - ifNode.thenBranch = new NBlock(nextNodeId++, tokens[current].pos, null); - ifNode.thenBranch.body = parseStatementBlock(); + ifNode.thenBranch = new NBlock(nextNodeId++, currentPos(), null); + ifNode.thenBranch.body = []; + ifNode.thenBranch.style = parseStatementBlock(ifNode.thenBranch.body); // Handle optional else clause var elseToken = tokens[current]; - if (elseToken.type == KwElse) { + if (elseToken != null && elseToken.type == KwElse) { advance(); while (match(LineBreak)) {} attachElseComments(ifNode, elseToken); if (check(KwIf)) { - ifNode.elseBranch = new NBlock(nextNodeId++, tokens[current].pos, null); + ifNode.elseBranch = new NBlock(nextNodeId++, currentPos(), null); ifNode.elseBranch.body = [parseIfStatement()]; + ifNode.elseBranch.style = Plain; } else { - ifNode.elseBranch = new NBlock(nextNodeId++, tokens[current].pos, null); - ifNode.elseBranch.body = parseStatementBlock(); + ifNode.elseBranch = new NBlock(nextNodeId++, currentPos(), null); + ifNode.elseBranch.body = []; + ifNode.elseBranch.style = parseStatementBlock(ifNode.elseBranch.body); } } + // Update statement position to wrap all its sub expressions + ifNode.pos = ifNode.pos.extendedTo(prevNonWhitespaceOrComment().pos); + return ifNode; } @@ -686,16 +766,16 @@ class Parser { * @return Transition node */ function parseTransition():NTransition { - final startPos = tokens[current].pos; + final startPos = currentPos(); expect(Arrow); - // Handle "end of stream" (->.) + // Handle "end of stream" (-> .) if (match(Dot)) { - return attachComments(new NTransition(nextNodeId++, startPos, ".")); + return attachComments(new NTransition(nextNodeId++, startPos.extendedTo(prevNonWhitespaceOrComment().pos), ".")); } final target = expectIdentifier(); - return attachComments(new NTransition(nextNodeId++, startPos, target)); + return attachComments(new NTransition(nextNodeId++, startPos.extendedTo(prevNonWhitespaceOrComment().pos), target)); } /** @@ -712,6 +792,7 @@ class Parser { advance(); final assignment = attachComments(new NAssign(nextNodeId++, expr.pos, expr, op, null)); assignment.value = parseExpression(); + assignment.pos = assignment.pos.extendedTo(assignment.value.pos); return assignment; } @@ -725,10 +806,11 @@ class Parser { function parseLogicalOr():NExpr { var expr = parseLogicalAnd(); - while (match(OpOr)) { + while (match(OpOr(false))) { final op = previous().type; final binary = attachComments(new NBinary(nextNodeId++, expr.pos, expr, op, null)); binary.right = parseLogicalAnd(); + binary.pos = binary.pos.extendedTo(binary.right.pos); expr = binary; } @@ -742,10 +824,11 @@ class Parser { function parseLogicalAnd():NExpr { var expr = parseEquality(); - while (match(OpAnd)) { + while (match(OpAnd(false))) { final op = previous().type; final binary = attachComments(new NBinary(nextNodeId++, expr.pos, expr, op, null)); binary.right = parseEquality(); + binary.pos = binary.pos.extendedTo(binary.right.pos); expr = binary; } @@ -763,6 +846,7 @@ class Parser { final op = previous().type; final binary = attachComments(new NBinary(nextNodeId++, expr.pos, expr, op, null)); binary.right = parseComparison(); + binary.pos = binary.pos.extendedTo(binary.right.pos); expr = binary; } @@ -780,6 +864,7 @@ class Parser { final op = previous().type; final binary = attachComments(new NBinary(nextNodeId++, expr.pos, expr, op, null)); binary.right = parseAdditive(); + binary.pos = binary.pos.extendedTo(binary.right.pos); expr = binary; } @@ -797,6 +882,7 @@ class Parser { final op = previous().type; final binary = attachComments(new NBinary(nextNodeId++, expr.pos, expr, op, null)); binary.right = parseMultiplicative(); + binary.pos = binary.pos.extendedTo(binary.right.pos); expr = binary; } @@ -814,6 +900,7 @@ class Parser { final op = previous().type; final binary = attachComments(new NBinary(nextNodeId++, expr.pos, expr, op, null)); binary.right = parseUnary(); + binary.pos = binary.pos.extendedTo(binary.right.pos); expr = binary; } @@ -827,8 +914,9 @@ class Parser { function parseUnary():NExpr { if (match(OpNot) || match(OpMinus)) { final op = previous().type; - final unary = attachComments(new NUnary(nextNodeId++, tokens[current].pos, op, null)); + final unary = attachComments(new NUnary(nextNodeId++, previous().pos, op, null)); unary.operand = parseUnary(); + unary.pos = unary.pos.extendedTo(unary.operand.pos); return unary; } @@ -840,7 +928,7 @@ class Parser { * @return Expression node */ function parsePrimary():NExpr { - final startPos = tokens[current].pos; + final startPos = currentPos(); return switch (tokens[current].type) { case LString(_, _): @@ -875,7 +963,7 @@ class Parser { expr; case _: - throw new ParseError("Unexpected token in expression", tokens[current].pos); + throw new ParseError("Unexpected token in expression", currentPos()); } } @@ -885,11 +973,18 @@ class Parser { * @return String literal node */ function parseStringLiteral():NStringLiteral { - final startPos = tokens[current].pos; + final stringLiteralPos = currentPos(); final parts = new Array(); switch (tokens[current].type) { case LString(quotes, content, attachments): + final startPos = if (quotes != Unquoted) { + stringLiteralPos.withOffset(content, 1, stringLiteralPos.length - 2); + } + else { + stringLiteralPos; + } + var currentPos = 0; // Handle simple strings without attachments @@ -900,10 +995,10 @@ class Parser { final partId = nextNodeId++; parts.push(new NStringPart(partId, partPos, Raw(content))); advance(); - return attachComments(new NStringLiteral(literalId, startPos, quotes, parts)); + return attachComments(new NStringLiteral(literalId, stringLiteralPos, quotes, parts)); } - final stringLiteral = attachComments(new NStringLiteral(nextNodeId++, startPos, quotes, parts)); + final stringLiteral = attachComments(new NStringLiteral(nextNodeId++, stringLiteralPos, quotes, parts)); // Process string with attachments (interpolations and tags) for (i in 0...attachments.length) { @@ -918,7 +1013,7 @@ class Parser { final partPos = makeStringPartPosition(startPos, content, currentPos); partPos.length = start - currentPos; parts.push(new NStringPart(nextNodeId++, partPos, Raw( - content.substr(currentPos, start - currentPos) + content.uSubstr(currentPos, start - currentPos) ))); } @@ -944,7 +1039,7 @@ class Parser { final partPos = makeStringPartPosition(startPos, content, currentPos); partPos.length = start - currentPos; parts.push(new NStringPart(nextNodeId++, partPos, Raw( - content.substr(currentPos, start - currentPos) + content.uSubstr(currentPos, start - currentPos) ))); } @@ -963,11 +1058,11 @@ class Parser { } // Add remaining text after last attachment - if (currentPos < content.length) { + if (currentPos < content.uLength()) { final partPos = makeStringPartPosition(startPos, content, currentPos); - partPos.length = content.length - currentPos; + partPos.length = content.uLength() - currentPos; parts.push(new NStringPart(nextNodeId++, partPos, Raw( - content.substr(currentPos) + content.uSubstr(currentPos) ))); } @@ -975,13 +1070,13 @@ class Parser { return stringLiteral; case _: - throw new ParseError('Expected string, got ${tokens[current].type}', tokens[current].pos); + throw new ParseError('Expected string, got ${tokens[current].type}', currentPos()); } } /** * Creates a Position object for a part of a string literal. - * @param stringStart Starting position of the entire string + * @param stringStart Starting position of the entire string content * @param content String content * @param offset Offset within the string * @return Position object for the string part @@ -992,7 +1087,7 @@ class Parser { // Track line and column numbers for (i in 0...offset) { - if (content.charCodeAt(i) == "\n".code) { + if (content.uCharCodeAt(i) == "\n".code) { line++; column = 1; } @@ -1011,12 +1106,28 @@ class Parser { * @param name Identifier name * @return Access expression node */ - function makeAccess(pos:Position, target:Null, name:String):NAccess { - if (pos.length == 0 && name != null && name.length > 0) { - pos = new Position( - pos.line, pos.column, pos.offset, - name.length - ); + function makeAccess(pos:Position, target:Null, name:String, namePos:Position):NAccess { + if (name != null && name.uLength() > 0) { + if (target != null) { + if (namePos != null) { + pos = new Position( + target.pos.line, target.pos.column, target.pos.offset, + namePos.offset + name.uLength() - target.pos.offset + ); + } + else { + throw new ParseError("Invalid access (missing name position)", pos); + } + } + else if (pos.length == 0) { + pos = new Position( + pos.line, pos.column, pos.offset, + namePos?.length ?? name.uLength() + ); + } + } + else { + throw new ParseError("Invalid access: " + (name != null ? "'" + name + "'" : "null"), pos); } return new NAccess(nextNodeId++, pos, target, name); @@ -1050,10 +1161,10 @@ class Parser { switch (tokens[i].type) { case Identifier(name): if (target == null) { - target = attachComments(makeAccess(tokens[i].pos, null, name)); + target = attachComments(makeAccess(tokens[i].pos, null, name, null)); } else if (prevIsDot) { - target = attachComments(makeAccess(tokens[i].pos, target, name)); + target = attachComments(makeAccess(tokens[i].pos, target, name, tokens[i].pos)); } else { throw new ParseError("Missing dot in simple interpolation", tokens[i].pos); @@ -1070,7 +1181,9 @@ class Parser { // Handle complex interpolation with braces (${expression}) else { final tempParser = new Parser(tokens); + tempParser.nextNodeId = nextNodeId; expr = tempParser.parseExpression(); + nextNodeId = tempParser.nextNodeId; if (!tempParser.isAtEnd()) { throw new ParseError("Unexpected tokens after interpolation expression", tempParser.tokens[tempParser.current].pos); @@ -1096,9 +1209,13 @@ class Parser { * @return StringPart representing the tag */ function parseStringTag(closing:Bool, start:Int, length:Int, content:String, quotes:Quotes, attachments:Array):NStringPart { - final pos = makeStringPartPosition(tokens[current].pos, content, start); + var pos = makeStringPartPosition(currentPos(), content, start); pos.length = length; + if (quotes != Unquoted) { + pos = pos.withOffset(content, 1, pos.length); + } + // Calculate tag content boundaries final offsetStart = (closing ? 2 : 1); final innerStart = start + offsetStart; // Skip < and optional / @@ -1134,15 +1251,15 @@ class Parser { attachComments(new NStringLiteral( literalId, partPos, - quotes, - [new NStringPart(partId, partPos, Raw(content.substr(innerStart, innerLength)))] + Unquoted, + [new NStringPart(partId, partPos, Raw(content.uSubstr(innerStart, innerLength)))] )) )); } // Process tag with interpolations final parts = new Array(); - final stringLiteral = attachComments(new NStringLiteral(nextNodeId++, pos, quotes, parts)); + final stringLiteral = attachComments(new NStringLiteral(nextNodeId++, pos, Unquoted, parts)); var currentPos = innerStart; // Process each attachment within tag bounds @@ -1154,10 +1271,10 @@ class Parser { if (aStart >= innerStart && aEnd <= innerEnd) { // Add raw text before interpolation if (aStart > currentPos) { - final partPos = makeStringPartPosition(pos, content.substr(start), currentPos - start + offsetStart); + final partPos = makeStringPartPosition(pos, content.uSubstr(start), currentPos - start + offsetStart); partPos.length = aStart - currentPos; parts.push(new NStringPart(nextNodeId++, partPos, Raw( - content.substr(currentPos, aStart - currentPos) + content.uSubstr(currentPos, aStart - currentPos) ))); } @@ -1181,10 +1298,10 @@ class Parser { // Add remaining raw text if (currentPos < innerEnd) { - final partPos = makeStringPartPosition(pos, content.substr(start), currentPos - start + offsetStart); + final partPos = makeStringPartPosition(pos, content.uSubstr(start), currentPos - start + offsetStart); partPos.length = (innerStart + innerEnd) - currentPos; parts.push(new NStringPart(nextNodeId++, partPos, Raw( - content.substr(currentPos, innerEnd - currentPos) + content.uSubstr(currentPos, innerEnd - currentPos) ))); } @@ -1201,22 +1318,22 @@ class Parser { * @return Expression node */ function parseIdentifierExpression(startPos:Position, name:String):NExpr { - var expr:NExpr = attachComments(makeAccess(startPos, null, name)); + var expr:NExpr = attachComments(makeAccess(startPos, null, name, null)); // Parse chained accesses (., [], and ()) while (true) { if (match(Dot)) { final prop = expectIdentifier(); - expr = attachComments(makeAccess(startPos, expr, prop)); + expr = attachComments(makeAccess(startPos, expr, prop, prevIdentifier().pos)); } else if (match(LBracket)) { final index = parseExpression(); expect(RBracket); - expr = attachComments(new NArrayAccess(nextNodeId++, startPos, expr, index)); + expr = attachComments(new NArrayAccess(nextNodeId++, startPos.extendedTo(previous().pos), expr, index)); } else if (match(LParen)) { final args = parseCallArguments(); - expr = attachComments(new NCall(nextNodeId++, startPos, expr, args)); + expr = attachComments(new NCall(nextNodeId++, startPos.extendedTo(previous().pos), expr, args)); } else { break; @@ -1231,7 +1348,7 @@ class Parser { * @return Expression node for array literal */ function parseArrayLiteral():NExpr { - final startPos = tokens[current].pos; + final startPos = currentPos(); final elements = []; final literal = new NLiteral(nextNodeId++, startPos, elements, Array); expect(LBracket); @@ -1255,7 +1372,7 @@ class Parser { } if (!check(RBracket) && needsSeparator) { - throw new ParseError("Expected comma or line break between elements", tokens[current].pos); + throw new ParseError("Expected comma or line break between elements", currentPos()); } while (match(LineBreak)) {} @@ -1279,11 +1396,12 @@ class Parser { * @return Expression node for object literal */ function parseObjectLiteral():NExpr { - final startPos = tokens[current].pos; + final startPos = currentPos(); final fields = []; - final literal = new NLiteral(nextNodeId++, startPos, fields, Object); final blockEnd:TokenType = parseBlockStart().type == Indent ? Unindent : RBrace; + final style:BlockStyle = (blockEnd == RBrace) ? Braces : Plain; + final literal = new NLiteral(nextNodeId++, startPos, fields, Object(style)); attachComments(literal); @@ -1304,7 +1422,7 @@ class Parser { } if (!check(blockEnd) && needsSeparator) { - throw new ParseError("Expected comma or line break between fields", tokens[current].pos); + throw new ParseError("Expected comma or line break between fields", currentPos()); } while (match(LineBreak) || (blockEnd != Unindent && match(Unindent))) {} @@ -1396,7 +1514,7 @@ class Parser { advance(); name; case _: - throw new ParseError('Expected identifier, got ${tokens[current].type}', tokens[current].pos); + throw new ParseError('Expected identifier, got ${tokens[current].type}', currentPos()); } } diff --git a/src/loreline/Position.hx b/src/loreline/Position.hx index b512ff0..96c9881 100644 --- a/src/loreline/Position.hx +++ b/src/loreline/Position.hx @@ -1,5 +1,7 @@ package loreline; +using loreline.Utf8; + /** * Represents a position within source code, tracking line number, column, and offset information. * Used throughout the compiler to pinpoint locations of tokens, nodes, and error messages. @@ -49,7 +51,7 @@ class Position { * @return String in format "(line X col Y)" */ public function toString():String { - return '(line $line col $column)'; + return '($line:$column:$offset:$length)'; } /** @@ -69,4 +71,90 @@ class Position { return json; } + /** + * Creates a new position that is offset from this position. + * This maintains the same line/column tracking but with an adjusted offset. + * Supports both positive and negative offsets. + * @param content String content to analyze for line/column tracking + * @param additionalOffset Number of characters to offset from current position (can be negative) + * @param newLength Optional new length for the offset position (default: 0) + * @return New Position object at the offset location + */ + public function withOffset(content:String, additionalOffset:Int, newLength:Int = 0):Position { + // Handle zero offset + if (additionalOffset == 0) { + return new Position(line, column, offset, newLength); + } + + var currentLine = line; + var currentColumn = column; + var currentOffset = offset; + + if (additionalOffset > 0) { + // Moving forward in the text + var chars = 0; + while (chars < additionalOffset) { + if (currentOffset < content.uLength() && content.uCharCodeAt(currentOffset) == '\n'.code) { + currentLine++; + currentColumn = 1; + } else { + currentColumn++; + } + chars++; + currentOffset++; + } + } else { + // Moving backward in the text + var chars = 0; + while (chars > additionalOffset) { + currentOffset--; + if (currentOffset >= 0 && content.uCharCodeAt(currentOffset) == '\n'.code) { + currentLine--; + // Need to scan backward to find the previous line's length + var col = 1; + var scanPos = currentOffset - 1; + while (scanPos >= 0) { + var c = content.uCharCodeAt(scanPos); + if (c == '\n'.code) break; + col++; + scanPos--; + } + currentColumn = col; + } else { + currentColumn--; + } + chars--; + } + } + + // Ensure we don't go before the start of the file + if (currentOffset < 0) { + currentOffset = 0; + currentLine = 1; + currentColumn = 1; + } + + return new Position( + currentLine, + currentColumn, + currentOffset, + newLength + ); + } + + /** + * Creates a new position that extends from this position's start to another position's end. + * Useful for creating spans that encompass multiple tokens or nodes. + * @param endPos Position marking the end of the span + * @return New Position object representing the extended span + */ + public function extendedTo(endPos:Position):Position { + return new Position( + line, + column, + offset, + (endPos.offset + endPos.length) - offset + ); + } + } \ No newline at end of file diff --git a/src/loreline/Printer.hx b/src/loreline/Printer.hx index a6c1c94..d54c75f 100644 --- a/src/loreline/Printer.hx +++ b/src/loreline/Printer.hx @@ -3,6 +3,8 @@ package loreline; import loreline.Lexer; import loreline.Node; +using loreline.Utf8; + /** * A code printer that converts AST nodes back into formatted Loreline source code. * Handles indentation, newlines, and pretty-printing of all node types. @@ -44,6 +46,11 @@ class Printer { */ final _newline:String; + /** + * Set to `false` to ignore comments. + */ + public var enableComments:Bool = true; + /** * Creates a new code printer with customizable formatting options. * @param indent String used for each level of indentation (default: 4 spaces) @@ -77,16 +84,16 @@ class Printer { * @return This printer instance for chaining */ public function write(s:String) { - if (s.length > 0) { + if (s.uLength() > 0) { if (_beginLine > 0) { tab(); _beginLine = 0; } _buf.add(s); - _lastChar = s.charCodeAt(s.length - 1); - var i = s.length - 1; + _lastChar = s.uCharCodeAt(s.uLength() - 1); + var i = s.uLength() - 1; while (i >= 0) { - final c = s.charCodeAt(i); + final c = s.uCharCodeAt(i); if (c != ' '.code && c != '\n'.code && c != '\r'.code && c != '\t'.code) { _lastVisibleChar = c; break; @@ -180,7 +187,7 @@ class Printer { */ public inline function clear() { _level = 0; - _buf = new StringBuf(); + _buf = new loreline.Utf8.Utf8Buf(); } /** @@ -255,7 +262,7 @@ class Printer { * @param node Node with potential comments */ function printLeadingComments(node:AstNode) { - if (node.leadingComments != null) { + if (enableComments && node.leadingComments != null) { for (comment in node.leadingComments) { if (comment.multiline) { writeln('/*${comment.content}*/'); @@ -272,7 +279,7 @@ class Printer { * @param node Node with potential comments */ function printTrailingComments(node:AstNode) { - if (node.trailingComments != null) { + if (enableComments && node.trailingComments != null) { for (comment in node.trailingComments) { if (_lastChar != ' '.code && _beginLine == 0) { write(' '); @@ -523,7 +530,8 @@ class Printer { * @param str String literal to print * @param surroundWithQuotes Whether to add quotation marks */ - function printStringLiteral(str:NStringLiteral, surroundWithQuotes:Bool = true) { + function printStringLiteral(str:NStringLiteral) { + final surroundWithQuotes = (str.quotes == DoubleQuotes); if (surroundWithQuotes) { printLeadingComments(str); write('"'); @@ -540,7 +548,7 @@ class Printer { if (needsBraces) write('}'); case Tag(closing, content): write(closing ? ''); } } @@ -572,7 +580,7 @@ class Printer { printNode(elem); } write(']'); - case Object: + case Object(_): writeln('{'); indent(); var first = true; @@ -647,7 +655,10 @@ class Printer { */ function printBinary(binary:NBinary, skipParen:Bool = false) { printLeadingComments(binary); - final needsParens = !skipParen && binary.op.match(OpAnd | OpOr); + final needsParens = !skipParen && switch binary.op { + case OpAnd(word) | OpOr(word): true; + case _: false; + }; if (needsParens) write('('); printNode(binary.left); write(' ${getOperator(binary.op)} '); @@ -718,8 +729,10 @@ class Printer { case OpLess: "<"; case OpGreaterEq: ">="; case OpLessEq: "<="; - case OpAnd: "&&"; - case OpOr: "||"; + case OpAnd(false): "&&"; + case OpOr(false): "||"; + case OpAnd(true): "and"; + case OpOr(true): "or"; case OpNot: "!"; case _: throw 'Unsupported operator: $op'; } diff --git a/src/loreline/Quotes.hx b/src/loreline/Quotes.hx index 496c102..3de77b5 100644 --- a/src/loreline/Quotes.hx +++ b/src/loreline/Quotes.hx @@ -6,4 +6,11 @@ enum abstract Quotes(Int) { var DoubleQuotes = 1; + public function toString() { + return switch abstract { + case Unquoted: "Unquoted"; + case DoubleQuotes: "DoubleQuotes"; + } + } + } diff --git a/src/loreline/Utf8.hx b/src/loreline/Utf8.hx new file mode 100644 index 0000000..8059f81 --- /dev/null +++ b/src/loreline/Utf8.hx @@ -0,0 +1,161 @@ +package loreline; + +#if neko +import neko.Utf8; +#end + +/** + * UTF-8 aware string operations that can be used as extension methods. + * Use with: using loreline.Utf8; + */ +class Utf8 { + + /** + * UTF-8 aware string length + */ + public static inline function uLength(str:String):Int { + #if neko + return neko.Utf8.length(str); + #else + return str.length; + #end + } + + /** + * UTF-8 aware substring extraction + */ + public static inline function uSubstr(str:String, pos:Int, ?len:Int):String { + #if neko + return neko.Utf8.sub(str, pos, len != null ? len : (neko.Utf8.length(str) - pos)); + #else + return str.substr(pos, len); + #end + } + + /** + * UTF-8 aware substring with end position + */ + public static #if !neko inline #end function uSubstring(str:String, startIndex:Int, ?endIndex:Int):String { + #if neko + final len = neko.Utf8.length(str); + final end = (endIndex != null) ? endIndex : len; + return neko.Utf8.sub(str, startIndex, end - startIndex); + #else + return str.substring(startIndex, endIndex); + #end + } + + /** + * UTF-8 aware character code at position + */ + public static #if !neko inline #end function uCharCodeAt(str:String, pos:Int):Null { + #if neko + if (pos < 0 || pos >= neko.Utf8.length(str)) return null; + return neko.Utf8.charCodeAt(str, pos); + #else + return str.charCodeAt(pos); + #end + } + + /** + * UTF-8 aware indexOf + */ + public static #if !neko inline #end function uIndexOf(str:String, substr:String, ?startIndex:Int):Int { + #if neko + if (startIndex == null) startIndex = 0; + final strLen = neko.Utf8.length(str); + final subLen = neko.Utf8.length(substr); + if (subLen == 0) return startIndex; + if (startIndex < 0) startIndex = 0; + if (startIndex >= strLen) return -1; + + for (i in startIndex...(strLen - subLen + 1)) { + if (neko.Utf8.sub(str, i, subLen) == substr) return i; + } + return -1; + #else + return str.indexOf(substr, startIndex); + #end + } + + /** + * UTF-8 aware lastIndexOf + */ + public static #if !neko inline #end function uLastIndexOf(str:String, substr:String, ?startIndex:Int):Int { + #if neko + final strLen = neko.Utf8.length(str); + final subLen = neko.Utf8.length(substr); + if (subLen == 0) return startIndex ?? strLen; + if (startIndex == null || startIndex >= strLen) startIndex = strLen - subLen; + if (startIndex < 0) return -1; + + var i = startIndex; + while (i >= 0) { + if (neko.Utf8.sub(str, i, subLen) == substr) return i; + i--; + } + return -1; + #else + return str.lastIndexOf(substr, startIndex); + #end + } + + /** + * UTF-8 aware string to char array + */ + public static #if !neko inline #end function uToChars(str:String):Array { + #if neko + final len = neko.Utf8.length(str); + return [for (i in 0...len) neko.Utf8.sub(str, i, 1)]; + #else + return str.split(""); + #end + } + + /** + * UTF-8 aware charAt + */ + public static inline function uCharAt(str:String, pos:Int):String { + #if neko + if (pos < 0 || pos >= neko.Utf8.length(str)) return ""; + return neko.Utf8.sub(str, pos, 1); + #else + return str.charAt(pos); + #end + } + +} + +#if neko +class Utf8Buf { + public var length(get, never):Int; + inline function get_length():Int { + return codes.length; + } + final codes:Array = []; + + public function new() {} + + public inline function add(x:Dynamic):Void { + final str = Std.string(x); + final len = neko.Utf8.length(str); + for (i in 0...len) { + addChar(neko.Utf8.charCodeAt(str, i)); + } + } + + public inline function addChar(c:Int):Void { + codes.push(c); + } + + public function toString():String { + final buf = new neko.Utf8(); + for (code in codes) { + buf.addChar(code); + } + return buf.toString(); + } +} +#else +typedef Utf8Buf = StringBuf; +#end diff --git a/cli/loreline/Cli.hx b/src/loreline/cli/Cli.hx similarity index 98% rename from cli/loreline/Cli.hx rename to src/loreline/cli/Cli.hx index 4b04dbd..04d0f43 100644 --- a/cli/loreline/Cli.hx +++ b/src/loreline/cli/Cli.hx @@ -1,4 +1,4 @@ -package loreline; +package loreline.cli; import haxe.CallStack.StackItem; import haxe.CallStack; @@ -6,11 +6,12 @@ import haxe.Json; import haxe.io.Path; import loreline.Error; import loreline.Interpreter; +import loreline.Lens; import sys.FileSystem; import sys.io.File; using StringTools; -using loreline.Colors; +using loreline.cli.CliColors; enum CliCommand { @@ -42,7 +43,7 @@ class Cli { var errorInStdOut:Bool = false; - var typeDelay:Float = 0.0075; + var typeDelay:Float = 0.0075;//#if eval 0 #else 0.0075 #end; var sentenceDelay:Float = 0.5; @@ -397,9 +398,11 @@ class Cli { Sys.sleep(delay); } Sys.stdout().writeString('\n'); + Sys.stdout().flush(); } else { print(str); + Sys.stdout().flush(); } } @@ -442,7 +445,7 @@ class Cli { function printStackTrace(returnOnly:Bool = false, ?stack:Array):String { - var result = new StringBuf(); + var result = new loreline.Utf8.Utf8Buf(); inline function print(data:Dynamic) { if (!returnOnly) { diff --git a/cli/loreline/Colors.hx b/src/loreline/cli/CliColors.hx similarity index 99% rename from cli/loreline/Colors.hx rename to src/loreline/cli/CliColors.hx index 02c1b5a..0ce975d 100644 --- a/cli/loreline/Colors.hx +++ b/src/loreline/cli/CliColors.hx @@ -1,6 +1,6 @@ -package loreline; +package loreline.cli; -class Colors { +class CliColors { public static var enabled:Bool = true; diff --git a/cli/loreline/CliMacros.hx b/src/loreline/cli/CliMacros.hx similarity index 98% rename from cli/loreline/CliMacros.hx rename to src/loreline/cli/CliMacros.hx index 5adc89b..bc7bd22 100644 --- a/cli/loreline/CliMacros.hx +++ b/src/loreline/cli/CliMacros.hx @@ -1,4 +1,4 @@ -package loreline; +package loreline.cli; #if macro import haxe.macro.Context; diff --git a/src/loreline/lsp/Protocol.hx b/src/loreline/lsp/Protocol.hx new file mode 100644 index 0000000..57a48f6 --- /dev/null +++ b/src/loreline/lsp/Protocol.hx @@ -0,0 +1,476 @@ +package loreline.lsp; + +/** + * LSP Message types + */ +enum abstract MessageType(Int) { + var Error = 1; + var Warning = 2; + var Info = 3; + var Log = 4; +} + +/** + * Response error codes + */ +enum abstract ErrorCodes(Int) { + var ParseError = -32700; + var InvalidRequest = -32600; + var MethodNotFound = -32601; + var InvalidParams = -32602; + var InternalError = -32603; + var ServerNotInitialized = -32002; + var UnknownErrorCode = -32001; +} + +/** + * Either type helper + */ +abstract EitherType(Dynamic) from T1 from T2 to T1 to T2 {} + +/** + * Base message type + */ +typedef Message = { + var jsonrpc:String; +} + +/** + * Request ID type (can be number or string) + */ +abstract RequestId(Dynamic) from Int from String to Int to String {} + +/** + * Request message + */ +typedef RequestMessage = Message & { + var ?id:RequestId; + var method:String; + var ?params:Any; +} + +/** + * Response message + */ +typedef ResponseMessage = Message & { + var id:RequestId; + var ?result:Any; + var ?error:ResponseError; +} + +/** + * Response error + */ +typedef ResponseError = { + var code:ErrorCodes; + var message:String; + var ?data:Any; +} + +/** + * Notification message + */ +typedef NotificationMessage = Message & { + var method:String; + var ?params:Any; +} + +/** + * Position in a text document + */ +typedef Position = { + var line:Int; + var character:Int; +} + +/** + * A range in a text document + */ +typedef Range = { + var start:Position; + var end:Position; +} + +/** + * Location in a text document + */ +typedef Location = { + var uri:String; + var range:Range; +} + +/** + * Text document identifier + */ +typedef TextDocumentIdentifier = { + var uri:String; +} + +/** + * Versioned text document identifier + */ +typedef VersionedTextDocumentIdentifier = TextDocumentIdentifier & { + var version:Int; +} + +/** + * Text document item + */ +typedef TextDocumentItem = { + var uri:String; + var languageId:String; + var version:Int; + var text:String; +} + +/** + * Text document content change event + */ +typedef TextDocumentContentChangeEvent = { + var ?range:Range; + var ?rangeLength:Int; + var text:String; +} + +/** + * Initialization parameters + */ +typedef InitializeParams = { + var processId:Null; + var ?rootPath:String; + var ?rootUri:String; + var capabilities:ClientCapabilities; + var ?trace:String; + var ?workspaceFolders:Array; +} + +/** + * Client capabilities + */ +typedef ClientCapabilities = { + var ?workspace:WorkspaceClientCapabilities; + var ?textDocument:TextDocumentClientCapabilities; + var ?experimental:Any; +} + +/** + * Server capabilities + */ +typedef ServerCapabilities = { + var ?textDocumentSync:TextDocumentSyncOptions; + var ?completionProvider:CompletionOptions; + var ?hoverProvider:Bool; + var ?definitionProvider:Bool; + var ?referencesProvider:Bool; + var ?documentSymbolProvider:Bool; + var ?documentFormattingProvider:Bool; +} + +/** + * Text document sync options + */ +typedef TextDocumentSyncOptions = { + var ?openClose:Bool; + var ?change:Int; + var ?save:SaveOptions; +} + +/** + * Save options + */ +typedef SaveOptions = { + var ?includeText:Bool; +} + +/** + * Completion provider options + */ +typedef CompletionOptions = { + var ?resolveProvider:Bool; + var ?triggerCharacters:Array; +} + +/** + * Completion item kinds + */ +enum abstract CompletionItemKind(Int) { + var Text = 1; + var Method = 2; + var Function = 3; + var Constructor = 4; + var Field = 5; + var Variable = 6; + var Class = 7; + var Interface = 8; + var Module = 9; + var Property = 10; + var Unit = 11; + var Value = 12; + var Enum = 13; + var Keyword = 14; + var Snippet = 15; + var Color = 16; + var File = 17; + var Reference = 18; + var Folder = 19; + var EnumMember = 20; + var Constant = 21; + var Struct = 22; + var Event = 23; + var Operator = 24; + var TypeParameter = 25; +} + +/** + * Completion item + */ +typedef CompletionItem = { + var label:String; + var ?kind:CompletionItemKind; + var ?detail:String; + var ?documentation:String; + var ?sortText:String; + var ?filterText:String; + var ?insertText:String; + var ?textEdit:TextEdit; + var ?additionalTextEdits:Array; + var ?commitCharacters:Array; + var ?command:Command; + var ?data:Any; +} + +/** + * Text edit + */ +typedef TextEdit = { + var range:Range; + var newText:String; +} + +/** + * Command + */ +typedef Command = { + var title:String; + var command:String; + var ?arguments:Array; +} + +/** + * Markup content + */ +typedef MarkupContent = { + var kind:MarkupKind; + var value:String; +} + +/** + * Markup kinds + */ +enum abstract MarkupKind(String) { + var PlainText = "plaintext"; + var Markdown = "markdown"; +} + +/** + * Hover + */ +typedef Hover = { + var contents:EitherType>>; + var ?range:Range; +} + +/** + * Marked string (deprecated, use MarkupContent instead) + */ +typedef MarkedString = EitherType; + +/** + * Diagnostic severity + */ +enum abstract DiagnosticSeverity(Int) { + var Error = 1; + var Warning = 2; + var Information = 3; + var Hint = 4; +} + +/** + * Diagnostic information + */ +typedef Diagnostic = { + var range:Range; + var ?severity:DiagnosticSeverity; + var ?code:String; + var ?source:String; + var message:String; + var ?relatedInformation:Array; +} + +/** + * Related diagnostic information + */ +typedef DiagnosticRelatedInformation = { + var location:Location; + var message:String; +} + +/** + * Workspace folder + */ +typedef WorkspaceFolder = { + var uri:String; + var name:String; +} + +/** + * Workspace client capabilities + */ +typedef WorkspaceClientCapabilities = { + var ?applyEdit:Bool; + var ?workspaceEdit:WorkspaceEditCapabilities; + var ?didChangeConfiguration:DynamicRegistrationCapabilities; + var ?didChangeWatchedFiles:DynamicRegistrationCapabilities; + var ?symbol:DynamicRegistrationCapabilities; + var ?executeCommand:DynamicRegistrationCapabilities; +} + +/** + * Text document client capabilities + */ +typedef TextDocumentClientCapabilities = { + var ?synchronization:TextDocumentSyncClientCapabilities; + var ?completion:CompletionClientCapabilities; + var ?hover:HoverClientCapabilities; + var ?definition:DefinitionClientCapabilities; +} + +/** + * Dynamic registration capabilities + */ +typedef DynamicRegistrationCapabilities = { + var ?dynamicRegistration:Bool; +} + +/** + * Workspace edit capabilities + */ +typedef WorkspaceEditCapabilities = { + var ?documentChanges:Bool; +} + +/** + * Completion client capabilities + */ +typedef CompletionClientCapabilities = DynamicRegistrationCapabilities & { + var ?completionItem:{ + var ?snippetSupport:Bool; + var ?commitCharactersSupport:Bool; + var ?documentationFormat:Array; + var ?deprecatedSupport:Bool; + var ?preselectSupport:Bool; + }; + var ?completionItemKind:{ + var ?valueSet:Array; + }; + var ?contextSupport:Bool; +} + +/** + * Hover client capabilities + */ +typedef HoverClientCapabilities = DynamicRegistrationCapabilities & { + var ?contentFormat:Array; +} + +/** + * Definition client capabilities + */ +typedef DefinitionClientCapabilities = DynamicRegistrationCapabilities & { + var ?linkSupport:Bool; +} + +/** + * Text document sync client capabilities + */ +typedef TextDocumentSyncClientCapabilities = DynamicRegistrationCapabilities & { + var ?willSave:Bool; + var ?willSaveWaitUntil:Bool; + var ?didSave:Bool; +} + +/** + * Completion trigger kinds + */ +enum abstract CompletionTriggerKind(Int) { + var Invoked = 1; + var TriggerCharacter = 2; + var TriggerForIncompleteCompletions = 3; +} + +/** + * Completion context + */ +typedef CompletionContext = { + var triggerKind:CompletionTriggerKind; + var ?triggerCharacter:String; +} + +/** + * Document symbol kinds + */ +enum abstract SymbolKind(Int) { + var File = 1; + var Module = 2; + var Namespace = 3; + var Package = 4; + var Class = 5; + var Method = 6; + var Property = 7; + var Field = 8; + var Constructor = 9; + var Enum = 10; + var Interface = 11; + var Function = 12; + var Variable = 13; + var Constant = 14; + var String = 15; + var Number = 16; + var Boolean = 17; + var Array = 18; + var Object = 19; + var Key = 20; + var Null = 21; + var EnumMember = 22; + var Struct = 23; + var Event = 24; + var Operator = 25; + var TypeParameter = 26; +} + +/** + * Document symbol + */ +typedef DocumentSymbol = { + var name:String; + var detail:String; + var kind:SymbolKind; + var deprecated:Bool; + var range:Range; + var selectionRange:Range; + var ?children:Array; +} + +/** + * Formatting options + */ +typedef FormattingOptions = { + var tabSize:Int; + var insertSpaces:Bool; + var ?trimTrailingWhitespace:Bool; + var ?insertFinalNewline:Bool; + var ?trimFinalNewlines:Bool; +} diff --git a/src/loreline/lsp/Server.hx b/src/loreline/lsp/Server.hx new file mode 100644 index 0000000..097a44c --- /dev/null +++ b/src/loreline/lsp/Server.hx @@ -0,0 +1,824 @@ +package loreline.lsp; + +import haxe.Json; +import loreline.Error; +import loreline.Lexer; +import loreline.Node; +import loreline.Parser; +import loreline.lsp.Protocol; + +using StringTools; +using loreline.Utf8; + +/** + * Main LSP server implementation for Loreline + */ +class Server { + + // Map of document URIs to their parsed ASTs + final documents:Map = []; + + // Map of document URIs to their text content + final documentContents:Map = []; + + // Map of document URIs to their diagnostics + final documentDiagnostics:Map> = []; + + // Client capabilities from initialize request + var clientCapabilities:ClientCapabilities; + + // Track server state + var initialized:Bool = false; + var shutdown:Bool = false; + + public dynamic function onLog(message:Any, ?pos:haxe.PosInfos) { + js.Node.console.log(message); + } + + public dynamic function onNotification(message:NotificationMessage) { + // Needs to be replaced by proper handler + } + + public function new() {} + + /** + * Handle incoming JSON-RPC message + */ + public function handleMessage(msg:Message):Null { + try { + // Check message type and dispatch accordingly + if (Reflect.hasField(msg, "method")) { + if (Reflect.hasField(msg, "id")) { + // Request message + return handleRequest(cast msg); + } else { + // Notification message + handleNotification(cast msg); + return null; + } + } + + return createErrorResponse(null, ErrorCodes.InvalidRequest, "Invalid message"); + + } catch (e:Dynamic) { + return createErrorResponse(null, ErrorCodes.ParseError, Std.string(e)); + } + } + + /** + * Create error response + */ + function createErrorResponse(id:RequestId, code:ErrorCodes, message:String, ?data:Any):ResponseMessage { + final response:ResponseMessage = { + jsonrpc: "2.0", + id: id, + error: { + code: code, + message: message, + data: data + } + }; + return response; + } + + /** + * Create success response + */ + function createResponse(id:RequestId, result:Any):ResponseMessage { + final response:ResponseMessage = { + jsonrpc: "2.0", + id: id, + result: result + }; + return response; + } + + /** + * Handle a request message + */ + function handleRequest(request:RequestMessage):ResponseMessage { + try { + if (!initialized && request.method != "initialize") { + throw { code: ErrorCodes.ServerNotInitialized, message: "Server not initialized" }; + } + + final result = switch (request.method) { + case "initialize": + handleInitialize(cast request.params); + + case "shutdown": + handleShutdown(); + + case "textDocument/completion": + handleCompletion(cast request.params); + + case "textDocument/definition": + handleDefinition(cast request.params); + + case "textDocument/hover": + handleHover(cast request.params); + + case "textDocument/documentSymbol": + handleDocumentSymbol(cast request.params); + + case "textDocument/references": + null; + // TODO + //handleReferences(cast request.params); + + case "textDocument/formatting": + handleDocumentFormatting(cast request.params); + + case _: + throw { code: ErrorCodes.MethodNotFound, message: 'Method not found: ${request.method}'}; + } + + return createResponse(request.id, result); + + } catch (e:Any) { + if (Reflect.hasField(e, 'code') && Reflect.hasField(e, 'message')) { + final err:ResponseError = e; + return createErrorResponse(request.id, err.code, err.message); + } + else { + return createErrorResponse(request.id, ErrorCodes.InternalError, Std.string(e)); + } + } + return null; + } + + /** + * Handle a notification message + */ + function handleNotification(notification:NotificationMessage) { + try { + if (!initialized && notification.method != "initialized") return; + + switch (notification.method) { + case "initialized": + initialized = true; + + case "textDocument/didOpen": + handleDidOpenTextDocument(cast notification.params); + + case "textDocument/didChange": + handleDidChangeTextDocument(cast notification.params); + + case "textDocument/didClose": + handleDidCloseTextDocument(cast notification.params); + + case "exit": + handleExit(); + + case _: + // Ignore unknown notifications + } + } catch (e:Dynamic) { + // Log error but don't respond to notifications + } + } + + /** + * Handle initialize request + */ + function handleInitialize(params:InitializeParams):{ capabilities: ServerCapabilities } { + if (initialized) { + throw { code: ErrorCodes.InvalidRequest, message: "Server already initialized" }; + } + + clientCapabilities = params.capabilities; + + return { + capabilities: { + // Full document sync means we'll get the entire document content on changes + textDocumentSync: { + openClose: true, + change: 1 // TextDocumentSyncKind.Full + }, + // Enable completion with specific trigger characters + completionProvider: { + resolveProvider: false, // We don't provide additional resolution + triggerCharacters: [ + ".", // For field access (character.name) + "$", // For interpolation ($variable) + "<", // For tags () + "\"" // For quoted strings + ] + }, + definitionProvider: true, // For go-to-definition + hoverProvider: true, // For hover tooltips + documentSymbolProvider: true, // For document outline + documentFormattingProvider: true // For code formatting + } + }; + } + + /** + * Handle shutdown request + */ + function handleShutdown():Null { + if (shutdown) { + throw { code: ErrorCodes.InvalidRequest, message: "Server already shut down" }; + } + shutdown = true; + return null; + } + + /** + * Handle exit notification + */ + function handleExit() { + Sys.exit(shutdown ? 0 : 1); + } + + /** + * Handle document open + */ + function handleDidOpenTextDocument(params:{textDocument:TextDocumentItem}) { + final doc = params.textDocument; + updateDocument(doc.uri, doc.text); + } + + /** + * Handle document change + */ + function handleDidChangeTextDocument(params:{ + textDocument:VersionedTextDocumentIdentifier, + contentChanges:Array + }) { + if (params.contentChanges.length > 0) { + // We're using full document sync, so just take the last change + final change = params.contentChanges[params.contentChanges.length - 1]; + updateDocument(params.textDocument.uri, change.text); + } + } + + /** + * Handle document close + */ + function handleDidCloseTextDocument(params:{textDocument:TextDocumentIdentifier}) { + documents.remove(params.textDocument.uri); + documentContents.remove(params.textDocument.uri); + documentDiagnostics.remove(params.textDocument.uri); + } + + /** + * Update document content and parse + */ + function updateDocument(uri:String, content:String) { + documentContents.set(uri, content); + documentDiagnostics.set(uri, []); + + try { + // Parse document and update AST + final lexer = new Lexer(content); + final parser = new Parser(lexer.tokenize()); + final ast = parser.parse(); + documents.set(uri, ast); + + // Check for parser errors + final errors = parser.getErrors(); + if (errors != null && errors.length > 0) { + for (error in errors) { + addDiagnostic(uri, error.pos, error.message, DiagnosticSeverity.Error); + } + } + + // TODO: Add semantic validation + validateDocument(uri, ast); + + } catch (e:Error) { + // Handle lexer/parser errors + addDiagnostic(uri, e.pos, e.message, DiagnosticSeverity.Error); + } catch (e:Dynamic) { + // Handle unexpected errors + addDiagnostic(uri, null, Std.string(e), DiagnosticSeverity.Error); + } + + // Publish diagnostics + publishDiagnostics(uri); + } + + /** + * Add a diagnostic + */ + function addDiagnostic(uri:String, pos:Null, message:String, severity:DiagnosticSeverity) { + final diagnostics = documentDiagnostics.get(uri); + if (diagnostics == null) return; + + final range = if (pos != null) { + { + start: {line: pos.line - 1, character: pos.column - 1}, + end: {line: pos.line - 1, character: pos.column - 1 + (pos.length > 0 ? pos.length : 1)} + } + } else { + { + start: {line: 0, character: 0}, + end: {line: 0, character: 0} + } + }; + + diagnostics.push({ + range: range, + severity: severity, + source: "loreline", + message: message + }); + } + + /** + * Publish diagnostics + */ + function publishDiagnostics(uri:String) { + final diagnostics = documentDiagnostics.get(uri); + if (diagnostics == null) return; + + final params = { + uri: uri, + diagnostics: diagnostics + }; + + // Send notification + final notification:NotificationMessage = { + jsonrpc: "2.0", + method: "textDocument/publishDiagnostics", + params: params + }; + + onNotification(notification); + } + + /** + * Validate document semantics + */ + function validateDocument(uri:String, ast:Script) { + // TODO: Add semantic validation: + // - Check for undefined beats in transitions + // - Check for undefined characters in dialogues + // - Check for undefined variables in interpolations + // - Check for undefined tags + } + + /** + * Handle completion request + */ + function handleCompletion(params:{ + textDocument:TextDocumentIdentifier, + position:Position, + ?context:CompletionContext + }):Array { + final ast = documents.get(params.textDocument.uri); + if (ast == null) return []; + + // TODO: Calculate completions based on context: + // - Beat names after -> + // - Character names before : + // - Variable names after $ or ${ + // - Tag names after < or { + final result = []; + + onLog("handleDefinition"); + final uri = params.textDocument.uri; + + final ast = documents.get(uri); + if (ast == null) return result; + + final content = documentContents.get(uri); + final lorelinePos = toLorelinePosition(params.position, content); + + final lens = new Lens(ast); + final node = lens.getNodeAtPosition(lorelinePos); + onLog("DEF NODE: " + node != null ? Type.getClassName(Type.getClass(node)) : '-'); + + if (node != null) { + switch Type.getClass(node) { + case NTransition: + final transition:NTransition = cast node; + final beatDecl = lens.findReferencedBeat(transition); + if (beatDecl != null) { + result.push({ + uri: uri, + range: firstLineRange(rangeFromLorelinePosition(beatDecl.pos, content), content) + }); + } + } + } + + // TODO: Find definitions: + // - Go to beat definition from transition + // - Go to character definition from dialogue + // - Go to variable definition from interpolation + return result; + } + + function makeHover(title:String, description:Array, content:String, node:Node, ?pos:loreline.Position, code:String = 'loreline'):Hover { + + final value:Array = []; + + if (title != null) { + if (code != null && code.length > 0) { + value.push("```" + code); + } + value.push(title); + if (code != null && code.length > 0) { + value.push("```"); + } + } + + if (description != null && description.length > 0) { + if (title != null) { + value.push(""); + value.push("---"); + value.push(""); + } + + value.push(description.join("\n")); + } + + return { + contents: { + kind: MarkupKind.Markdown, + value: value.join("\n") + }, + range: rangeFromLorelinePosition(pos ?? node.pos, content) + } + + } + + /** + * Handle hover request + */ + function handleHover(params:{ + textDocument:TextDocumentIdentifier, + position:Position + }):Null { + final uri = params.textDocument.uri; + + final ast = documents.get(uri); + if (ast == null) return null; + + final content = documentContents.get(uri); + final lorelinePos = toLorelinePosition(params.position, content); + + final lens = new Lens(ast); + final node = lens.getNodeAtPosition(lorelinePos); + + onLog("POS " + Json.stringify(params.position)); + onLog("LOR POS " + lorelinePos + ' offset=${lorelinePos.offset} column=${lorelinePos.column}'); + onLog('NODE: ' + (node != null ? Type.getClassName(Type.getClass(node)) : null)); + if (node != null) { + return makeNodeHover(lens, content, node); + } + + // TODO: Show hover information: + // - Beat summary/contents + // - Character properties + // - Variable type/value + // - Tag documentation + return null; + } + + function makeNodeHover(lens:Lens, content:String, node:Node):Null { + + onLog(Json.stringify(node.pos.toJson())); + + switch Type.getClass(node) { + case NBeatDecl: + return makeBeatDeclHover(cast node, content); + case NChoiceStatement: + return makeChoiceHover(cast node, content); + case NTransition: + return makeExprHover(cast node, content); + case NAccess: + var access:NAccess = cast node; + var parent = lens.getParentNode(access); + if (parent is NCall) { + return makeExprHover(cast parent, content); + } + else if (parent is NArrayAccess) { + return makeExprHover(cast parent, content); + } + return makeExprHover(cast node, content); + case NLiteral | NExpr | NAssign | NUnary | NBinary: + return makeExprHover(cast node, content); + case NIfStatement: + return makeHover("if", null, content, node, null, "loreline.expression"); + case NStringLiteral | NDialogueStatement: + return makeStatementHover(cast node, content); + case NStringPart: + var parent = node; + do { + parent = lens.getParentNode(parent); + // if (parent != null) { + // onLog("PARENT: " + Type.getClassName(Type.getClass(parent))); + // } + } + while (parent != null); + + final stringPart:NStringPart = cast node; + + switch stringPart.type { + case Raw(text): + case Expr(expr): + return makeNodeHover(lens, content, expr); + case Tag(closing, expr): + return makeHover("<" + printLoreline(expr) + ">", hoverDescriptionForNode(expr), content, stringPart); + } + + final literal = lens.getParentOfType(node, NStringLiteral); + + if (literal != null) { + final literalParent = lens.getParentNode(literal); + if (literalParent != null && literalParent is NStringPart) { + final parentStringPart:NStringPart = cast literalParent; + switch parentStringPart.type { + case Raw(text): + case Expr(expr): + return makeNodeHover(lens, content, expr); + case Tag(closing, expr): + return makeHover("<" + printLoreline(expr) + ">", hoverDescriptionForNode(expr), content, parentStringPart); + } + } + + final partIndex = literal.parts.indexOf(stringPart); + if (literal.quotes == Unquoted) { + if (partIndex > 0) { + // In unquoted strings, we ignore space between leading tags and actual text + var keepWhitespace = true; + for (i in 0...partIndex) { + switch literal.parts[i].type { + case Raw(text): + if (text.trim().uLength() > 0) { + keepWhitespace = false; + break; + } + case Expr(expr): + keepWhitespace = false; + break; + case Tag(closing, expr): + } + } + if (keepWhitespace) { + switch literal.parts[partIndex].type { + case Raw(text): + final spaces = text.uLength() - text.ltrim().uLength(); + if (spaces > 0) { + return makeHover(text.trim(), hoverDescriptionForNode(literal), content, stringPart, stringPart.pos.withOffset(content, spaces, stringPart.pos.length - spaces)); + } + case _: + } + } + } + } + else if (literal.quotes == DoubleQuotes) { + if (literal.parts.length == 1) { + switch literal.parts[0].type { + case Raw(text): + return makeHover(text.trim(), hoverDescriptionForNode(literal), content, stringPart, stringPart.pos.withOffset(content, -1, stringPart.pos.length + 2)); + case Expr(expr): + case Tag(closing, expr): + } + } + else if (partIndex == 0) { + switch literal.parts[0].type { + case Raw(text): + return makeHover(text.trim(), hoverDescriptionForNode(literal), content, stringPart, stringPart.pos.withOffset(content, -1, stringPart.pos.length + 1)); + case Expr(expr): + case Tag(closing, expr): + } + } + else if (partIndex == literal.parts.length - 1) { + switch literal.parts[literal.parts.length - 1].type { + case Raw(text): + return makeHover(text.trim(), hoverDescriptionForNode(literal), content, stringPart, stringPart.pos.withOffset(content, 0, stringPart.pos.length + 1)); + case Expr(expr): + case Tag(closing, expr): + } + } + } + } + + switch stringPart.type { + case Raw(text): + return makeHover(text.trim(), hoverDescriptionForNode(cast node), content, stringPart); + case Expr(expr): + case Tag(closing, expr): + } + + case _: + } + return null; + + } + + function hoverDescriptionForNode(node:AstNode):Array { + + final description:Array = []; + + if (node.leadingComments != null) { + for (comment in node.leadingComments) { + description.push(comment.content); + } + } + + if (node.trailingComments != null) { + if (description.length > 0) { + description.push(''); + description.push('---'); + description.push(''); + } + for (comment in node.trailingComments) { + description.push(comment.content); + } + } + + return description; + + } + + function makeBeatDeclHover(beatDecl:NBeatDecl, content:String):Hover { + + return makeHover('beat ${beatDecl.name}', hoverDescriptionForNode(beatDecl), content, beatDecl); + + } + + function makeChoiceHover(choice:NChoiceStatement, content:String):Hover { + + return makeHover('choice', hoverDescriptionForNode(choice), content, choice); + + } + + function makeExprHover(expr:NExpr, content:String):Hover { + + return makeHover(printLoreline(expr), hoverDescriptionForNode(expr), content, expr, null, 'loreline.expression'); + + } + + function makeStatementHover(expr:NExpr, content:String):Hover { + + return makeHover(printLoreline(expr), hoverDescriptionForNode(expr), content, expr); + + } + + function printLoreline(node:Node):String { + + final printer = new Printer(); + + printer.enableComments = false; + + return printer.print(node).trim(); + + } + + /** + * Handle document symbol request + */ + function handleDocumentSymbol(params:{ + textDocument:TextDocumentIdentifier + }):Array { + + final ast = documents.get(params.textDocument.uri); + if (ast == null) return []; + + final content = documentContents.get(params.textDocument.uri); + if (content == null) return []; + + final printer = new SymbolPrinter(content); + return printer.print(ast); + + } + + /** + * Handle document formatting request + */ + function handleDocumentFormatting(params:{ + textDocument:TextDocumentIdentifier, + options:FormattingOptions + }):Array { + final ast = documents.get(params.textDocument.uri); + if (ast == null) return []; + + // TODO: Format document: + // - Proper indentation + // - Consistent spacing + // - Line breaks between blocks + return []; + } + + /** + * Convert an LSP Protocol Position to a Loreline Position + * + * LSP positions are 0-based for both line and character + * Loreline positions are 1-based for both line and column + * + * @param protocolPos The LSP Protocol position with line and character fields + * @param content document content string to compute offset + * @param length Optional length to include in position + * @return A Loreline Position instance + */ + function toLorelinePosition(protocolPos:loreline.lsp.Protocol.Position, content:String, ?length:Int = 0):loreline.Position { + // Convert from 0-based to 1-based indexing + final line = protocolPos.line + 1; + final column = protocolPos.character + 1; + + // Calculate absolute offset if content is provided + final offset = computeLorelineOffset(line, column, content); + + return new loreline.Position(line, column, offset, length); + } + + function computeLorelineOffset(line:Int, column:Int, content:String) { + + var offset = 0; + + if (content != null) { + var currentLine = 1; + var currentCol = 1; + var i = 0; + + while (i < content.length) { + if (currentLine == line && currentCol == column) { + offset = i; + break; + } + + if (content.uCharCodeAt(i) == '\n'.code) { + currentLine++; + currentCol = 1; + } else { + currentCol++; + } + i++; + } + + // Handle position at end of content + if (currentLine == line && currentCol == column) { + offset = i; + } + } + + return offset; + + } + + /** + * Convert a Loreline Position to an LSP Protocol Position + * + * Loreline positions are 1-based for both line and column + * LSP positions are 0-based for both line and character + * + * @param lorelinePos The Loreline position instance + * @return An LSP Protocol position + */ + function fromLorelinePosition(lorelinePos:loreline.Position):loreline.lsp.Protocol.Position { + return { + // Convert from 1-based to 0-based indexing + line: lorelinePos.line - 1, + character: lorelinePos.column - 1 + }; + } + + function rangeFromLorelinePosition(lorelinePos:loreline.Position, content:String):loreline.lsp.Protocol.Range { + final start = fromLorelinePosition(lorelinePos); + final end = fromLorelinePosition(lorelinePos.withOffset(content, lorelinePos.length)); + return { + start: start, + end: end + }; + } + + function firstLineRange(range:Range, content:String):Range { + // Skip preceding whitespace lines + var lineStart = range.start.line; + final lines = content.split("\n"); + while (lineStart < lines.length && lines[lineStart].trim().length == 0) { + lineStart++; + } + + // Find end of first non-empty line + var lineEnd = lineStart; + var charEnd = lines[lineEnd].length; + + return { + start: { + line: lineStart, + character: range.start.character + }, + end: { + line: lineEnd, + character: charEnd + } + }; + } + +} \ No newline at end of file diff --git a/src/loreline/lsp/SymbolPrinter.hx b/src/loreline/lsp/SymbolPrinter.hx new file mode 100644 index 0000000..6b59427 --- /dev/null +++ b/src/loreline/lsp/SymbolPrinter.hx @@ -0,0 +1,328 @@ +package loreline.lsp; + +import loreline.Node; +import loreline.Position; +import loreline.Printer; +import loreline.lsp.Protocol; + +using StringTools; + +/** + * A printer that converts AST nodes into LSP document symbols. + * Similar to the Printer class but outputs DocumentSymbol array. + */ +class SymbolPrinter { + + /** Reference to the document content for range calculations */ + final content:String; + + /** Current parent symbol being built */ + var currentSymbol:DocumentSymbol; + + /** + * Creates a new symbol printer. + * @param content Document content for range calculation + */ + public function new(content:String) { + this.content = content; + } + + /** + * Main entry point for converting a node to document symbols. + * @param node Root node to process + * @return Array of document symbols + */ + public function print(node:Node):Array { + return switch Type.getClass(node) { + case Script: printScript(cast node); + case _: []; + } + } + + /** + * Process a complete script node. + * @param script Script node to process + * @return Array of document symbols for top-level declarations + */ + function printScript(script:Script):Array { + final symbols:Array = []; + + for (decl in script.declarations) { + switch Type.getClass(decl) { + case NChoiceStatement: + symbols.push(printChoice(cast decl)); + case NIfStatement: + for (symbol in printIf(cast decl)) { + symbols.push(symbol); + } + case NStateDecl: + symbols.push(printStateDecl(cast decl)); + case NBeatDecl: + symbols.push(printBeatDecl(cast decl)); + case NCharacterDecl: + symbols.push(printCharacterDecl(cast decl)); + case _: + // Skip other top-level nodes + } + } + + return symbols; + } + + /** + * Process a beat declaration. + * @param beat Beat node to process + * @return Document symbol for the beat + */ + function printBeatDecl(beat:NBeatDecl):DocumentSymbol { + final children:Array = []; + + for (node in beat.body) { + switch Type.getClass(node) { + case NChoiceStatement: + children.push(printChoice(cast node)); + case NIfStatement: + for (symbol in printIf(cast node)) { + children.push(symbol); + } + case NStateDecl: + children.push(printStateDecl(cast node)); + case NBeatDecl: + children.push(printBeatDecl(cast node)); + case _: + // Skip other node types + } + } + + return { + name: beat.name, + detail: "beat", + kind: SymbolKind.Class, + deprecated: false, + range: rangeFromPosition(beat.pos), + selectionRange: rangeFromPosition(beat.pos), + children: children + }; + } + + /** + * Process a character declaration. + * @param char Character node to process + * @return Document symbol for the character + */ + function printCharacterDecl(char:NCharacterDecl):DocumentSymbol { + final children:Array = []; + + for (prop in char.properties) { + children.push({ + name: prop.name, + detail: printValue(prop.value), + kind: SymbolKind.Property, + deprecated: false, + range: rangeFromPosition(prop.pos), + selectionRange: rangeFromPosition(prop.pos) + }); + } + + return { + name: char.name, + detail: "character", + kind: SymbolKind.Object, + deprecated: false, + range: rangeFromPosition(char.pos), + selectionRange: rangeFromPosition(char.pos), + children: children + }; + } + + /** + * Process a state declaration. + * @param state State node to process + * @return Document symbol for the state + */ + function printStateDecl(state:NStateDecl):DocumentSymbol { + final children:Array = []; + final fields = state.fields.value; + + for (field in (fields:Array)) { + children.push({ + name: field.name, + detail: printValue(field.value), + kind: SymbolKind.Variable, + deprecated: false, + range: rangeFromPosition(field.pos), + selectionRange: rangeFromPosition(field.pos) + }); + } + + return { + name: state.temporary ? "new state" : "state", + detail: children.length + " fields", + kind: SymbolKind.Namespace, + deprecated: false, + range: rangeFromPosition(state.pos), + selectionRange: rangeFromPosition(state.pos), + children: children + }; + } + + /** + * Process a choice statement. + * @param choice Choice node to process + * @return Document symbol for the choice + */ + function printChoice(choice:NChoiceStatement):DocumentSymbol { + final children:Array = []; + + for (option in choice.options) { + final label = switch (option.text) { + case null: "(empty)"; + case text: printValue(text); + } + + final optionSymbol:DocumentSymbol = { + name: label, + detail: option.condition != null ? 'if ${printValue(option.condition)}' : "", + kind: SymbolKind.EnumMember, + deprecated: false, + range: rangeFromPosition(option.pos), + selectionRange: rangeFromPosition(option.pos), + children: [] + }; + + for (node in option.body) { + switch Type.getClass(node) { + case NChoiceStatement: + optionSymbol.children.push(printChoice(cast node)); + case NIfStatement: + for (symbol in printIf(cast node)) { + optionSymbol.children.push(symbol); + } + case NStateDecl: + optionSymbol.children.push(printStateDecl(cast node)); + case _: + // Skip other node types + } + } + + children.push(optionSymbol); + } + + return { + name: "choice", + detail: choice.options.length + " options", + kind: SymbolKind.Enum, + deprecated: false, + range: rangeFromPosition(choice.pos), + selectionRange: rangeFromPosition(choice.pos), + children: children + }; + } + + /** + * Process an if statement. + * @param ifStmt If statement node to process + * @return Document symbol for the if statement + */ + function printIf(ifStmt:NIfStatement):Array { + final result:Array = []; + + if (ifStmt.thenBranch != null) { + final blockSymbol = printBlock('then', null, ifStmt.thenBranch); + result.push({ + name: printValue(ifStmt.condition), + detail: "condition", + kind: SymbolKind.Boolean, + deprecated: false, + range: rangeFromPosition(ifStmt.pos), + selectionRange: rangeFromPosition(ifStmt.pos), + children: blockSymbol.children + }); + } + + if (ifStmt.elseBranch != null && ifStmt.elseBranch.body.length == 1) { + if (Type.getClass(ifStmt.elseBranch.body[0]) == NIfStatement) { + for (symbol in printIf(cast ifStmt.elseBranch.body[0])) { + result.push(symbol); + } + } + } + + return result; + } + + /** + * Process a block of nodes. + * @param name Block name + * @param detail Block detail text + * @param block Block node to process + * @return Document symbol for the block + */ + function printBlock(name:String, detail:String, block:NBlock):DocumentSymbol { + final children:Array = []; + + for (node in block.body) { + switch Type.getClass(node) { + case NChoiceStatement: + children.push(printChoice(cast node)); + case NIfStatement: + for (symbol in printIf(cast node)) { + children.push(symbol); + } + case NStateDecl: + children.push(printStateDecl(cast node)); + case _: + // Skip other node types + } + } + + return { + name: name, + detail: detail, + kind: SymbolKind.Namespace, + deprecated: false, + range: rangeFromPosition(block.pos), + selectionRange: rangeFromPosition(block.pos), + children: children + }; + } + + /** + * Convert a node to a string value for display. + * @param node Node to convert + * @return String representation + */ + function printValue(node:Node):String { + final printer = new Printer(); + printer.enableComments = false; + return printer.print(node).trim(); + } + + /** + * Convert a Loreline position to an LSP Range. + * @param pos Loreline position + * @return LSP Range + */ + function rangeFromPosition(pos:loreline.Position):Range { + final start = fromLorelinePosition(pos); + final end = fromLorelinePosition(pos.withOffset(content, pos.length)); + return { + start: start, + end: end + }; + } + + /** + * Convert a Loreline position to an LSP Position. + * @param pos Loreline position + * @return LSP Position + */ + function fromLorelinePosition(pos:loreline.Position):loreline.lsp.Protocol.Position { + return { + // Convert from 1-based to 0-based indexing + line: pos.line - 1, + character: pos.column - 1 + }; + } + +} \ No newline at end of file