From d1b6a0ec9f508d4e2d0b3820bca6af75ec86a8f3 Mon Sep 17 00:00:00 2001 From: Evan Hedbor Date: Wed, 22 May 2024 11:38:22 +0200 Subject: [PATCH 1/3] Add incremental parsing --- server/src/analyzer.ts | 6 +-- server/src/project/document.ts | 70 +++++++++++++++++++++++++++------- server/src/project/project.ts | 19 +++++---- server/src/server.ts | 5 ++- server/src/util/tree-sitter.ts | 9 +++++ 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/server/src/analyzer.ts b/server/src/analyzer.ts index 4810a54..4c28edb 100644 --- a/server/src/analyzer.ts +++ b/server/src/analyzer.ts @@ -47,9 +47,9 @@ import * as fsSync from 'node:fs'; import * as url from 'node:url'; import { ModelicaDocument, ModelicaLibrary, ModelicaProject } from './project'; -import { uriToPath } from "./util"; import { getAllDeclarationsInTree } from './util/declarations'; import { logger } from './util/logger'; +import { uriToPath } from './util'; export default class Analyzer { #project: ModelicaProject; @@ -110,8 +110,8 @@ export default class Analyzer { * @param text the modification * @param range range to update, or `undefined` to replace the whole file */ - public async updateDocument(uri: LSP.DocumentUri, text: string): Promise { - await this.#project.updateDocument(uriToPath(uri), text); + public updateDocument(uri: LSP.DocumentUri, text: string, range?: LSP.Range): void { + this.#project.updateDocument(url.fileURLToPath(uri), text, range); } /** diff --git a/server/src/project/document.ts b/server/src/project/document.ts index ac17f42..81afeb0 100644 --- a/server/src/project/document.ts +++ b/server/src/project/document.ts @@ -33,16 +33,18 @@ * */ -import { TextDocument } from 'vscode-languageserver-textdocument'; -import * as LSP from 'vscode-languageserver/node'; -import Parser from 'web-tree-sitter'; -import * as fs from 'node:fs/promises'; -import * as TreeSitterUtil from '../util/tree-sitter'; - +import { TextDocument } from "vscode-languageserver-textdocument"; +import * as LSP from "vscode-languageserver/node"; +import Parser from "web-tree-sitter"; +import * as fs from "node:fs/promises"; +import * as url from "node:url"; +import * as TreeSitterUtil from "../util/tree-sitter"; + +import { logger } from "../util/logger"; +import { ModelicaLibrary } from "./library"; +import { ModelicaProject } from "./project"; +import { positionToPoint } from '../util/tree-sitter'; import { pathToUri, uriToPath } from '../util'; -import { logger } from '../util/logger'; -import { ModelicaLibrary } from './library'; -import { ModelicaProject } from './project'; export class ModelicaDocument implements TextDocument { readonly #project: ModelicaProject; @@ -90,12 +92,54 @@ export class ModelicaDocument implements TextDocument { /** * Updates a document. + * * @param text the modification + * @param range the range to update, or `undefined` to replace the whole file */ - public async update(text: string): Promise { - TextDocument.update(this.#document, [{ text }], this.version + 1); - this.#tree = this.project.parser.parse(text); - return; + public async update(text: string, range?: LSP.Range): Promise { + if (range === undefined) { + TextDocument.update(this.#document, [{ text }], this.version + 1); + this.#tree = this.project.parser.parse(text); + return; + } + + const startIndex = this.offsetAt(range.start); + const startPosition = positionToPoint(range.start); + const oldEndIndex = this.offsetAt(range.end); + const oldEndPosition = positionToPoint(range.end); + const newEndIndex = startIndex + text.length; + + TextDocument.update(this.#document, [{ text, range }], this.version + 1); + const newEndPosition = positionToPoint(this.positionAt(newEndIndex)); + + this.#tree.edit({ + startIndex, + startPosition, + oldEndIndex, + oldEndPosition, + newEndIndex, + newEndPosition, + }); + + this.#tree = this.project.parser.parse((index: number, position?: Parser.Point) => { + if (position) { + return this.getText({ + start: { + character: position.column, + line: position.row, + }, + end: { + character: position.column + 1, + line: position.row, + }, + }); + } else { + return this.getText({ + start: this.positionAt(index), + end: this.positionAt(index + 1), + }); + } + }, this.#tree); } public getText(range?: LSP.Range | undefined): string { diff --git a/server/src/project/project.ts b/server/src/project/project.ts index 8c06622..53adec4 100644 --- a/server/src/project/project.ts +++ b/server/src/project/project.ts @@ -33,14 +33,14 @@ * */ -import Parser from "web-tree-sitter"; -import * as LSP from "vscode-languageserver"; -import url from "node:url"; -import path from "node:path"; +import Parser from 'web-tree-sitter'; +import * as LSP from 'vscode-languageserver'; +import url from 'node:url'; +import path from 'node:path'; -import { ModelicaLibrary } from "./library"; +import { ModelicaLibrary } from './library'; import { ModelicaDocument } from './document'; -import { logger } from "../util/logger"; +import { logger } from '../util/logger'; /** Options for {@link ModelicaProject.getDocument} */ export interface GetDocumentOptions { @@ -117,7 +117,7 @@ export class ModelicaProject { for (const library of this.#libraries) { const relative = path.relative(library.path, documentPath); - const isSubdirectory = relative && !relative.startsWith("..") && !path.isAbsolute(relative); + const isSubdirectory = relative && !relative.startsWith('..') && !path.isAbsolute(relative); // Assume that files can't be inside multiple libraries at the same time if (!isSubdirectory) { @@ -155,12 +155,12 @@ export class ModelicaProject { * @param text the modification * @returns if the document was updated */ - public async updateDocument(documentPath: string, text: string): Promise { + public async updateDocument(documentPath: string, text: string, range?: LSP.Range): Promise { logger.debug(`Updating document at '${documentPath}'...`); const doc = await this.getDocument(documentPath, { load: true }); if (doc) { - doc.update(text); + doc.update(text, range); logger.debug(`Updated document '${documentPath}'`); return true; } else { @@ -191,5 +191,4 @@ export class ModelicaProject { public get parser(): Parser { return this.#parser; } - } diff --git a/server/src/server.ts b/server/src/server.ts index a4ec4ea..912a78d 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -102,7 +102,7 @@ export class ModelicaServer { documentSymbolProvider: true, colorProvider: false, semanticTokensProvider: undefined, - textDocumentSync: LSP.TextDocumentSyncKind.Full, + textDocumentSync: LSP.TextDocumentSyncKind.Incremental, workspace: { workspaceFolders: { supported: true, @@ -150,7 +150,8 @@ export class ModelicaServer { private async onDidChangeTextDocument(params: LSP.DidChangeTextDocumentParams): Promise { logger.debug('onDidChangeTextDocument'); for (const change of params.contentChanges) { - await this.#analyzer.updateDocument(params.textDocument.uri, change.text); + const range = 'range' in change ? change.range : undefined; + this.#analyzer.updateDocument(params.textDocument.uri, change.text, range); } } diff --git a/server/src/util/tree-sitter.ts b/server/src/util/tree-sitter.ts index f159ce5..601f632 100644 --- a/server/src/util/tree-sitter.ts +++ b/server/src/util/tree-sitter.ts @@ -39,6 +39,7 @@ * ----------------------------------------------------------------------------- */ +import Parser from 'web-tree-sitter'; import * as LSP from 'vscode-languageserver/node'; import { SyntaxNode } from 'web-tree-sitter'; @@ -169,3 +170,11 @@ export function getClassPrefixes(node: SyntaxNode): string | null { return classPrefixNode.text; } + +export function positionToPoint(position: LSP.Position): Parser.Point { + return { row: position.line, column: position.character }; +} + +export function pointToPosition(point: Parser.Point): LSP.Position { + return { line: point.row, character: point.column }; +} From bfe7c140c8dd0ed2d593d4bef4636f8e40e49972 Mon Sep 17 00:00:00 2001 From: PaddiM8 Date: Thu, 23 May 2024 14:43:27 +0200 Subject: [PATCH 2/3] Add test for incremental parsing --- server/src/analyzer.ts | 8 +++--- server/src/project/document.ts | 32 +++++++++++++----------- server/src/project/test/document.test.ts | 21 ++++++++++++++++ server/src/server.ts | 4 +-- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/server/src/analyzer.ts b/server/src/analyzer.ts index 4c28edb..789b0f8 100644 --- a/server/src/analyzer.ts +++ b/server/src/analyzer.ts @@ -98,8 +98,8 @@ export default class Analyzer { * @param uri uri to document to add * @throws if the document does not belong to a library */ - public addDocument(uri: LSP.DocumentUri): void { - this.#project.addDocument(uriToPath(uri)); + public async addDocument(uri: LSP.DocumentUri): Promise { + await this.#project.addDocument(uriToPath(uri)); } /** @@ -110,8 +110,8 @@ export default class Analyzer { * @param text the modification * @param range range to update, or `undefined` to replace the whole file */ - public updateDocument(uri: LSP.DocumentUri, text: string, range?: LSP.Range): void { - this.#project.updateDocument(url.fileURLToPath(uri), text, range); + public async updateDocument(uri: LSP.DocumentUri, text: string, range?: LSP.Range): Promise { + await this.#project.updateDocument(uriToPath(uri), text, range); } /** diff --git a/server/src/project/document.ts b/server/src/project/document.ts index 81afeb0..ea7e26c 100644 --- a/server/src/project/document.ts +++ b/server/src/project/document.ts @@ -33,16 +33,15 @@ * */ -import { TextDocument } from "vscode-languageserver-textdocument"; -import * as LSP from "vscode-languageserver/node"; -import Parser from "web-tree-sitter"; -import * as fs from "node:fs/promises"; -import * as url from "node:url"; -import * as TreeSitterUtil from "../util/tree-sitter"; - -import { logger } from "../util/logger"; -import { ModelicaLibrary } from "./library"; -import { ModelicaProject } from "./project"; +import { TextDocument } from 'vscode-languageserver-textdocument'; +import * as LSP from 'vscode-languageserver/node'; +import Parser from 'web-tree-sitter'; +import * as fs from 'node:fs/promises'; +import * as TreeSitterUtil from '../util/tree-sitter'; + +import { logger } from '../util/logger'; +import { ModelicaLibrary } from './library'; +import { ModelicaProject } from './project'; import { positionToPoint } from '../util/tree-sitter'; import { pathToUri, uriToPath } from '../util'; @@ -52,7 +51,12 @@ export class ModelicaDocument implements TextDocument { readonly #document: TextDocument; #tree: Parser.Tree; - public constructor(project: ModelicaProject, library: ModelicaLibrary | null, document: TextDocument, tree: Parser.Tree) { + public constructor( + project: ModelicaProject, + library: ModelicaLibrary | null, + document: TextDocument, + tree: Parser.Tree, + ) { this.#project = project; this.#library = library; this.#document = document; @@ -182,7 +186,7 @@ export class ModelicaDocument implements TextDocument { public get within(): string[] { const withinClause = this.#tree.rootNode.children .find((node) => node.type === 'within_clause') - ?.childForFieldName("name"); + ?.childForFieldName('name'); if (!withinClause) { return []; } @@ -190,11 +194,11 @@ export class ModelicaDocument implements TextDocument { // TODO: Use a helper function from TreeSitterUtil const identifiers: string[] = []; TreeSitterUtil.forEach(withinClause, (node) => { - if (node.type === "name") { + if (node.type === 'name') { return true; } - if (node.type === "IDENT") { + if (node.type === 'IDENT') { identifiers.push(node.text); } diff --git a/server/src/project/test/document.test.ts b/server/src/project/test/document.test.ts index 09f7431..24472ed 100644 --- a/server/src/project/test/document.test.ts +++ b/server/src/project/test/document.test.ts @@ -82,6 +82,27 @@ describe('ModelicaDocument', () => { assert.equal(document.getText().trim(), UPDATED_TEST_PACKAGE_CONTENT.trim()); }); + it('can update incrementally', () => { + const textDocument = createTextDocument('.', TEST_PACKAGE_CONTENT); + const tree = project.parser.parse(TEST_PACKAGE_CONTENT); + const document = new ModelicaDocument(project, library, textDocument, tree); + document.update( + '1.0.1', + { + start: { + line: 1, + character: 22, + }, + end: { + line: 1, + character: 27, + }, + } + ); + + assert.equal(document.getText().trim(), UPDATED_TEST_PACKAGE_CONTENT.trim()); + }); + it('a file with no `within` clause has the correct package path', () => { const textDocument = createTextDocument('./package.mo', TEST_PACKAGE_CONTENT); const tree = project.parser.parse(TEST_PACKAGE_CONTENT); diff --git a/server/src/server.ts b/server/src/server.ts index 912a78d..5ac20d0 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -151,7 +151,7 @@ export class ModelicaServer { logger.debug('onDidChangeTextDocument'); for (const change of params.contentChanges) { const range = 'range' in change ? change.range : undefined; - this.#analyzer.updateDocument(params.textDocument.uri, change.text, range); + await this.#analyzer.updateDocument(params.textDocument.uri, change.text, range); } } @@ -161,7 +161,7 @@ export class ModelicaServer { for (const change of params.changes) { switch (change.type) { case LSP.FileChangeType.Created: - this.#analyzer.addDocument(change.uri); + await this.#analyzer.addDocument(change.uri); break; case LSP.FileChangeType.Changed: { // TODO: incremental? From 1dbabcf4b4e8bdea0d4101361f0d2cc52768a319 Mon Sep 17 00:00:00 2001 From: PaddiM8 Date: Wed, 29 May 2024 15:43:09 +0200 Subject: [PATCH 3/3] Test incremental insertion of a new model --- server/src/project/test/document.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/src/project/test/document.test.ts b/server/src/project/test/document.test.ts index 24472ed..760d777 100644 --- a/server/src/project/test/document.test.ts +++ b/server/src/project/test/document.test.ts @@ -101,6 +101,25 @@ describe('ModelicaDocument', () => { ); assert.equal(document.getText().trim(), UPDATED_TEST_PACKAGE_CONTENT.trim()); + + document.update( + '\n model A\n end A;', + { + start: { + line: 1, + character: 30, + }, + end: { + line: 1, + character: 30, + }, + } + ); + + const model = document.tree.rootNode.descendantsOfType("class_definition")[1]; + assert.equal(model.type, "class_definition"); + assert.equal(model.descendantsOfType("IDENT")[0].text, "A"); + assert.equal(document.tree.rootNode.descendantsOfType("annotation_clause").length, 1); }); it('a file with no `within` clause has the correct package path', () => {