Export textmate highlighting from CLI (#1434)

eclipse-langium · Apr 22, 2024 · 39e55c4 · 39e55c4
1 parent 26cca7a
commit 39e55c4
Show file tree

Hide file tree

Showing 41 changed files with 200 additions and 177 deletions.
diff --git a/examples/arithmetics/package.json b/examples/arithmetics/package.json
@@ -74,7 +74,7 @@
         "vscode-languageserver": "~9.0.1"
     },
     "devDependencies": {
-        "langium-cli": "~3.0.1"
+        "langium-cli": "~3.0.2"
     },
     "volta": {
         "node": "18.19.1",

diff --git a/examples/arithmetics/src/language-server/generated/ast.ts b/examples/arithmetics/src/language-server/generated/ast.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/arithmetics/src/language-server/generated/grammar.ts b/examples/arithmetics/src/language-server/generated/grammar.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/arithmetics/src/language-server/generated/module.ts b/examples/arithmetics/src/language-server/generated/module.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/domainmodel/package.json b/examples/domainmodel/package.json
@@ -66,7 +66,7 @@
         "vscode-languageserver": "~9.0.1"
     },
     "devDependencies": {
-        "langium-cli": "~3.0.1"
+        "langium-cli": "~3.0.2"
     },
     "volta": {
         "node": "18.19.1",

diff --git a/examples/domainmodel/src/language-server/generated/ast.ts b/examples/domainmodel/src/language-server/generated/ast.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/domainmodel/src/language-server/generated/grammar.ts b/examples/domainmodel/src/language-server/generated/grammar.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/domainmodel/src/language-server/generated/module.ts b/examples/domainmodel/src/language-server/generated/module.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/requirements/package.json b/examples/requirements/package.json
@@ -76,7 +76,7 @@
         "vscode-languageserver": "~9.0.1"
     },
     "devDependencies": {
-        "langium-cli": "~3.0.1"
+        "langium-cli": "~3.0.2"
     },
     "volta": {
         "node": "18.19.1",

diff --git a/examples/requirements/src/language-server/generated/ast.ts b/examples/requirements/src/language-server/generated/ast.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/requirements/src/language-server/generated/grammar.ts b/examples/requirements/src/language-server/generated/grammar.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/requirements/src/language-server/generated/module.ts b/examples/requirements/src/language-server/generated/module.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/statemachine/package.json b/examples/statemachine/package.json
@@ -64,7 +64,7 @@
         "vscode-languageserver": "~9.0.1"
     },
     "devDependencies": {
-        "langium-cli": "~3.0.1"
+        "langium-cli": "~3.0.2"
     },
     "volta": {
         "node": "18.19.1",

diff --git a/examples/statemachine/src/language-server/generated/ast.ts b/examples/statemachine/src/language-server/generated/ast.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/statemachine/src/language-server/generated/grammar.ts b/examples/statemachine/src/language-server/generated/grammar.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/examples/statemachine/src/language-server/generated/module.ts b/examples/statemachine/src/language-server/generated/module.ts
@@ -1,5 +1,5 @@
 /******************************************************************************
- * This file was generated by langium-cli 3.0.1.
+ * This file was generated by langium-cli 3.0.2.
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/generator-langium/templates/core/.package.json b/packages/generator-langium/templates/core/.package.json
@@ -22,7 +22,7 @@
         "@typescript-eslint/parser": "~7.3.1",
         "@typescript-eslint/eslint-plugin": "~7.3.1",
         "eslint": "~8.57.0",
-        "langium-cli": "~3.0.1",
+        "langium-cli": "~3.0.2",
         "typescript": "~5.1.6"
     },
     "volta": {

diff --git a/packages/langium-cli/CHANGELOG.md b/packages/langium-cli/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Change Log of `langium-cli`
 
+## v3.0.2 (Apr. 2024)
+
+Exported the textmate syntax highlighting under `langium-cli/textmate`. This enables importing the highlighting code for the Langium playground.
+
 ## v3.0.1 (Feb. 2024)
 
 Fixed a minor issue related to generated code for projects that don't use the `langium/lsp` import ([#1393](https://github.com/eclipse-langium/langium/pull/1393)).

diff --git a/packages/langium-cli/package.json b/packages/langium-cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "langium-cli",
-  "version": "3.0.1",
+  "version": "3.0.2",
   "description": "CLI for Langium - the language engineering tool",
   "homepage": "https://langium.org",
   "engines": {
@@ -27,6 +27,10 @@
     ".": {
       "import": "./lib/index.js",
       "types": "./lib/index.d.ts"
+    },
+    "./textmate": {
+      "import": "./lib/generator/highlighting/textmate-generator.js",
+      "types": "./lib/generator/highlighting/textmate-generator.d.ts"
     }
   },
   "type": "module",

diff --git a/packages/langium-cli/src/generate.ts b/packages/langium-cli/src/generate.ts
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 import type { AstNode, Grammar, LangiumDocument, Mutable } from 'langium';
-import type { LangiumConfig, LangiumLanguageConfig} from './package.js';
+import type { LangiumConfig, LangiumLanguageConfig} from './package-types.js';
 import { URI } from 'langium';
 import { loadConfig } from './package.js';
 import { AstUtils, GrammarAST } from 'langium';
@@ -17,8 +17,10 @@ import { generateModule } from './generator/module-generator.js';
 import { generateTextMate } from './generator/highlighting/textmate-generator.js';
 import { generateMonarch } from './generator/highlighting/monarch-generator.js';
 import { generatePrismHighlighting } from './generator/highlighting/prism-generator.js';
-import { elapsedTime, getTime, getUserChoice, log, schema } from './generator/util.js';
-import { getFilePath, RelativePath } from './package.js';
+import { elapsedTime, getTime, log } from './generator/langium-util.js';
+import { getUserChoice, schema } from './generator/node-util.js';
+import { RelativePath } from './package-types.js';
+import { getFilePath } from './package.js';
 import { validateParser } from './parser-validation.js';
 import { generateTypesFile } from './generator/types-generator.js';
 import { createGrammarDiagramHtml, createGrammarDiagramSvg } from 'langium-railroad';

diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts
@@ -6,10 +6,11 @@
 import type { Grammar, LangiumCoreServices } from 'langium';
 import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate';
 import type { AstTypes, Property, PropertyDefaultValue } from 'langium/grammar';
-import type { LangiumConfig } from '../package.js';
+import type { LangiumConfig } from '../package-types.js';
 import { AstUtils, MultiMap, GrammarAST } from 'langium';
 import { collectAst, collectTypeHierarchy, findReferenceTypes, isAstType, mergeTypesAndInterfaces } from 'langium/grammar';
-import { collectTerminalRegexps, generatedHeader } from './util.js';
+import { generatedHeader } from './node-util.js';
+import { collectTerminalRegexps } from './langium-util.js';
 
 export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string {
     const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments);

diff --git a/packages/langium-cli/src/generator/grammar-serializer.ts b/packages/langium-cli/src/generator/grammar-serializer.ts
@@ -7,8 +7,8 @@
 import type { Grammar, LangiumCoreServices, Reference } from 'langium';
 import { expandToNode, joinToNode, normalizeEOL, toString } from 'langium/generate';
 import type { URI } from 'vscode-uri';
-import type { LangiumConfig } from '../package.js';
-import { generatedHeader } from './util.js';
+import type { LangiumConfig } from '../package-types.js';
+import { generatedHeader } from './node-util.js';
 
 export function serializeGrammar(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string {
     const node = expandToNode`

diff --git a/packages/langium-cli/src/generator/highlighting/monarch-generator.ts b/packages/langium-cli/src/generator/highlighting/monarch-generator.ts
@@ -6,8 +6,8 @@
 
 import { type Grammar, GrammarAST, GrammarUtils, RegExpUtils } from 'langium';
 import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate';
-import type { LangiumLanguageConfig } from '../../package.js';
-import { collectKeywords } from '../util.js';
+import type { LangiumLanguageConfig } from '../../package-types.js';
+import { collectKeywords } from '../langium-util.js';
 
 /**
  * Monarch Language Definition, describes aspects & token categories of target language

diff --git a/packages/langium-cli/src/generator/highlighting/prism-generator.ts b/packages/langium-cli/src/generator/highlighting/prism-generator.ts
@@ -6,8 +6,8 @@
 import { GrammarAST, type Grammar, GrammarUtils, RegExpUtils } from 'langium';
 import { expandToNode, joinToNode, toString, type Generated } from 'langium/generate';
 import _ from 'lodash';
-import type { LangiumLanguageConfig } from '../../package.js';
-import { collectKeywords } from '../util.js';
+import type { LangiumLanguageConfig } from '../../package-types.js';
+import { collectKeywords } from '../langium-util.js';
 
 interface HighlightElement {
     pattern: string;

diff --git a/packages/langium-cli/src/generator/highlighting/textmate-generator.ts b/packages/langium-cli/src/generator/highlighting/textmate-generator.ts
@@ -5,12 +5,12 @@
 ******************************************************************************/
 import type { Grammar } from 'langium';
 import { GrammarAST, GrammarUtils, RegExpUtils, stream } from 'langium';
-import type { LangiumLanguageConfig } from '../../package.js';
-import { collectKeywords } from '../util.js';
+import type { LangiumLanguageConfig } from '../../package-types.js';
+import { collectKeywords } from '../langium-util.js';
 
 /* eslint-disable dot-notation */
 
-export interface TextMateGrammar {
+interface TextMateGrammar {
     repository: Repository;
     readonly scopeName: string;
     readonly patterns: Pattern[];
@@ -21,11 +21,11 @@ export interface TextMateGrammar {
     readonly firstLineMatch?: string;
 }
 
-export interface Repository {
+interface Repository {
     [name: string]: Pattern;
 }
 
-export interface Pattern {
+interface Pattern {
     id?: number;
     readonly include?: string;
     readonly name?: string;
@@ -43,7 +43,7 @@ export interface Pattern {
     readonly applyEndPatternLast?: boolean;
 }
 
-export interface Captures {
+interface Captures {
     [captureId: string]: Pattern;
 }
 

diff --git a/packages/langium-cli/src/generator/langium-util.ts b/packages/langium-cli/src/generator/langium-util.ts
@@ -0,0 +1,57 @@
+/******************************************************************************
+ * Copyright 2021 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+import { AstUtils, type Grammar, GrammarAST, GrammarUtils, stream } from 'langium';
+import chalk from 'chalk';
+
+//eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function log(level: 'log' | 'warn' | 'error', options: { watch?: boolean }, message: string, ...args: any[]): void {
+    if (options.watch) {
+        console[level](getTime() + message, ...args);
+    } else {
+        console[level](message, ...args);
+    }
+}
+
+let start = process.hrtime();
+
+export function elapsedTime(): string {
+    const elapsed = process.hrtime(start)[1] / 1000000; // divide by a million to get nano to milli
+    start = process.hrtime(); // reset the timer
+    return elapsed.toFixed();
+}
+
+export function getTime(): string {
+    const date = new Date();
+    return `[${chalk.gray(`${padZeroes(date.getHours())}:${padZeroes(date.getMinutes())}:${padZeroes(date.getSeconds())}`)}] `;
+}
+
+function padZeroes(i: number): string {
+    return i.toString().padStart(2, '0');
+}
+
+export function collectKeywords(grammar: Grammar): string[] {
+    const keywords = new Set<string>();
+    const reachableRules = GrammarUtils.getAllReachableRules(grammar, false);
+
+    for (const keyword of stream(reachableRules)
+        .filter(GrammarAST.isParserRule)
+        .flatMap(rule => AstUtils.streamAllContents(rule).filter(GrammarAST.isKeyword))) {
+        keywords.add(keyword.value);
+    }
+
+    return Array.from(keywords).sort();
+}
+
+export function collectTerminalRegexps(grammar: Grammar): Record<string, RegExp> {
+    const result: Record<string, RegExp> = {};
+    const reachableRules = GrammarUtils.getAllReachableRules(grammar, false);
+    for (const terminalRule of stream(reachableRules).filter(GrammarAST.isTerminalRule)) {
+        const name = terminalRule.name;
+        const regexp = GrammarUtils.terminalRegex(terminalRule);
+        result[name] = regexp;
+    }
+    return result;
+}
diff --git a/packages/langium-cli/src/generator/module-generator.ts b/packages/langium-cli/src/generator/module-generator.ts
@@ -6,8 +6,8 @@
 
 import type { Grammar, IParserConfig } from 'langium';
 import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate';
-import type { LangiumConfig, LangiumLanguageConfig } from '../package.js';
-import { generatedHeader } from './util.js';
+import type { LangiumConfig, LangiumLanguageConfig } from '../package-types.js';
+import { generatedHeader } from './node-util.js';
 
 export function generateModule(grammars: Grammar[], config: LangiumConfig, grammarConfigMap: Map<Grammar, LangiumLanguageConfig>): string {
     const grammarsWithName = grammars.filter(grammar => !!grammar.name);