Fix for incorrect highlighting in verbatim strings

Azure · Mar 20, 2024 · 3d54b9a · 3d54b9a
1 parent 96f6c3a
commit 3d54b9a
Show file tree

Hide file tree

Showing 22 changed files with 229 additions and 94 deletions.
diff --git a/src/highlightjs/dist/bicep.min.js b/src/highlightjs/dist/bicep.min.js
diff --git a/src/highlightjs/package.json b/src/highlightjs/package.json
@@ -26,6 +26,7 @@
   "scripts": {
     "build": "webpack",
     "test": "jest",
+    "test:fix": "BASELINE_RECORD=true jest",
     "lint": "eslint src",
     "lint:fix": "eslint src"
   }

diff --git a/src/highlightjs/src/bicep.ts b/src/highlightjs/src/bicep.ts
@@ -83,7 +83,7 @@ const escapeChar: Mode = {
 const stringVerbatim: Mode = {
   className: 'string',
   begin: `'''`,
-  end: `'''`,
+  end: `'''${notBefore(`'`)}`,
 }
 
 const stringSubstitution: Mode = {

diff --git a/src/highlightjs/test/baselines.test.ts b/src/highlightjs/test/baselines.test.ts
@@ -2,26 +2,23 @@
 // Licensed under the MIT License.
 
 import { readdirSync, existsSync } from 'fs';
-import { readFile, writeFile } from 'fs/promises';
+import { readFile } from 'fs/promises';
 import path, { dirname, basename, extname } from 'path';
 import { env } from 'process';
 import { spawnSync } from 'child_process';
 import hljs from 'highlight.js';
-import bicep, { default as bicepLanguage } from '../src/bicep';
+import { default as bicepLanguage } from '../src/bicep';
+import { expectFileContents, baselineRecordEnabled } from './utils';
 
-async function writeBaseline(filePath: string) {
+async function generateBaseline(filePath: string) {
   const baselineBaseName = basename(filePath, extname(filePath));
   const baselineFilePath = path.join(dirname(filePath), `${baselineBaseName}.html`);
 
-  let diffBefore = '';
   const bicepFile = await readFile(filePath, { encoding: 'utf-8' });
-  try {
-    diffBefore = await readFile(baselineFilePath, { encoding: 'utf-8' });
-  } catch {} // ignore and create the baseline file anyway
 
   hljs.registerLanguage('bicep', bicepLanguage);
   const result = hljs.highlight(bicepFile, { language: 'bicep' });
-  const diffAfter = `
+  const expected = `
 <!--
   Preview this file by prepending http://htmlpreview.github.io/? to its URL
   e.g. http://htmlpreview.github.io/?https://raw.githubusercontent.com/Azure/bicep/main/src/highlightjs/test/baselines/${baselineBaseName}.html
@@ -37,15 +34,10 @@ ${result.value}
   </body>
 </html>`;
 
-  const output = {
-    diffBefore: diffBefore.replace(/\r\n/g, '\n'),
-    diffAfter: diffAfter.replace(/\r\n/g, '\n'),
+  return {
+    expected: expected.replace(/\r\n/g, '\n'),
     baselineFilePath,
   };
-
-  await writeFile(baselineFilePath, output.diffAfter, { encoding: 'utf-8' });
-
-  return output;
 }
 
 const baselinesDir = `${__dirname}/baselines`;
@@ -56,16 +48,6 @@ const baselineFiles = readdirSync(baselinesDir)
 
 for (const filePath of baselineFiles) {
   describe(filePath, () => {
-    let result = {
-      baselineFilePath: '',
-      diffBefore: '',
-      diffAfter: ''
-    };
-
-    beforeAll(async () => {
-      result = await writeBaseline(filePath);
-    });
-
     if (!basename(filePath).startsWith('invalid_')) {
       // skip the invalid files - we don't expect them to compile
 
@@ -90,8 +72,17 @@ for (const filePath of baselineFiles) {
       });
     }
 
-    it('baseline matches expected', () => {
-      expect(result.diffBefore).toEqual(result.diffAfter);
+    it('baseline matches expected', async () => {
+      const { expected, baselineFilePath } = await generateBaseline(filePath);
+
+      await expectFileContents(baselineFilePath, expected);
     });
   });
 }
+
+describe('Test suite', () => {
+  it('should not succeed if BASELINE_RECORD is set to true', () => {
+    // This test just ensures the suite doesn't pass in 'record' mode
+    expect(baselineRecordEnabled).toBeFalsy();
+  });
+});
diff --git a/src/highlightjs/test/baselines/strings.bicep b/src/highlightjs/test/baselines/strings.bicep
@@ -0,0 +1,9 @@
+// evaluates to '\'abc\''
+var multilineExtraQuotes = ''''abc''''
+
+// evaluates to '\'\nabc\n\''
+var multilineExtraQuotesNewlines = ''''
+abc
+''''
+
+var multilineSingleLine = '''hello!'''
diff --git a/src/highlightjs/test/baselines/strings.html b/src/highlightjs/test/baselines/strings.html
@@ -0,0 +1,24 @@
+
+<!--
+  Preview this file by prepending http://htmlpreview.github.io/? to its URL
+  e.g. http://htmlpreview.github.io/?https://raw.githubusercontent.com/Azure/bicep/main/src/highlightjs/test/baselines/strings.html
+-->
+<html>
+  <head>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.2/styles/default.min.css">
+  </head>
+  <body>
+    <pre class="hljs">
+<span class="hljs-comment">// evaluates to &#x27;\&#x27;abc\&#x27;&#x27;</span>
+<span class="hljs-variable"><span class="hljs-keyword">var</span></span> <span class="hljs-variable">multilineExtraQuotes</span> = <span class="hljs-string">&#x27;&#x27;&#x27;&#x27;abc&#x27;&#x27;&#x27;&#x27;</span>
+
+<span class="hljs-comment">// evaluates to &#x27;\&#x27;\nabc\n\&#x27;&#x27;</span>
+<span class="hljs-variable"><span class="hljs-keyword">var</span></span> <span class="hljs-variable">multilineExtraQuotesNewlines</span> = <span class="hljs-string">&#x27;&#x27;&#x27;&#x27;
+abc
+&#x27;&#x27;&#x27;&#x27;</span>
+
+<span class="hljs-variable"><span class="hljs-keyword">var</span></span> <span class="hljs-variable">multilineSingleLine</span> = <span class="hljs-string">&#x27;&#x27;&#x27;hello!&#x27;&#x27;&#x27;</span>
+
+    </pre>
+  </body>
+</html>
diff --git a/src/highlightjs/test/grammar.test.ts b/src/highlightjs/test/grammar.test.ts
@@ -4,6 +4,7 @@
 import { existsSync } from "fs";
 import { readFile, rm } from "fs/promises";
 import { spawnSync } from "child_process";
+import { expectFileContents } from "./utils";
 
 const root = `${__dirname}/..`;
 
@@ -27,9 +28,8 @@ describe('grammar tests', () => {
   });
 
   it('should be up-to-date', async () => {
-    const savedGrammar = await readFile(grammarPath, { encoding: 'utf8' });
-
     const generatedGrammar = await generateGrammar();
-    expect(generatedGrammar).toStrictEqual(savedGrammar);
+
+    await expectFileContents(grammarPath, generatedGrammar);
   }, webpackTestTimeout);
 });
diff --git a/src/highlightjs/test/utils.ts b/src/highlightjs/test/utils.ts
@@ -0,0 +1,18 @@
+import { existsSync } from "fs";
+import { writeFile, mkdir, readFile } from "fs/promises";
+import path from "path";
+
+export const baselineRecordEnabled = (process.env['BASELINE_RECORD']?.toLowerCase() === 'true');
+
+export async function expectFileContents(filePath: string, contents: string) {
+  if (baselineRecordEnabled) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, contents, 'utf-8');
+  } else {
+    // If these assertions fail, use 'npm run test:fix' to replace the baseline files
+    expect(existsSync(filePath)).toBeTruthy();
+
+    const readContents = await readFile(filePath, { encoding: 'utf8' });
+    expect(contents).toBe(readContents);
+  }
+}
diff --git a/src/monarch/package.json b/src/monarch/package.json
@@ -24,6 +24,7 @@
   "scripts": {
     "build": "tsc -p ./",
     "test": "jest",
+    "test:fix": "BASELINE_RECORD=true jest",
     "lint": "eslint src",
     "lint:fix": "eslint src"
   },

diff --git a/src/monarch/src/bicep.ts b/src/monarch/src/bicep.ts
@@ -4,6 +4,10 @@
 import { languages } from 'monaco-editor-core';
 
 const bounded = (text: string) => `\\b${text}\\b`;
+const after = (regex: string) => `(?<=${regex})`;
+const notAfter = (regex: string) => `(?<!${regex})`;
+const before = (regex: string) => `(?=${regex})`;
+const notBefore = (regex: string) => `(?!${regex})`;
 
 const identifierStart = "[_a-zA-Z]";
 const identifierContinue = "[_a-zA-Z0-9]";
@@ -67,7 +71,7 @@ export const BicepLanguage: languages.IMonarchLanguage = {
 
     stringVerbatim: [
       { regex: `(|'|'')[^']`, action: { token: 'string' } },
-      { regex: `'''`, action: { token: 'string.quote', next: '@pop' } },
+      { regex: `'''${notBefore(`'`)}`, action: { token: 'string.quote', next: '@pop' } },
     ],
 
     stringLiteral: [
@@ -98,7 +102,7 @@ export const BicepLanguage: languages.IMonarchLanguage = {
 
     expression: [
       { regex: `'''`, action: { token: 'string.quote', next: '@stringVerbatim' } },
-      { regex: `'`, action: { token: 'string.quote', next: '@stringLiteral' } },
+      { regex: `'${notBefore(`''`)}`, action: { token: 'string.quote', next: '@stringLiteral' } },
       { regex: numericLiteral, action: { token: 'number' } },
       { regex: identifier, action: {
         cases: {

diff --git a/src/monarch/test/baselines.test.ts b/src/monarch/test/baselines.test.ts
@@ -27,6 +27,7 @@ import { BicepLanguage } from '../src/bicep';
 import { editor, languages } from 'monaco-editor-core';
 import { escape } from 'html-escaper';
 import { env } from 'process';
+import { expectFileContents, baselineRecordEnabled } from './utils';
 
 const tokenToHljsClass: Record<string, string | null> = {
   'string.bicep': 'string',
@@ -51,16 +52,11 @@ async function getTokensByLine(content: string) {
   }));
 }
 
-async function writeBaseline(filePath: string) {
-  const baselineBaseName = basename(filePath, extname(filePath));
-  const baselineFilePath = path.join(dirname(filePath), `${baselineBaseName}.html`);
+async function generateBaseline(inputFilePath: string) {
+  const baselineBaseName = basename(inputFilePath, extname(inputFilePath));
+  const baselineFilePath = path.join(dirname(inputFilePath), `${baselineBaseName}.html`);
 
-  let diffBefore = '';
-  const bicepFile = await readFile(filePath, { encoding: 'utf-8' });
-  try {
-    diffBefore = await readFile(baselineFilePath, { encoding: 'utf-8' });
-  // eslint-disable-next-line no-empty
-  } catch {} // ignore and create the baseline file anyway
+  const bicepFile = await readFile(inputFilePath, { encoding: 'utf-8' });
 
   let html = '';
   const tokensByLine = await getTokensByLine(bicepFile);
@@ -95,7 +91,7 @@ async function writeBaseline(filePath: string) {
     html += '\n';
   }
 
-  const diffAfter = `
+  const expected = `
 <!--
   Preview this file by prepending http://htmlpreview.github.io/? to its URL
   e.g. http://htmlpreview.github.io/?https://raw.githubusercontent.com/Azure/bicep/main/src/monarch/test/baselines/${baselineBaseName}.html
@@ -111,15 +107,10 @@ ${html}
   </body>
 </html>`;
 
-  const output = {
-    diffBefore: diffBefore.replace(/\r\n/g, '\n'),
-    diffAfter: diffAfter.replace(/\r\n/g, '\n'),
+  return {
+    expected: expected.replace(/\r\n/g, '\n'),
     baselineFilePath,
   };
-
-  await writeFile(baselineFilePath, output.diffAfter, { encoding: 'utf-8' });
-
-  return output;
 }
 
 const baselinesDir = `${__dirname}/baselines`;
@@ -130,16 +121,6 @@ const baselineFiles = readdirSync(baselinesDir)
 
 for (const filePath of baselineFiles) {
   describe(`Baseline: ${filePath}`, () => {
-    let result: {
-      baselineFilePath: string;
-      diffBefore: string;
-      diffAfter: string;
-    };
-
-    beforeAll(async () => {
-      result = await writeBaseline(filePath);
-    });
-
     if (!basename(filePath).startsWith('invalid_')) {
       // skip the invalid files - we don't expect them to compile
 
@@ -164,8 +145,17 @@ for (const filePath of baselineFiles) {
       });
     }
 
-    it('baseline matches expected', () => {
-      expect(result.diffBefore).toStrictEqual(result.diffAfter);
+    it('baseline matches expected', async () => {
+      const { expected, baselineFilePath } = await generateBaseline(filePath);
+
+      await expectFileContents(baselineFilePath, expected);
     });
   });
 }
+
+describe('Test suite', () => {
+  it('should not succeed if BASELINE_RECORD is set to true', () => {
+    // This test just ensures the suite doesn't pass in 'record' mode
+    expect(baselineRecordEnabled).toBeFalsy();
+  });
+});
diff --git a/src/monarch/test/baselines/strings.bicep b/src/monarch/test/baselines/strings.bicep
@@ -0,0 +1,9 @@
+// evaluates to '\'abc\''
+var multilineExtraQuotes = ''''abc''''
+
+// evaluates to '\'\nabc\n\''
+var multilineExtraQuotesNewlines = ''''
+abc
+''''
+
+var multilineSingleLine = '''hello!'''
diff --git a/src/monarch/test/baselines/strings.html b/src/monarch/test/baselines/strings.html
@@ -0,0 +1,25 @@
+
+<!--
+  Preview this file by prepending http://htmlpreview.github.io/? to its URL
+  e.g. http://htmlpreview.github.io/?https://raw.githubusercontent.com/Azure/bicep/main/src/monarch/test/baselines/strings.html
+-->
+<html>
+  <head>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.2/styles/default.min.css">
+  </head>
+  <body>
+    <pre class="hljs">
+<span class="hljs-comment">// evaluates to &#39;\&#39;abc\&#39;&#39;</span>
+<span class="hljs-keyword">var</span> <span class="hljs-variable">multilineExtraQuotes</span> = <span class="hljs-string">&#39;&#39;&#39;</span><span class="hljs-string">&#39;abc</span>&#39;<span class="hljs-string">&#39;&#39;&#39;</span>
+
+<span class="hljs-comment">// evaluates to &#39;\&#39;\nabc\n\&#39;&#39;</span>
+<span class="hljs-keyword">var</span> <span class="hljs-variable">multilineExtraQuotesNewlines</span> = <span class="hljs-string">&#39;&#39;&#39;</span>&#39;
+<span class="hljs-string">abc</span>
+&#39;<span class="hljs-string">&#39;&#39;&#39;</span>
+
+<span class="hljs-keyword">var</span> <span class="hljs-variable">multilineSingleLine</span> = <span class="hljs-string">&#39;&#39;&#39;</span><span class="hljs-string">hello!</span><span class="hljs-string">&#39;&#39;&#39;</span>
+
+
+    </pre>
+  </body>
+</html>
diff --git a/src/monarch/test/utils.ts b/src/monarch/test/utils.ts
@@ -0,0 +1,18 @@
+import { existsSync } from "fs";
+import { writeFile, mkdir, readFile } from "fs/promises";
+import path from "path";
+
+export const baselineRecordEnabled = (process.env['BASELINE_RECORD']?.toLowerCase() === 'true');
+
+export async function expectFileContents(filePath: string, contents: string) {
+  if (baselineRecordEnabled) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, contents, 'utf-8');
+  } else {
+    // If these assertions fail, use 'npm run test:fix' to replace the baseline files
+    expect(existsSync(filePath)).toBeTruthy();
+
+    const readContents = await readFile(filePath, { encoding: 'utf8' });
+    expect(contents).toBe(readContents);
+  }
+}
diff --git a/src/textmate/bicep.tmlanguage b/src/textmate/bicep.tmlanguage
@@ -361,7 +361,7 @@
         <key>begin</key>
         <string>'''</string>
         <key>end</key>
-        <string>'''</string>
+        <string>'''(?!')</string>
         <key>patterns</key>
         <array/>
       </dict>

diff --git a/src/textmate/package.json b/src/textmate/package.json
@@ -28,6 +28,7 @@
   "scripts": {
     "build": "ts-node src/cmd/build.ts",
     "test": "jest",
+    "test:fix": "BASELINE_RECORD=true jest",
     "lint": "eslint src",
     "lint:fix": "eslint src"
   }

diff --git a/src/textmate/src/bicep.ts b/src/textmate/src/bicep.ts
@@ -111,7 +111,7 @@ const stringVerbatim: BeginEndRule = {
   key: "string-verbatim",
   scope: "string.quoted.multi.bicep",
   begin: `'''`,
-  end: `'''`,
+  end: `'''${notBefore(`'`)}`,
   patterns: [],
 }