fix: recover syntax highlighting after incomplete new (#435)

Author: levivilet

packages/e2e-integration/prepare.mjs

@@ -8,7 +8,7 @@ if (!process.argv[2])
   throw new Error('Pass the path to a disposable LVCE checkout')
 const application = resolve(process.argv[2])
 const manifest = JSON.parse(
-  await readFile(join(application, 'package.json'), 'utf8'),
+  await readFile(join(application, 'package.json'), 'utf8')
 )
 if (manifest.name !== 'lvce-editor')
   throw new Error('Expected an LVCE application checkout')
@@ -30,7 +30,7 @@ try {
 for (const path of config.scripts) {
   await cp(
     join(here, 'scripts', path.split('/').at(-1)),
-    join(application, path),
+    join(application, path)
   )
 }
 // Exercise this repository's build in the pinned application runtime.

packages/e2e-integration/src/viewlet.editor-typescript-destructured-async-default-parameters.ts

@@ -46,20 +46,20 @@ const nextValue = 123
 
   await expect(Locator('.Token.Text')).toHaveCount(0)
   await expect(Locator('.Token.Type', { hasText: 'EditorStatus' })).toHaveText(
-    'EditorStatus',
+    'EditorStatus'
   )
   await expect(Locator('.Token.Type', { hasText: 'StatusBarItem' })).toHaveText(
-    'StatusBarItem',
+    'StatusBarItem'
   )
   await expect(
-    Locator('.Token.Type', { hasText: 'GetBuiltinStatusBarItemsOptions' }),
+    Locator('.Token.Type', { hasText: 'GetBuiltinStatusBarItemsOptions' })
   ).toHaveCount(2)
   await expect(Locator('.Token.Class', { hasText: 'Promise' })).toHaveText(
-    'Promise',
+    'Promise'
   )
   await expect(Locator('.Token.KeywordReturn')).toHaveCount(1)
   await expect(
-    Locator('.Token.Function', { hasText: 'getEditorStatusBarItems' }),
+    Locator('.Token.Function', { hasText: 'getEditorStatusBarItems' })
   ).toHaveText('getEditorStatusBarItems')
   await expect(Locator('.Token.Numeric', { hasText: '123' })).toHaveText('123')
 

packages/e2e-integration/src/viewlet.editor-typescript-generic-function-type-object-member.ts

@@ -50,7 +50,7 @@ export const createRpc = async (url = new URL('.tmp/cacheWorkerMain.js', import.
   )
   await expect(Locator('.Token.Punctuation', { hasText: '[' })).toHaveCount(1)
 
-  const genericLine = source.split('\n')[1]
+  const genericLine = source.split('\n', 2)[1]
   const genericDefaultColumn = genericLine.indexOf('unknown') + 'unknown'.length
   await Editor.setCursor(1, genericDefaultColumn)
   await Editor.type('[]')

packages/e2e-integration/src/viewlet.editor-typescript-readonly-union-parameters.ts

@@ -26,20 +26,20 @@ export const test = async ({
   return process.children
 }
 const nextValue = 123
-`,
+`
   )
   await Settings.update({ 'editor.combineWhitespaceTokens': false })
   await Workspace.setUri(tmpDir)
   await Main.openUri(filePath)
 
   await expect(
-    Locator('.Token.TypePrimitive', { hasText: 'number' }),
+    Locator('.Token.TypePrimitive', { hasText: 'number' })
   ).toHaveText('number')
   await expect(Locator('.Token.Type', { hasText: 'ProcessInfo' })).toHaveText(
-    'ProcessInfo',
+    'ProcessInfo'
   )
   await expect(
-    Locator('.Token.Type', { hasText: 'VisibleProcess' }),
+    Locator('.Token.Type', { hasText: 'VisibleProcess' })
   ).toHaveText('VisibleProcess')
   await expect(Locator('.Token.KeywordReturn')).toHaveCount(3)
   await expect(Locator('.Token.Numeric', { hasText: '123' })).toHaveText('123')

src/tokenizeTypeScript.js

@@ -2975,6 +2975,8 @@ export const tokenizeLine = (line, lineState) => {
           stack.push(state)
           token = TokenType.Comment
           state = State.InsideBlockComment
+        } else if ((next = part.match(RE_LINE_COMMENT))) {
+          token = TokenType.Comment
         } else {
           throw new Error('no')
         }

test/incompleteNew.test.js

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import {
+  initialLineState,
+  TokenMap,
+  tokenizeLine,
+} from '../src/tokenizeTypeScript.js'
+
+const tokenize = (source) => {
+  let state = structuredClone(initialLineState)
+  return source.split('\n').map((line) => {
+    state = tokenizeLine(line, state)
+    const tokens = []
+    let offset = 0
+    for (let index = 0; index < state.tokens.length; index += 2) {
+      const type = TokenMap[state.tokens[index]]
+      const length = state.tokens[index + 1]
+      tokens.push([type, line.slice(offset, offset + length)])
+      offset += length
+    }
+    assert.equal(offset, line.length)
+    return tokens
+  })
+}
+
+test('recovers after a standalone new and a following line comment', () => {
+  const lines = tokenize(
+    `const instance = new  \n// constructor to be added\nconst later = new Float32Array(9)`
+  )
+
+  assert.ok(
+    lines[0].some(([type, text]) => type === 'KeywordNew' && text === 'new')
+  )
+  assert.ok(
+    lines[1].some(([type, text]) => type === 'Comment' && text.startsWith('//'))
+  )
+  assert.ok(
+    lines[2].some(([type, text]) => type === 'VariableName' && text === 'later')
+  )
+  assert.ok(
+    lines[2].some(([type, text]) => type === 'Class' && text === 'Float32Array')
+  )
+})
+
+test('recovers after an incomplete constructor name', () => {
+  const lines = tokenize(
+    `const instance = new Floa\n// finish the constructor\nconst later = 1`
+  )
+
+  assert.ok(
+    lines[0].some(([type, text]) => type === 'Class' && text === 'Floa')
+  )
+  assert.ok(
+    lines[1].some(([type, text]) => type === 'Comment' && text.startsWith('//'))
+  )
+  assert.ok(
+    lines[2].some(([type, text]) => type === 'VariableName' && text === 'later')
+  )
+})
+
+test('keeps constructor continuation after a comment following new', () => {
+  const lines = tokenize(
+    `const instance = new\n// constructor continues below\nFloat32Array(9)`
+  )
+
+  assert.ok(
+    lines[2].some(([type, text]) => type === 'Class' && text === 'Float32Array')
+  )
+})