|
| 1 | +import assert from 'node:assert/strict' |
| 2 | +import { test } from 'node:test' |
| 3 | +import { |
| 4 | + initialLineState, |
| 5 | + TokenMap, |
| 6 | + tokenizeLine, |
| 7 | +} from '../src/tokenizeTypeScript.js' |
| 8 | + |
| 9 | +const tokenize = (source) => { |
| 10 | + let state = structuredClone(initialLineState) |
| 11 | + const result = [] |
| 12 | + for (const line of source.split('\n')) { |
| 13 | + state = tokenizeLine(line, state) |
| 14 | + let offset = 0 |
| 15 | + for (let index = 0; index < state.tokens.length; index += 2) { |
| 16 | + const length = state.tokens[index + 1] |
| 17 | + result.push([ |
| 18 | + TokenMap[state.tokens[index]], |
| 19 | + line.slice(offset, offset + length), |
| 20 | + ]) |
| 21 | + offset += length |
| 22 | + } |
| 23 | + assert.equal(offset, line.length) |
| 24 | + } |
| 25 | + return result |
| 26 | +} |
| 27 | + |
| 28 | +test('preserves parameter and generic type classifications across scan fast paths', () => { |
| 29 | + const tokens = tokenize(`const value = 1 |
| 30 | +function first(value: number) {} |
| 31 | +const plain = value |
| 32 | +const table: Map<string, User> = new Map() |
| 33 | +const identity = (value: User): User => value |
| 34 | +/* open |
| 35 | +comment */ |
| 36 | +function last(value: boolean) {}`) |
| 37 | + assert.ok( |
| 38 | + tokens.some(([type, text]) => type === 'TypePrimitive' && text === 'number') |
| 39 | + ) |
| 40 | + assert.ok( |
| 41 | + tokens.some( |
| 42 | + ([type, text]) => type === 'TypePrimitive' && text === 'boolean' |
| 43 | + ) |
| 44 | + ) |
| 45 | + assert.ok(tokens.some(([type, text]) => type === 'Type' && text === 'User')) |
| 46 | + assert.ok( |
| 47 | + tokens.some( |
| 48 | + ([type, text]) => type === 'Comment' && text.includes('comment') |
| 49 | + ) |
| 50 | + ) |
| 51 | +}) |
0 commit comments