265 lines
12 KiB
JavaScript
265 lines
12 KiB
JavaScript
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const { createRequire } = require('node:module');
|
|
const test = require('node:test');
|
|
|
|
const root = path.resolve(__dirname, '../..');
|
|
const extension = path.join(root, 'editor/vscode');
|
|
const extensionRequire = createRequire(path.join(extension, 'package.json'));
|
|
const textmate = extensionRequire('vscode-textmate');
|
|
const oniguruma = extensionRequire('vscode-oniguruma');
|
|
const cliArguments = process.argv.slice(2);
|
|
assert.equal(cliArguments.length, 2, 'pass --markdown-grammar followed by the real VS Code grammar path');
|
|
assert.equal(cliArguments[0], '--markdown-grammar');
|
|
const markdownPath = path.resolve(cliArguments[1]);
|
|
const manifest = JSON.parse(fs.readFileSync(path.join(extension, 'package.json'), 'utf8'));
|
|
const configuration = JSON.parse(fs.readFileSync(path.join(extension, 'language-configuration.json'), 'utf8'));
|
|
|
|
async function loadGrammars() {
|
|
const wasm = fs.readFileSync(extensionRequire.resolve('vscode-oniguruma/release/onig.wasm'));
|
|
await oniguruma.loadWASM(wasm.buffer.slice(wasm.byteOffset, wasm.byteOffset + wasm.byteLength));
|
|
const catalog = new Map();
|
|
for (const contribution of manifest.contributes.grammars) {
|
|
const filename = path.join(extension, contribution.path);
|
|
catalog.set(contribution.scopeName, textmate.parseRawGrammar(fs.readFileSync(filename, 'utf8'), filename));
|
|
}
|
|
const markdown = textmate.parseRawGrammar(fs.readFileSync(markdownPath, 'utf8'), markdownPath);
|
|
assert.equal(markdown.scopeName, 'text.html.markdown');
|
|
catalog.set(markdown.scopeName, markdown);
|
|
const registry = new textmate.Registry({
|
|
onigLib: Promise.resolve({
|
|
createOnigScanner: patterns => new oniguruma.OnigScanner(patterns),
|
|
createOnigString: value => new oniguruma.OnigString(value)
|
|
}),
|
|
loadGrammar: scope => catalog.get(scope) || null,
|
|
getInjections: scope => manifest.contributes.grammars
|
|
.filter(contribution => contribution.injectTo?.includes(scope))
|
|
.map(contribution => contribution.scopeName)
|
|
});
|
|
return {
|
|
source: await registry.loadGrammarWithConfiguration('source.dever', 1, {}),
|
|
markdown: await registry.loadGrammarWithConfiguration('text.html.markdown', 2, {
|
|
embeddedLanguages: { 'meta.embedded.block.dever': 1 },
|
|
tokenTypes: { 'meta.embedded.block.dever': 0 }
|
|
})
|
|
};
|
|
}
|
|
|
|
const grammars = loadGrammars();
|
|
|
|
function tokenize(grammar, document) {
|
|
let state = textmate.INITIAL;
|
|
return document.split(/\r\n|\r|\n/).map(line => {
|
|
const encoded = grammar.tokenizeLine2(line, state);
|
|
const result = grammar.tokenizeLine(line, state);
|
|
state = result.ruleStack;
|
|
return { line, tokens: result.tokens, encoded: encoded.tokens };
|
|
});
|
|
}
|
|
|
|
function tokenAt(rows, line, fragment) {
|
|
const row = rows[line];
|
|
const offset = row.line.indexOf(fragment);
|
|
assert.ok(offset >= 0, `line ${line + 1} does not contain ${fragment}`);
|
|
const token = row.tokens.find(candidate => candidate.startIndex <= offset && candidate.endIndex > offset);
|
|
assert.ok(token, `no token for ${fragment} on line ${line + 1}`);
|
|
return token;
|
|
}
|
|
|
|
function hasScope(rows, line, fragment, scope) {
|
|
const actual = tokenAt(rows, line, fragment).scopes;
|
|
assert.ok(actual.includes(scope), `${fragment}: expected ${scope}, received ${actual.join(' ')}`);
|
|
}
|
|
|
|
function sourceFiles(directory, suffix) {
|
|
return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
|
|
const filename = path.join(directory, entry.name);
|
|
return entry.isDirectory() ? sourceFiles(filename, suffix) : filename.endsWith(suffix) ? [filename] : [];
|
|
});
|
|
}
|
|
|
|
test('types, fields and named function inputs/outputs receive Dever scopes', async () => {
|
|
const rows = tokenize((await grammars).source, [
|
|
'type Profile {',
|
|
' id: Int?',
|
|
' name: Text',
|
|
'}',
|
|
'greet(name: Text) (message: Text) {',
|
|
' message = app.format(name)',
|
|
'}'
|
|
].join('\n'));
|
|
hasScope(rows, 0, 'type', 'storage.type.dever');
|
|
hasScope(rows, 0, 'Profile', 'entity.name.type.dever');
|
|
hasScope(rows, 1, 'id', 'variable.other.member.dever');
|
|
hasScope(rows, 1, 'Int', 'support.type.dever');
|
|
hasScope(rows, 4, 'greet', 'entity.name.function.dever');
|
|
hasScope(rows, 4, 'name', 'variable.other.member.dever');
|
|
hasScope(rows, 4, 'message', 'variable.other.member.dever');
|
|
hasScope(rows, 5, 'app', 'entity.name.namespace.dever');
|
|
hasScope(rows, 5, 'format', 'entity.name.function.dever');
|
|
});
|
|
|
|
test('API, Model, Job and external Adapter declarations are covered', async () => {
|
|
const source = (await grammars).source;
|
|
for (const method of ['get', 'post', 'put', 'delete', 'cmd', 'rest']) {
|
|
const rows = tokenize(source, `${method} profile = app.profile`);
|
|
hasScope(rows, 0, method, 'keyword.control.api.dever');
|
|
}
|
|
const rows = tokenize(source, [
|
|
'public get profile = app.profile',
|
|
' private owner user_id: user.account.model.id = dever.auth.user_id()',
|
|
'index(user_id, created_at)',
|
|
'sql owned_articles(id: Int) (articles: List<Article>) {',
|
|
'job send(message: Text) () fails app.Error {',
|
|
'schedule send_every_minute {',
|
|
'external npm "worker.js" {}'
|
|
].join('\n'));
|
|
hasScope(rows, 0, 'public', 'storage.modifier.dever');
|
|
hasScope(rows, 1, 'owner', 'storage.modifier.dever');
|
|
hasScope(rows, 2, 'index', 'keyword.other.declaration.dever');
|
|
hasScope(rows, 3, 'sql', 'keyword.other.declaration.dever');
|
|
hasScope(rows, 4, 'job', 'storage.modifier.dever');
|
|
hasScope(rows, 4, 'fails', 'storage.modifier.dever');
|
|
hasScope(rows, 5, 'schedule', 'keyword.other.declaration.dever');
|
|
hasScope(rows, 6, 'npm', 'keyword.other.external.dever');
|
|
});
|
|
|
|
test('hash comments, quoted text and escapes do not leak into subsequent lines', async () => {
|
|
const rows = tokenize((await grammars).source, [
|
|
'# type Fake and return',
|
|
String.raw`message = "中文 # and type \n \u{4E2D} \""`,
|
|
String.raw`invalid = "\q"`,
|
|
'broken = "unfinished',
|
|
'type Next { value: Text }'
|
|
].join('\n'));
|
|
hasScope(rows, 0, 'type', 'comment.line.number-sign.dever');
|
|
hasScope(rows, 1, '#', 'string.quoted.double.dever');
|
|
hasScope(rows, 1, '\\n', 'constant.character.escape.dever');
|
|
hasScope(rows, 1, '\\u{4E2D}', 'constant.character.escape.dever');
|
|
hasScope(rows, 2, '\\q', 'invalid.illegal.escape.dever');
|
|
hasScope(rows, 4, 'type', 'storage.type.dever');
|
|
assert.ok(!tokenAt(rows, 4, 'type').scopes.includes('string.quoted.double.dever'));
|
|
});
|
|
|
|
test('integer division is an operator, not a JavaScript line comment', async () => {
|
|
const rows = tokenize((await grammars).source, 'count = 12 // 3\namount = 1.25e-2\nvalid = true and not false');
|
|
hasScope(rows, 0, '//', 'keyword.operator.dever');
|
|
hasScope(rows, 0, '3', 'constant.numeric.dever');
|
|
assert.ok(!tokenAt(rows, 0, '3').scopes.some(scope => scope.startsWith('comment.')));
|
|
hasScope(rows, 1, '1.25e-2', 'constant.numeric.dever');
|
|
hasScope(rows, 2, 'true', 'constant.language.dever');
|
|
hasScope(rows, 2, 'and', 'keyword.operator.logical.dever');
|
|
});
|
|
|
|
test('excluded control-flow syntax is not presented as a valid function', async () => {
|
|
const rows = tokenize((await grammars).source, 'if(value)\nreturn value\nimport other\npackage main');
|
|
for (const [line, keyword] of ['if', 'return', 'import', 'package'].entries()) {
|
|
hasScope(rows, line, keyword, 'invalid.illegal.keyword.dever');
|
|
}
|
|
});
|
|
|
|
test('Markdown fences reuse source.dever while prose keeps Markdown scopes', async () => {
|
|
for (const marker of ['```', '~~~']) {
|
|
for (const language of ['dever', 'typescript dever']) {
|
|
const rows = tokenize((await grammars).markdown, [
|
|
'# 文档', '', '## 类型', `${marker}${language}`,
|
|
'type Profile { name: Text }', marker, '', '正文 type Profile'
|
|
].join('\n'));
|
|
hasScope(rows, 4, 'type', 'storage.type.dever');
|
|
hasScope(rows, 4, 'Text', 'support.type.dever');
|
|
hasScope(rows, 4, 'Profile', 'meta.embedded.block.dever');
|
|
assert.ok(!tokenAt(rows, 7, 'type').scopes.includes('source.dever'));
|
|
}
|
|
}
|
|
});
|
|
|
|
test('embedded tokens carry the Dever language ID and are not string tokens', async () => {
|
|
const rows = tokenize((await grammars).markdown, '```dever\ntype Profile {}\n```');
|
|
const metadata = rows[1].encoded[1];
|
|
assert.equal(metadata & 0xff, 1);
|
|
assert.equal((metadata >>> 8) & 3, 0);
|
|
});
|
|
|
|
test('fence closing honors its character and minimum opening length', async () => {
|
|
const rows = tokenize((await grammars).markdown, [
|
|
'````dever', 'type First {}', '```', 'type Second {}',
|
|
'~~~~', 'type Third {}', '`````', '正文 type Outside'
|
|
].join('\n'));
|
|
for (const line of [1, 3, 5]) hasScope(rows, line, 'type', 'storage.type.dever');
|
|
assert.ok(!tokenAt(rows, 7, 'type').scopes.includes('source.dever'));
|
|
const waves = tokenize((await grammars).markdown, '~~~dever\ntype Inside {}\n~~~~\n正文 type Outside');
|
|
assert.ok(!tokenAt(waves, 3, 'type').scopes.includes('source.dever'));
|
|
});
|
|
|
|
test('valid indentation and CRLF work without taking over other language fences', async () => {
|
|
const rows = tokenize((await grammars).markdown, ' ```dever\r\ntype Profile {}\r\n ```\r\n正文');
|
|
hasScope(rows, 1, 'type', 'storage.type.dever');
|
|
for (const language of ['typescript', 'javascript', 'dever extra', 'typescript dever']) {
|
|
const other = tokenize((await grammars).markdown, `\`\`\`${language}\ntype Profile {}\n\`\`\``);
|
|
assert.ok(!tokenAt(other, 1, 'type').scopes.includes('source.dever'), language);
|
|
}
|
|
});
|
|
|
|
test('quotes, lists, showcase fences, indented code and comments retain Markdown scopes', async () => {
|
|
for (const [document, line] of [
|
|
['> ```dever\n> type Quoted {}\n> ```', 1],
|
|
['- 示例\n\n ```dever\n type Nested {}\n ```', 3],
|
|
['````markdown\n```dever\ntype Showcase {}\n```\n````', 2],
|
|
[' ```dever\n type Indented {}\n ```', 1],
|
|
['<!--\n```dever\ntype Comment {}\n```\n-->', 2]
|
|
]) {
|
|
const rows = tokenize((await grammars).markdown, document);
|
|
assert.ok(!tokenAt(rows, line, 'type').scopes.includes('source.dever'), document);
|
|
}
|
|
});
|
|
|
|
test('language configuration uses Dever comments, pairs and indentation', () => {
|
|
assert.equal(configuration.comments.lineComment, '#');
|
|
assert.equal(configuration.comments.blockComment, undefined);
|
|
assert.deepEqual(configuration.brackets, [['{', '}'], ['[', ']'], ['(', ')']]);
|
|
const increase = new RegExp(configuration.indentationRules.increaseIndentPattern);
|
|
const decrease = new RegExp(configuration.indentationRules.decreaseIndentPattern);
|
|
for (const line of ['greet(name: Text) (message: Text) {', ' values = [', 'type Profile { # 字段']) {
|
|
assert.ok(increase.test(line), line);
|
|
}
|
|
for (const line of ['# example {', 'message = "literal {"', 'message = "value" # {']) {
|
|
assert.ok(!increase.test(line), line);
|
|
}
|
|
assert.ok(decrease.test(' }'));
|
|
assert.ok(decrease.test(' ]'));
|
|
assert.ok(configuration.autoClosingPairs.every(pair => pair.notIn.includes('string') && pair.notIn.includes('comment')));
|
|
assert.deepEqual(manifest.contributes.languages[0].extensions, ['.dever']);
|
|
const injection = manifest.contributes.grammars.find(contribution => contribution.injectTo?.includes('text.html.markdown'));
|
|
assert.deepEqual(injection.embeddedLanguages, { 'meta.embedded.block.dever': 'dever' });
|
|
assert.deepEqual(injection.tokenTypes, { 'meta.embedded.block.dever': 'other' });
|
|
assert.equal(manifest.main, undefined);
|
|
assert.equal(manifest.dependencies, undefined);
|
|
});
|
|
|
|
test('maintained plain and Markdown CMS files tokenize without illegal tokens', async () => {
|
|
const loaded = await grammars;
|
|
for (const [directory, suffix, grammar] of [
|
|
['examples/cms/dever', '.dever', loaded.source],
|
|
['examples/cms/md', '.dever.md', loaded.markdown]
|
|
]) {
|
|
const files = sourceFiles(path.join(root, directory), suffix);
|
|
assert.ok(files.length > 0, directory);
|
|
let highlighted = 0;
|
|
for (const filename of files) {
|
|
const rows = tokenize(grammar, fs.readFileSync(filename, 'utf8'));
|
|
let fileHighlighted = 0;
|
|
for (const row of rows) {
|
|
for (const token of row.tokens) {
|
|
assert.ok(!token.scopes.some(scope => scope.startsWith('invalid.illegal.')), `${filename}: ${row.line}`);
|
|
if (token.scopes.includes('source.dever')) fileHighlighted += 1;
|
|
}
|
|
}
|
|
assert.ok(fileHighlighted > 0, `${filename} must actually enter the Dever grammar`);
|
|
highlighted += fileHighlighted;
|
|
}
|
|
assert.ok(highlighted > files.length, `${directory} must actually enter the Dever grammar`);
|
|
process.stdout.write(`CMS ${directory}: ${files.length} files, ${highlighted} Dever tokens\n`);
|
|
}
|
|
});
|