feat: update contribute ts and update unit test

Author: zhaoge

src/_.contribution.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { languages } from './fillers/monaco-editor-core';
+import { editor, languages } from './fillers/monaco-editor-core';
 
 interface ILang extends languages.ILanguageExtensionPoint {
 	loader: () => Promise<ILangImpl>;
@@ -39,11 +39,6 @@ class LazyLanguageLoader {
 			this._lazyLoadPromiseReject = reject;
 		});
 	}
-
-	public whenLoaded(): Promise<ILangImpl> {
-		return this._lazyLoadPromise;
-	}
-
 	public load(): Promise<ILangImpl> {
 		if (!this._loadingTriggered) {
 			this._loadingTriggered = true;
@@ -58,22 +53,24 @@ class LazyLanguageLoader {
 
 export async function loadLanguage(languageId: string): Promise<void> {
 	await LazyLanguageLoader.getOrCreate(languageId).load();
+	// trigger tokenizer creation by instantiating a model
+	const model = editor.createModel('', languageId);
+	model.dispose();
 }
 
 export function registerLanguage(def: ILang): void {
 	const languageId = def.id;
-
 	languageDefinitions[languageId] = def;
 	languages.register(def);
 
 	const lazyLanguageLoader = LazyLanguageLoader.getOrCreate(languageId);
-
-	languages.setMonarchTokensProvider(
-		languageId,
-		lazyLanguageLoader.whenLoaded().then((mod) => mod.language)
-	);
-
-	languages.onLanguage(languageId, async () => {
+	languages.registerTokensProviderFactory(languageId, {
+		create: async (): Promise<languages.IMonarchLanguage> => {
+			const mod = await lazyLanguageLoader.load();
+			return mod.language;
+		}
+	});
+	languages.onLanguageEncountered(languageId, async () => {
 		const mod = await lazyLanguageLoader.load();
 		languages.setLanguageConfiguration(languageId, mod.conf);
 	});

src/languages/hive/hive.test.ts

@@ -3,8 +3,10 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import './hive.contribution';
+
+import { postfixTokenClass, TokenClassConsts } from '../../common/constants';
 import { testTokenization } from '../../test/testRunner';
-import { TokenClassConsts, postfixTokenClass } from '../../common/constants';
 
 testTokenization('hivesql', [
 	// Comments

src/test/testRunner.ts

@@ -4,8 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 
 import '../all.contributions';
-import { loadLanguage } from '../_.contribution';
+
 import * as assert from 'assert';
+
+import { loadLanguage } from '../_.contribution';
 import { editor } from '../fillers/monaco-editor-core';
 
 export interface IRelaxedToken {
@@ -59,5 +61,10 @@ function runTest(languageId: string, test: ITestItem[]): void {
 		};
 	});
 
-	assert.deepStrictEqual(actual, test);
+	try {
+		assert.deepStrictEqual(actual, test);
+	} catch (err) {
+		console.error(`❌ Tokenization mismatch for ${languageId}:`);
+		throw err;
+	}
 }

test/all.js

@@ -2,14 +2,15 @@ const requirejs = require('requirejs');
 const jsdom = require('jsdom');
 const glob = require('fast-glob');
 const path = require('path');
+const Mocha = require('mocha');
 
 requirejs.config({
-	baseUrl: '',
+	baseUrl: path.join(__dirname, '../'),
 	paths: {
+		vs: 'node_modules/monaco-editor/dev/vs',
 		'vs/css': 'test/css.mock',
 		'vs/nls': 'test/nls.mock',
-		'out/amd/fillers/monaco-editor-core': 'out/amd/fillers/monaco-editor-core-amd',
-		vs: 'node_modules/monaco-editor/dev/vs'
+		'out/amd/fillers/monaco-editor-core': 'out/amd/fillers/monaco-editor-core-amd'
 	},
 	nodeRequire: require
 });
@@ -24,6 +25,9 @@ global.document.queryCommandSupported = function () {
 };
 global.UIEvent = tmp.window.UIEvent;
 global.define = requirejs.define;
+
+// 添加完整的DOM环境支持
+global.Element = tmp.window.Element;
 global.window = {
 	location: {},
 	navigator: tmp.window.navigator,
@@ -32,8 +36,18 @@ global.window = {
 			matches: false,
 			addEventListener: function () {}
 		};
-	}
+	},
+	setInterval: setInterval,
+	clearInterval: clearInterval,
+	setTimeout: setTimeout,
+	clearTimeout: clearTimeout,
+	document: tmp.window.document,
+	Element: tmp.window.Element
 };
+if (!document.body) {
+	const body = document.createElement('body');
+	document.appendChild(body);
+}
 
 requirejs(
 	['./test/setup'],
@@ -52,10 +66,37 @@ requirejs(
 		requirejs(
 			files.map((f) => f.replace(/\.js$/, '')),
 			function () {
-				run(); // We can launch the tests!
+				// 初始化Mocha
+				const mocha = new Mocha({
+					ui: 'bdd',
+					reporter: 'spec',
+					timeout: 5000
+				});
+
+				// 手动添加测试到Mocha的suite
+				const Suite = require('mocha/lib/suite');
+				const Test = require('mocha/lib/test');
+
+				// 创建一个根suite
+				const rootSuite = new Suite('Root Suite');
+				mocha.suite.addSuite(rootSuite);
+
+				// 添加存储的测试
+				if (global._pendingTests && global._pendingTests.length > 0) {
+					global._pendingTests.forEach(function (test) {
+						const mochaTest = new Test(test.name, test.fn);
+						rootSuite.addTest(mochaTest);
+					});
+				}
+
+				// 运行测试
+				mocha.run(function (failures) {
+					process.exit(failures ? 1 : 0);
+				});
 			},
 			function (err) {
-				console.log(err);
+				console.log('Error loading test files:', err);
+				process.exit(1);
 			}
 		);
 	},

test/setup.js

@@ -10,6 +10,9 @@ define('vs/nls', [], {
 			localize: function () {
 				return 'NO_LOCALIZATION_FOR_YOU';
 			},
+			localize2: function () {
+				return 'NO_LOCALIZATION_FOR_YOU';
+			},
 			getConfiguredDefaultLocale: function () {
 				return undefined;
 			}
@@ -18,8 +21,8 @@ define('vs/nls', [], {
 	localize: function () {
 		return 'NO_LOCALIZATION_FOR_YOU';
 	},
-	localize2: function () {
-		return 'NO_LOCALIZATION_FOR_YOU';
+	localize2: function (key, message) {
+		return { value: 'NO_LOCALIZATION_FOR_YOU', original: message };
 	},
 	load: function (name, req, load) {
 		load({});
@@ -29,3 +32,52 @@ define('vs/nls', [], {
 define(['vs/editor/editor.main'], function (api) {
 	global.monaco = api;
 });
+
+// 定义Mocha全局函数
+global.test = function (name, fn) {
+	// 检查函数是否返回Promise（包括编译后的异步函数）
+	const isPromiseFunction =
+		fn &&
+		typeof fn === 'function' &&
+		((fn.constructor && fn.constructor.name === 'AsyncFunction') ||
+			fn.toString().includes('__awaiter') ||
+			fn.toString().includes('__generator') ||
+			fn.toString().includes('Promise'));
+
+	if (isPromiseFunction) {
+		// 异步函数需要包装为Mocha能理解的格式
+		const wrappedFn = function (done) {
+			const result = fn();
+			if (result && typeof result.then === 'function') {
+				result
+					.then(() => {
+						if (done) done();
+					})
+					.catch((err) => {
+						if (done) done(err);
+					});
+			} else {
+				if (done) done();
+			}
+		};
+		global._pendingTests = global._pendingTests || [];
+		global._pendingTests.push({ name, fn: wrappedFn });
+	} else {
+		global._pendingTests = global._pendingTests || [];
+		global._pendingTests.push({ name, fn });
+	}
+};
+
+global.describe =
+	global.describe ||
+	function (name, fn) {
+		global._pendingSuites = global._pendingSuites || [];
+		global._pendingSuites.push({ name, fn });
+	};
+
+global.it =
+	global.it ||
+	function (name, fn) {
+		global._pendingTests = global._pendingTests || [];
+		global._pendingTests.push({ name, fn });
+	};

tsconfig.amd.json

@@ -4,6 +4,9 @@
 		"module": "amd",
 		"outDir": "out/amd/",
 		"target": "es5",
-		"noEmit": false
+		"noEmit": false,
+		"sourceMap": true,
+		"allowJs": true,
+		"checkJs": false
 	}
 }