diff --git a/src/PowerShellEditorServices/Server/PsesLanguageServer.cs b/src/PowerShellEditorServices/Server/PsesLanguageServer.cs
index 82abc2b72..52eb2dec4 100644
--- a/src/PowerShellEditorServices/Server/PsesLanguageServer.cs
+++ b/src/PowerShellEditorServices/Server/PsesLanguageServer.cs
@@ -92,6 +92,7 @@ public async Task StartAsync()
.WithHandler<GetCommandHandler>()
.WithHandler<ShowHelpHandler>()
.WithHandler<ExpandAliasHandler>()
+ .WithHandler<PsesSemanticTokensHandler>()
.OnInitialize(
async (languageServer, request, cancellationToken) =>
{
diff --git a/src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs b/src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs
new file mode 100644
index 000000000..a5a520783
--- /dev/null
+++ b/src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs
@@ -0,0 +1,168 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Collections.Generic;
+using System.Management.Automation.Language;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.PowerShell.EditorServices.Services;
+using Microsoft.PowerShell.EditorServices.Services.TextDocument;
+using Microsoft.PowerShell.EditorServices.Utility;
+using OmniSharp.Extensions.LanguageServer.Protocol;
+using OmniSharp.Extensions.LanguageServer.Protocol.Document.Proposals;
+using OmniSharp.Extensions.LanguageServer.Protocol.Models;
+using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
+
+namespace Microsoft.PowerShell.EditorServices.Handlers
+{
+ internal class PsesSemanticTokensHandler : SemanticTokensHandler
+ {
+ private static readonly SemanticTokensRegistrationOptions s_registrationOptions = new SemanticTokensRegistrationOptions
+ {
+ DocumentSelector = LspUtils.PowerShellDocumentSelector,
+ Legend = new SemanticTokensLegend(),
+ DocumentProvider = new Supports<SemanticTokensDocumentProviderOptions>(
+ isSupported: true,
+ new SemanticTokensDocumentProviderOptions
+ {
+ Edits = true
+ }),
+ RangeProvider = true
+ };
+
+ private readonly ILogger _logger;
+ private readonly WorkspaceService _workspaceService;
+
+ public PsesSemanticTokensHandler(ILogger<PsesSemanticTokensHandler> logger, WorkspaceService workspaceService)
+ : base(s_registrationOptions)
+ {
+ _logger = logger;
+ _workspaceService = workspaceService;
+ }
+
+ protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier,
+ CancellationToken cancellationToken)
+ {
+ ScriptFile file = _workspaceService.GetFile(identifier.TextDocument.Uri);
+ foreach (Token token in file.ScriptTokens)
+ {
+ PushToken(token, builder);
+ }
+ return Task.CompletedTask;
+ }
+
+ private static void PushToken(Token token, SemanticTokensBuilder builder)
+ {
+ foreach (SemanticToken sToken in ConvertToSemanticTokens(token))
+ {
+ builder.Push(
+ sToken.Line,
+ sToken.Column,
+ length: sToken.Text.Length,
+ sToken.Type,
+ tokenModifiers: sToken.TokenModifiers);
+ }
+ }
+
+ internal static IEnumerable<SemanticToken> ConvertToSemanticTokens(Token token)
+ {
+ if (token is StringExpandableToken stringExpandableToken)
+ {
+ // Try parsing tokens within the string
+ if (stringExpandableToken.NestedTokens != null)
+ {
+ foreach (Token t in stringExpandableToken.NestedTokens)
+ {
+ foreach (SemanticToken subToken in ConvertToSemanticTokens(t))
+ yield return subToken;
+ }
+ yield break;
+ }
+ }
+
+ SemanticTokenType mappedType = MapSemanticTokenType(token);
+ if (mappedType == null)
+ {
+ yield break;
+ }
+
+ //Note that both column and line numbers are 0-based
+ yield return new SemanticToken(
+ token.Text,
+ mappedType,
+ line: token.Extent.StartLineNumber - 1,
+ column: token.Extent.StartColumnNumber - 1,
+ tokenModifiers: Array.Empty<string>());
+ }
+
+ private static SemanticTokenType MapSemanticTokenType(Token token)
+ {
+ // First check token flags
+ if ((token.TokenFlags & TokenFlags.Keyword) != 0)
+ {
+ return SemanticTokenType.Keyword;
+ }
+
+ if ((token.TokenFlags & TokenFlags.CommandName) != 0)
+ {
+ return SemanticTokenType.Function;
+ }
+
+ if (token.Kind != TokenKind.Generic && (token.TokenFlags &
+ (TokenFlags.BinaryOperator | TokenFlags.UnaryOperator | TokenFlags.AssignmentOperator)) != 0)
+ {
+ return SemanticTokenType.Operator;
+ }
+
+ if ((token.TokenFlags & TokenFlags.TypeName) != 0)
+ {
+ return SemanticTokenType.Type;
+ }
+
+ if ((token.TokenFlags & TokenFlags.MemberName) != 0)
+ {
+ return SemanticTokenType.Member;
+ }
+
+ // Only check token kind after checking flags
+ switch (token.Kind)
+ {
+ case TokenKind.Comment:
+ return SemanticTokenType.Comment;
+
+ case TokenKind.Parameter:
+ case TokenKind.Generic when token is StringLiteralToken slt && slt.Text.StartsWith("--"):
+ return SemanticTokenType.Parameter;
+
+ case TokenKind.Variable:
+ case TokenKind.SplattedVariable:
+ return SemanticTokenType.Variable;
+
+ case TokenKind.StringExpandable:
+ case TokenKind.StringLiteral:
+ case TokenKind.HereStringExpandable:
+ case TokenKind.HereStringLiteral:
+ return SemanticTokenType.String;
+
+ case TokenKind.Number:
+ return SemanticTokenType.Number;
+
+ case TokenKind.Generic:
+ return SemanticTokenType.Function;
+ }
+
+ return null;
+ }
+
+ protected override Task<SemanticTokensDocument> GetSemanticTokensDocument(
+ ITextDocumentIdentifierParams @params,
+ CancellationToken cancellationToken)
+ {
+ return Task.FromResult(new SemanticTokensDocument(GetRegistrationOptions().Legend));
+ }
+ }
+}
diff --git a/src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs b/src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs
new file mode 100644
index 000000000..a687690b2
--- /dev/null
+++ b/src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs
@@ -0,0 +1,27 @@
+using System.Collections.Generic;
+using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
+
+namespace Microsoft.PowerShell.EditorServices.Services.TextDocument
+{
+ internal class SemanticToken
+ {
+ public SemanticToken(string text, SemanticTokenType type, int line, int column, IEnumerable<string> tokenModifiers)
+ {
+ Line = line;
+ Text = text;
+ Column = column;
+ Type = type;
+ TokenModifiers = tokenModifiers;
+ }
+
+ public string Text { get; set ;}
+
+ public int Line { get; set; }
+
+ public int Column { get; set; }
+
+ public SemanticTokenType Type { get; set; }
+
+ public IEnumerable<string> TokenModifiers { get; set; }
+ }
+}
diff --git a/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs b/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs
index 05826f514..a6047542f 100644
--- a/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs
+++ b/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs
@@ -19,6 +19,7 @@
using OmniSharp.Extensions.LanguageServer.Protocol.Document;
using OmniSharp.Extensions.LanguageServer.Protocol.Models;
using OmniSharp.Extensions.LanguageServer.Protocol.Workspace;
+using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
using Xunit;
using Xunit.Abstractions;
using Range = OmniSharp.Extensions.LanguageServer.Protocol.Models.Range;
@@ -1136,5 +1137,35 @@ await PsesLanguageClient
Assert.Equal("Get-ChildItem", expandAliasResult.Text);
}
+
+ [Fact]
+ public async Task CanSendSemanticTokenRequest()
+ {
+ string scriptContent = "function";
+ string scriptPath = NewTestFile(scriptContent);
+
+ SemanticTokens result =
+ await PsesLanguageClient
+ .SendRequest<SemanticTokensParams>(
+ "textDocument/semanticTokens",
+ new SemanticTokensParams
+ {
+ TextDocument = new TextDocumentIdentifier
+ {
+ Uri = new Uri(scriptPath)
+ }
+ })
+ .Returning<SemanticTokens>(CancellationToken.None);
+
+ // More information about how this data is generated can be found at
+ // https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
+ var expectedArr = new int[5]
+ {
+ // line, index, token length, token type, token modifiers
+ 0, 0, scriptContent.Length, 2, 0 //function token: line 0, index 0, length, type 2 = keyword, no modifiers
+ };
+
+ Assert.Equal(expectedArr, result.Data.ToArray());
+ }
}
}
diff --git a/test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs b/test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs
new file mode 100644
index 000000000..3dbd66e7d
--- /dev/null
+++ b/test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs
@@ -0,0 +1,188 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Management.Automation.Language;
+using System.Threading.Tasks;
+using Microsoft.PowerShell.EditorServices.Services.TextDocument;
+using Microsoft.PowerShell.EditorServices.Handlers;
+using OmniSharp.Extensions.LanguageServer.Protocol;
+using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
+using Xunit;
+
+namespace Microsoft.PowerShell.EditorServices.Test.Language
+{
+ public class SemanticTokenTest
+ {
+ [Fact]
+ public async Task TokenizesFunctionElements()
+ {
+ string text = @"
+function Get-Sum {
+ param( [int]$a, [int]$b )
+ return $a + $b
+}
+";
+ ScriptFile scriptFile = new ScriptFile(
+ // Use any absolute path. Even if it doesn't exist.
+ DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")),
+ text,
+ Version.Parse("5.0"));
+
+ foreach (Token t in scriptFile.ScriptTokens)
+ {
+ List<SemanticToken> mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(t));
+ switch (t.Text)
+ {
+ case "function":
+ case "param":
+ case "return":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Keyword == sToken.Type);
+ break;
+ case "Get-Sum":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Function == sToken.Type);
+ break;
+ case "$a":
+ case "$b":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Variable == sToken.Type);
+ break;
+ case "[int]":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Type == sToken.Type);
+ break;
+ case "+":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Operator == sToken.Type);
+ break;
+ }
+ }
+ }
+
+ [Fact]
+ public async Task TokenizesStringExpansion()
+ {
+ string text = "Write-Host \"$(Test-Property Get-Whatever) $(Get-Whatever)\"";
+ ScriptFile scriptFile = new ScriptFile(
+ // Use any absolute path. Even if it doesn't exist.
+ DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")),
+ text,
+ Version.Parse("5.0"));
+
+ Token commandToken = scriptFile.ScriptTokens[0];
+ List<SemanticToken> mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(commandToken));
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Function == sToken.Type);
+
+ Token stringExpandableToken = scriptFile.ScriptTokens[1];
+ mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(stringExpandableToken));
+ Assert.Collection(mappedTokens,
+ sToken => Assert.Equal(SemanticTokenType.Function, sToken.Type),
+ sToken => Assert.Equal(SemanticTokenType.Function, sToken.Type),
+ sToken => Assert.Equal(SemanticTokenType.Function, sToken.Type)
+ );
+ }
+
+ [Fact]
+ public async Task RecognizesTokensWithAsterisk()
+ {
+ string text = @"
+function Get-A*A {
+}
+Get-A*A
+";
+ ScriptFile scriptFile = new ScriptFile(
+ // Use any absolute path. Even if it doesn't exist.
+ DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")),
+ text,
+ Version.Parse("5.0"));
+
+ foreach (Token t in scriptFile.ScriptTokens)
+ {
+ List<SemanticToken> mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(t));
+ switch (t.Text)
+ {
+ case "function":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Keyword == sToken.Type);
+ break;
+ case "Get-A*A":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Function == sToken.Type);
+ break;
+ }
+ }
+ }
+
+ [Fact]
+ public async Task RecognizesArrayMemberInExpandableString()
+ {
+ string text = "\"$(@($Array).Count) OtherText\"";
+ ScriptFile scriptFile = new ScriptFile(
+ // Use any absolute path. Even if it doesn't exist.
+ DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")),
+ text,
+ Version.Parse("5.0"));
+
+ foreach (Token t in scriptFile.ScriptTokens)
+ {
+ List<SemanticToken> mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(t));
+ switch (t.Text)
+ {
+ case "$Array":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Variable == sToken.Type);
+ break;
+ case "Count":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Member == sToken.Type);
+ break;
+ }
+ }
+ }
+
+ [Fact]
+ public async Task RecognizesCurlyQuotedString()
+ {
+ string text = "“^[-'a-z]*”";
+ ScriptFile scriptFile = new ScriptFile(
+ // Use any absolute path. Even if it doesn't exist.
+ DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")),
+ text,
+ Version.Parse("5.0"));
+
+ List<SemanticToken> mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(scriptFile.ScriptTokens[0]));
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.String == sToken.Type);
+ }
+
+ [Fact]
+ public async Task RecognizeEnum()
+ {
+ string text = @"
+enum MyEnum{
+ one
+ two
+ three
+}
+";
+ ScriptFile scriptFile = new ScriptFile(
+ // Use any absolute path. Even if it doesn't exist.
+ DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")),
+ text,
+ Version.Parse("5.0"));
+
+ foreach (Token t in scriptFile.ScriptTokens)
+ {
+ List<SemanticToken> mappedTokens = new List<SemanticToken>(PsesSemanticTokensHandler.ConvertToSemanticTokens(t));
+ switch (t.Text)
+ {
+ case "enum":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Keyword == sToken.Type);
+ break;
+ case "MyEnum":
+ case "one":
+ case "two":
+ case "three":
+ Assert.Single(mappedTokens, sToken => SemanticTokenType.Member == sToken.Type);
+ break;
+ }
+ }
+ }
+ }
+}