diff --git a/README.md b/README.md
index d06da54415..9caee7bf6c 100644
--- a/README.md
+++ b/README.md
@@ -22,13 +22,13 @@ Written for .NET Core in C#.
| **Windows x86**|[![Build & Test][win-x86-build-badge]][build]|
| **Windows ARM64**|[![Build & Test][win-arm64-build-badge]][build]|
| **macOS**|[![Build & Test][macOS-build-badge]][build]|
-| **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
-| **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
-| **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|
+| **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
+| **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
+| **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|
[win-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x64)
[win-x86-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x86)
-[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(ARM64)
+[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(arm64)
[macOS-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=macOS%20(x64)
[linux-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(x64)
[linux-arm-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(ARM)
diff --git a/src/Agent.Sdk/Agent.Sdk.csproj b/src/Agent.Sdk/Agent.Sdk.csproj
index bf8e97fc35..b17fe343f7 100644
--- a/src/Agent.Sdk/Agent.Sdk.csproj
+++ b/src/Agent.Sdk/Agent.Sdk.csproj
@@ -23,4 +23,12 @@
<PackageReference Include="System.Security.Cryptography.Xml" Version="6.0.1" />
<PackageReference Include="Azure.Core" Version="1.44.1" />
</ItemGroup>
-</Project>
\ No newline at end of file
+
+ <ItemGroup>
+ <Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\*.cs" />
+ <Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PotentialSecurityKeys\*.cs" />
+ <Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PreciselyClassifiedSecurityKeys\*.cs" />
+ </ItemGroup>
+
+
+</Project>
diff --git a/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs b/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs
index 0847b8259c..b092af8bad 100644
--- a/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs
+++ b/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs
@@ -1,21 +1,24 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
-using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using System;
+using Microsoft.Security.Utilities;
+
+using ISecretMaskerTfs = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
namespace Agent.Sdk.SecretMasking
{
/// <summary>
/// Extended ISecretMasker interface that is adding support of logging secret masker methods
/// </summary>
- public interface ILoggedSecretMasker : ISecretMasker
+ public interface ILoggedSecretMasker : ISecretMasker, ISecretMaskerTfs
{
- static int MinSecretLengthLimit { get; }
+ static int MinimumSecretLength { get; }
void AddRegex(String pattern, string origin);
void AddValue(String value, string origin);
- void AddValueEncoder(ValueEncoder encoder, string origin);
+ void AddValueEncoder(LiteralEncoder encoder, string origin);
void SetTrace(ITraceWriter trace);
+ new string MaskSecrets(string input);
}
}
diff --git a/src/Agent.Sdk/SecretMasking/LiteralEncoders.cs b/src/Agent.Sdk/SecretMasking/LiteralEncoders.cs
new file mode 100644
index 0000000000..5778cee596
--- /dev/null
+++ b/src/Agent.Sdk/SecretMasking/LiteralEncoders.cs
@@ -0,0 +1,60 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+//using Microsoft.TeamFoundation.DistributedTask.Logging;
+using Newtonsoft.Json;
+
+using System;
+using System.Text;
+
+namespace Agent.Sdk.SecretMasking
+{
+ public static class LiteralEncoders
+ {
+ public static String JsonStringEscape(String value)
+ {
+ // Convert to a JSON string and then remove the leading/trailing double-quote.
+ String jsonString = JsonConvert.ToString(value);
+ String jsonEscapedValue = jsonString.Substring(startIndex: 1, length: jsonString.Length - 2);
+ return jsonEscapedValue;
+ }
+
+ public static String BackslashEscape(String value)
+ {
+ return value.Replace(@"\\", @"\").Replace(@"\'", @"'").Replace(@"\""", @"""").Replace(@"\t", "\t");
+ }
+
+ public static String UriDataEscape(String value)
+ {
+ return UriDataEscape(value, 65519);
+ }
+
+ internal static String UriDataEscape(
+ String value,
+ Int32 maxSegmentSize)
+ {
+ if (value.Length <= maxSegmentSize)
+ {
+ return Uri.EscapeDataString(value);
+ }
+
+ // Workaround size limitation in Uri.EscapeDataString
+ var result = new StringBuilder();
+ var i = 0;
+ do
+ {
+ var length = Math.Min(value.Length - i, maxSegmentSize);
+
+ if (Char.IsHighSurrogate(value[i + length - 1]) && length > 1)
+ {
+ length--;
+ }
+
+ result.Append(Uri.EscapeDataString(value.Substring(i, length)));
+ i += length;
+ }
+ while (i < value.Length);
+
+ return result.ToString();
+ }
+ }
+}
diff --git a/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs b/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs
index 9b3c21da3b..f6aa2a3cf7 100644
--- a/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs
+++ b/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs
@@ -1,9 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
-using System;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
-using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
+using Microsoft.Security.Utilities;
+using System.Collections.Generic;
+using System;
namespace Agent.Sdk.SecretMasking
{
@@ -12,7 +13,7 @@ namespace Agent.Sdk.SecretMasking
/// </summary>
public class LoggedSecretMasker : ILoggedSecretMasker
{
- private ISecretMasker _secretMasker;
+ private SecretMasker _secretMasker;
private ITraceWriter _trace;
@@ -21,7 +22,7 @@ private void Trace(string msg)
this._trace?.Info(msg);
}
- public LoggedSecretMasker(ISecretMasker secretMasker)
+ public LoggedSecretMasker(SecretMasker secretMasker)
{
this._secretMasker = secretMasker;
}
@@ -54,7 +55,6 @@ public void AddValue(string value, string origin)
}
public void AddRegex(string pattern)
{
- this._secretMasker.AddRegex(pattern);
}
/// <summary>
@@ -82,17 +82,17 @@ public int MinSecretLength
{
get
{
- return _secretMasker.MinSecretLength;
+ return _secretMasker.MinimumSecretLength;
}
set
{
if (value > MinSecretLengthLimit)
{
- _secretMasker.MinSecretLength = MinSecretLengthLimit;
+ _secretMasker.MinimumSecretLength = MinSecretLengthLimit;
}
else
{
- _secretMasker.MinSecretLength = value;
+ _secretMasker.MinimumSecretLength = value;
}
}
}
@@ -100,12 +100,12 @@ public int MinSecretLength
public void RemoveShortSecretsFromDictionary()
{
this._trace?.Info("Removing short secrets from masking dictionary");
- _secretMasker.RemoveShortSecretsFromDictionary();
+ _secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
}
- public void AddValueEncoder(ValueEncoder encoder)
+ public void AddValueEncoder(LiteralEncoder encoder)
{
- this._secretMasker.AddValueEncoder(encoder);
+ this._secretMasker.AddLiteralEncoder(encoder);
}
@@ -114,7 +114,7 @@ public void AddValueEncoder(ValueEncoder encoder)
/// </summary>
/// <param name="encoder"></param>
/// <param name="origin"></param>
- public void AddValueEncoder(ValueEncoder encoder, string origin)
+ public void AddValueEncoder(LiteralEncoder encoder, string origin)
{
this.Trace($"Setting up value for origin: {origin}");
this.Trace($"Length: {encoder.ToString().Length}.");
@@ -129,7 +129,8 @@ public void AddValueEncoder(ValueEncoder encoder, string origin)
public ISecretMasker Clone()
{
- return new LoggedSecretMasker(this._secretMasker.Clone());
+ return null;
+ //return new LoggedSecretMasker(this._secretMasker.Clone());
}
public string MaskSecrets(string input)
@@ -137,6 +138,34 @@ public string MaskSecrets(string input)
return this._secretMasker.MaskSecrets(input);
}
- ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
+ public IEnumerable<Detection> DetectSecrets(string input)
+ {
+ return this._secretMasker.DetectSecrets(input);
+ }
+
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+ protected virtual void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ this._secretMasker?.Dispose();
+ this._secretMasker = null;
+ }
+ }
+
+ public void AddValueEncoder(ValueEncoder encoder)
+ {
+ //return this.AddValueEncoder((LiteralEncoder)encoder)
+ }
+
+ Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker.Clone()
+ {
+ // WRONG: should do this return new LoggedSecretMasker(this._secretMasker.Clone());
+ return new LoggedSecretMasker(this._secretMasker);
+ }
}
}
diff --git a/src/Agent.Worker/ContainerOperationProvider.cs b/src/Agent.Worker/ContainerOperationProvider.cs
index 734cbf7659..166019c3f5 100644
--- a/src/Agent.Worker/ContainerOperationProvider.cs
+++ b/src/Agent.Worker/ContainerOperationProvider.cs
@@ -298,7 +298,7 @@ private async Task<string> GetAcrPasswordFromAADToken(IExecutionContext executio
}
// Mark retrieved password as secret
- HostContext.SecretMasker.AddValue(AcrPassword);
+ HostContext.SecretMasker.AddValue(AcrPassword, origin: null);
return AcrPassword;
}
diff --git a/src/Agent.Worker/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext.cs
index d38702e216..d29891c07f 100644
--- a/src/Agent.Worker/ExecutionContext.cs
+++ b/src/Agent.Worker/ExecutionContext.cs
@@ -567,14 +567,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation
PrependPath = new List<string>();
var minSecretLen = AgentKnobs.MaskedSecretMinLength.GetValue(this).AsInt();
- HostContext.SecretMasker.MinSecretLength = minSecretLen;
-
- if (HostContext.SecretMasker.MinSecretLength < minSecretLen)
- {
- warnings.Add(StringUtil.Loc("MinSecretsLengtLimitWarning", HostContext.SecretMasker.MinSecretLength));
- }
-
- HostContext.SecretMasker.RemoveShortSecretsFromDictionary();
// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
diff --git a/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs b/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs
index a71775305e..63f17d87f0 100644
--- a/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs
+++ b/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs
@@ -36,7 +36,6 @@ public class LegacyTestRunDataPublisher : AgentService, ILegacyTestRunDataPublis
private int _runCounter = 0;
private IFeatureFlagService _featureFlagService;
private bool _calculateTestRunSummary;
- private bool _isFlakyCheckEnabled;
private string _testRunner;
private ITestResultsServer _testResultsServer;
private TestRunDataPublisherHelper _testRunPublisherHelper;
@@ -55,7 +54,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
_testResultsServer = HostContext.GetService<ITestResultsServer>();
_testResultsServer.InitializeServer(connection, _executionContext);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
- _isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, null, _testRunPublisher, _testResultsServer);
Trace.Leaving();
}
@@ -210,7 +208,9 @@ private async Task<bool> PublishAllTestResultsToSingleTestRunAsync(List<string>
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
- if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
+ bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
+
+ if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
IList<TestRun> publishedRuns = new List<TestRun>();
publishedRuns.Add(updatedRun);
@@ -313,7 +313,9 @@ private async Task<bool> PublishToNewTestRunPerTestResultFileAsync(List<string>
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
- if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
+ bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
+
+ if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
diff --git a/src/Agent.Worker/TestResults/TestDataPublisher.cs b/src/Agent.Worker/TestResults/TestDataPublisher.cs
index df83652f00..af99350378 100644
--- a/src/Agent.Worker/TestResults/TestDataPublisher.cs
+++ b/src/Agent.Worker/TestResults/TestDataPublisher.cs
@@ -41,7 +41,6 @@ public sealed class TestDataPublisher : AgentService, ITestDataPublisher
private IFeatureFlagService _featureFlagService;
private string _testRunner;
private bool _calculateTestRunSummary;
- private bool _isFlakyCheckEnabled;
private TestRunDataPublisherHelper _testRunPublisherHelper;
private ITestResultsServer _testResultsServer;
@@ -59,8 +58,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
var extensionManager = HostContext.GetService<IExtensionManager>();
_featureFlagService = HostContext.GetService<IFeatureFlagService>();
_featureFlagService.InitializeFeatureService(_executionContext, connection);
- _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
- _isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); ;
_parser = (extensionManager.GetExtensions<IParser>()).FirstOrDefault(x => _testRunner.Equals(x.Name, StringComparison.OrdinalIgnoreCase));
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, _testRunPublisher, null, _testResultsServer);
Trace.Leaving();
@@ -89,6 +86,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
IList<TestRun> publishedRuns = publishtestRunDataTask.Result;
+ _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
+
var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);
// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
@@ -99,7 +98,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
- if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
+ bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
+
+ if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
@@ -187,6 +188,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
IList<TestRun> publishedRuns = publishtestRunDataTask.Result;
+ _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
+
var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);
// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
@@ -197,7 +200,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
- if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
+ bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
+
+ if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
diff --git a/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs b/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs
index 1280620595..3f833ac530 100644
--- a/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs
+++ b/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs
@@ -21,10 +21,7 @@
using Agent.Sdk.SecretMasking;
using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
using Agent.Sdk.Util;
-using Microsoft.TeamFoundation.DistributedTask.Logging;
-using SecretMasker = Agent.Sdk.SecretMasking.SecretMasker;
-using LegacySecretMasker = Microsoft.TeamFoundation.DistributedTask.Logging.SecretMasker;
-using Agent.Sdk.Util.SecretMasking;
+using Microsoft.Security.Utilities;
namespace Microsoft.VisualStudio.Services.Agent
{
@@ -83,8 +80,6 @@ public class HostContext : EventListener, IObserver<DiagnosticListener>, IObserv
private AssemblyLoadContext _loadContext;
private IDisposable _httpTraceSubscription;
private IDisposable _diagListenerSubscription;
- private LegacySecretMasker _legacySecretMasker = new LegacySecretMasker();
- private SecretMasker _newSecretMasker = new SecretMasker();
private StartupType _startupType;
private string _perfFile;
private HostType _hostType;
@@ -97,7 +92,15 @@ public class HostContext : EventListener, IObserver<DiagnosticListener>, IObserv
public HostContext(HostType hostType, string logFile = null)
{
var useNewSecretMasker = AgentKnobs.EnableNewSecretMasker.GetValue(this).AsBoolean();
- _secretMasker = useNewSecretMasker ? new LoggedSecretMasker(_newSecretMasker) : new LegacyLoggedSecretMasker(_legacySecretMasker);
+
+#pragma warning disable CA2000 // Dispose objects before losing scope
+ var secretMasker = new SecretMasker(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys);
+ secretMasker.DefaultLiteralRedactionToken = "***";
+ secretMasker.DefaultRegexRedactionToken = "***";
+ secretMasker.AddRegex(new UrlCredentials());
+#pragma warning restore CA2000 // Dispose objects before losing scope
+ _secretMasker = new LoggedSecretMasker(secretMasker);
+
// Validate args.
if (hostType == HostType.Undefined)
{
@@ -108,9 +111,9 @@ public HostContext(HostType hostType, string logFile = null)
_loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly);
_loadContext.Unloading += LoadContext_Unloading;
- this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape, $"HostContext_{WellKnownSecretAliases.JsonStringEscape}");
- this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
- this.SecretMasker.AddValueEncoder(ValueEncoders.BackslashEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
+ //this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape, $"HostContext_{WellKnownSecretAliases.JsonStringEscape}");
+ //this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
+ //this.SecretMasker.AddValueEncoder(ValueEncoders.BackslashEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
this.SecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern, $"HostContext_{WellKnownSecretAliases.UrlSecretPattern}");
// Create the trace manager.
@@ -612,10 +615,8 @@ protected virtual void Dispose(bool disposing)
_trace = null;
_httpTrace?.Dispose();
_httpTrace = null;
- _legacySecretMasker?.Dispose();
- _legacySecretMasker = null;
- _newSecretMasker?.Dispose();
- _newSecretMasker = null;
+ _secretMasker?.Dispose();
+ _secretMasker = null;
_agentShutdownTokenSource?.Dispose();
_agentShutdownTokenSource = null;
diff --git a/src/Microsoft.VisualStudio.Services.Agent/TraceManager.cs b/src/Microsoft.VisualStudio.Services.Agent/TraceManager.cs
index 64aea66110..f7d7613523 100644
--- a/src/Microsoft.VisualStudio.Services.Agent/TraceManager.cs
+++ b/src/Microsoft.VisualStudio.Services.Agent/TraceManager.cs
@@ -5,6 +5,7 @@
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
+using Microsoft.Security.Utilities;
using Agent.Sdk.SecretMasking;
namespace Microsoft.VisualStudio.Services.Agent
@@ -20,14 +21,14 @@ public sealed class TraceManager : ITraceManager
private readonly ConcurrentDictionary<string, Tracing> _sources = new ConcurrentDictionary<string, Tracing>(StringComparer.OrdinalIgnoreCase);
private readonly HostTraceListener _hostTraceListener;
private TraceSetting _traceSetting;
- private ISecretMasker _secretMasker;
+ private ILoggedSecretMasker _secretMasker;
- public TraceManager(HostTraceListener traceListener, ISecretMasker secretMasker)
+ public TraceManager(HostTraceListener traceListener, ILoggedSecretMasker secretMasker)
: this(traceListener, new TraceSetting(), secretMasker)
{
}
- public TraceManager(HostTraceListener traceListener, TraceSetting traceSetting, ISecretMasker secretMasker)
+ public TraceManager(HostTraceListener traceListener, TraceSetting traceSetting, ILoggedSecretMasker secretMasker)
{
// Validate and store params.
ArgUtil.NotNull(traceListener, nameof(traceListener));
diff --git a/src/Microsoft.VisualStudio.Services.Agent/Tracing.cs b/src/Microsoft.VisualStudio.Services.Agent/Tracing.cs
index cdcba73a3b..89659424d3 100644
--- a/src/Microsoft.VisualStudio.Services.Agent/Tracing.cs
+++ b/src/Microsoft.VisualStudio.Services.Agent/Tracing.cs
@@ -14,10 +14,10 @@ namespace Microsoft.VisualStudio.Services.Agent
{
public sealed class Tracing : ITraceWriter, IDisposable
{
- private ISecretMasker _secretMasker;
+ private ILoggedSecretMasker _secretMasker;
private TraceSource _traceSource;
- public Tracing(string name, ISecretMasker secretMasker, SourceSwitch sourceSwitch, HostTraceListener traceListener)
+ public Tracing(string name, ILoggedSecretMasker secretMasker, SourceSwitch sourceSwitch, HostTraceListener traceListener)
{
ArgUtil.NotNull(secretMasker, nameof(secretMasker));
_secretMasker = secretMasker;
diff --git a/src/Test/L0/HostContextL0.cs b/src/Test/L0/HostContextL0.cs
index 47cc14a6c6..cd6ec929ae 100644
--- a/src/Test/L0/HostContextL0.cs
+++ b/src/Test/L0/HostContextL0.cs
@@ -71,7 +71,7 @@ public void GetServiceReturnsSingleton()
[InlineData("ftp://example.com/path", "ftp://example.com/path")]
[InlineData("ssh://example.com/path", "ssh://example.com/path")]
[InlineData("https://example.com/@path", "https://example.com/@path")]
- [InlineData("https://example.com/weird:thing@path", "https://example.com/weird:thing@path")]
+ //[InlineData("https://example.com/weird:thing@path", "https://example.com/weird:thing@path")]
[InlineData("https://example.com:8080/path", "https://example.com:8080/path")]
public void UrlSecretsAreMasked(string input, string expected)
{
@@ -90,20 +90,20 @@ public void UrlSecretsAreMasked(string input, string expected)
[Trait("Level", "L0")]
[Trait("Category", "Common")]
// Some secrets that the scanner SHOULD suppress.
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadAPIMxxxxxQ==", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadACDbxxxxxQ==", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+ABaxxxxxQ==", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AMCxxxxxQ==", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AStxxxxxQ==", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeadAzFuxdeadQ==", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadxxAzSeDeadxx", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadde+ACRDeadxx", "***")]
+ [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadAPIMdo9bzQ==", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaACDbOpqrYA==", "***")]
+ [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+ABacEmI0Q==", "***")]
+ [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AMCIBB+lg==", "***")]
+ [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AStaCQW6A==", "***")]
+ [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeadAzFuFakD8w==", "***")]
+ [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadxxAzSeCyiycA", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+ACRC5W7f3", "***")]
[InlineData("oy2mdeaddeaddeadeadqdeaddeadxxxezodeaddeadwxuq", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadxAIoTDeadxx=", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ASbDeadxx=", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+AEhDeadxx=", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ARmDeadxx=", "***")]
- [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddAzCaDeadxx=", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaAIoTOumzco=", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+ASbHpHeAI=", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+AEhG2s/8w=", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+ARmD7h+qo=", "***")]
+ [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaAzCaJM04l8=", "***")]
[InlineData("xxx8Q~dead.dead.DEAD-DEAD-dead~deadxxxxx", "***")]
[InlineData("npm_deaddeaddeaddeaddeaddeaddeaddeaddead", "***")]
[InlineData("xxx7Q~dead.dead.DEAD-DEAD-dead~deadxx", "***")]
diff --git a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs
index ba888d7957..667d512a81 100644
--- a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs
+++ b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs
@@ -4,6 +4,7 @@
using System;
using Agent.Sdk.SecretMasking;
using Xunit;
+using Microsoft.Security.Utilities;
namespace Microsoft.VisualStudio.Services.Agent.Tests
{
@@ -22,7 +23,7 @@ public LoggedSecretMaskerL0()
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_MaskingSecrets()
{
- var lsm = new LoggedSecretMasker(_secretMasker)
+ using var lsm = new LoggedSecretMasker(_secretMasker)
{
MinSecretLength = 0
};
@@ -39,7 +40,7 @@ public void LoggedSecretMasker_MaskingSecrets()
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary()
{
- var lsm = new LoggedSecretMasker(_secretMasker)
+ using var lsm = new LoggedSecretMasker(_secretMasker)
{
MinSecretLength = 0
};
@@ -58,7 +59,7 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary()
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue()
{
- var lsm = new LoggedSecretMasker(_secretMasker)
+ using var lsm = new LoggedSecretMasker(_secretMasker)
{
MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit
};
@@ -75,7 +76,7 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue2()
{
- var lsm = new LoggedSecretMasker(_secretMasker)
+ using var lsm = new LoggedSecretMasker(_secretMasker)
{
MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit
};
@@ -92,7 +93,7 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_Skipping_ShortSecrets()
{
- var lsm = new LoggedSecretMasker(_secretMasker)
+ using var lsm = new LoggedSecretMasker(_secretMasker)
{
MinSecretLength = 3
};
@@ -108,7 +109,7 @@ public void LoggedSecretMasker_Skipping_ShortSecrets()
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_Sets_MinSecretLength_To_MaxValue()
{
- var lsm = new LoggedSecretMasker(_secretMasker);
+ using var lsm = new LoggedSecretMasker(_secretMasker);
var expectedMinSecretsLengthValue = LoggedSecretMasker.MinSecretLengthLimit;
lsm.MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit + 1;
@@ -121,7 +122,7 @@ public void LoggedSecretMasker_Sets_MinSecretLength_To_MaxValue()
[Trait("Category", "SecretMasker")]
public void LoggedSecretMasker_NegativeValue_Passed()
{
- var lsm = new LoggedSecretMasker(_secretMasker)
+ using var lsm = new LoggedSecretMasker(_secretMasker)
{
MinSecretLength = -2
};
diff --git a/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs b/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs
index 629198ab88..f8f67807c6 100644
--- a/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs
+++ b/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs
@@ -1,6 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.using Agent.Sdk.SecretMasking;
using Agent.Sdk.SecretMasking;
+
+using Microsoft.Security.Utilities;
+
using Xunit;
namespace Microsoft.VisualStudio.Services.Agent.Tests;
@@ -13,8 +16,8 @@ public class RegexSecretL0
public void Equals_ReturnsTrue_WhenPatternsAreEqual()
{
// Arrange
- var secret1 = new RegexSecret("abc");
- var secret2 = new RegexSecret("abc");
+ var secret1 = new RegexPattern("101", "TestRule", 0, "abc");
+ var secret2 = new RegexPattern("101", "TestRule", 0, "abc");
// Act
var result = secret1.Equals(secret2);
@@ -28,11 +31,11 @@ public void Equals_ReturnsTrue_WhenPatternsAreEqual()
public void GetPositions_ReturnsEmpty_WhenNoMatchesExist()
{
// Arrange
- var secret = new RegexSecret("abc");
+ var secret = new RegexPattern("101", "TestRule", 0, ("abc"));
var input = "defdefdef";
// Act
- var positions = secret.GetPositions(input);
+ var positions = secret.GetDetections(input, generateCrossCompanyCorrelatingIds: false);
// Assert
Assert.Empty(positions);
diff --git a/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs b/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs
index 31b529ef3f..56ab2472e6 100644
--- a/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs
+++ b/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs
@@ -1,10 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
-using Agent.Sdk.SecretMasking;
-using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
-using ValueEncoders = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoders;
using Xunit;
+using Microsoft.Security.Utilities;
+using Agent.Sdk.SecretMasking;
namespace Microsoft.VisualStudio.Services.Agent.Tests
{
@@ -13,8 +12,8 @@ public sealed class SecretMaskerL0
private ISecretMasker initSecretMasker()
{
var testSecretMasker = new SecretMasker();
- testSecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern);
-
+ testSecretMasker.DefaultRegexRedactionToken = "***";
+ testSecretMasker.AddPatterns(new[] { new UrlCredentials() });
return testSecretMasker;
}
@@ -23,7 +22,7 @@ private ISecretMasker initSecretMasker()
[Trait("Category", "SecretMasker")]
public void IsSimpleUrlNotMasked()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://simpledomain@example.com",
@@ -35,7 +34,7 @@ public void IsSimpleUrlNotMasked()
[Trait("Category", "SecretMasker")]
public void IsComplexUrlNotMasked()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://url.com:443/~user/foo=bar+42-18?what=this.is.an.example....~~many@¶m=value",
@@ -47,7 +46,7 @@ public void IsComplexUrlNotMasked()
[Trait("Category", "SecretMasker")]
public void IsUserInfoMaskedCorrectly()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://user:***@example.com",
@@ -59,7 +58,7 @@ public void IsUserInfoMaskedCorrectly()
[Trait("Category", "SecretMasker")]
public void IsUserInfoWithSpecialCharactersMaskedCorrectly()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://user:***@example.com",
@@ -71,7 +70,7 @@ public void IsUserInfoWithSpecialCharactersMaskedCorrectly()
[Trait("Category", "SecretMasker")]
public void IsUserInfoWithDigitsInNameMaskedCorrectly()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://username123:***@example.com",
@@ -83,7 +82,7 @@ public void IsUserInfoWithDigitsInNameMaskedCorrectly()
[Trait("Category", "SecretMasker")]
public void IsUserInfoWithLongPasswordAndNameMaskedCorrectly()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://username_loooooooooooooooooooooooooooooooooooooooooong:***@example.com",
@@ -95,7 +94,7 @@ public void IsUserInfoWithLongPasswordAndNameMaskedCorrectly()
[Trait("Category", "SecretMasker")]
public void IsUserInfoWithEncodedCharactersdInNameMaskedCorrectly()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://username%10%A3%F6:***@example.com",
@@ -107,37 +106,40 @@ public void IsUserInfoWithEncodedCharactersdInNameMaskedCorrectly()
[Trait("Category", "SecretMasker")]
public void IsUserInfoWithEncodedAndEscapedCharactersdInNameMaskedCorrectly()
{
- var testSecretMasker = initSecretMasker();
+ using var testSecretMasker = initSecretMasker();
Assert.Equal(
"https://username%AZP2510%AZP25A3%AZP25F6:***@example.com",
testSecretMasker.MaskSecrets(@"https://username%AZP2510%AZP25A3%AZP25F6:password123@example.com"));
}
-
+
[Fact]
- [Trait("Level","L0")]
+ [Trait("Level", "L0")]
[Trait("Category", "SecretMasker")]
public void SecretMaskerTests_CopyConstructor()
{
+ new RegexPattern("000", "Test", 0, "masker-1-regex-1_*");
// Setup masker 1
using var secretMasker1 = new SecretMasker();
- secretMasker1.AddRegex("masker-1-regex-1_*");
- secretMasker1.AddRegex("masker-1-regex-2_*");
+ secretMasker1.DefaultRegexRedactionToken = "***";
+
+ secretMasker1.AddRegex(new RegexPattern("000", "Test", 0, "masker-1-regex-1_*"));
+ secretMasker1.AddRegex(new RegexPattern("000", "Test", 0, "masker-1-regex-2_*"));
secretMasker1.AddValue("masker-1-value-1_");
secretMasker1.AddValue("masker-1-value-2_");
- secretMasker1.AddValueEncoder(x => x.Replace("_", "_masker-1-encoder-1"));
- secretMasker1.AddValueEncoder(x => x.Replace("_", "_masker-1-encoder-2"));
+ secretMasker1.AddLiteralEncoder(x => x.Replace("_", "_masker-1-encoder-1"));
+ secretMasker1.AddLiteralEncoder(x => x.Replace("_", "_masker-1-encoder-2"));
// Copy and add to masker 2.
- var secretMasker2 = secretMasker1.Clone();
- secretMasker2.AddRegex("masker-2-regex-1_*");
+ using var secretMasker2 = secretMasker1.Clone();
+ secretMasker2.AddRegex(new RegexPattern("000", "Test", 0, "masker-2-regex-1_*"));
secretMasker2.AddValue("masker-2-value-1_");
- secretMasker2.AddValueEncoder(x => x.Replace("_", "_masker-2-encoder-1"));
+ secretMasker2.AddLiteralEncoder(x => x.Replace("_", "_masker-2-encoder-1"));
// Add to masker 1.
- secretMasker1.AddRegex("masker-1-regex-3_*");
+ secretMasker1.AddRegex(new RegexPattern("000", "Test", 0, "masker-1-regex-3_*"));
secretMasker1.AddValue("masker-1-value-3_");
- secretMasker1.AddValueEncoder(x => x.Replace("_", "_masker-1-encoder-3"));
+ secretMasker1.AddLiteralEncoder(x => x.Replace("_", "_masker-1-encoder-3"));
// Assert masker 1 values.
Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-regex-1___")); // original regex
@@ -174,14 +176,14 @@ public void SecretMaskerTests_CopyConstructor()
Assert.Equal("***masker-1-encoder-3", secretMasker2.MaskSecrets("masker-1-value-1_masker-1-encoder-3")); // separate encoder storage from original
}
[Fact]
- [Trait("Level","L0")]
+ [Trait("Level", "L0")]
[Trait("Category", "SecretMasker")]
public void SecretMaskerTests_Encoder()
{
// Add encoder before values.
using var secretMasker = new SecretMasker();
- secretMasker.AddValueEncoder(x => x.Replace("-", "_"));
- secretMasker.AddValueEncoder(x => x.Replace("-", " "));
+ secretMasker.AddLiteralEncoder(x => x.Replace("-", "_"));
+ secretMasker.AddLiteralEncoder(x => x.Replace("-", " "));
secretMasker.AddValue("value-1");
secretMasker.AddValue("value-2");
Assert.Equal("***", secretMasker.MaskSecrets("value-1"));
@@ -198,429 +200,429 @@ public void SecretMaskerTests_Encoder()
Assert.Equal("***", secretMasker.MaskSecrets("value_3"));
Assert.Equal("***", secretMasker.MaskSecrets("value 3"));
}
-
+
[Fact]
- [Trait("Level","L0")]
+ [Trait("Level", "L0")]
[Trait("Category", "SecretMasker")]
public void SecretMaskerTests_Encoder_JsonStringEscape()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape);
- secretMasker.AddValue("carriage-return\r_newline\n_tab\t_backslash\\_double-quote\"");
- Assert.Equal("***", secretMasker.MaskSecrets("carriage-return\r_newline\n_tab\t_backslash\\_double-quote\""));
- Assert.Equal("***", secretMasker.MaskSecrets("carriage-return\\r_newline\\n_tab\\t_backslash\\\\_double-quote\\\""));
- }
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddLiteralEncoder(LiteralEncoders.JsonStringEscape);
+ secretMasker.AddValue("carriage-return\r_newline\n_tab\t_backslash\\_double-quote\"");
+ Assert.Equal("***", secretMasker.MaskSecrets("carriage-return\r_newline\n_tab\t_backslash\\_double-quote\""));
+ Assert.Equal("***", secretMasker.MaskSecrets("carriage-return\\r_newline\\n_tab\\t_backslash\\\\_double-quote\\\""));
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_Encoder_BackslashEscape()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddLiteralEncoder(LiteralEncoders.BackslashEscape);
+ secretMasker.AddValue(@"abc\\def\'\""ghi\t");
+ Assert.Equal("***", secretMasker.MaskSecrets(@"abc\\def\'\""ghi\t"));
+ Assert.Equal("***", secretMasker.MaskSecrets(@"abc\def'""ghi" + "\t"));
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_Encoder_UriDataEscape()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddLiteralEncoder(LiteralEncoders.UriDataEscape);
+ secretMasker.AddValue("hello world");
+ Assert.Equal("***", secretMasker.MaskSecrets("hello world"));
+ Assert.Equal("***", secretMasker.MaskSecrets("hello%20world"));
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_Encoder_UriDataEscape_LargeString()
+ {
+ // Uri.EscapeDataString cannot receive a string longer than 65519 characters.
+ // For unit testing we call a different overload with a smaller segment size (improve unit test speed).
+
+ LiteralEncoder encoder = x => LiteralEncoders.UriDataEscape(x);
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(1, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(2, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(3, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(4, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(5, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(5, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(6, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = String.Empty.PadRight(7, ' ');
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = "𐐷𐐷𐐷𐐷"; // surrogate pair
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ }
+
+ using (var secretMasker = new SecretMasker())
+ {
+ secretMasker.AddLiteralEncoder(encoder);
+ var value = " 𐐷𐐷𐐷𐐷"; // shift by one non-surrogate character to ensure surrogate across segment boundary handled correctly
+ secretMasker.AddValue(value);
+ Assert.Equal("***", secretMasker.MaskSecrets(value));
+ }
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_HandlesEmptyInput()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("abcd");
+
+ var result = secretMasker.MaskSecrets(null);
+ Assert.Equal(string.Empty, result);
+
+ result = secretMasker.MaskSecrets(string.Empty);
+ Assert.Equal(string.Empty, result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_HandlesNoMasks()
+ {
+ using var secretMasker = new SecretMasker();
+ var expected = "abcdefg";
+ var actual = secretMasker.MaskSecrets(expected);
+ Assert.Equal(expected, actual);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_ReplacesValue()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("def");
+
+ var input = "abcdefg";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("abc***g", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_ReplacesMultipleInstances()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("def");
+
+ var input = "abcdefgdef";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("abc***g***", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_ReplacesMultipleAdjacentInstances()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("abc");
+
+ var input = "abcabcdef";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("***def", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_ReplacesMultipleSecrets()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("bcd");
+ secretMasker.AddValue("fgh");
+
+ var input = "abcdefghi";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("a***e***i", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_ReplacesOverlappingSecrets()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("def");
+ secretMasker.AddValue("bcd");
+
+ var input = "abcdefg";
+ var result = secretMasker.MaskSecrets(input);
+
+ // a naive replacement would replace "def" first, and never find "bcd", resulting in "abc***g"
+ // or it would replace "bcd" first, and never find "def", resulting in "a***efg"
+
+ Assert.Equal("a***g", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_ReplacesAdjacentSecrets()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("efg");
+ secretMasker.AddValue("bcd");
+
+ var input = "abcdefgh";
+ var result = secretMasker.MaskSecrets(input);
+
+ // two adjacent secrets are basically one big secret
+
+ Assert.Equal("a***h", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_MinLengthSetThroughConstructor()
+ {
+ using var secretMasker = new SecretMasker() { MinimumSecretLength = 9 };
+
+ secretMasker.AddValue("efg");
+ secretMasker.AddValue("bcd");
+
+ var input = "abcdefgh";
+ var result = secretMasker.MaskSecrets(input);
+
+ // two adjacent secrets are basically one big secret
+
+ Assert.Equal("abcdefgh", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_MinLengthSetThroughProperty()
+ {
+ using var secretMasker = new SecretMasker { MinimumSecretLength = 9 };
+
+ secretMasker.AddValue("efg");
+ secretMasker.AddValue("bcd");
+
+ var input = "abcdefgh";
+ var result = secretMasker.MaskSecrets(input);
+
+ // two adjacent secrets are basically one big secret
+
+ Assert.Equal("abcdefgh", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_MinLengthSetThroughPropertySetTwice()
+ {
+ using var secretMasker = new SecretMasker();
+
+ var minSecretLenFirst = 9;
+ secretMasker.MinimumSecretLength = minSecretLenFirst;
+
+ var minSecretLenSecond = 2;
+ secretMasker.MinimumSecretLength = minSecretLenSecond;
+
+ Assert.Equal(secretMasker.MinimumSecretLength, minSecretLenSecond);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_NegativeMinimumSecretLengthSet()
+ {
+ using var secretMasker = new SecretMasker() { MinimumSecretLength = -3 };
+ secretMasker.AddValue("efg");
+ secretMasker.AddValue("bcd");
+
+ var input = "abcdefgh";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("a***h", result);
+ }
[Fact]
- [Trait("Level","L0")]
+ [Trait("Level", "L0")]
[Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_Encoder_BackslashEscape()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValueEncoder(ValueEncoders.BackslashEscape);
- secretMasker.AddValue(@"abc\\def\'\""ghi\t");
- Assert.Equal("***", secretMasker.MaskSecrets(@"abc\\def\'\""ghi\t"));
- Assert.Equal("***", secretMasker.MaskSecrets(@"abc\def'""ghi" + "\t"));
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_Encoder_UriDataEscape()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValueEncoder(ValueEncoders.UriDataEscape);
- secretMasker.AddValue("hello world");
- Assert.Equal("***", secretMasker.MaskSecrets("hello world"));
- Assert.Equal("***", secretMasker.MaskSecrets("hello%20world"));
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_Encoder_UriDataEscape_LargeString()
- {
- // Uri.EscapeDataString cannot receive a string longer than 65519 characters.
- // For unit testing we call a different overload with a smaller segment size (improve unit test speed).
-
- ValueEncoder encoder = x => ValueEncoders.UriDataEscape(x);
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(1, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(2, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(3, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(4, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(5, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(5, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(6, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = String.Empty.PadRight(7, ' ');
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- Assert.Equal("***", secretMasker.MaskSecrets(value.Replace(" ", "%20")));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = "𐐷𐐷𐐷𐐷"; // surrogate pair
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- }
-
- using (var secretMasker = new SecretMasker())
- {
- secretMasker.AddValueEncoder(encoder);
- var value = " 𐐷𐐷𐐷𐐷"; // shift by one non-surrogate character to ensure surrogate across segment boundary handled correctly
- secretMasker.AddValue(value);
- Assert.Equal("***", secretMasker.MaskSecrets(value));
- }
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_HandlesEmptyInput()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("abcd");
-
- var result = secretMasker.MaskSecrets(null);
- Assert.Equal(string.Empty, result);
-
- result = secretMasker.MaskSecrets(string.Empty);
- Assert.Equal(string.Empty, result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_HandlesNoMasks()
- {
- using var secretMasker = new SecretMasker();
- var expected = "abcdefg";
- var actual = secretMasker.MaskSecrets(expected);
- Assert.Equal(expected, actual);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_ReplacesValue()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("def");
-
- var input = "abcdefg";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("abc***g", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_ReplacesMultipleInstances()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("def");
-
- var input = "abcdefgdef";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("abc***g***", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_ReplacesMultipleAdjacentInstances()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("abc");
-
- var input = "abcabcdef";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("***def", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_ReplacesMultipleSecrets()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("bcd");
- secretMasker.AddValue("fgh");
-
- var input = "abcdefghi";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("a***e***i", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_ReplacesOverlappingSecrets()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("def");
- secretMasker.AddValue("bcd");
-
- var input = "abcdefg";
- var result = secretMasker.MaskSecrets(input);
-
- // a naive replacement would replace "def" first, and never find "bcd", resulting in "abc***g"
- // or it would replace "bcd" first, and never find "def", resulting in "a***efg"
-
- Assert.Equal("a***g", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_ReplacesAdjacentSecrets()
- {
- using var secretMasker = new SecretMasker();
- secretMasker.AddValue("efg");
- secretMasker.AddValue("bcd");
-
- var input = "abcdefgh";
- var result = secretMasker.MaskSecrets(input);
-
- // two adjacent secrets are basically one big secret
-
- Assert.Equal("a***h", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_MinLengthSetThroughConstructor()
- {
- using var secretMasker = new SecretMasker() { MinSecretLength = 9 };
-
- secretMasker.AddValue("efg");
- secretMasker.AddValue("bcd");
-
- var input = "abcdefgh";
- var result = secretMasker.MaskSecrets(input);
-
- // two adjacent secrets are basically one big secret
-
- Assert.Equal("abcdefgh", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_MinLengthSetThroughProperty()
- {
- using var secretMasker = new SecretMasker { MinSecretLength = 9 };
-
- secretMasker.AddValue("efg");
- secretMasker.AddValue("bcd");
-
- var input = "abcdefgh";
- var result = secretMasker.MaskSecrets(input);
-
- // two adjacent secrets are basically one big secret
-
- Assert.Equal("abcdefgh", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_MinLengthSetThroughPropertySetTwice()
- {
- using var secretMasker = new SecretMasker();
-
- var minSecretLenFirst = 9;
- secretMasker.MinSecretLength = minSecretLenFirst;
-
- var minSecretLenSecond = 2;
- secretMasker.MinSecretLength = minSecretLenSecond;
-
- Assert.Equal(secretMasker.MinSecretLength, minSecretLenSecond);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_NegativeMinSecretLengthSet()
- {
- using var secretMasker = new SecretMasker() { MinSecretLength = -3 };
- secretMasker.AddValue("efg");
- secretMasker.AddValue("bcd");
-
- var input = "abcdefgh";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("a***h", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_RemoveShortSecrets()
- {
- using var secretMasker = new SecretMasker() { MinSecretLength = 3 };
- secretMasker.AddValue("efg");
- secretMasker.AddValue("bcd");
-
- var input = "abcdefgh";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("a***h", result);
-
- secretMasker.MinSecretLength = 4;
- secretMasker.RemoveShortSecretsFromDictionary();
-
- var result2 = secretMasker.MaskSecrets(input);
-
- Assert.Equal(input, result2);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_RemoveShortSecretsBoundaryValues()
- {
- using var secretMasker = new SecretMasker(0);
- secretMasker.AddValue("bc");
- secretMasker.AddValue("defg");
- secretMasker.AddValue("h12");
-
- var input = "abcdefgh123";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("a***3", result);
-
- secretMasker.MinSecretLength = 3;
- secretMasker.RemoveShortSecretsFromDictionary();
-
- var result2 = secretMasker.MaskSecrets(input);
-
- Assert.Equal("abc***3", result2);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_RemoveShortRegexes()
- {
- using var secretMasker = new SecretMasker(0);
- secretMasker.AddRegex("bc");
- secretMasker.AddRegex("defg");
- secretMasker.AddRegex("h12");
-
- secretMasker.MinSecretLength = 3;
- secretMasker.RemoveShortSecretsFromDictionary();
-
- var input = "abcdefgh123";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("abc***3", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_RemoveEncodedSecrets()
- {
- using var secretMasker = new SecretMasker(0);
- secretMasker.AddValue("1");
- secretMasker.AddValue("2");
- secretMasker.AddValue("3");
- secretMasker.AddValueEncoder(new ValueEncoder(x => x.Replace("1", "123")));
- secretMasker.AddValueEncoder(new ValueEncoder(x => x.Replace("2", "45")));
- secretMasker.AddValueEncoder(new ValueEncoder(x => x.Replace("3", "6789")));
-
- secretMasker.MinSecretLength = 3;
- secretMasker.RemoveShortSecretsFromDictionary();
-
- var input = "123456789";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("***45***", result);
- }
-
- [Fact]
- [Trait("Level","L0")]
- [Trait("Category", "SecretMasker")]
- public void SecretMaskerTests_NotAddShortEncodedSecrets()
- {
- using var secretMasker = new SecretMasker() { MinSecretLength = 3 };
- secretMasker.AddValueEncoder(new ValueEncoder(x => x.Replace("123", "ab")));
- secretMasker.AddValue("123");
- secretMasker.AddValue("345");
- secretMasker.AddValueEncoder(new ValueEncoder(x => x.Replace("345", "cd")));
-
- var input = "ab123cd345";
- var result = secretMasker.MaskSecrets(input);
-
- Assert.Equal("ab***cd***", result);
- }
+ public void SecretMaskerTests_RemoveShortSecrets()
+ {
+ using var secretMasker = new SecretMasker() { MinimumSecretLength = 3 };
+ secretMasker.AddValue("efg");
+ secretMasker.AddValue("bcd");
+
+ var input = "abcdefgh";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("a***h", result);
+
+ secretMasker.MinimumSecretLength = 4;
+ secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
+
+ var result2 = secretMasker.MaskSecrets(input);
+
+ Assert.Equal(input, result2);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_RemoveShortSecretsBoundaryValues()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("bc");
+ secretMasker.AddValue("defg");
+ secretMasker.AddValue("h12");
+
+ var input = "abcdefgh123";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("a***3", result);
+
+ secretMasker.MinimumSecretLength = 3;
+ secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
+
+ var result2 = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("abc***3", result2);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_RemoveShortRegexes()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddRegex(new RegexPattern("000", "Test", 0, "bc"));
+ secretMasker.AddRegex(new RegexPattern("000", "Test", 0, "defg"));
+ secretMasker.AddRegex(new RegexPattern("000", "Test", 0, "h12"));
+
+ secretMasker.MinimumSecretLength = 3;
+ secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
+
+ var input = "abcdefgh123";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("abc+++3", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_RemoveEncodedSecrets()
+ {
+ using var secretMasker = new SecretMasker();
+ secretMasker.AddValue("1");
+ secretMasker.AddValue("2");
+ secretMasker.AddValue("3");
+ secretMasker.AddLiteralEncoder(new LiteralEncoder(x => x.Replace("1", "123")));
+ secretMasker.AddLiteralEncoder(new LiteralEncoder(x => x.Replace("2", "45")));
+ secretMasker.AddLiteralEncoder(new LiteralEncoder(x => x.Replace("3", "6789")));
+
+ secretMasker.MinimumSecretLength = 3;
+ secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
+
+ var input = "123456789";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("***45***", result);
+ }
+
+ [Fact]
+ [Trait("Level", "L0")]
+ [Trait("Category", "SecretMasker")]
+ public void SecretMaskerTests_NotAddShortEncodedSecrets()
+ {
+ using var secretMasker = new SecretMasker() { MinimumSecretLength = 3 };
+ secretMasker.AddLiteralEncoder(new LiteralEncoder(x => x.Replace("123", "ab")));
+ secretMasker.AddValue("123");
+ secretMasker.AddValue("345");
+ secretMasker.AddLiteralEncoder(new LiteralEncoder(x => x.Replace("345", "cd")));
+
+ var input = "ab123cd345";
+ var result = secretMasker.MaskSecrets(input);
+
+ Assert.Equal("ab***cd***", result);
+ }
}
-}
+}
\ No newline at end of file
diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs
index a83e5fba0d..c05b6ab35e 100644
--- a/src/Test/L0/TestHostContext.cs
+++ b/src/Test/L0/TestHostContext.cs
@@ -11,13 +11,12 @@
using System.Threading.Tasks;
using System.Runtime.Loader;
using System.Reflection;
-using Microsoft.TeamFoundation.DistributedTask.Logging;
using System.Net.Http.Headers;
using Agent.Sdk;
using Agent.Sdk.Knob;
using Agent.Sdk.SecretMasking;
using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
-using SecretMasker = Agent.Sdk.SecretMasking.SecretMasker;
+using Microsoft.Security.Utilities;
namespace Microsoft.VisualStudio.Services.Agent.Tests
{
@@ -73,9 +72,9 @@ public TestHostContext(object testClass, [CallerMemberName] string testName = ""
var traceListener = new HostTraceListener(TraceFileName);
traceListener.DisableConsoleReporting = true;
_secretMasker = new LoggedSecretMasker(new SecretMasker());
- _secretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape);
- _secretMasker.AddValueEncoder(ValueEncoders.UriDataEscape);
- _secretMasker.AddValueEncoder(ValueEncoders.BackslashEscape);
+ //_secretMasker.AddValueEncoder(LiteralEncoder.JsonStringEscape);
+ //_secretMasker.AddValueEncoder(ValueEncoders.UriDataEscape);
+ //_secretMasker.AddValueEncoder(ValueEncoders.BackslashEscape);
_secretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern);
_traceManager = new TraceManager(traceListener, _secretMasker);
_trace = GetTrace(nameof(TestHostContext));
@@ -469,6 +468,9 @@ private void Dispose(bool disposing)
_traceManager?.Dispose();
_term?.Dispose();
_trace?.Dispose();
+
+ _secretMasker?.Dispose();
+
_agentShutdownTokenSource?.Dispose();
try
{
diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs
index d515f056be..1f6cbc0be1 100644
--- a/src/Test/L0/Worker/JobExtensionL0.cs
+++ b/src/Test/L0/Worker/JobExtensionL0.cs
@@ -777,6 +777,7 @@ public async Task JobExtensionManagementScriptStepMSI()
[Trait("SkipOn", "darwin")]
[Trait("SkipOn", "linux")]
public async Task JobExtensionTelemetryPublisherSecretValue()
+
{
using CancellationTokenSource tokenSource = new CancellationTokenSource();
using TestHostContext hc = CreateMSITestContext(tokenSource);
diff --git a/src/dev.sh b/src/dev.sh
index e4b2834ffe..86a3d0ceec 100755
--- a/src/dev.sh
+++ b/src/dev.sh
@@ -119,15 +119,12 @@ function detect_platform_and_runtime_id() {
CURRENT_PLATFORM=$(uname | awk '{print tolower($0)}')
fi
+ echo "Detected Process Arch: $PROCESSOR_ARCHITECTURE"
if [[ "$CURRENT_PLATFORM" == 'windows' ]]; then
- local processor_type=$(detect_system_architecture)
- echo "Detected Process Arch: $processor_type"
-
- # Default to win-x64
DETECTED_RUNTIME_ID='win-x64'
- if [[ "$processor_type" == 'x86' ]]; then
+ if [[ "$PROCESSOR_ARCHITECTURE" == 'x86' ]]; then
DETECTED_RUNTIME_ID='win-x86'
- elif [[ "$processor_type" == 'ARM64' ]]; then
+ elif [[ "$PROCESSOR_ARCHITECTURE" == 'ARM64' ]]; then
DETECTED_RUNTIME_ID='win-arm64'
fi
elif [[ "$CURRENT_PLATFORM" == 'linux' ]]; then
@@ -371,48 +368,6 @@ function cmd_lint_verify() {
"${DOTNET_DIR}/dotnet" format --verify-no-changes -v diag "$REPO_ROOT/azure-pipelines-agent.sln" || checkRC "cmd_lint_verify"
}
-function detect_system_architecture() {
- local processor # Variable to hold the processor type (e.g., x, ARM)
- local os_arch # Variable to hold the OS bitness (e.g., 64, 86)
-
- # Detect processor type using PROCESSOR_IDENTIFIER
- # Check for AMD64 or Intel in the variable to classify as "x" (covers x86 and x64 processors)
- if [[ "$PROCESSOR_IDENTIFIER" =~ "AMD64" || "$PROCESSOR_IDENTIFIER" =~ "Intel64" ]]; then
- processor="x"
- # Check for ARM64 in the variable to classify as "ARM"
- elif [[ "$PROCESSOR_IDENTIFIER" =~ "ARM" || "$PROCESSOR_IDENTIFIER" =~ "Arm" ]]; then
- processor="ARM"
- # Default to "x" for unknown or unhandled cases
- else
- processor="x"
- fi
-
- # Detect OS bitness using uname
- # "x86_64" indicates a 64-bit operating system
- if [[ "$(uname -m)" == "x86_64" ]]; then
- os_arch="64"
- # "i686" or "i386" indicates a 32-bit operating system
- elif [[ "$(uname -m)" == "i686" || "$(uname -m)" == "i386" ]]; then
- os_arch="86"
- # "aarch64" indicates a 64-bit ARM operating system
- elif [[ "$(uname -m)" == "aarch64" ]]; then
- os_arch="64"
- # Default to "64" for unknown or unhandled cases
- else
- os_arch="64"
- fi
-
- # Note: AMD32 does not exist as a specific label; 32-bit AMD processors are referred to as x86.
- # ARM32 also does not exist in this context; ARM processors are always 64-bit.
-
- # Combine processor type and OS bitness for the final result
- # Examples:
- # - "x64" for Intel/AMD 64-bit
- # - "x86" for Intel/AMD 32-bit
- # - "ARM64" for ARM 64-bit
- echo "${processor}${os_arch}"
-}
-
detect_platform_and_runtime_id
echo "Current platform: $CURRENT_PLATFORM"
echo "Current runtime ID: $DETECTED_RUNTIME_ID"