Skip to content

Add support for pipeline resources with sourceId in YAML parsing #5283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Add support for pipeline resources with sourceId in YAML parsing #5283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
82d35db
Initial plan
Copilot Aug 6, 2025
38b0f7e
Add pipeline resource support with source and sourceId
Copilot Aug 6, 2025
eb5bc69
Refine pipeline resource implementation - nested structure only
Copilot Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 145 additions & 48 deletions src/Agent.Listener/DistributedTask.Pipelines/Yaml/TypeConverters/ConverterUtil.processes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,40 +19,107 @@ internal static partial class ConverterUtil
internal static IList<ProcessResource> ReadProcessResources(IParser parser)
{
var result = new List<ProcessResource>();
parser.Expect<SequenceStart>();
while (parser.Allow<SequenceEnd>() == null)

// Check if this is a sequence (flat structure) or mapping (nested structure)
if (parser.Accept<SequenceStart>())
{
parser.Expect<MappingStart>();
Scalar scalar = parser.Expect<Scalar>();
switch (scalar.Value ?? String.Empty)
// Flat structure: resources: - repo: self
parser.Expect<SequenceStart>();
while (parser.Allow<SequenceEnd>() == null)
{
case YamlConstants.Endpoint:
case YamlConstants.Repo:
break;
default:
throw new SyntaxErrorException(scalar.Start, scalar.End, $"Unexpected resource type: '{scalar.Value}'");
}

var resource = new ProcessResource { Type = scalar.Value };
resource.Name = ReadNonEmptyString(parser);
while (parser.Allow<MappingEnd>() == null)
{
string dataKey = ReadNonEmptyString(parser);
if (parser.Accept<MappingStart>())
parser.Expect<MappingStart>();
Scalar scalar = parser.Expect<Scalar>();
switch (scalar.Value ?? String.Empty)
{
resource.Data[dataKey] = ReadMapping(parser);
case YamlConstants.Endpoint:
case YamlConstants.Repo:
break;
case YamlConstants.Pipeline:
throw new SyntaxErrorException(scalar.Start, scalar.End,
$"Pipeline resources must be defined under 'pipelines:' section, not as flat resources");
default:
throw new SyntaxErrorException(scalar.Start, scalar.End, $"Unexpected resource type: '{scalar.Value}'");
}
else if (parser.Accept<SequenceStart>())

var resource = new ProcessResource { Type = scalar.Value };
resource.Name = ReadNonEmptyString(parser);
while (parser.Allow<MappingEnd>() == null)
{
resource.Data[dataKey] = ReadSequence(parser);
string dataKey = ReadNonEmptyString(parser);
if (parser.Accept<MappingStart>())
{
resource.Data[dataKey] = ReadMapping(parser);
}
else if (parser.Accept<SequenceStart>())
{
resource.Data[dataKey] = ReadSequence(parser);
}
else
{
resource.Data[dataKey] = parser.Expect<Scalar>().Value ?? String.Empty;
}
}
else

result.Add(resource);
}
}
else if (parser.Accept<MappingStart>())
{
// Nested structure: resources: pipelines: - pipeline: my
parser.Expect<MappingStart>();
while (parser.Allow<MappingEnd>() == null)
{
Scalar sectionScalar = parser.Expect<Scalar>();
switch (sectionScalar.Value ?? String.Empty)
{
resource.Data[dataKey] = parser.Expect<Scalar>().Value ?? String.Empty;
case YamlConstants.Pipelines:
// Read pipeline resources
parser.Expect<SequenceStart>();
while (parser.Allow<SequenceEnd>() == null)
{
parser.Expect<MappingStart>();
Scalar pipelineTypeScalar = parser.Expect<Scalar>();

if (pipelineTypeScalar.Value != YamlConstants.Pipeline)
{
throw new SyntaxErrorException(pipelineTypeScalar.Start, pipelineTypeScalar.End,
$"Expected 'pipeline' but found: '{pipelineTypeScalar.Value}'");
}

var pipelineResource = new ProcessResource { Type = YamlConstants.Pipeline };
pipelineResource.Name = ReadNonEmptyString(parser);

while (parser.Allow<MappingEnd>() == null)
{
string dataKey = ReadNonEmptyString(parser);
if (parser.Accept<MappingStart>())
{
pipelineResource.Data[dataKey] = ReadMapping(parser);
}
else if (parser.Accept<SequenceStart>())
{
pipelineResource.Data[dataKey] = ReadSequence(parser);
}
else
{
pipelineResource.Data[dataKey] = parser.Expect<Scalar>().Value ?? String.Empty;
}
}

result.Add(pipelineResource);
}
break;
// TODO: Add support for other resource sections (repositories, etc.) if needed
default:
throw new SyntaxErrorException(sectionScalar.Start, sectionScalar.End,
$"Unexpected resource section: '{sectionScalar.Value}'");
}
}

result.Add(resource);
}
else
{
throw new SyntaxErrorException(parser.Current.Start, parser.Current.End,
"Expected resources to be either a sequence or mapping");
}

return result;
Expand All @@ -74,36 +141,66 @@ internal static ProcessTemplateReference ReadProcessTemplateReference(IParser pa

internal static void WriteProcessResources(IEmitter emitter, IList<ProcessResource> resources)
{
emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block));
foreach (ProcessResource resource in resources)
// Separate pipeline resources from other resources
var pipelineResources = resources.Where(r => r.Type == YamlConstants.Pipeline).ToList();
var otherResources = resources.Where(r => r.Type != YamlConstants.Pipeline).ToList();

// If we only have other resources (repo, endpoint), use flat structure
if (pipelineResources.Count == 0)
{
emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block));
foreach (ProcessResource resource in otherResources)
{
WriteProcessResource(emitter, resource);
}
emitter.Emit(new SequenceEnd());
}
// If we only have pipeline resources, use nested structure
else if (otherResources.Count == 0)
{
emitter.Emit(new MappingStart());
emitter.Emit(new Scalar(resource.Type));
emitter.Emit(new Scalar(resource.Name));
if (resource.Data != null && resource.Data.Count > 0)
emitter.Emit(new Scalar(YamlConstants.Pipelines));
emitter.Emit(new SequenceStart(null, null, true, SequenceStyle.Block));
foreach (ProcessResource resource in pipelineResources)
{
foreach (KeyValuePair<String, Object> pair in resource.Data)
{
emitter.Emit(new Scalar(pair.Key));
if (pair.Value is String)
{
emitter.Emit(new Scalar(pair.Value as string));
}
else if (pair.Value is Dictionary<String, Object>)
{
WriteMapping(emitter, pair.Value as Dictionary<String, Object>);
}
else
{
WriteSequence(emitter, pair.Value as List<Object>);
}
}
WriteProcessResource(emitter, resource);
}

emitter.Emit(new SequenceEnd());
emitter.Emit(new MappingEnd());
}
// If we have both, pipeline resources should not exist in flat structure, this is an error
else
{
throw new InvalidOperationException("Pipeline resources cannot be mixed with other resource types. " +
"Pipeline resources must be defined under 'pipelines:' section.");
}
}

emitter.Emit(new SequenceEnd());
private static void WriteProcessResource(IEmitter emitter, ProcessResource resource)
{
emitter.Emit(new MappingStart());
emitter.Emit(new Scalar(resource.Type));
emitter.Emit(new Scalar(resource.Name));
if (resource.Data != null && resource.Data.Count > 0)
{
foreach (KeyValuePair<String, Object> pair in resource.Data)
{
emitter.Emit(new Scalar(pair.Key));
if (pair.Value is String)
{
emitter.Emit(new Scalar(pair.Value as string));
}
else if (pair.Value is Dictionary<String, Object>)
{
WriteMapping(emitter, pair.Value as Dictionary<String, Object>);
}
else
{
WriteSequence(emitter, pair.Value as List<Object>);
}
}
}
emitter.Emit(new MappingEnd());
}
}
}
4 changes: 4 additions & 0 deletions src/Agent.Listener/DistributedTask.Pipelines/Yaml/TypeConverters/YamlConstants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,14 @@ internal static class YamlConstants
internal const String Parameters = "parameters";
internal const String Percentage = "percentage";
internal const String Phases = "phases";
internal const String Pipeline = "pipeline";
internal const String Pipelines = "pipelines";
internal const String PowerShell = "powershell";
internal const String Queue = "queue";
internal const String Repo = "repo";
internal const String Resources = "resources";
internal const String Source = "source";
internal const String SourceId = "sourceId";
internal const String Script = "script";
internal const String Self = "self";
internal const String Server = "server";
Expand Down
132 changes: 132 additions & 0 deletions src/Test/L0/Listener/PipelineParserL0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,138 @@ public void PhaseVariables_NameValue()
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Agent")]
public void PipelineResources_Source()
{
using (CreateTestContext())
{
// Arrange.
String expected = @"
resources:
pipelines:
- pipeline: my
source: MyFolder\My Pipeline
steps:
- script: echo hello
";
m_fileProvider.FileContent[Path.Combine(c_defaultRoot, "pipelineResources_source.yml")] = expected;

// Act.
String actual = m_pipelineParser.DeserializeAndSerialize(
c_defaultRoot,
"pipelineResources_source.yml",
mustacheContext: null,
cancellationToken: CancellationToken.None);

// Assert.
Assert.Equal(expected.Trim(), actual.Trim());
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Agent")]
public void PipelineResources_SourceId()
{
using (CreateTestContext())
{
// Arrange.
String expected = @"
resources:
pipelines:
- pipeline: my
sourceId: 123
steps:
- script: echo hello
";
m_fileProvider.FileContent[Path.Combine(c_defaultRoot, "pipelineResources_sourceId.yml")] = expected;

// Act.
String actual = m_pipelineParser.DeserializeAndSerialize(
c_defaultRoot,
"pipelineResources_sourceId.yml",
mustacheContext: null,
cancellationToken: CancellationToken.None);

// Assert.
Assert.Equal(expected.Trim(), actual.Trim());
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Agent")]
public void PipelineResources_Multiple()
{
using (CreateTestContext())
{
// Arrange.
String expected = @"
resources:
pipelines:
- pipeline: my
source: MyFolder\My Pipeline
- pipeline: other
sourceId: 123
- pipeline: third
source: AnotherFolder\Another Pipeline
branch: main
steps:
- script: echo hello
";
m_fileProvider.FileContent[Path.Combine(c_defaultRoot, "pipelineResources_multiple.yml")] = expected;

// Act.
String actual = m_pipelineParser.DeserializeAndSerialize(
c_defaultRoot,
"pipelineResources_multiple.yml",
mustacheContext: null,
cancellationToken: CancellationToken.None);

// Assert.
Assert.Equal(expected.Trim(), actual.Trim());
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Agent")]
public void PipelineResources_FlatStructure_ThrowsError()
{
using (CreateTestContext())
{
// Arrange.
String content = @"
resources:
- pipeline: my
source: MyFolder\My Pipeline
steps:
- script: echo hello
";
m_fileProvider.FileContent[Path.Combine(c_defaultRoot, "pipelineResources_flatError.yml")] = content;

try
{
// Act.
m_pipelineParser.DeserializeAndSerialize(
c_defaultRoot,
"pipelineResources_flatError.yml",
mustacheContext: null,
cancellationToken: CancellationToken.None);

// Assert.
Assert.True(false, "Should have thrown syntax error exception");
}
catch (SyntaxErrorException ex)
{
// Assert.
Assert.Contains("Pipeline resources must be defined under 'pipelines:' section", ex.Message);
}
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Agent")]
Expand Down