Skip to content

Commit 6a2137e

Browse files
banlorbanlor
committed
Reconcile GitRepository includes on artifact content changes
Index local include references and enqueue dependants when an included artifact revision or digest changes. Ignore metadata-only updates and skip suspended, deleting and cyclic dependants. Add envtest coverage for nested includes without changing the parent Git revision. Assisted-by: Hermes Agent (gpt-6-astra / openai-codex) Signed-off-by: Michael Basov <124149774+banlor@users.noreply.github.com>
1 parent 143c11a commit 6a2137e

7 files changed

Lines changed: 425 additions & 1 deletion

‎docs/spec/v1/gitrepositories.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -852,6 +852,13 @@ multiple benefits over regular submodules:
852852
- Multiple `GitRepository` objects could include the same repository, which
853853
decreases the amount of cloning done compared to using submodules.
854854

855+
Changes to an included Artifact's revision or digest trigger reconciliation
856+
outside the interval window. The including Artifact retains its own Git revision;
857+
its digest reflects the combined contents. This also propagates changes through
858+
nested includes. Include references are local to the GitRepository's namespace.
859+
Avoid circular includes: to prevent immediate rebuild loops, changes are not
860+
propagated along cyclic include edges. Those repositories remain interval-driven.
861+
855862
```yaml
856863
---
857864
apiVersion: source.toolkit.fluxcd.io/v1

‎internal/controller/gitrepository_controller.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"sigs.k8s.io/controller-runtime/pkg/client"
4646
"sigs.k8s.io/controller-runtime/pkg/controller"
4747
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
48+
"sigs.k8s.io/controller-runtime/pkg/handler"
4849
"sigs.k8s.io/controller-runtime/pkg/predicate"
4950
"sigs.k8s.io/controller-runtime/pkg/reconcile"
5051

@@ -196,10 +197,18 @@ func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager, opts GitRep
196197
r.features = features.FeatureGates()
197198
}
198199

200+
if err := mgr.GetCache().IndexField(context.Background(), &sourcev1.GitRepository{},
201+
indexKeyGitRepositoryInclude, indexGitRepositoryIncludes); err != nil {
202+
return fmt.Errorf("failed indexing GitRepository includes: %w", err)
203+
}
204+
199205
return ctrl.NewControllerManagedBy(mgr).
200206
For(&sourcev1.GitRepository{}, builder.WithPredicates(
201207
predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}),
202208
)).
209+
Watches(&sourcev1.GitRepository{},
210+
handler.EnqueueRequestsFromMapFunc(r.requestsForIncludeChange),
211+
builder.WithPredicates(SourceRevisionChangePredicate{})).
203212
WithOptions(controller.Options{
204213
RateLimiter: opts.RateLimiter,
205214
}).

‎internal/controller/gitrepository_include.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
Copyright 2026 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
22+
apierrors "k8s.io/apimachinery/pkg/api/errors"
23+
"k8s.io/apimachinery/pkg/util/sets"
24+
ctrl "sigs.k8s.io/controller-runtime"
25+
"sigs.k8s.io/controller-runtime/pkg/client"
26+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
27+
28+
sourcev1 "github.com/fluxcd/source-controller/api/v1"
29+
)
30+
31+
const indexKeyGitRepositoryInclude = ".metadata.gitRepositoryInclude"
32+
33+
// indexGitRepositoryIncludes indexes local references, independent of artifact availability.
34+
func indexGitRepositoryIncludes(o client.Object) []string {
35+
repo, ok := o.(*sourcev1.GitRepository)
36+
if !ok {
37+
return nil
38+
}
39+
refs := sets.New[string]()
40+
for _, incl := range repo.Spec.Include {
41+
refs.Insert(incl.GitRepositoryRef.Name)
42+
}
43+
return sets.List(refs)
44+
}
45+
46+
// requestsForIncludeChange enqueues repositories including the changed artifact.
47+
func (r *GitRepositoryReconciler) requestsForIncludeChange(ctx context.Context, o client.Object) []reconcile.Request {
48+
repo, ok := o.(*sourcev1.GitRepository)
49+
if !ok || repo.GetArtifact() == nil {
50+
return nil
51+
}
52+
var list sourcev1.GitRepositoryList
53+
if err := r.List(ctx, &list, client.InNamespace(repo.Namespace), client.MatchingFields{
54+
indexKeyGitRepositoryInclude: repo.Name,
55+
}); err != nil {
56+
ctrl.LoggerFrom(ctx).Error(err, "failed to list GitRepositories for include change")
57+
return nil
58+
}
59+
if len(list.Items) == 0 {
60+
return nil
61+
}
62+
63+
// Do not turn an existing include cycle into an event-driven rebuild loop.
64+
// Each repository is visited at most once, through the manager's scoped cache.
65+
ancestors, err := r.includeDependencies(ctx, repo)
66+
if err != nil {
67+
ctrl.LoggerFrom(ctx).Error(err, "failed to check GitRepository include dependencies")
68+
return nil
69+
}
70+
var requests []reconcile.Request
71+
for i := range list.Items {
72+
dependent := &list.Items[i]
73+
if dependent.Spec.Suspend || !dependent.DeletionTimestamp.IsZero() || ancestors.Has(dependent.Name) {
74+
continue
75+
}
76+
requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(dependent)})
77+
}
78+
return requests
79+
}
80+
81+
// includeDependencies returns transitive includes and the repository itself.
82+
// Cyclic edges remain interval-driven, as they were before include watches.
83+
func (r *GitRepositoryReconciler) includeDependencies(ctx context.Context, repo *sourcev1.GitRepository) (sets.Set[string], error) {
84+
seen := sets.New(repo.Name)
85+
pending := append([]sourcev1.GitRepositoryInclude(nil), repo.Spec.Include...)
86+
for len(pending) > 0 {
87+
name := pending[0].GitRepositoryRef.Name
88+
pending = pending[1:]
89+
if seen.Has(name) {
90+
continue
91+
}
92+
seen.Insert(name)
93+
var dep sourcev1.GitRepository
94+
if err := r.Get(ctx, client.ObjectKey{Namespace: repo.Namespace, Name: name}, &dep); err != nil {
95+
if apierrors.IsNotFound(err) {
96+
continue
97+
}
98+
return nil, err
99+
}
100+
pending = append(pending, dep.Spec.Include...)
101+
}
102+
return seen, nil
103+
}

‎internal/controller/gitrepository_include_propagation_test.go‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
Copyright 2026 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"os"
21+
"path/filepath"
22+
"testing"
23+
"time"
24+
25+
gogit "github.com/go-git/go-git/v5"
26+
. "github.com/onsi/gomega"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"sigs.k8s.io/controller-runtime/pkg/client"
29+
30+
"github.com/fluxcd/pkg/apis/meta"
31+
"github.com/fluxcd/pkg/git"
32+
"github.com/fluxcd/pkg/gittestserver"
33+
"github.com/fluxcd/pkg/runtime/conditions"
34+
sourcev1 "github.com/fluxcd/source-controller/api/v1"
35+
)
36+
37+
// TestGitRepositoryIncludePropagation exercises the manager watch, checkout,
38+
// composition and artifact publication through two include edges.
39+
func TestGitRepositoryIncludePropagation(t *testing.T) {
40+
g := NewWithT(t)
41+
server, err := gittestserver.NewTempGitServer()
42+
g.Expect(err).NotTo(HaveOccurred())
43+
t.Cleanup(func() { g.Expect(os.RemoveAll(server.Root())).To(Succeed()) })
44+
server.AutoCreate()
45+
g.Expect(server.StartHTTP()).To(Succeed())
46+
t.Cleanup(server.StopHTTP)
47+
48+
leafDir, mainDir := t.TempDir(), t.TempDir()
49+
g.Expect(os.WriteFile(filepath.Join(leafDir, "value.txt"), []byte("before"), 0o644)).To(Succeed())
50+
g.Expect(os.WriteFile(filepath.Join(mainDir, "root.txt"), []byte("unchanged"), 0o644)).To(Succeed())
51+
leafGit, err := initGitRepo(server, leafDir, git.DefaultBranch, "/leaf.git")
52+
g.Expect(err).NotTo(HaveOccurred())
53+
_, err = initGitRepo(server, mainDir, git.DefaultBranch, "/main.git")
54+
g.Expect(err).NotTo(HaveOccurred())
55+
56+
newSource := func(path string, include *sourcev1.GitRepository) *sourcev1.GitRepository {
57+
repo := &sourcev1.GitRepository{
58+
ObjectMeta: metav1.ObjectMeta{GenerateName: "include-chain-", Namespace: "default"},
59+
Spec: sourcev1.GitRepositorySpec{
60+
URL: server.HTTPAddress() + path,
61+
Interval: metav1.Duration{Duration: time.Hour},
62+
},
63+
}
64+
if include != nil {
65+
repo.Spec.Include = []sourcev1.GitRepositoryInclude{{GitRepositoryRef: meta.LocalObjectReference{Name: include.Name}, ToPath: "included"}}
66+
}
67+
g.Expect(k8sClient.Create(ctx, repo)).To(Succeed())
68+
t.Cleanup(func() { g.Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, repo))).To(Succeed()) })
69+
g.Eventually(func() bool {
70+
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(repo), repo); err != nil {
71+
return false
72+
}
73+
return conditions.IsReady(repo) && repo.GetArtifact() != nil
74+
}, 30*time.Second, 100*time.Millisecond).Should(BeTrue())
75+
return repo
76+
}
77+
leaf := newSource("/leaf.git", nil)
78+
middle := newSource("/main.git", leaf)
79+
outer := newSource("/main.git", middle)
80+
middleBefore, outerBefore := middle.Status.Artifact.DeepCopy(), outer.Status.Artifact.DeepCopy()
81+
82+
// Only the leaf's Git revision changes. The other objects have hour-long
83+
// intervals, so their updates must arrive via the include watches.
84+
g.Expect(os.WriteFile(filepath.Join(leafDir, "value.txt"), []byte("after"), 0o644)).To(Succeed())
85+
g.Expect(commitFromFixture(leafGit, leafDir)).To(Succeed())
86+
g.Expect(leafGit.Push(&gogit.PushOptions{})).To(Succeed())
87+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(leaf), leaf)).To(Succeed())
88+
leaf.Annotations = map[string]string{meta.ReconcileRequestAnnotation: "leaf-update"}
89+
g.Expect(k8sClient.Update(ctx, leaf)).To(Succeed())
90+
for _, obj := range []*sourcev1.GitRepository{middle, outer} {
91+
before := middleBefore
92+
if obj == outer {
93+
before = outerBefore
94+
}
95+
g.Eventually(func() bool {
96+
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil {
97+
return false
98+
}
99+
return conditions.IsReady(obj) && obj.GetArtifact() != nil && obj.GetArtifact().Digest != before.Digest
100+
}, 30*time.Second, 100*time.Millisecond).Should(BeTrue())
101+
g.Expect(obj.Status.Artifact.Revision).To(Equal(before.Revision))
102+
}
103+
g.Expect(middle.Status.IncludedArtifacts[0].Digest).NotTo(BeEmpty())
104+
g.Expect(outer.Status.IncludedArtifacts[0].Digest).To(Equal(middle.Status.Artifact.Digest))
105+
extracted := filepath.Join(t.TempDir(), "artifact")
106+
g.Expect(testStorage.CopyToPath(outer.Status.Artifact, ".", extracted)).To(Succeed())
107+
content, err := os.ReadFile(filepath.Join(extracted, "included", "included", "value.txt"))
108+
g.Expect(err).NotTo(HaveOccurred())
109+
g.Expect(string(content)).To(Equal("after"))
110+
111+
// A forced no-op reconciliation keeps the artifact stable.
112+
before := outer.Status.Artifact.DeepCopy()
113+
outer.Annotations = map[string]string{meta.ReconcileRequestAnnotation: "no-op"}
114+
g.Expect(k8sClient.Update(ctx, outer)).To(Succeed())
115+
g.Eventually(func() string {
116+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(outer), outer)).To(Succeed())
117+
return outer.Status.LastHandledReconcileAt
118+
}, 30*time.Second, 100*time.Millisecond).Should(Equal("no-op"))
119+
g.Expect(outer.Status.Artifact).To(Equal(before))
120+
121+
// Timestamp-only source events do not requeue either dependent.
122+
middleVersion, outerVersion := middle.ResourceVersion, outer.ResourceVersion
123+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(leaf), leaf)).To(Succeed())
124+
leaf.Status.Artifact.LastUpdateTime = metav1.Now()
125+
g.Expect(k8sClient.Status().Update(ctx, leaf)).To(Succeed())
126+
g.Consistently(func() []string {
127+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(middle), middle)).To(Succeed())
128+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(outer), outer)).To(Succeed())
129+
return []string{middle.ResourceVersion, outer.ResourceVersion}
130+
}, time.Second, 100*time.Millisecond).Should(Equal([]string{middleVersion, outerVersion}))
131+
}
132+
133+
func TestGitRepositoryIncludeArchiveStability(t *testing.T) {
134+
g := NewWithT(t)
135+
dir := t.TempDir()
136+
file := filepath.Join(dir, "value")
137+
g.Expect(os.WriteFile(file, []byte("unchanged"), 0o644)).To(Succeed())
138+
obj := &sourcev1.GitRepository{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "archive-stability"}}
139+
artifact := testStorage.NewArtifactFor(sourcev1.GitRepositoryKind, obj, "same", "test.tar.gz")
140+
g.Expect(testStorage.MkdirAll(artifact)).To(Succeed())
141+
t.Cleanup(func() { _, err := testStorage.RemoveAll(artifact); g.Expect(err).NotTo(HaveOccurred()) })
142+
g.Expect(testStorage.Archive(&artifact, dir, nil)).To(Succeed())
143+
digest := artifact.Digest
144+
g.Expect(os.Chtimes(file, time.Unix(1000, 0), time.Unix(1000, 0))).To(Succeed())
145+
g.Expect(testStorage.Archive(&artifact, dir, nil)).To(Succeed())
146+
g.Expect(artifact.Digest).To(Equal(digest))
147+
}

‎internal/controller/gitrepository_include_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
Copyright 2026 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
"testing"
22+
23+
. "github.com/onsi/gomega"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25+
"k8s.io/apimachinery/pkg/runtime"
26+
"sigs.k8s.io/controller-runtime/pkg/client"
27+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
28+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
29+
30+
"github.com/fluxcd/pkg/apis/meta"
31+
sourcev1 "github.com/fluxcd/source-controller/api/v1"
32+
)
33+
34+
func TestGitRepositoryIncludeMapping(t *testing.T) {
35+
g := NewWithT(t)
36+
ctx := context.Background()
37+
scheme := runtime.NewScheme()
38+
g.Expect(sourcev1.AddToScheme(scheme)).To(Succeed())
39+
newRepo := func(ns, name string, includes ...string) *sourcev1.GitRepository {
40+
r := &sourcev1.GitRepository{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}}
41+
for _, ref := range includes {
42+
r.Spec.Include = append(r.Spec.Include, sourcev1.GitRepositoryInclude{GitRepositoryRef: meta.LocalObjectReference{Name: ref}})
43+
}
44+
return r
45+
}
46+
leaf := newRepo("one", "leaf")
47+
leaf.Status.Artifact = &meta.Artifact{Revision: "same", Digest: "sha256:new"}
48+
parent := newRepo("one", "parent", "leaf", "leaf")
49+
otherNS := newRepo("two", "parent", "leaf")
50+
suspended := newRepo("one", "suspended", "leaf")
51+
suspended.Spec.Suspend = true
52+
c := fake.NewClientBuilder().WithScheme(scheme).
53+
WithObjects(leaf, parent, otherNS, suspended).
54+
WithIndex(&sourcev1.GitRepository{}, indexKeyGitRepositoryInclude, indexGitRepositoryIncludes).Build()
55+
r := &GitRepositoryReconciler{Client: c}
56+
request := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(parent)}
57+
g.Expect(indexGitRepositoryIncludes(parent)).To(Equal([]string{"leaf"}))
58+
g.Expect(r.requestsForIncludeChange(ctx, leaf)).To(Equal([]reconcile.Request{request}))
59+
60+
// Updating the specification removes old references and adds new ones.
61+
g.Expect(c.Get(ctx, client.ObjectKeyFromObject(parent), parent)).To(Succeed())
62+
parent.Spec.Include = newRepo("one", "parent", "replacement").Spec.Include
63+
g.Expect(c.Update(ctx, parent)).To(Succeed())
64+
g.Expect(r.requestsForIncludeChange(ctx, leaf)).To(BeEmpty())
65+
replacement := newRepo("one", "replacement")
66+
replacement.Status.Artifact = leaf.Status.Artifact.DeepCopy()
67+
g.Expect(r.requestsForIncludeChange(ctx, replacement)).To(Equal([]reconcile.Request{request}))
68+
replacement.Status.Artifact = nil
69+
g.Expect(r.requestsForIncludeChange(ctx, replacement)).To(BeEmpty())
70+
}
71+
72+
func TestGitRepositoryIncludeCycles(t *testing.T) {
73+
for _, size := range []int{1, 2, 3} {
74+
t.Run(string(rune('0'+size)), func(t *testing.T) {
75+
g := NewWithT(t)
76+
scheme := runtime.NewScheme()
77+
g.Expect(sourcev1.AddToScheme(scheme)).To(Succeed())
78+
var objs []client.Object
79+
for i := 0; i < size; i++ {
80+
objs = append(objs, &sourcev1.GitRepository{
81+
ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: string(rune('a' + i))},
82+
Spec: sourcev1.GitRepositorySpec{Include: []sourcev1.GitRepositoryInclude{{GitRepositoryRef: meta.LocalObjectReference{Name: string(rune('a' + (i+1)%size))}}}},
83+
Status: sourcev1.GitRepositoryStatus{Artifact: &meta.Artifact{Revision: "same", Digest: "sha256:new"}},
84+
})
85+
}
86+
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).
87+
WithIndex(&sourcev1.GitRepository{}, indexKeyGitRepositoryInclude, indexGitRepositoryIncludes).Build()
88+
r := &GitRepositoryReconciler{Client: c}
89+
for _, obj := range objs {
90+
g.Expect(r.requestsForIncludeChange(context.Background(), obj)).To(BeEmpty())
91+
}
92+
})
93+
}
94+
}

0 commit comments

Comments
 (0)