Skip to content

Commit d36eacb

Browse files
authored
Merge pull request #2818 from redpanda-data/add-migration-to-cloud
Add migration components to all package and allow list
2 parents c390014 + 53341da commit d36eacb

File tree

14 files changed

+184
-66
lines changed

14 files changed

+184
-66
lines changed

CHANGELOG.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ Changelog
33

44
All notable changes to this project will be documented in this file.
55

6-
## 4.33.1 - TBD
6+
## 4.34.0 - 2024-08-29
77

88
### Fixed
99

1010
- The `schema_registry` output now allows pushing schemas if the target Schema Registry instance is in `IMPORT` mode. (@mihaitodor)
1111
- Fixed an issue where the `azure_blob_storage` input would fail to delete blobs when using `targets_input` with `delete_objects: true`. (@mihaitodor)
12+
- New experimental `gcp_vertex_ai_chat` processor. (@rockwotj)
13+
- New experimental `aws_bedrock_chat` processor. (@rockwotj)
1214

1315
## 4.33.0 - 2024-08-13
1416

docs/modules/components/pages/inputs/schema_registry.adoc

+21
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,27 @@ xref:configuration:interpolation.adoc#bloblang-queries[function interpolation].
8383
8484
8585
86+
== Examples
87+
88+
[tabs]
89+
======
90+
Read schemas::
91+
+
92+
--
93+
94+
Read all schemas (including deleted) from a Schema Registry instance which are associated with subjects matching the `^foo.*` filter.
95+
96+
```yaml
97+
input:
98+
schema_registry:
99+
url: http://localhost:8081
100+
include_deleted: true
101+
subject_filter: ^foo.*
102+
```
103+
104+
--
105+
======
106+
86107
== Fields
87108
88109
=== `url`

docs/modules/components/pages/outputs/schema_registry.adoc

+31-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,37 @@ output:
7474
7575
This output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`.
7676
77-
This output benefits from sending messages as a batch for improved performance. Batches can be formed at both the input and output level. You can find out more xref:configuration:batching.adoc[in this doc].
77+
== Examples
78+
79+
[tabs]
80+
======
81+
Write schemas::
82+
+
83+
--
84+
85+
Write schemas to a Schema Registry instance and log errors for schemas which already exist.
86+
87+
```yaml
88+
output:
89+
fallback:
90+
- schema_registry:
91+
url: http://localhost:8082
92+
subject: ${! @schema_registry_subject }
93+
- switch:
94+
cases:
95+
- check: '@fallback_error == "request returned status: 422"'
96+
output:
97+
drop: {}
98+
processors:
99+
- log:
100+
message: |
101+
Subject '${! @schema_registry_subject }' version ${! @schema_registry_version } already has schema: ${! content() }
102+
- output:
103+
reject: ${! @fallback_error }
104+
```
105+
106+
--
107+
======
78108
79109
== Fields
80110

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ require (
9696
github.com/rabbitmq/amqp091-go v1.10.0
9797
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
9898
github.com/redis/go-redis/v9 v9.6.1
99-
github.com/redpanda-data/benthos/v4 v4.35.0
99+
github.com/redpanda-data/benthos/v4 v4.36.0
100100
github.com/redpanda-data/connect/public/bundle/free/v4 v4.31.0
101101
github.com/rs/xid v1.5.0
102102
github.com/sijms/go-ora/v2 v2.8.19

go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -1023,8 +1023,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
10231023
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
10241024
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
10251025
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
1026-
github.com/redpanda-data/benthos/v4 v4.35.0 h1:NlicpomoZzUDuBmpdYF2EftN5VXHya4+Vcvarltucoc=
1027-
github.com/redpanda-data/benthos/v4 v4.35.0/go.mod h1:A5izknIGyzs16rCU0qliFVgdCLn2yyvLM4Hltx+s+TI=
1026+
github.com/redpanda-data/benthos/v4 v4.36.0 h1:FQDw/pupNCfpr+sF7q7jYBlXhO3hb10lOB5UtPxUe9M=
1027+
github.com/redpanda-data/benthos/v4 v4.36.0/go.mod h1:A5izknIGyzs16rCU0qliFVgdCLn2yyvLM4Hltx+s+TI=
10281028
github.com/redpanda-data/connect/public/bundle/free/v4 v4.31.0 h1:Qiz4Q8ZO17n8797hgDdJ2f1XN7wh6J2hIRgeeSw4F24=
10291029
github.com/redpanda-data/connect/public/bundle/free/v4 v4.31.0/go.mod h1:ISgO+/kuuSW0Z7sJo1rWe/rYKIv1rDPHTQ/bSLQEog0=
10301030
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=

public/bundle/enterprise/package.go

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
// Copyright 2024 Redpanda Data, Inc.
22
//
3-
// Licensed under the Apache License, Version 2.0 (the "License");
4-
// you may not use this file except in compliance with the License.
5-
// You may obtain a copy of the License at
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
8-
//
9-
// Unless required by applicable law or agreed to in writing, software
10-
// distributed under the License is distributed on an "AS IS" BASIS,
11-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// See the License for the specific language governing permissions and
13-
// limitations under the License.
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
148

159
// Package enterprise imports all enterprise licensed plugin implementations
1610
// that ship with Redpanda Connect, along with all free plugin implementations.

public/components/all/package.go

+5-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
// Copyright 2024 Redpanda Data, Inc.
22
//
3-
// Licensed under the Apache License, Version 2.0 (the "License");
4-
// you may not use this file except in compliance with the License.
5-
// You may obtain a copy of the License at
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
8-
//
9-
// Unless required by applicable law or agreed to in writing, software
10-
// distributed under the License is distributed on an "AS IS" BASIS,
11-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// See the License for the specific language governing permissions and
13-
// limitations under the License.
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
148

159
// Package all imports all enterprise and FOSS component implementations that
1610
// ship with Redpanda Connect. This is a convenient way of importing every
@@ -25,6 +19,7 @@ import (
2519
// Import all enterprise components.
2620
_ "github.com/redpanda-data/connect/v4/public/components/aws/enterprise"
2721
_ "github.com/redpanda-data/connect/v4/public/components/gcp/enterprise"
22+
_ "github.com/redpanda-data/connect/v4/public/components/kafka/enterprise"
2823
_ "github.com/redpanda-data/connect/v4/public/components/ollama"
2924
_ "github.com/redpanda-data/connect/v4/public/components/openai"
3025
_ "github.com/redpanda-data/connect/v4/public/components/snowflake"

public/components/cloud/package.go

+8-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
// Copyright 2024 Redpanda Data, Inc.
22
//
3-
// Licensed under the Apache License, Version 2.0 (the "License");
4-
// you may not use this file except in compliance with the License.
5-
// You may obtain a copy of the License at
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
8-
//
9-
// Unless required by applicable law or agreed to in writing, software
10-
// distributed under the License is distributed on an "AS IS" BASIS,
11-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// See the License for the specific language governing permissions and
13-
// limitations under the License.
7+
// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md
148

159
// Package cloud imports all enterprise and FOSS component implementations that
1610
// ship with Redpanda Connect in the cloud.
@@ -24,9 +18,11 @@ import (
2418
_ "github.com/redpanda-data/connect/v4/public/components/changelog"
2519
_ "github.com/redpanda-data/connect/v4/public/components/confluent"
2620
_ "github.com/redpanda-data/connect/v4/public/components/crypto"
21+
_ "github.com/redpanda-data/connect/v4/public/components/dgraph"
2722
_ "github.com/redpanda-data/connect/v4/public/components/gcp/enterprise"
2823
_ "github.com/redpanda-data/connect/v4/public/components/io"
2924
_ "github.com/redpanda-data/connect/v4/public/components/kafka"
25+
_ "github.com/redpanda-data/connect/v4/public/components/kafka/enterprise"
3026
_ "github.com/redpanda-data/connect/v4/public/components/maxmind"
3127
_ "github.com/redpanda-data/connect/v4/public/components/memcached"
3228
_ "github.com/redpanda-data/connect/v4/public/components/msgpack"
@@ -41,6 +37,8 @@ import (
4137
_ "github.com/redpanda-data/connect/v4/public/components/qdrant"
4238
_ "github.com/redpanda-data/connect/v4/public/components/redis"
4339
_ "github.com/redpanda-data/connect/v4/public/components/sftp"
40+
_ "github.com/redpanda-data/connect/v4/public/components/snowflake"
41+
_ "github.com/redpanda-data/connect/v4/public/components/splunk"
4442
_ "github.com/redpanda-data/connect/v4/public/components/sql/base"
4543

4644
// Import all (supported) sql drivers.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright 2024 Redpanda Data, Inc.
2+
//
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
6+
//
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
8+
9+
package enterprise
10+
11+
import (
12+
// Bring in the internal plugin definitions.
13+
_ "github.com/redpanda-data/connect/v4/internal/impl/kafka/enterprise"
14+
)

public/components/ollama/package.go

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
// Copyright 2024 Redpanda Data, Inc.
22
//
3-
// Licensed under the Apache License, Version 2.0 (the "License");
4-
// you may not use this file except in compliance with the License.
5-
// You may obtain a copy of the License at
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
8-
//
9-
// Unless required by applicable law or agreed to in writing, software
10-
// distributed under the License is distributed on an "AS IS" BASIS,
11-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// See the License for the specific language governing permissions and
13-
// limitations under the License.
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
148

159
package ollama
1610

public/components/snowflake/package.go

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
// Copyright 2024 Redpanda Data, Inc.
22
//
3-
// Licensed under the Apache License, Version 2.0 (the "License");
4-
// you may not use this file except in compliance with the License.
5-
// You may obtain a copy of the License at
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
8-
//
9-
// Unless required by applicable law or agreed to in writing, software
10-
// distributed under the License is distributed on an "AS IS" BASIS,
11-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// See the License for the specific language governing permissions and
13-
// limitations under the License.
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
148

159
package snowflake
1610

public/components/splunk/package.go

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
// Copyright 2024 Redpanda Data, Inc.
22
//
3-
// Licensed under the Apache License, Version 2.0 (the "License");
4-
// you may not use this file except in compliance with the License.
5-
// You may obtain a copy of the License at
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
8-
//
9-
// Unless required by applicable law or agreed to in writing, software
10-
// distributed under the License is distributed on an "AS IS" BASIS,
11-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// See the License for the specific language governing permissions and
13-
// limitations under the License.
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
148

159
package splunk
1610

public/schema/cloud_allow_list.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ nats_kv
3838
nats_request_reply
3939
ollama_chat
4040
ollama_embeddings
41-
openai_chat
4241
openai_chat_completion
4342
openai_embeddings
4443
openai_image_generation
@@ -62,6 +61,7 @@ redis_streams
6261
ristretto
6362
schema_registry_decode
6463
schema_registry_encode
64+
schema_registry
6565
select_parts
6666
sftp
6767
sleep

public/schema/schema_test.go

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright 2024 Redpanda Data, Inc.
2+
//
3+
// Licensed as a Redpanda Enterprise file under the Redpanda Community
4+
// License (the "License"); you may not use this file except in compliance with
5+
// the License. You may obtain a copy of the License at
6+
//
7+
// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md
8+
9+
package schema_test
10+
11+
import (
12+
// Only import a subset of components for execution.
13+
"strings"
14+
"testing"
15+
16+
"github.com/redpanda-data/benthos/v4/public/service"
17+
"github.com/stretchr/testify/assert"
18+
19+
_ "embed"
20+
21+
_ "github.com/redpanda-data/connect/v4/public/components/cloud"
22+
)
23+
24+
//go:embed cloud_allow_list.txt
25+
var allowList string
26+
27+
func TestImportsMatch(t *testing.T) {
28+
t.Skip("Skipping until we have a structured allow list")
29+
30+
var allowSlice []string
31+
for _, s := range strings.Split(allowList, "\n") {
32+
s = strings.TrimSpace(s)
33+
if s == "" || strings.HasPrefix(s, "#") {
34+
continue
35+
}
36+
allowSlice = append(allowSlice, s)
37+
}
38+
39+
env := service.GlobalEnvironment()
40+
41+
seen := map[string]struct{}{}
42+
43+
env.WalkBuffers(func(name string, config *service.ConfigView) {
44+
seen[name] = struct{}{}
45+
})
46+
47+
env.WalkCaches(func(name string, config *service.ConfigView) {
48+
seen[name] = struct{}{}
49+
})
50+
51+
env.WalkInputs(func(name string, config *service.ConfigView) {
52+
seen[name] = struct{}{}
53+
})
54+
55+
env.WalkMetrics(func(name string, config *service.ConfigView) {
56+
seen[name] = struct{}{}
57+
})
58+
59+
env.WalkOutputs(func(name string, config *service.ConfigView) {
60+
seen[name] = struct{}{}
61+
})
62+
63+
env.WalkProcessors(func(name string, config *service.ConfigView) {
64+
seen[name] = struct{}{}
65+
})
66+
67+
env.WalkRateLimits(func(name string, config *service.ConfigView) {
68+
seen[name] = struct{}{}
69+
})
70+
71+
env.WalkScanners(func(name string, config *service.ConfigView) {
72+
seen[name] = struct{}{}
73+
})
74+
75+
env.WalkTracers(func(name string, config *service.ConfigView) {
76+
seen[name] = struct{}{}
77+
})
78+
79+
for _, k := range allowSlice {
80+
assert.Contains(t, seen, k)
81+
}
82+
}

0 commit comments

Comments
 (0)