Skip to content

Commit 98912aa

Browse files
jon-weijon-wei
committed
Allow broker to use catalog for datasource schemas
1 parent 9f3b266 commit 98912aa

File tree

57 files changed

+2705
-350
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+2705
-350
lines changed

‎extensions-core/druid-catalog/pom.xml

+30-82
Original file line numberDiff line numberDiff line change
@@ -48,44 +48,22 @@
4848
<version>${project.parent.version}</version>
4949
<scope>provided</scope>
5050
</dependency>
51-
<dependency>
52-
<groupId>org.apache.druid</groupId>
53-
<artifactId>druid-indexing-service</artifactId>
54-
<version>${project.parent.version}</version>
55-
<scope>provided</scope>
56-
</dependency>
5751
<dependency>
5852
<groupId>org.apache.druid</groupId>
5953
<artifactId>druid-sql</artifactId>
6054
<version>${project.parent.version}</version>
6155
<scope>provided</scope>
6256
</dependency>
63-
<dependency>
64-
<groupId>org.apache.druid</groupId>
65-
<artifactId>druid-services</artifactId>
66-
<version>${project.parent.version}</version>
67-
<scope>provided</scope>
68-
</dependency>
6957
<dependency>
7058
<groupId>com.google.inject</groupId>
7159
<artifactId>guice</artifactId>
7260
<scope>provided</scope>
7361
</dependency>
74-
<dependency>
75-
<groupId>com.google.inject.extensions</groupId>
76-
<artifactId>guice-multibindings</artifactId>
77-
<scope>provided</scope>
78-
</dependency>
7962
<dependency>
8063
<groupId>com.google.guava</groupId>
8164
<artifactId>guava</artifactId>
8265
<scope>provided</scope>
8366
</dependency>
84-
<dependency>
85-
<groupId>com.opencsv</groupId>
86-
<artifactId>opencsv</artifactId>
87-
<scope>provided</scope>
88-
</dependency>
8967
<dependency>
9068
<groupId>com.fasterxml.jackson.core</groupId>
9169
<artifactId>jackson-databind</artifactId>
@@ -142,11 +120,6 @@
142120
<artifactId>curator-client</artifactId>
143121
<scope>provided</scope>
144122
</dependency>
145-
<dependency>
146-
<groupId>com.fasterxml.jackson.dataformat</groupId>
147-
<artifactId>jackson-dataformat-smile</artifactId>
148-
<scope>provided</scope>
149-
</dependency>
150123
<dependency>
151124
<groupId>org.jdbi</groupId>
152125
<artifactId>jdbi</artifactId>
@@ -162,11 +135,6 @@
162135
<artifactId>jsr311-api</artifactId>
163136
<scope>provided</scope>
164137
</dependency>
165-
<dependency>
166-
<groupId>org.apache.commons</groupId>
167-
<artifactId>commons-lang3</artifactId>
168-
<scope>provided</scope>
169-
</dependency>
170138
<dependency>
171139
<groupId>javax.servlet</groupId>
172140
<artifactId>javax.servlet-api</artifactId>
@@ -177,68 +145,18 @@
177145
<artifactId>jersey-server</artifactId>
178146
<scope>provided</scope>
179147
</dependency>
180-
<dependency>
181-
<groupId>com.google.errorprone</groupId>
182-
<artifactId>error_prone_annotations</artifactId>
183-
<scope>provided</scope>
184-
</dependency>
185-
<dependency>
186-
<groupId>org.lz4</groupId>
187-
<artifactId>lz4-java</artifactId>
188-
<scope>provided</scope>
189-
</dependency>
190-
<dependency>
191-
<groupId>org.apache.datasketches</groupId>
192-
<artifactId>datasketches-java</artifactId>
193-
<scope>provided</scope>
194-
</dependency>
195-
<dependency>
196-
<groupId>org.apache.datasketches</groupId>
197-
<artifactId>datasketches-memory</artifactId>
198-
<scope>provided</scope>
199-
</dependency>
200-
<dependency>
201-
<groupId>it.unimi.dsi</groupId>
202-
<artifactId>fastutil-core</artifactId>
203-
<scope>provided</scope>
204-
</dependency>
205-
<dependency>
206-
<groupId>commons-io</groupId>
207-
<artifactId>commons-io</artifactId>
208-
<scope>provided</scope>
209-
</dependency>
210148

211149
<!-- Tests -->
212150
<dependency>
213151
<groupId>org.easymock</groupId>
214152
<artifactId>easymock</artifactId>
215153
<scope>test</scope>
216154
</dependency>
217-
<dependency>
218-
<groupId>org.hamcrest</groupId>
219-
<artifactId>hamcrest-all</artifactId>
220-
<scope>test</scope>
221-
</dependency>
222-
<dependency>
223-
<groupId>org.hamcrest</groupId>
224-
<artifactId>hamcrest-core</artifactId>
225-
<scope>test</scope>
226-
</dependency>
227155
<dependency>
228156
<groupId>junit</groupId>
229157
<artifactId>junit</artifactId>
230158
<scope>test</scope>
231159
</dependency>
232-
<dependency>
233-
<groupId>org.mockito</groupId>
234-
<artifactId>mockito-core</artifactId>
235-
<scope>test</scope>
236-
</dependency>
237-
<dependency>
238-
<groupId>nl.jqno.equalsverifier</groupId>
239-
<artifactId>equalsverifier</artifactId>
240-
<scope>test</scope>
241-
</dependency>
242160
<dependency>
243161
<groupId>org.apache.druid</groupId>
244162
<artifactId>druid-processing</artifactId>
@@ -262,4 +180,34 @@
262180
</dependency>
263181

264182
</dependencies>
183+
184+
<build>
185+
<plugins>
186+
<plugin>
187+
<groupId>org.apache.maven.plugins</groupId>
188+
<artifactId>maven-jar-plugin</artifactId>
189+
<executions>
190+
<execution>
191+
<goals>
192+
<goal>test-jar</goal>
193+
</goals>
194+
</execution>
195+
</executions>
196+
</plugin>
197+
<plugin>
198+
<groupId>org.apache.maven.plugins</groupId>
199+
<artifactId>maven-dependency-plugin</artifactId>
200+
<configuration>
201+
<!-- tool gets confused between these two items. -->
202+
<usedDependencies>
203+
<dependency>javax.inject:javax.inject</dependency>
204+
</usedDependencies>
205+
<ignoredUsedUndeclaredDependencies>
206+
<ignoredUsedUndeclaredDependency>javax.inject:javax.inject</ignoredUsedUndeclaredDependency>
207+
<ignoredUsedUndeclaredDependency>jakarta.inject:jakarta.inject-api</ignoredUsedUndeclaredDependency>
208+
</ignoredUsedUndeclaredDependencies>
209+
</configuration>
210+
</plugin>
211+
</plugins>
212+
</build>
265213
</project>

‎extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogBrokerModule.java

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.catalog.guice;
21+
22+
import com.google.inject.Binder;
23+
import org.apache.druid.catalog.http.CatalogListenerResource;
24+
import org.apache.druid.catalog.model.SchemaRegistry;
25+
import org.apache.druid.catalog.model.SchemaRegistryImpl;
26+
import org.apache.druid.catalog.sql.LiveCatalogResolver;
27+
import org.apache.druid.catalog.sync.CachedMetadataCatalog;
28+
import org.apache.druid.catalog.sync.CatalogClient;
29+
import org.apache.druid.catalog.sync.CatalogUpdateListener;
30+
import org.apache.druid.catalog.sync.CatalogUpdateReceiver;
31+
import org.apache.druid.catalog.sync.MetadataCatalog;
32+
import org.apache.druid.catalog.sync.MetadataCatalog.CatalogSource;
33+
import org.apache.druid.discovery.NodeRole;
34+
import org.apache.druid.guice.Jerseys;
35+
import org.apache.druid.guice.LazySingleton;
36+
import org.apache.druid.guice.LifecycleModule;
37+
import org.apache.druid.guice.ManageLifecycle;
38+
import org.apache.druid.guice.annotations.LoadScope;
39+
import org.apache.druid.initialization.DruidModule;
40+
import org.apache.druid.sql.calcite.planner.CatalogResolver;
41+
42+
/**
43+
* Configures the metadata catalog on the Broker to use a cache
44+
* and network communications for pull and push updates.
45+
*/
46+
@LoadScope(roles = NodeRole.BROKER_JSON_NAME)
47+
public class CatalogBrokerModule implements DruidModule
48+
{
49+
@Override
50+
public void configure(Binder binder)
51+
{
52+
// The Broker (catalog client) uses a cached metadata catalog.
53+
binder
54+
.bind(CachedMetadataCatalog.class)
55+
.in(LazySingleton.class);
56+
57+
// Broker code accesses he catalog through the
58+
// MetadataCatalog interface.
59+
binder
60+
.bind(MetadataCatalog.class)
61+
.to(CachedMetadataCatalog.class)
62+
.in(LazySingleton.class);
63+
64+
// The cached metadata catalog needs a "pull" source,
65+
// which is the network client.
66+
binder
67+
.bind(CatalogSource.class)
68+
.to(CatalogClient.class)
69+
.in(LazySingleton.class);
70+
71+
// The cached metadata catalog is the listener for"push" events.
72+
binder
73+
.bind(CatalogUpdateListener.class)
74+
.to(CachedMetadataCatalog.class)
75+
.in(LazySingleton.class);
76+
77+
// At present, the set of schemas is fixed.
78+
binder
79+
.bind(SchemaRegistry.class)
80+
.to(SchemaRegistryImpl.class)
81+
.in(LazySingleton.class);
82+
83+
// Lifecycle-managed class to prime the metadata cache
84+
binder
85+
.bind(CatalogUpdateReceiver.class)
86+
.in(ManageLifecycle.class);
87+
LifecycleModule.register(binder, CatalogUpdateReceiver.class);
88+
89+
// Catalog resolver for the planner. This will override the
90+
// base binding.
91+
binder
92+
.bind(CatalogResolver.class)
93+
.to(LiveCatalogResolver.class)
94+
.in(LazySingleton.class);
95+
96+
// The listener resource sends to the catalog
97+
// listener (the cached catalog.)
98+
Jerseys.addResource(binder, CatalogListenerResource.class);
99+
}
100+
}

‎extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogListenerResource.java

+18
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,22 @@ public Response syncTable(final UpdateEvent event)
6767
listener.updated(event);
6868
return Response.status(Response.Status.ACCEPTED).build();
6969
}
70+
71+
@POST
72+
@Path("flush")
73+
@ResourceFilters(ConfigResourceFilter.class)
74+
public Response flush()
75+
{
76+
listener.flush();
77+
return Response.status(Response.Status.ACCEPTED).build();
78+
}
79+
80+
@POST
81+
@Path("resync")
82+
@ResourceFilters(ConfigResourceFilter.class)
83+
public Response resync()
84+
{
85+
listener.resync();
86+
return Response.status(Response.Status.ACCEPTED).build();
87+
}
7088
}

‎extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,7 @@ public Response editTable(
278278
// Retrieval
279279

280280
/**
281-
* Retrieves the list of all Druid schema names, all table names, or
282-
* all table metadata.
281+
* Retrieves the list of all Druid schema names.
283282
*
284283
* @param format the format of the response. See the code for the
285284
* available formats

0 commit comments

Comments
 (0)