Skip to content

Commit 85c8301

Browse files
Release 2.0.7
1 parent 7b02f26 commit 85c8301

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+9699
-123
lines changed

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Event Hub SDK is a library that helps connect an application to [Event Hub](http
1010
- [Closing Connection](#closing-connection)
1111
- [Automatic Reconnection](#automatic-reconnection)
1212
- [OAuth2 token management](#oauth2-token-management)
13+
- [Debugging](#debugging)
1314
- [Building SDK from repository](#building-sdk-from-repository)
1415
- [Common Issues](#building-sdk-from-repository)
1516

@@ -22,7 +23,7 @@ The most simple way to use this SDK is to pull from the Predix Snapshot artifact
2223
<dependency>
2324
<groupId>com.ge.predix.eventhub</groupId>
2425
<artifactId>predix-event-hub-sdk</artifactId>
25-
<version>2.0.1</version>
26+
<version>2.0.7</version>
2627
</dependency>
2728
...
2829
</dependencies>
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.idea/
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
applications:
3+
- name: event-hub-java-sample-spark-app
4+
memory: 512M
5+
instances: 1
6+
timeout: 180
7+
path: target/event-hub-java-spark-sample-app-1.0-SNAPSHOT.jar
8+
buildpack: java-buildpack
9+
env:
10+
UAA_INSTANCE_NAME: <uaa>
11+
EVENTHUB_INSTANCE_NAME: <event_hub>
12+
CLIENT_ID: <client_id>
13+
CLIENT_SECRET: <client_password>
14+
EVENTHUB_ENABLE_DEBUG: true
15+
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# event-hub-java-spark-sample-app
2+
3+
## Background:
4+
Built on top of EventHub Java Sample App (https://github.com/PredixDev//predix-event-hub-java-sdk/examples) but this app adds a simple Spark job.
5+
6+
## Summary:
7+
Writes all received messages in the Subscriber Callback to a file.
8+
9+
A GET request to "/spark" will trigger a simple Spark job that counts how many times unique words appear in the written file of received messages.
10+
11+
The results of the Spark job are then written to "output/part-00000".
12+
13+
14+
[![Analytics](https://ga-beacon.appspot.com/UA-82773213-1/predix-event-hub-sdk/readme?pixel)](https://github.com/PredixDev)
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>event-hub-java-spark-sample-app</groupId>
6+
<artifactId>event-hub-java-spark-sample-app</artifactId>
7+
<version>1.0.0</version>
8+
9+
<parent>
10+
<groupId>org.springframework.boot</groupId>
11+
<artifactId>spring-boot-starter-parent</artifactId>
12+
<version>1.4.0.RELEASE</version>
13+
<relativePath /> <!-- lookup parent from repository -->
14+
</parent>
15+
16+
<build>
17+
<plugins>
18+
<plugin>
19+
<groupId>org.apache.maven.plugins</groupId>
20+
<artifactId>maven-compiler-plugin</artifactId>
21+
<version>2.0.2</version>
22+
<configuration>
23+
<source>1.8</source>
24+
<target>1.8</target>
25+
</configuration>
26+
</plugin>
27+
<plugin>
28+
<groupId>org.apache.maven.plugins</groupId>
29+
<artifactId>maven-jar-plugin</artifactId>
30+
<configuration>
31+
<archive>
32+
<manifest>
33+
<addClasspath>true</addClasspath>
34+
<classpathPrefix>lib/</classpathPrefix>
35+
<mainClass>com.ge.predix.eventhub.spark.sample.Application</mainClass>
36+
</manifest>
37+
</archive>
38+
</configuration>
39+
</plugin>
40+
<plugin>
41+
<groupId>org.apache.maven.plugins</groupId>
42+
<artifactId>maven-dependency-plugin</artifactId>
43+
<executions>
44+
<execution>
45+
<id>copy</id>
46+
<phase>install</phase>
47+
<goals>
48+
<goal>copy-dependencies</goal>
49+
</goals>
50+
<configuration>
51+
<outputDirectory>${project.build.directory}/lib</outputDirectory>
52+
</configuration>
53+
</execution>
54+
</executions>
55+
</plugin>
56+
<plugin>
57+
<groupId>org.springframework.boot</groupId>
58+
<artifactId>spring-boot-maven-plugin</artifactId>
59+
</plugin>
60+
</plugins>
61+
</build>
62+
63+
<properties>
64+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
65+
<jackson.version>2.6.5</jackson.version>
66+
</properties>
67+
68+
<dependencies>
69+
<dependency>
70+
<groupId>com.neovisionaries</groupId>
71+
<artifactId>nv-websocket-client</artifactId>
72+
<version>1.3</version>
73+
</dependency>
74+
<dependency>
75+
<groupId>com.ge.predix.eventhub</groupId>
76+
<artifactId>predix-event-hub-sdk</artifactId>
77+
<version>2.0.7</version>
78+
</dependency>
79+
<dependency>
80+
<groupId>org.springframework.boot</groupId>
81+
<artifactId>spring-boot</artifactId>
82+
</dependency>
83+
<dependency>
84+
<groupId>org.springframework.boot</groupId>
85+
<artifactId>spring-boot-starter</artifactId>
86+
</dependency>
87+
<dependency>
88+
<groupId>org.springframework.boot</groupId>
89+
<artifactId>spring-boot-starter-web</artifactId>
90+
</dependency>
91+
<dependency>
92+
<groupId>org.springframework.cloud</groupId>
93+
<artifactId>spring-cloud-core</artifactId>
94+
</dependency>
95+
<dependency>
96+
<groupId>org.springframework.boot</groupId>
97+
<artifactId>spring-boot-starter-websocket</artifactId>
98+
</dependency>
99+
100+
<dependency>
101+
<groupId>org.apache.spark</groupId>
102+
<artifactId>spark-core_2.11</artifactId>
103+
<version>2.1.0</version>
104+
</dependency>
105+
<dependency>
106+
<groupId>com.fasterxml.jackson.core</groupId>
107+
<artifactId>jackson-databind</artifactId>
108+
<version>${jackson.version}</version>
109+
</dependency>
110+
<dependency>
111+
<groupId>junit</groupId>
112+
<artifactId>junit</artifactId>
113+
<version>3.8.1</version>
114+
<scope>test</scope>
115+
</dependency>
116+
</dependencies>
117+
</project>
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package com.ge.predix.eventhub.spark.sample;
2+
3+
import org.springframework.boot.SpringApplication;
4+
import org.springframework.boot.SpringBootConfiguration;
5+
import org.springframework.context.annotation.ComponentScan;
6+
import org.springframework.context.annotation.Configuration;
7+
8+
@ComponentScan
9+
@Configuration
10+
@SpringBootConfiguration
11+
public class Application {
12+
public static void main(String[] args) {
13+
SpringApplication.run(Application.class, args);
14+
}
15+
}
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
package com.ge.predix.eventhub.spark.sample;
2+
3+
import com.ge.predix.eventhub.*;
4+
import com.ge.predix.eventhub.client.Client;
5+
import com.ge.predix.eventhub.configuration.EventHubConfiguration;
6+
import com.ge.predix.eventhub.configuration.PublishConfiguration;
7+
import com.ge.predix.eventhub.configuration.SubscribeConfiguration;
8+
import com.google.protobuf.ByteString;
9+
import org.apache.spark.SparkConf;
10+
import org.apache.spark.api.java.JavaPairRDD;
11+
import org.apache.spark.api.java.JavaRDD;
12+
import org.apache.spark.api.java.JavaSparkContext;
13+
import org.json.JSONArray;
14+
import org.json.JSONObject;
15+
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
16+
import org.springframework.web.bind.annotation.*;
17+
import scala.Tuple2;
18+
19+
import javax.annotation.PostConstruct;
20+
import javax.annotation.PreDestroy;
21+
import java.io.BufferedWriter;
22+
import java.io.FileWriter;
23+
import java.io.IOException;
24+
import java.util.ArrayList;
25+
import java.util.Arrays;
26+
import java.util.Collections;
27+
import java.util.List;
28+
import java.util.concurrent.atomic.AtomicInteger;
29+
import java.util.logging.Logger;
30+
31+
@RestController
32+
@EnableAutoConfiguration
33+
public class Controller {
34+
private static final Logger logger = Logger.getLogger(Controller.class.getName());
35+
private static Client eventHub;
36+
private static EventHubConfiguration eventHubConfiguration;
37+
private static List<Message> rxMessages = Collections.synchronizedList(new ArrayList<Message>());
38+
private static List<String> rxErrors = Collections.synchronizedList(new ArrayList<String>());
39+
private static AtomicInteger rxErrorCount = new AtomicInteger();
40+
41+
private static final String subscriberName = "java-sdk-spark-sample-app-subscriber";
42+
private static final String subscriberInstance = "spark-sample-app-instance";
43+
private static BufferedWriter writer = null;
44+
private static SparkConf conf = null;
45+
private static JavaSparkContext sc = null;
46+
47+
48+
@PostConstruct
49+
public void makeClient() throws Exception {
50+
writer = new BufferedWriter(new FileWriter("messages.txt", true));
51+
conf = new SparkConf().setMaster("local").setAppName("Work Count App");
52+
sc = new JavaSparkContext(conf);
53+
System.out.println("******************************* Using environment variables *******************************");
54+
fromExplicit();
55+
56+
class SubCallback implements Client.SubscribeCallback {
57+
@Override
58+
public void onMessage(Message message) {
59+
rxMessages.add(message);
60+
try {
61+
writer.write(message.toString());
62+
writer.flush();
63+
} catch (IOException e) {
64+
e.printStackTrace();
65+
}
66+
}
67+
@Override
68+
public void onFailure(Throwable throwable) {
69+
rxErrors.add(throwable.getMessage());
70+
rxErrorCount.incrementAndGet();
71+
}
72+
}
73+
eventHub.subscribe(new SubCallback());
74+
}
75+
76+
@PreDestroy
77+
void closeWriter() throws IOException {
78+
writer.close();
79+
}
80+
81+
@RequestMapping(value = "/spark", method = RequestMethod.GET)
82+
void spark() {
83+
JavaRDD<String> lines = sc.textFile("messages.txt");
84+
JavaPairRDD<String, Integer> pairs = lines.mapToPair(s -> new Tuple2<>(s,1));
85+
JavaPairRDD<String, Integer> counts = pairs.reduceByKey((a, b) -> a + b);
86+
counts.saveAsTextFile("output");
87+
sc.close();
88+
}
89+
90+
91+
private void fromExplicit() throws EventHubClientException {
92+
try {
93+
EventHubConfiguration configuration = new EventHubConfiguration.Builder()
94+
.host(System.getenv("EVENTHUB_URI"))
95+
.port(Integer.parseInt(System.getenv("EVENTHUB_PORT")))
96+
.authURL(System.getenv("AUTH_URL"))
97+
.clientID(System.getenv("CLIENT_ID"))
98+
.clientSecret(System.getenv("CLIENT_SECRET"))
99+
.zoneID(System.getenv("ZONE_ID"))
100+
.publishConfiguration(new PublishConfiguration.Builder().publisherType(PublishConfiguration.PublisherType.SYNC).build())
101+
.subscribeConfiguration(new SubscribeConfiguration.Builder().subscriberName(subscriberName).subscriberInstance(subscriberInstance).build())
102+
.build();
103+
104+
eventHub = new Client(configuration);
105+
logger.info("** logging Client credentials **");
106+
logger.info(configuration.getAuthURL());
107+
logger.info(configuration.getClientID());
108+
logger.info(configuration.getHost());
109+
logger.info(configuration.getZoneID());
110+
logger.info(configuration.getPort() + "");
111+
logger.info(configuration.isAutomaticTokenRenew() + "");
112+
logger.info("** logging Client details done**");
113+
eventHub.forceRenewToken();
114+
eventHubConfiguration = configuration;
115+
116+
} catch(EventHubClientException.InvalidConfigurationException e) {
117+
logger.info(e.getMessage());
118+
System.out.println("Could not create client");
119+
}
120+
}
121+
122+
@RequestMapping(value = "/subscribe", method = RequestMethod.GET)
123+
String subscribe() throws EventHubClientException, IOException {
124+
JSONObject responses = new JSONObject();
125+
JSONArray messages = new JSONArray();
126+
JSONArray errors = new JSONArray();
127+
128+
129+
while(rxMessages.size() != 0) {
130+
Message message = rxMessages.remove(0);
131+
messages.put(new JSONObject(String.format("{\"id\":\"%s\", \"body\":%s}", message.getId(), message.getBody().toStringUtf8())));
132+
}
133+
134+
135+
for (String error : rxErrors) {
136+
errors.put(error);
137+
}
138+
139+
responses.put("messages", messages);
140+
responses.put("errors", errors);
141+
142+
return responses.toString();
143+
}
144+
145+
@RequestMapping(value = "/publish", method = RequestMethod.POST)
146+
String publish(@RequestBody String input, @RequestParam(value = "id") String id, @RequestParam(value = "count", required = false) Integer count ) throws EventHubClientException {
147+
List<Ack> acks;
148+
if(count != null && count >= 1){
149+
Messages.Builder msgBuilder = Messages.newBuilder();
150+
for(int i=0;i<count;i++){
151+
Message msg = Message.newBuilder().setId(String.format("%s-%d", id, i))
152+
.setZoneId(eventHubConfiguration.getZoneID())
153+
.setBody(ByteString.copyFromUtf8(String.format("{message:\"%s\"}", input))).build();
154+
155+
msgBuilder.addMsg(msg);
156+
}
157+
acks = eventHub.addMessages(msgBuilder.build()).flush();
158+
}
159+
else{
160+
count = 1;
161+
acks = eventHub.addMessage(id, input, null).flush();
162+
}
163+
164+
JSONArray array = new JSONArray();
165+
for(Ack a : acks){
166+
array.put(ackToJSON(a));
167+
}
168+
return array.toString();
169+
}
170+
171+
172+
@RequestMapping("/")
173+
String home() {
174+
return "Hello Event Hub!";
175+
}
176+
177+
private JSONObject ackToJSON(Ack a) {
178+
JSONObject j = new JSONObject();
179+
if(!a.getId().equals(a.getDefaultInstanceForType().getId()))
180+
j.put("id", a.getId());
181+
j.put("status_code", a.getStatusCode());
182+
if(a.getStatusCode() == AckStatus.ACCEPTED){
183+
j.put("offset", a.getOffset());
184+
j.put("partition", a.getPartition());
185+
j.put("topic", a.getTopic());
186+
}else{
187+
j.put("desc", a.getDesc());
188+
}
189+
return j;
190+
}
191+
}
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
rm -r output
4+
rm messages.txt
5+
mvn clean install
6+
java -jar target/event-hub-java-spark-sample-app-1.0-SNAPSHOT.jar --server.port=58621

0 commit comments

Comments
 (0)