Adding Yahoo Quotes Source App

pom.xml

@@ -59,6 +59,7 @@
 		<module>triggertask</module>
 		<module>twitter</module>
 		<module>websocket</module>
+		<module>yahoo-quotes</module>
 		<module>starter-parent</module>
 		<module>spring-cloud-stream-app-documentation-maven-plugin</module>
 	</modules>

spring-cloud-stream-app-dependencies/pom.xml

@@ -311,6 +311,11 @@
 				<artifactId>spring-cloud-starter-stream-source-mail</artifactId>
 				<version>1.0.3.BUILD-SNAPSHOT</version>
 			</dependency>
+			<dependency>
+				<groupId>org.springframework.cloud.stream.app</groupId>
+				<artifactId>spring-cloud-starter-stream-source-yahoo-quotes</artifactId>
+				<version>1.0.3.BUILD-SNAPSHOT</version>
+			</dependency>
 
 			<dependency>
 				<groupId>org.springframework.cloud</groupId>
@@ -350,6 +355,7 @@
 				<scope>test</scope>
 			</dependency>
 
+
 			<!-- Core dependencies needed for the starter libs -->
 
 			<dependency>

spring-cloud-stream-app-generator/pom.xml

@@ -164,6 +164,7 @@
 							<extraTestConfigClass>org.springframework.cloud.stream.app.test.twitter.TwitterTestConfiguration.class</extraTestConfigClass>
 						</twitterstream-source>
 						<websocket-sink />
+						<yahoo-quotes-source/>
 						<!-- Test Modules -->
 						<integration-test-processor>
 						</integration-test-processor>

yahoo-quotes/pom.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+		 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>spring-cloud-stream-yahoo-quotes-parent</artifactId>
+	<packaging>pom</packaging>
+
+	<parent>
+		<artifactId>spring-cloud-stream-app-starters</artifactId>
+		<groupId>org.springframework.cloud.stream.app</groupId>
+		<version>1.0.3.BUILD-SNAPSHOT</version>
+		<relativePath>../starter-parent</relativePath>
+	</parent>
+
+
+	<modules>
+		<module>spring-cloud-starter-stream-source-yahoo-quotes</module>
+	</modules>
+
+
+</project>

yahoo-quotes/spring-cloud-starter-stream-source-yahoo-quotes/README.adoc

@@ -0,0 +1,136 @@
+//tag::ref-doc[]
+= Yahoo Quotes Stream Source
+
+This source ingests data from https://developer.yahoo.com/yql/[Yahoo! YQL finance quotes] . It pulls symbols from the API using a configurable cron expression. You can select which fields are going to be returned, or fetch all the fields (83 at the moment).
+
+YQL has a limit of 2000 calls per hour, so the source does its best on batching calls. It's tuned to fetch up to 400 symbols on one single call. This can be changed by modifying the `yahoo.quotes.batchSize` property.
+
+With it's default configuration, the source makes 20 (7925/400) calls  per minute to the API.
+
+The source splits the batch calls and send one message per symbol. The content is the same as the one received from Yahoo!
+
+You can play with a `yahoo.quotes.throttle` property to avoid bursts (for example, avoid sending 7900 message at a burst).
+
+[source,javascript]
+----
+{
+    "symbol": "VMW",
+    "Ask": "68.49",
+    "AverageDailyVolume": "2161690",
+    "Bid": "68.16",
+    "AskRealtime": null,
+    "BidRealtime": null,
+    "BookValue": "19.50",
+    "Change_PercentChange": "+0.50 - +0.81%",
+    "Change": "+0.50",
+    "Commission": null,
+    "Currency": "USD",
+    "ChangeRealtime": null,
+    "AfterHoursChangeRealtime": null,
+    "DividendShare": null,
+    "LastTradeDate": "7/18/2016",
+    "TradeDate": null,
+    "EarningsShare": "2.27",
+    "ErrorIndicationreturnedforsymbolchangedinvalid": null,
+    "EPSEstimateCurrentYear": "4.14",
+    "EPSEstimateNextYear": "4.41",
+    "EPSEstimateNextQuarter": "1.05",
+    "DaysLow": "62.14",
+    "DaysHigh": "63.00",
+    "YearLow": "43.25",
+    "YearHigh": "93.43",
+    "HoldingsGainPercent": null,
+    "AnnualizedGain": null,
+    "HoldingsGain": null,
+    "HoldingsGainPercentRealtime": null,
+    "HoldingsGainRealtime": null,
+    "MoreInfo": null,
+    "OrderBookRealtime": null,
+    "MarketCapitalization": "26.55B",
+    "MarketCapRealtime": null,
+    "EBITDA": "1.76B",
+    "ChangeFromYearLow": "19.32",
+    "PercentChangeFromYearLow": "+44.67%",
+    "LastTradeRealtimeWithTime": null,
+    "ChangePercentRealtime": null,
+    "ChangeFromYearHigh": "-30.86",
+    "PercebtChangeFromYearHigh": "-33.03%",
+    "LastTradeWithTime": "4:02pm - <b>62.57</b>",
+    "LastTradePriceOnly": "62.57",
+    "HighLimit": null,
+    "LowLimit": null,
+    "DaysRange": "62.14 - 63.00",
+    "DaysRangeRealtime": null,
+    "FiftydayMovingAverage": "60.39",
+    "TwoHundreddayMovingAverage": "54.17",
+    "ChangeFromTwoHundreddayMovingAverage": "8.40",
+    "PercentChangeFromTwoHundreddayMovingAverage": "+15.52%",
+    "ChangeFromFiftydayMovingAverage": "2.18",
+    "PercentChangeFromFiftydayMovingAverage": "+3.60%",
+    "Name": "Vmware, Inc. Common stock, Clas",
+    "Notes": null,
+    "Open": "62.46",
+    "PreviousClose": "62.07",
+    "PricePaid": null,
+    "ChangeinPercent": "+0.81%",
+    "PriceSales": "3.92",
+    "PriceBook": "3.18",
+    "ExDividendDate": null,
+    "PERatio": "27.54",
+    "DividendPayDate": null,
+    "PERatioRealtime": null,
+    "PEGRatio": "1.11",
+    "PriceEPSEstimateCurrentYear": "15.11",
+    "PriceEPSEstimateNextYear": "14.19",
+    "Symbol": "VMW",
+    "SharesOwned": null,
+    "ShortRatio": "8.75",
+    "LastTradeTime": "4:02pm",
+    "TickerTrend": null,
+    "OneyrTargetPrice": "63.12",
+    "Volume": "5474990",
+    "HoldingsValue": null,
+    "HoldingsValueRealtime": null,
+    "YearRange": "43.25 - 93.43",
+    "DaysValueChange": null,
+    "DaysValueChangeRealtime": null,
+    "StockExchange": "NYQ",
+    "DividendYield": null,
+    "PercentChange": "+0.81%"
+   }
+
+----
+
+
+== Options
+
+The **$$yahooquotes$$** $$source$$ has the following options:
+
+//tag::configuration-properties[]
+$$yahoo.quotes.symbols$$:: $$Comma delimited list of symbols$$ *($$String$$, default: `A list of 7925 symbols from NASDAQ and NYSE`)*
+$$yahoo.quotes.fields$$:: $$Field selector to filter which fields should be returned$$ *($$String$$, default: `*` all fields)*
+$$yahoo.quotes.batchSize$$:: $$How many symbols to fetch per call$$ *($$Integer$$, default: `400`)*
+$$yahoo.quotes.cronExpression:: $$Cron expression to determine when to poll for quotes$$ *($$String$$, default: `0 * 9-17 * * MON-FRI` Only polls during exchange hours)*
+$$yahoo.quotes.zone:: $$Timezone used for the cron expression$$ *($$String$$, default: `EST`)*
+$$yahoo.quotes.throttle$$:: $$How many messages per second should the source emmit$$ *($$Integer$$, default: `null`, if set the source will throttle message output instead of sending a burst. Be mindful of the number of symbols and frequency when setting this in order to avoid blocking the output queue)*
+//end::configuration-properties[]
+
+== Example : Query for Google and Apple stocks every 5 seconds fetching only the symbol, bid and ask price:
+
+[source,yaml]
+----
+yahoo:
+  quotes:
+    symbol: GOOG,AAPL
+    fields: Symbol,Ask,Bid
+    cronExpression: 0/5 * * * * *
+
+----
+
+//end::ref-doc[]
+== Build
+
+```
+$ mvn clean package
+```
+

yahoo-quotes/spring-cloud-starter-stream-source-yahoo-quotes/pom.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		 xmlns="http://maven.apache.org/POM/4.0.0"
+		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<artifactId>spring-cloud-starter-stream-source-yahoo-quotes</artifactId>
+	<packaging>jar</packaging>
+	<description>Yahoo YQL quotes source</description>
+
+	<parent>
+		<artifactId>spring-cloud-stream-yahoo-quotes-parent</artifactId>
+		<groupId>org.springframework.cloud.stream.app</groupId>
+		<version>1.0.3.BUILD-SNAPSHOT</version>
+	</parent>
+
+
+	<dependencies>
+		<dependency>
+			<groupId>org.isomorphism</groupId>
+			<artifactId>token-bucket</artifactId>
+			<version>1.6</version>
+		</dependency>
+			<dependency>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-starter-test</artifactId>
+				<scope>test</scope>
+			</dependency>
+	</dependencies>
+</project>

yahoo-quotes/spring-cloud-starter-stream-source-yahoo-quotes/src/main/java/org/springframework/cloud/stream/app/yahoo/quotes/source/YahooQuotesClient.java

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2016 original author or authors.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.stream.app.yahoo.quotes.source;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author Vinicius Carvalho
+ */
+public interface YahooQuotesClient {
+	List<Map<String, Object>> fetchQuotes(List<String> symbols, String filter);
+}

yahoo-quotes/spring-cloud-starter-stream-source-yahoo-quotes/src/main/java/org/springframework/cloud/stream/app/yahoo/quotes/source/YahooQuotesClientImpl.java

@@ -0,0 +1,98 @@
+/*
+ *  Copyright 2016 original author or authors.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.stream.app.yahoo.quotes.source;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.util.StringUtils;
+import org.springframework.web.client.RestTemplate;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * @author Vinicius Carvalho
+ */
+public class YahooQuotesClientImpl implements YahooQuotesClient {
+
+	private final String endpoint = "https://query.yahooapis.com/v1/public/yql?q={query}&format=json&env={env}&callback=";
+
+	private final String queryTemplate = "select %s from yahoo.finance.quotes(%d) where symbol in (%s)";
+
+	private RestTemplate client;
+
+	private ObjectMapper mapper = new ObjectMapper();
+
+	final DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
+
+	public YahooQuotesClientImpl(RestTemplate client) {
+		this.client = client;
+	}
+
+
+
+	@Override
+	public List<Map<String, Object>> fetchQuotes(List<String> symbols, String filter) {
+		String fields = (StringUtils.isEmpty(filter)) ? "*" : filter;
+		String query = String.format(queryTemplate, fields, symbols.size(),
+				wouldbeSimplerWithLambdas(symbols));
+		Map<String, String> vars = new HashMap<>();
+		vars.put("query", query);
+		vars.put("env", "store://datatables.org/alltableswithkeys");
+		ResponseEntity<JsonNode> response = client.getForEntity(endpoint, JsonNode.class,
+				vars);
+		return processResponse(response);
+	}
+
+	private List<Map<String, Object>> processResponse(ResponseEntity<JsonNode> response) {
+		if (!response.getStatusCode().is2xxSuccessful())
+			return Collections.emptyList();
+		String timestamp = df.format(new Date());
+		JsonNode node = response.getBody();
+		JsonNode resultsNode = node.get("query").get("results").get("quote");
+		List<Map<String, Object>> results = new LinkedList<>();
+		if (resultsNode.isArray()) {
+			for (JsonNode child : resultsNode) {
+				((ObjectNode)child).put("timestamp",timestamp);
+				results.add(mapper.convertValue(child, Map.class));
+			}
+		}
+		else {
+			((ObjectNode)resultsNode).put("timestamp",timestamp);
+			results.add(mapper.convertValue(resultsNode, Map.class));
+		}
+		return results;
+	}
+
+	private String wouldbeSimplerWithLambdas(List<String> symbols) {
+		for (int i = 0; i < symbols.size(); i++) {
+			if (!symbols.get(i).startsWith("'") && !symbols.get(i).startsWith("\\"))
+				symbols.set(i, StringUtils.quote(symbols.get(i)));
+		}
+		return StringUtils.collectionToCommaDelimitedString(symbols);
+	}
+}