From 735f543c6ebcde59d46c681c3405070dafe80ae9 Mon Sep 17 00:00:00 2001 From: gsingers Date: Sat, 2 Nov 2013 12:21:11 -0400 Subject: [PATCH] some cleanup --- data/sp500List.txt | 14 +++---- src/main/python/fields.py | 5 ++- src/main/python/python.py | 3 +- src/main/python/setup.py | 46 +++++++++++++-------- src/main/python/templates/facet_ranges.html | 2 +- 5 files changed, 41 insertions(+), 29 deletions(-) diff --git a/data/sp500List.txt b/data/sp500List.txt index c230c968..0f6dead3 100644 --- a/data/sp500List.txt +++ b/data/sp500List.txt @@ -1,10 +1,10 @@ -#Symbol,Company,Industry,City,State -A,Agilent Technologies Inc,Health Care,Santa Clara, California -AA,Alcoa Inc,Materials,New York, New York -AAPL,Apple Inc.,Information Technology,Cupertino, California -ABC,AmerisourceBergen Corp,Health Care,Chesterbrook, Pennsylvania -ABT,Abbott Laboratories,Health Care,North Chicago, Illinois -ACE,ACE Limited,Financials,Zurich, Switzerland +#Symbol,Company,Industry,City,State,URL +A,Agilent Technologies Inc,Health Care,Santa Clara, California,http://www.agilent.com +AA,Alcoa Inc,Materials,New York, New York,http://www.alcoa.com +AAPL,Apple Inc.,Information Technology,Cupertino, California,http://www.apple.com +ABC,AmerisourceBergen Corp,Health Care,Chesterbrook, Pennsylvania,http://www.amerisourcebergen.com/ +ABT,Abbott Laboratories,Health Care,North Chicago, Illinois,http://www.abbott.com/ +ACE,ACE Limited,Financials,Zurich, Switzerland,http://www.acegroup.com/ ACN,Accenture,Information Technology,Dublin, Ireland ADBE,Adobe Systems Inc,Information Technology,San Jose, California ADI,Analog Devices Inc,Information Technology,Norwood, Massachusetts diff --git a/src/main/python/fields.py b/src/main/python/fields.py index b64f9ed3..d61c508b 100644 --- a/src/main/python/fields.py +++ b/src/main/python/fields.py @@ -1,9 +1,10 @@ #!/usr/bin/python import sys -from lweutils import COL_URL, json_http, pretty_json, parse_opts +from lweutils import json_http, pretty_json, parse_opts +import lweutils -FIELDS_URL = COL_URL + '/fields' +FIELDS_URL = lweutils.COL_URL + '/fields' ## def help(args): diff --git a/src/main/python/python.py b/src/main/python/python.py index ae672ee2..c2db15a2 100644 --- a/src/main/python/python.py +++ b/src/main/python/python.py @@ -61,9 +61,10 @@ def standard(name=None): source_filters.append(dsn_results) group = "true" - +# &facet.date=timestamp&facet.date.start=2013-10-08T14:17:49.04Z&facet.date.end=NOW/DAY%2B1DAY&facet.date.gap=%2B1HOUR app.logger.info("Query: " + query) kwargs = {"qt": "/lucid", "facet": "true", "start": start, "fl":"*,score", + "facet.date":"timestamp", "facet.date.start":"NOW/DAY-1DAY", "facet.date.end":"NOW/DAY+1DAY", "facet.date.gap":"+15MINUTE", "facet.range":["open", "close", "volume"], "facet.range.start":"0", "facet.range.end":"1000", diff --git a/src/main/python/setup.py b/src/main/python/setup.py index 68d6eb0d..35a0e08e 100755 --- a/src/main/python/setup.py +++ b/src/main/python/setup.py @@ -6,6 +6,7 @@ import ds import fields +import traceback twitter_fields = {"batch_id": {"name": "batch_id"}, "body": {"name": "body"}, @@ -15,7 +16,7 @@ "parsing": {"name": "parsing"}, "retweetCount": {"name": "retweetCount", "ft":"int", "facet":"true"}, "source": {"name": "creator"}, "userId": {"name": "userId", "ft":"long"}, "content_length": {"name": "content_length", "ft":"int"}, - "userLang": {"name": "lang", "ft": "string", "facet":"true"}, + "userLang": {"name": "lang"}, "userLocation": {"name": "userLocation", "ft":"text_en"}, "location": {"name": "location", "ft":"point"}, "userName": {"name": "author"}, @@ -50,15 +51,15 @@ except ImportError: raise ImportError("No suitable ElementTree implementation was found.") -from lweutils import SOLR_URL, COL_URL, API_URL, COLLECTION, json_http +import lweutils ########### top level actions def setup(options, args): - solr = pysolr.Solr(SOLR_URL, timeout=10) + solr = pysolr.Solr(lweutils.SOLR_URL, timeout=10) stocks = load_stocks(options.stocks_file) if options.collection: - create_collection(COLLECTION) + create_collection(lweutils.COLLECTION) if options.fields: create_fields(args) if options.twitter: @@ -113,8 +114,12 @@ def index_stocks(solr, stocks, id): def create_collection(name): data = {"name": name} - rsp = json_http(API_URL + "/collections", method='POST', data=data) - print "Created New Collection: " + data['name'] + try: + print "Trying: " + name + rsp = lweutils.json_http(lweutils.API_URL + "/collections", method='POST', data=data) + print "Created New Collection: " + data['name'] + except Exception as e: + traceback.print_exc() #TODO: Add in Aggregate RequestHandler capability @@ -203,8 +208,9 @@ def add_twitter(i, stock_lists, stocks, access_token, consumer_key, consumer_sec args.append("filter_track=" + symbols[:len(symbols) - 1]) id = ds.create(args) data = create_twitter_mappings() - rsp = json_http(COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data) - rsp = json_http(COL_URL + "/datasources/" + id + "/job", method="PUT") + print lweutils.COL_URL + rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data) + rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/job", method="PUT") def create_historical_ds(): @@ -216,7 +222,7 @@ def create_historical_ds(): "trade_date":"trade_date", "volume": "volume", "adj_close": "adj_close"}} - rsp = json_http(COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data) + rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data) return id @@ -226,12 +232,13 @@ def create_company_ds(): data = { "mappings": {"symbol": "symbol", "company": "company", "industry": "industry", "city": "city", "state": "state"}} - rsp = json_http(COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data) + rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data) return id def create_fields(args): #Twitter + print lweutils.COLLECTION for field in twitter_fields: if twitter_fields[field] and 'ft' in twitter_fields[field]: facet = "false" @@ -409,17 +416,20 @@ def help(solr, options, args): opts, args = p.parse_args() action = opts.action -COLLECTION = opts.collection -LWS_URL = "http://" + opts.host + ":" + opts.api_port -API_URL = LWS_URL + "/api" -SOLR_URL = LWS_URL + "/solr/" + COLLECTION -COL_URL = API_URL + "/collections/" + COLLECTION +# TODO: FIX UP ALL THIS HACKY VARIABLE STUFF +lweutils.COLLECTION = opts.collection +lweutils.LWS_URL = "http://" + opts.host + ":" + opts.api_port +lweutils.API_URL = lweutils.LWS_URL + "/api" +lweutils.SOLR_URL = lweutils.LWS_URL + "/solr/" + lweutils.COLLECTION +lweutils.COL_URL = lweutils.API_URL + "/collections/" + lweutils.COLLECTION +fields.FIELDS_URL = lweutils.COL_URL + '/fields' #TODO: fix this +ds.DS_URL = lweutils.COL_URL + '/datasources' if (opts.ui_host and opts.ui_port): - UI_URL = "http://" + opts.ui_host + ":" + opts.ui_port + lweutils.UI_URL = "http://" + opts.ui_host + ":" + opts.ui_port else: - UI_URL = "http://" + opts.host + ":8989" -UI_API_URL = UI_URL + "/api" + lweutils.UI_URL = "http://" + opts.host + ":8989" +lweutils.UI_API_URL = lweutils.UI_URL + "/api" if opts.all: opts.external = True diff --git a/src/main/python/templates/facet_ranges.html b/src/main/python/templates/facet_ranges.html index d6aa56ba..01a272c5 100644 --- a/src/main/python/templates/facet_ranges.html +++ b/src/main/python/templates/facet_ranges.html @@ -9,7 +9,7 @@ {% set end = field['end'] %}
{{ facets|title|replace('_facet', '' )|replace('_', ' ') }}
{% if field['before'] > 0%}
Before ({{ field['before'] }})
{% endif %} -
Between ({{ field['between'] }})
+ {% if field['between'] and field['between'] > 0 %}
Between ({{ field['between'] }})
{% endif %}
{% for val in field['counts'] %} {% if loop.index0 % 2 == 0 %}