some cleanup

data/sp500List.txt

@@ -1,10 +1,10 @@
-#Symbol,Company,Industry,City,State
-A,Agilent Technologies Inc,Health Care,Santa Clara, California
-AA,Alcoa Inc,Materials,New York, New York
-AAPL,Apple Inc.,Information Technology,Cupertino, California
-ABC,AmerisourceBergen Corp,Health Care,Chesterbrook, Pennsylvania
-ABT,Abbott Laboratories,Health Care,North Chicago, Illinois
-ACE,ACE Limited,Financials,Zurich, Switzerland
+#Symbol,Company,Industry,City,State,URL
+A,Agilent Technologies Inc,Health Care,Santa Clara, California,http://www.agilent.com
+AA,Alcoa Inc,Materials,New York, New York,http://www.alcoa.com
+AAPL,Apple Inc.,Information Technology,Cupertino, California,http://www.apple.com
+ABC,AmerisourceBergen Corp,Health Care,Chesterbrook, Pennsylvania,http://www.amerisourcebergen.com/
+ABT,Abbott Laboratories,Health Care,North Chicago, Illinois,http://www.abbott.com/
+ACE,ACE Limited,Financials,Zurich, Switzerland,http://www.acegroup.com/
 ACN,Accenture,Information Technology,Dublin, Ireland
 ADBE,Adobe Systems Inc,Information Technology,San Jose, California
 ADI,Analog Devices Inc,Information Technology,Norwood, Massachusetts

src/main/python/fields.py

@@ -1,9 +1,10 @@
 #!/usr/bin/python
 
 import sys
-from lweutils import COL_URL, json_http, pretty_json, parse_opts
+from lweutils import json_http, pretty_json, parse_opts
+import lweutils
 
-FIELDS_URL = COL_URL + '/fields'
+FIELDS_URL = lweutils.COL_URL + '/fields'
 
 ##
 def help(args):

src/main/python/python.py

@@ -61,9 +61,10 @@ def standard(name=None):
         source_filters.append(dsn_results)
         group = "true"
 
-
+# &facet.date=timestamp&facet.date.start=2013-10-08T14:17:49.04Z&facet.date.end=NOW/DAY%2B1DAY&facet.date.gap=%2B1HOUR
     app.logger.info("Query: " + query)
     kwargs = {"qt": "/lucid", "facet": "true", "start": start, "fl":"*,score",
+            "facet.date":"timestamp", "facet.date.start":"NOW/DAY-1DAY", "facet.date.end":"NOW/DAY+1DAY", "facet.date.gap":"+15MINUTE",
               "facet.range":["open", "close", "volume"],
               "facet.range.start":"0",
               "facet.range.end":"1000",

src/main/python/setup.py

@@ -6,6 +6,7 @@
 
 import ds
 import fields
+import traceback
 
 twitter_fields = {"batch_id": {"name": "batch_id"},
                   "body": {"name": "body"},
@@ -15,7 +16,7 @@
                   "parsing": {"name": "parsing"},
                   "retweetCount": {"name": "retweetCount", "ft":"int", "facet":"true"}, "source": {"name": "creator"},
                   "userId": {"name": "userId", "ft":"long"}, "content_length": {"name": "content_length", "ft":"int"},
-                  "userLang": {"name": "lang", "ft": "string", "facet":"true"},
+                  "userLang": {"name": "lang"},
                   "userLocation": {"name": "userLocation", "ft":"text_en"},
                   "location": {"name": "location", "ft":"point"},
                   "userName": {"name": "author"},
@@ -50,15 +51,15 @@
     except ImportError:
         raise ImportError("No suitable ElementTree implementation was found.")
 
-from lweutils import SOLR_URL, COL_URL, API_URL, COLLECTION, json_http
+import lweutils
 ########### top level actions
 
 def setup(options, args):
-    solr = pysolr.Solr(SOLR_URL, timeout=10)
+    solr = pysolr.Solr(lweutils.SOLR_URL, timeout=10)
     stocks = load_stocks(options.stocks_file)
 
     if options.collection:
-        create_collection(COLLECTION)
+        create_collection(lweutils.COLLECTION)
     if options.fields:
         create_fields(args)
     if options.twitter:
@@ -113,8 +114,12 @@ def index_stocks(solr, stocks, id):
 
 def create_collection(name):
     data = {"name": name}
-    rsp = json_http(API_URL + "/collections", method='POST', data=data)
-    print "Created New Collection: " + data['name']
+    try:
+        print "Trying: " + name
+        rsp = lweutils.json_http(lweutils.API_URL + "/collections", method='POST', data=data)
+        print "Created New Collection: " + data['name']
+    except Exception as e:
+        traceback.print_exc()
     #TODO: Add in Aggregate RequestHandler capability
 
 
@@ -203,8 +208,9 @@ def add_twitter(i, stock_lists, stocks, access_token, consumer_key, consumer_sec
     args.append("filter_track=" + symbols[:len(symbols) - 1])
     id = ds.create(args)
     data = create_twitter_mappings()
-    rsp = json_http(COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data)
-    rsp = json_http(COL_URL + "/datasources/" + id + "/job", method="PUT")
+    print lweutils.COL_URL
+    rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data)
+    rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/job", method="PUT")
 
 
 def create_historical_ds():
@@ -216,7 +222,7 @@ def create_historical_ds():
                      "trade_date":"trade_date",
                      "volume": "volume",
                      "adj_close": "adj_close"}}
-    rsp = json_http(COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data)
+    rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data)
     return id
 
 
@@ -226,12 +232,13 @@ def create_company_ds():
     data = {
         "mappings": {"symbol": "symbol", "company": "company", "industry": "industry", "city": "city",
                      "state": "state"}}
-    rsp = json_http(COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data)
+    rsp = lweutils.json_http(lweutils.COL_URL + "/datasources/" + id + "/mapping", method="PUT", data=data)
     return id
 
 
 def create_fields(args):
     #Twitter
+    print lweutils.COLLECTION
     for field in twitter_fields:
         if twitter_fields[field] and 'ft' in twitter_fields[field]:
             facet = "false"
@@ -409,17 +416,20 @@ def help(solr, options, args):
 
 opts, args = p.parse_args()
 action = opts.action
-COLLECTION = opts.collection
-LWS_URL = "http://" + opts.host + ":" + opts.api_port
-API_URL = LWS_URL + "/api"
-SOLR_URL = LWS_URL + "/solr/" + COLLECTION
-COL_URL = API_URL + "/collections/" + COLLECTION
+# TODO: FIX UP ALL THIS HACKY VARIABLE STUFF
+lweutils.COLLECTION = opts.collection
+lweutils.LWS_URL = "http://" + opts.host + ":" + opts.api_port
+lweutils.API_URL = lweutils.LWS_URL + "/api"
+lweutils.SOLR_URL = lweutils.LWS_URL + "/solr/" + lweutils.COLLECTION
+lweutils.COL_URL = lweutils.API_URL + "/collections/" + lweutils.COLLECTION
+fields.FIELDS_URL = lweutils.COL_URL + '/fields'  #TODO: fix this
+ds.DS_URL = lweutils.COL_URL + '/datasources'
 
 if (opts.ui_host and opts.ui_port):
-    UI_URL = "http://" + opts.ui_host + ":" + opts.ui_port
+    lweutils.UI_URL = "http://" + opts.ui_host + ":" + opts.ui_port
 else:
-    UI_URL = "http://" + opts.host + ":8989"
-UI_API_URL = UI_URL + "/api"
+    lweutils.UI_URL = "http://" + opts.host + ":8989"
+lweutils.UI_API_URL = lweutils.UI_URL + "/api"
 
 if opts.all:
     opts.external = True

src/main/python/templates/facet_ranges.html

@@ -9,7 +9,7 @@
             {% set end = field['end'] %}
             <div class="facet-header">{{ facets|title|replace('_facet', '' )|replace('_', ' ') }}</div>
             {% if field['before'] > 0%}<div class="facet-val"><a href="{{ current_url }}&fq={{ facets }}:[* TO {{ start - 1 }}]">Before</a> ({{ field['before'] }})</div>{% endif %}
-            <div class="facet-val">Between ({{ field['between'] }})</div>
+            {% if field['between'] and field['between'] > 0 %}<div class="facet-val">Between ({{ field['between'] }})</div>{% endif %}
             <div class="facet-group">
             {% for val in field['counts'] %}
                 {% if loop.index0 % 2 == 0 %}