From a29a6aa08c4b41e86d4a203f1ba032bfc55c1d30 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Sun, 5 Jun 2016 15:46:13 +0200 Subject: [PATCH 01/35] Basic Cassandra framework --- lib/physiqual/data_services.rb | 1 + .../data_services/cassandra_data_service.rb | 41 +++++++++++++++++++ lib/physiqual/exporters/exporter.rb | 1 + 3 files changed, 43 insertions(+) create mode 100644 lib/physiqual/data_services/cassandra_data_service.rb diff --git a/lib/physiqual/data_services.rb b/lib/physiqual/data_services.rb index e398970..53b17db 100644 --- a/lib/physiqual/data_services.rb +++ b/lib/physiqual/data_services.rb @@ -7,6 +7,7 @@ require 'physiqual/data_services/mock_service' require 'physiqual/data_services/summarized_data_service' require 'physiqual/data_services/bucketeer_data_service' +require 'physiqual/data_services/cassandra_data_service' module Physiqual module DataServices diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb new file mode 100644 index 0000000..f8c5b35 --- /dev/null +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -0,0 +1,41 @@ +module Physiqual + module DataServices + class CassandraDataService < DataServiceDecorator + def initialize(data_service) + super(data_service) + end + + def service_name + "cassandra_#{data_service.service_name}" + end + + def heart_rate(from, to) + heart_rate = data_service.heart_rate(from, to) + end + + def sleep(from, to) + sleep = data_service.sleep(from, to) + end + + def calories(from, to) + calories_measured = data_service.calories(from, to) + end + + def distance(from, to) + distance = data_service.distance(from, to) + end + + def steps(from, to) + steps = data_service.steps(from, to) + end + + def activities(from, to) + activities = data_service.activities(from, to) + end + + private + + + end + end +end diff --git a/lib/physiqual/exporters/exporter.rb b/lib/physiqual/exporters/exporter.rb index 52991b2..413c63b 100644 --- a/lib/physiqual/exporters/exporter.rb +++ b/lib/physiqual/exporters/exporter.rb @@ -34,6 +34,7 @@ def create_service(token, bucket_generator) return [] unless token.complete? session = Sessions::TokenAuthorizedSession.new(token) service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) + service = DataServices::CassandraDataService.new(service) service = DataServices::BucketeerDataService.new(service, bucket_generator) service = DataServices::SummarizedDataService.new(service) DataServices::CachedDataService.new service From 278950185cb8f9ec608e24fbd5a1aaa38d239708 Mon Sep 17 00:00:00 2001 From: frbl Date: Mon, 6 Jun 2016 09:04:48 +0200 Subject: [PATCH 02/35] Add cassandra username / password to physiqual engine --- lib/physiqual/engine.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index 94131fd..a984bfc 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -9,6 +9,9 @@ class << self mattr_accessor :google_client_secret mattr_accessor :fitbit_client_id mattr_accessor :fitbit_client_secret + mattr_accessor :cassandra_username + mattr_accessor :cassandra_password + mattr_accessor :cassandra_host_url mattr_accessor :host_url mattr_accessor :host_protocol mattr_accessor :measurements_per_day From 8bdb368ec067f2ee5e6e444516561a78abbca9eb Mon Sep 17 00:00:00 2001 From: frbl Date: Mon, 6 Jun 2016 09:07:01 +0200 Subject: [PATCH 03/35] Updated dummy initializer to have cassandra config --- spec/dummy/config/initializers/physiqual.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/dummy/config/initializers/physiqual.rb b/spec/dummy/config/initializers/physiqual.rb index 47a451f..03d192a 100644 --- a/spec/dummy/config/initializers/physiqual.rb +++ b/spec/dummy/config/initializers/physiqual.rb @@ -11,6 +11,11 @@ config.host_url = ENV['HOST_URL'] || 'physiqual.dev' config.host_protocol = ENV['HOST_PROTOCOL'] || 'http' + # Cassandra settings + config.cassandra_username = ENV['CASSANDRA_USERNAME'] + config.cassandra_password = ENV['CASSANDRA_PASSWORD'] + config.cassandra_host_url = ENV['CASSANDRA_HOST_URL'] + # EMA Settings config.measurements_per_day = 1 # Number of measurements per day config.interval = 24 # Number of hours between measurements From 22421ee422226f5bd199ad1b7f5f8488fe1c1c08 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 9 Jun 2016 15:16:00 +0200 Subject: [PATCH 04/35] Commit for IDE transfer --- .../data_services/cassandra_connection.rb | 22 +++++++++++++++++++ physiqual.gemspec | 3 +++ spec/dummy/config/initializers/physiqual.rb | 7 +++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 lib/physiqual/data_services/cassandra_connection.rb diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb new file mode 100644 index 0000000..c09b83e --- /dev/null +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -0,0 +1,22 @@ +module Physiqual + module DataServices + class CassandraConnection + include Singleton + + def initialize() + @cluster = Cassandra.cluster( + username: config.cassandra_username, + password: config.cassandra_password, + hosts: config.cassandra_host_urls + ) + @session = cluster.connect(config.cassandra_keyspace) + @insert_heart_rate_statement = session.prepare('') + @insert_sleep_statement = session.prepare('') + @insert_calories_statement = session.prepare('') + @insert_distance_statement = session.prepare('') + @insert_steps_statement = session.prepare('') + @insert_activities_statement = session.prepare('') + end + end + end +end diff --git a/physiqual.gemspec b/physiqual.gemspec index fa68fe6..18f8fb6 100644 --- a/physiqual.gemspec +++ b/physiqual.gemspec @@ -37,6 +37,9 @@ Gem::Specification.new do |s| s.add_dependency 'omniauth-google-oauth2' s.add_dependency 'omniauth-fitbit-oauth2' + # Cassandra caching + s.add_dependency 'cassandra-driver' + s.add_development_dependency 'codeclimate-test-reporter' s.add_development_dependency 'sqlite3' s.add_development_dependency 'rspec-rails' diff --git a/spec/dummy/config/initializers/physiqual.rb b/spec/dummy/config/initializers/physiqual.rb index 03d192a..bde1dbc 100644 --- a/spec/dummy/config/initializers/physiqual.rb +++ b/spec/dummy/config/initializers/physiqual.rb @@ -12,9 +12,10 @@ config.host_protocol = ENV['HOST_PROTOCOL'] || 'http' # Cassandra settings - config.cassandra_username = ENV['CASSANDRA_USERNAME'] - config.cassandra_password = ENV['CASSANDRA_PASSWORD'] - config.cassandra_host_url = ENV['CASSANDRA_HOST_URL'] + config.cassandra_username = ENV['CASSANDRA_USERNAME'] || '' + config.cassandra_password = ENV['CASSANDRA_PASSWORD'] || '' + config.cassandra_host_urls = (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + config.cassandra_keyspace = ENV['CASSANDRA_KEYSPACE'] # EMA Settings config.measurements_per_day = 1 # Number of measurements per day From a87d4668749a679e26d20c90ac484885171bcfa5 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 9 Jun 2016 17:50:08 +0200 Subject: [PATCH 05/35] Commit for Frank check --- lib/physiqual/data_services/cassandra_data_service.rb | 3 ++- lib/physiqual/engine.rb | 3 ++- lib/physiqual/exporters/exporter.rb | 8 +++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index f8c5b35..ad11c8a 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -1,8 +1,9 @@ module Physiqual module DataServices class CassandraDataService < DataServiceDecorator - def initialize(data_service) + def initialize(data_service, user) super(data_service) + @user = user end def service_name diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index a984bfc..1987386 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -11,7 +11,8 @@ class << self mattr_accessor :fitbit_client_secret mattr_accessor :cassandra_username mattr_accessor :cassandra_password - mattr_accessor :cassandra_host_url + mattr_accessor :cassandra_host_urls + mattr_accessor :cassandra_keyspace mattr_accessor :host_url mattr_accessor :host_protocol mattr_accessor :measurements_per_day diff --git a/lib/physiqual/exporters/exporter.rb b/lib/physiqual/exporters/exporter.rb index 413c63b..2704086 100644 --- a/lib/physiqual/exporters/exporter.rb +++ b/lib/physiqual/exporters/exporter.rb @@ -9,7 +9,7 @@ def export_data(user_id, first_measurement, number_of_days) Physiqual.hours_before_first_measurement ) - service = create_service(user.physiqual_token, bucket_generator) + service = create_service(user, bucket_generator) data_imputer = DataImputer.new(service, Physiqual.imputers) from = from_time(first_measurement) @@ -30,11 +30,13 @@ def to_time(first_measurement, number_of_days) ((Physiqual.measurements_per_day - 1) * Physiqual.interval).hours end - def create_service(token, bucket_generator) + def create_service(user, bucket_generator) + puts 'token =' + (token = user.physiqual_token) return [] unless token.complete? + puts 'fixnum? ' + (user.user_id instance_of? Fixnum) session = Sessions::TokenAuthorizedSession.new(token) service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) - service = DataServices::CassandraDataService.new(service) + service = DataServices::CassandraDataService.new(service, user) service = DataServices::BucketeerDataService.new(service, bucket_generator) service = DataServices::SummarizedDataService.new(service) DataServices::CachedDataService.new service From a6b82e9bef58feb7098f9b69172d865e72c5efbb Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Sat, 11 Jun 2016 23:26:09 +0200 Subject: [PATCH 06/35] CassandraConnection creation and exporter prep work --- db/seeds.rb | 3 ++ .../data_services/cassandra_connection.rb | 48 ++++++++++++++++--- lib/physiqual/exporters/exporter.rb | 3 +- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index db24990..6f6f52c 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -12,4 +12,7 @@ module Physiqual User.find_or_create_by(user_id: 'b') User.find_or_create_by(user_id: 'c') end + puts 'Initializing Cassandra' + connection = CassandraConnection.instance + connection.init_db end diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index c09b83e..3ad7e9c 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -3,19 +3,53 @@ module DataServices class CassandraConnection include Singleton - def initialize() + def initialize @cluster = Cassandra.cluster( username: config.cassandra_username, password: config.cassandra_password, hosts: config.cassandra_host_urls ) @session = cluster.connect(config.cassandra_keyspace) - @insert_heart_rate_statement = session.prepare('') - @insert_sleep_statement = session.prepare('') - @insert_calories_statement = session.prepare('') - @insert_distance_statement = session.prepare('') - @insert_steps_statement = session.prepare('') - @insert_activities_statement = session.prepare('') + @insert_statement = @session.prepare(' + INSERT INTO ? ( + userid, year, time, ? + ) VALUES ( + ?, ?, ?, ? + ) + ') + @query_statement = @session.prepare(' + SELECT time, ? + FROM ? + WHERE userid = ? + AND year = ? + AND time IN (?, ?) + ORDER BY time ASC + ') + end + + def insert(table, userid, year, time, value) + @session.execute(@insert_statement, arguments: [table, table, userid, year, time, value]) + end + + def query(table, userid, year, from, to) + @session.execute(@query_statement, arguments: [table, table, userid, year, from, to]) + end + + def init_db + create_statement = @session.prepare(" + CREATE TABLE IF NOT EXISTS #{config.cassandra_keyspace}.? ( + userid text, + year int, + time timestamp, + ? ? + PRIMARY KEY ((userid, year), time) + ") + @session.execute(create_statement, arguments: ['heart_rate', 'heart_rate', 'decimal']) + @session.execute(create_statement, arguments: ['sleep', 'sleep', 'decimal']) + @session.execute(create_statement, arguments: ['calories', 'calories', 'decimal']) + @session.execute(create_statement, arguments: ['distance', 'distance', 'decimal']) + @session.execute(create_statement, arguments: ['steps', 'steps', 'decimal']) + @session.execute(create_statement, arguments: ['activities', 'activities', 'decimal']) end end end diff --git a/lib/physiqual/exporters/exporter.rb b/lib/physiqual/exporters/exporter.rb index 2704086..0311e67 100644 --- a/lib/physiqual/exporters/exporter.rb +++ b/lib/physiqual/exporters/exporter.rb @@ -31,9 +31,8 @@ def to_time(first_measurement, number_of_days) end def create_service(user, bucket_generator) - puts 'token =' + (token = user.physiqual_token) + token = user.physiqual_token return [] unless token.complete? - puts 'fixnum? ' + (user.user_id instance_of? Fixnum) session = Sessions::TokenAuthorizedSession.new(token) service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) service = DataServices::CassandraDataService.new(service, user) From b88334c5989b68a30c78e35372ec3faa00965402 Mon Sep 17 00:00:00 2001 From: frbl Date: Mon, 13 Jun 2016 09:29:56 +0200 Subject: [PATCH 07/35] Add comments explaining the properties --- app/models/physiqual/data_entry.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/models/physiqual/data_entry.rb b/app/models/physiqual/data_entry.rb index edacfa5..ddc5e52 100644 --- a/app/models/physiqual/data_entry.rb +++ b/app/models/physiqual/data_entry.rb @@ -4,9 +4,18 @@ class DataEntry DATE_FORMAT = '%Y-%m-%d'.freeze + # The start date property is the start of this dataentry object, meaning it + # denotes the actual start of the data in this measurment attribute :start_date, DateTime + + # The end date property denotes the end of this measurement (exclusing the time). + # Meaning, data in this data entry is upto, but not including this time point attribute :end_date, DateTime attribute :values, Array, default: [] + + # Measurement moment is used in the physiqual process. It is used to determine + # the actual 'measurement time' of the current dataentry object. Often it's + # precisely in the middle of the start and end date of this data entry object. attribute :measurement_moment, DateTime, default: :default_measurement_moment def default_measurement_moment From 1e12dac5daf1a7e787f3445d7aad1c85842f1a56 Mon Sep 17 00:00:00 2001 From: frbl Date: Mon, 13 Jun 2016 09:31:10 +0200 Subject: [PATCH 08/35] typo --- app/models/physiqual/data_entry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/physiqual/data_entry.rb b/app/models/physiqual/data_entry.rb index ddc5e52..b393d5a 100644 --- a/app/models/physiqual/data_entry.rb +++ b/app/models/physiqual/data_entry.rb @@ -5,7 +5,7 @@ class DataEntry DATE_FORMAT = '%Y-%m-%d'.freeze # The start date property is the start of this dataentry object, meaning it - # denotes the actual start of the data in this measurment + # denotes the actual start of the data in this measurement attribute :start_date, DateTime # The end date property denotes the end of this measurement (exclusing the time). From ace818028a5d79944fedd04346ea25ba2767ddd4 Mon Sep 17 00:00:00 2001 From: frbl Date: Mon, 13 Jun 2016 16:34:15 +0200 Subject: [PATCH 09/35] remove trailing whitespacing --- app/models/physiqual/data_entry.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/physiqual/data_entry.rb b/app/models/physiqual/data_entry.rb index b393d5a..6cdff5f 100644 --- a/app/models/physiqual/data_entry.rb +++ b/app/models/physiqual/data_entry.rb @@ -4,18 +4,18 @@ class DataEntry DATE_FORMAT = '%Y-%m-%d'.freeze - # The start date property is the start of this dataentry object, meaning it + # The start date property is the start of this dataentry object, meaning it # denotes the actual start of the data in this measurement attribute :start_date, DateTime - # The end date property denotes the end of this measurement (exclusing the time). + # The end date property denotes the end of this measurement (exclusing the time). # Meaning, data in this data entry is upto, but not including this time point attribute :end_date, DateTime attribute :values, Array, default: [] - # Measurement moment is used in the physiqual process. It is used to determine - # the actual 'measurement time' of the current dataentry object. Often it's - # precisely in the middle of the start and end date of this data entry object. + # Measurement moment is used in the physiqual process. It is used to determine + # the actual 'measurement time' of the current dataentry object. Often it's + # precisely in the middle of the start and end date of this data entry object. attribute :measurement_moment, DateTime, default: :default_measurement_moment def default_measurement_moment From e566fd9aeb4463bbe45a6a8c93c644f3fefbfdb1 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Tue, 14 Jun 2016 00:27:38 +0200 Subject: [PATCH 10/35] Caching first implementation --- app/workers/physiqual/cache_worker.rb | 24 ++ db/seeds.rb | 2 +- lib/physiqual/data_services.rb | 1 + .../data_services/cassandra_connection.rb | 354 ++++++++++++++++-- .../data_services/cassandra_data_service.rb | 191 +++++++++- lib/physiqual/exporters/exporter.rb | 2 +- physiqual.gemspec | 2 + spec/dummy/config/application.rb | 1 + spec/dummy/config/environments/development.rb | 2 +- spec/dummy/config/environments/test.rb | 2 +- 10 files changed, 537 insertions(+), 44 deletions(-) create mode 100644 app/workers/physiqual/cache_worker.rb diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb new file mode 100644 index 0000000..11734bb --- /dev/null +++ b/app/workers/physiqual/cache_worker.rb @@ -0,0 +1,24 @@ +require 'sidekiq' + +module Physiqual + class CacheWorker + include Sidekiq::Worker + def perform(table, user_id, year, times, start_dates, end_dates, values) + connection = DataServices::CassandraConnection.instance + case table + when 'heart_rate' + connection.insert_heart_rate(user_id, year, times, start_dates, end_dates, values) + when 'sleep' + connection.insert_sleep(user_id, year, times, start_dates, end_dates, values) + when 'calories' + connection.insert_calories(user_id, year, times, start_dates, end_dates, values) + when 'distance' + connection.insert_distance(user_id, year, times, start_dates, end_dates, values) + when 'steps' + connection.insert_steps(user_id, year, times, start_dates, end_dates, values) + when 'activities' + connection.insert_activities(user_id, year, times, start_dates, end_dates, values) + end + end + end +end \ No newline at end of file diff --git a/db/seeds.rb b/db/seeds.rb index 6f6f52c..ab4e2b0 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -13,6 +13,6 @@ module Physiqual User.find_or_create_by(user_id: 'c') end puts 'Initializing Cassandra' - connection = CassandraConnection.instance + connection = DataServices::CassandraConnection.instance connection.init_db end diff --git a/lib/physiqual/data_services.rb b/lib/physiqual/data_services.rb index 53b17db..ad58b70 100644 --- a/lib/physiqual/data_services.rb +++ b/lib/physiqual/data_services.rb @@ -8,6 +8,7 @@ require 'physiqual/data_services/summarized_data_service' require 'physiqual/data_services/bucketeer_data_service' require 'physiqual/data_services/cassandra_data_service' +require 'physiqual/data_services/cassandra_connection' module Physiqual module DataServices diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index 3ad7e9c..277ff90 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -1,55 +1,347 @@ +require 'singleton' +require 'cassandra' + module Physiqual module DataServices class CassandraConnection include Singleton def initialize - @cluster = Cassandra.cluster( - username: config.cassandra_username, - password: config.cassandra_password, - hosts: config.cassandra_host_urls - ) - @session = cluster.connect(config.cassandra_keyspace) - @insert_statement = @session.prepare(' - INSERT INTO ? ( - userid, year, time, ? + @cluster = nil + if (!ENV['CASSANDRA_USERNAME']) || (!ENV['CASSANDRA_PASSWORD']) || ENV['CASSANDRA_USERNAME'] == '' then + @cluster = Cassandra.cluster( + hosts: (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + ) + else + @cluster = Cassandra.cluster( + username: ENV['CASSANDRA_USERNAME'], + password: ENV['CASSANDRA_PASSWORD'], + hosts: (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + ) + end + @session = @cluster.connect('physiqual') + + init_db + + @insert_heart_rate = @session.prepare(' + INSERT INTO heart_rate ( + user_id, year, time, start_date, end_date, value ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ? ) ') - @query_statement = @session.prepare(' - SELECT time, ? - FROM ? - WHERE userid = ? + @insert_sleep = @session.prepare(' + INSERT INTO sleep ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_calories = @session.prepare(' + INSERT INTO calories ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_distance = @session.prepare(' + INSERT INTO distance ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_steps = @session.prepare(' + INSERT INTO steps ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_activities = @session.prepare(' + INSERT INTO activities ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + + @query_heart_rate = @session.prepare(' + SELECT time, start_date, end_date, value + FROM heart_rate + WHERE user_id = ? + AND year = ? + AND time >= ? + AND time < ? + ORDER BY time ASC + ') + @query_sleep = @session.prepare(' + SELECT time, start_date, end_date, value + FROM sleep + WHERE user_id = ? AND year = ? - AND time IN (?, ?) + AND time >= ? + AND time < ? ORDER BY time ASC ') + @query_calories = @session.prepare(' + SELECT time, start_date, end_date, value + FROM calories + WHERE user_id = ? + AND year = ? + AND time >= ? + AND time < ? + ORDER BY time ASC + ') + @query_distance = @session.prepare(' + SELECT time, start_date, end_date, value + FROM distance + WHERE user_id = ? + AND year = ? + AND time >= ? + AND time < ? + ORDER BY time ASC + ') + @query_steps = @session.prepare(' + SELECT time, start_date, end_date, value + FROM steps + WHERE user_id = ? + AND year = ? + AND time >= ? + AND time < ? + ORDER BY time ASC + ') + @query_activities = @session.prepare(' + SELECT time, start_date, end_date, value + FROM activities + WHERE user_id = ? + AND year = ? + AND time >= ? + AND time < ? + ORDER BY time ASC + ') + + end + + + + def insert_heart_rate(user_id, year, times, start_dates, end_dates, values) + times_slices = times.each_slice(100).to_a + start_dates_slices = start_dates.each_slice(100).to_a + end_dates_slices = end_dates.each_slice(100).to_a + values_slices = values.each_slice(100).to_a + + times_slices.each_with_index do |times_slice, i| + start_dates_slice = start_dates_slices[i] + end_dates_slice = end_dates_slices[i] + values_slice = values_slices[i] + + batch = @session.batch do |b| + times_slice.each_with_index do |time, i| + b.add(@insert_heart_rate, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + end + end + @session.execute(batch) + end + end + + def insert_sleep(user_id, year, times, start_dates, end_dates, values) + times_slices = times.each_slice(100).to_a + start_dates_slices = start_dates.each_slice(100).to_a + end_dates_slices = end_dates.each_slice(100).to_a + values_slices = values.each_slice(100).to_a + + times_slices.each_with_index do |times_slice, i| + start_dates_slice = start_dates_slices[i] + end_dates_slice = end_dates_slices[i] + values_slice = values_slices[i] + + batch = @session.batch do |b| + times_slice.each_with_index do |time, i| + b.add(@insert_sleep, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + end + end + @session.execute(batch) + end + end + + def insert_calories(user_id, year, times, start_dates, end_dates, values) + times_slices = times.each_slice(100).to_a + start_dates_slices = start_dates.each_slice(100).to_a + end_dates_slices = end_dates.each_slice(100).to_a + values_slices = values.each_slice(100).to_a + + times_slices.each_with_index do |times_slice, i| + start_dates_slice = start_dates_slices[i] + end_dates_slice = end_dates_slices[i] + values_slice = values_slices[i] + + batch = @session.batch do |b| + times_slice.each_with_index do |time, i| + b.add(@insert_calories, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + end + end + @session.execute(batch) + end end - def insert(table, userid, year, time, value) - @session.execute(@insert_statement, arguments: [table, table, userid, year, time, value]) + def insert_distance(user_id, year, times, start_dates, end_dates, values) + times_slices = times.each_slice(100).to_a + start_dates_slices = start_dates.each_slice(100).to_a + end_dates_slices = end_dates.each_slice(100).to_a + values_slices = values.each_slice(100).to_a + + times_slices.each_with_index do |times_slice, i| + start_dates_slice = start_dates_slices[i] + end_dates_slice = end_dates_slices[i] + values_slice = values_slices[i] + + batch = @session.batch do |b| + times_slice.each_with_index do |time, i| + b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + end + end + @session.execute(batch) + end end - def query(table, userid, year, from, to) - @session.execute(@query_statement, arguments: [table, table, userid, year, from, to]) + def insert_steps(user_id, year, times, start_dates, end_dates, values) + times_slices = times.each_slice(100).to_a + start_dates_slices = start_dates.each_slice(100).to_a + end_dates_slices = end_dates.each_slice(100).to_a + values_slices = values.each_slice(100).to_a + + times_slices.each_with_index do |times_slice, i| + start_dates_slice = start_dates_slices[i] + end_dates_slice = end_dates_slices[i] + values_slice = values_slices[i] + + batch = @session.batch do |b| + times_slice.each_with_index do |time, i| + b.add(@insert_steps, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + end + end + @session.execute(batch) + end end + def insert_activities(user_id, year, times, start_dates, end_dates, values) + times_slices = times.each_slice(100).to_a + start_dates_slices = start_dates.each_slice(100).to_a + end_dates_slices = end_dates.each_slice(100).to_a + values_slices = values.each_slice(100).to_a + + times_slices.each_with_index do |times_slice, i| + start_dates_slice = start_dates_slices[i] + end_dates_slice = end_dates_slices[i] + values_slice = values_slices[i] + + batch = @session.batch do |b| + times_slice.each_with_index do |time, i| + b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, values_slice[i]]) + end + end + @session.execute(batch) + end + end + + + + def query_heart_rate(user_id, year, from, to) + @session.execute(@query_heart_rate, arguments: [user_id, year, from, to]) + end + + def query_sleep(user_id, year, from, to) + @session.execute(@query_sleep, arguments: [user_id, year, from, to]) + end + + def query_calories(user_id, year, from, to) + @session.execute(@query_calories, arguments: [user_id, year, from, to]) + end + + def query_distance(user_id, year, from, to) + @session.execute(@query_distance, arguments: [user_id, year, from, to]) + end + + def query_steps(user_id, year, from, to) + @session.execute(@query_steps, arguments: [user_id, year, from, to]) + end + + def query_activities(user_id, year, from, to) + @session.execute(@query_activities, arguments: [user_id, year, from, to]) + end + + + + private + def init_db - create_statement = @session.prepare(" - CREATE TABLE IF NOT EXISTS #{config.cassandra_keyspace}.? ( - userid text, + @session.execute(' + CREATE TABLE IF NOT EXISTS heart_rate ( + user_id text, + year int, + time timestamp, + start_date timestamp, + end_date timestamp, + value decimal, + PRIMARY KEY ((user_id, year), time) + ) + ') + @session.execute(' + CREATE TABLE IF NOT EXISTS sleep ( + user_id text, + year int, + time timestamp, + start_date timestamp, + end_date timestamp, + value decimal, + PRIMARY KEY ((user_id, year), time) + ) + ') + @session.execute(' + CREATE TABLE IF NOT EXISTS calories ( + user_id text, + year int, + time timestamp, + start_date timestamp, + end_date timestamp, + value decimal, + PRIMARY KEY ((user_id, year), time) + ) + ') + @session.execute(' + CREATE TABLE IF NOT EXISTS distance ( + user_id text, + year int, + time timestamp, + start_date timestamp, + end_date timestamp, + value decimal, + PRIMARY KEY ((user_id, year), time) + ) + ') + @session.execute(' + CREATE TABLE IF NOT EXISTS steps ( + user_id text, year int, time timestamp, - ? ? - PRIMARY KEY ((userid, year), time) - ") - @session.execute(create_statement, arguments: ['heart_rate', 'heart_rate', 'decimal']) - @session.execute(create_statement, arguments: ['sleep', 'sleep', 'decimal']) - @session.execute(create_statement, arguments: ['calories', 'calories', 'decimal']) - @session.execute(create_statement, arguments: ['distance', 'distance', 'decimal']) - @session.execute(create_statement, arguments: ['steps', 'steps', 'decimal']) - @session.execute(create_statement, arguments: ['activities', 'activities', 'decimal']) + start_date timestamp, + end_date timestamp, + value decimal, + PRIMARY KEY ((user_id, year), time) + ) + ') + @session.execute(' + CREATE TABLE IF NOT EXISTS activities ( + user_id text, + year int, + time timestamp, + start_date timestamp, + end_date timestamp, + value varchar, + PRIMARY KEY ((user_id, year), time) + ) + ') end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index ad11c8a..7289b97 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -1,9 +1,10 @@ module Physiqual module DataServices class CassandraDataService < DataServiceDecorator - def initialize(data_service, user) + def initialize(data_service, user_id) super(data_service) - @user = user + @user_id = user_id + @connection = CassandraConnection.instance end def service_name @@ -11,32 +12,204 @@ def service_name end def heart_rate(from, to) - heart_rate = data_service.heart_rate(from, to) + entries = [] + years(from, to) do |year, from_per_year, to_per_year| + entries += make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + end + new_entries = [] + if entries.blank? + new_entries = data_service.heart_rate(from, to) + else + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service.heart_rate(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service.heart_rate(entries.last.end_date, to) + end + end + if new_entries.present? + cache('heart_rate', @user_id, new_entries) + end + entries + new_entries end def sleep(from, to) - sleep = data_service.sleep(from, to) + entries = [] + years(from, to) do |year, from_per_year, to_per_year| + entries += make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + end + new_entries = [] + if entries.blank? + new_entries = data_service.sleep(from, to) + else + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service.sleep(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service.sleep(entries.last.end_date, to) + end + end + if new_entries.present? + cache('sleep', @user_id, new_entries) + end + entries + new_entries end def calories(from, to) - calories_measured = data_service.calories(from, to) + entries = [] + years(from, to) do |year, from_per_year, to_per_year| + entries += make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) + end + new_entries = [] + if entries.blank? + new_entries = data_service.calories(from, to) + else + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service.calories(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service.calories(entries.last.end_date, to) + end + end + if new_entries.present? + cache('calories', @user_id, new_entries) + end + entries + new_entries end def distance(from, to) - distance = data_service.distance(from, to) + entries = [] + years(from, to) do |year, from_per_year, to_per_year| + entries += make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) + end + new_entries = [] + if entries.blank? + new_entries = data_service.distance(from, to) + else + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service.distance(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service.distance(entries.last.end_date, to) + end + end + if new_entries.present? + cache('distance', @user_id, new_entries) + end + entries + new_entries end def steps(from, to) - steps = data_service.steps(from, to) + entries = [] + years(from, to) do |year, from_per_year, to_per_year| + entries += make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) + end + new_entries = [] + if entries.blank? + new_entries = data_service.steps(from, to) + else + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service.steps(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service.steps(entries.last.end_date, to) + end + end + if new_entries.present? + cache('steps', @user_id, new_entries) + end + entries + new_entries end def activities(from, to) - activities = data_service.activities(from, to) + entries = [] + years(from, to) do |year, from_per_year, to_per_year| + entries += make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) + end + new_entries = [] + if entries.blank? + new_entries = data_service.activities(from, to) + else + if entries.first.start_date > from + new_entries += data_service.activities(from, entries.last.start_date) + end + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service.activities(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service.activities(entries.last.end_date, to) + end + end + if new_entries.present? + cache('activities', @user_id, new_entries) + end + entries + new_entries end private - + def years(from, to) + from_year = from.strftime('%Y').to_i + to_year = to.strftime('%Y').to_i + (from_year..to_year).each do |year| + if from_year < year + from_per_year = Time.zone.local(year, 1, 1, 0, 0, 0) + else + from_per_year = from + end + if to_year > year + to_per_year = Time.zone.local(year, 12, 31, 23, 59, 59) + else + to_per_year = to + end + yield(year, from_per_year, to_per_year) + end + end + + def find_gaps(entries) + entries.each_with_index do |entry, i| + break if i == entries.length - 1 + if entry.end_date != entries[i+1].start_date + yield(entry.end_date, entries[i+1].start_date) + end + end + end + + def make_data_entries(results) + return [] if results.blank? + entries = [] + results.each do |result| + entries << DataEntry.new(start_date: result['start_date'].in_time_zone, end_date: result['end_date'].in_time_zone, + values: result['value'], + measurement_moment: result['time'].in_time_zone) + end + entries + end + + def cache(table, user_id, new_entries) + year = new_entries.first.measurement_moment.strftime('%Y').to_i + times = [] + start_dates = [] + end_dates = [] + values = [] + new_entries.each do |entry| + if year != entry.measurement_moment.strftime('%Y').to_i + Physiqual::CacheWorker.perform_async(table, user_id, year, times, start_dates, end_dates, values) + year = entry.measurement_moment.strftime('%Y').to_i + times = [] + start_dates = [] + end_dates = [] + values = [] + end + times << entry.measurement_moment + start_dates << entry.start_date + end_dates << entry.end_date + values << entry.values.first + if entry == new_entries.last + Physiqual::CacheWorker.perform_async(table, user_id, year, times, start_dates, end_dates, values) + end + end + end end end end diff --git a/lib/physiqual/exporters/exporter.rb b/lib/physiqual/exporters/exporter.rb index 0311e67..184bb26 100644 --- a/lib/physiqual/exporters/exporter.rb +++ b/lib/physiqual/exporters/exporter.rb @@ -35,7 +35,7 @@ def create_service(user, bucket_generator) return [] unless token.complete? session = Sessions::TokenAuthorizedSession.new(token) service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) - service = DataServices::CassandraDataService.new(service, user) + service = DataServices::CassandraDataService.new(service, user.user_id) service = DataServices::BucketeerDataService.new(service, bucket_generator) service = DataServices::SummarizedDataService.new(service) DataServices::CachedDataService.new service diff --git a/physiqual.gemspec b/physiqual.gemspec index 18f8fb6..cd8588b 100644 --- a/physiqual.gemspec +++ b/physiqual.gemspec @@ -40,6 +40,8 @@ Gem::Specification.new do |s| # Cassandra caching s.add_dependency 'cassandra-driver' + s.add_dependency 'sidekiq' + s.add_development_dependency 'codeclimate-test-reporter' s.add_development_dependency 'sqlite3' s.add_development_dependency 'rspec-rails' diff --git a/spec/dummy/config/application.rb b/spec/dummy/config/application.rb index 919609a..a83b323 100644 --- a/spec/dummy/config/application.rb +++ b/spec/dummy/config/application.rb @@ -28,5 +28,6 @@ class Application < Rails::Application # Do not swallow errors in after_commit/after_rollback callbacks. config.active_record.raise_in_transactional_callbacks = true + config.eager_load_paths += %W(#{config.root}/app/workers) end end diff --git a/spec/dummy/config/environments/development.rb b/spec/dummy/config/environments/development.rb index b55e214..a6a43fa 100644 --- a/spec/dummy/config/environments/development.rb +++ b/spec/dummy/config/environments/development.rb @@ -7,7 +7,7 @@ config.cache_classes = false # Do not eager load code on boot. - config.eager_load = false + config.eager_load = true # Show full error reports and disable caching. config.consider_all_requests_local = true diff --git a/spec/dummy/config/environments/test.rb b/spec/dummy/config/environments/test.rb index b4b4b9f..ec95664 100644 --- a/spec/dummy/config/environments/test.rb +++ b/spec/dummy/config/environments/test.rb @@ -17,7 +17,7 @@ # Do not eager load code on boot. This avoids loading your whole application # just for the purpose of running a single test. If you are using a tool that # preloads Rails for running tests, you may have to set it to true. - config.eager_load = false + config.eager_load = true # Configure static file server for tests with Cache-Control for performance. config.serve_static_files = true From c8dd572b01d1ddef702b04219ed8d5bff92ee3b1 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 16 Jun 2016 13:58:56 +0200 Subject: [PATCH 11/35] Some refactoring --- app/workers/physiqual/cache_worker.rb | 15 +- .../data_services/cassandra_connection.rb | 121 ++---------- .../data_services/cassandra_data_service.rb | 174 ++++++------------ 3 files changed, 72 insertions(+), 238 deletions(-) diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb index 11734bb..fa961fe 100644 --- a/app/workers/physiqual/cache_worker.rb +++ b/app/workers/physiqual/cache_worker.rb @@ -5,20 +5,7 @@ class CacheWorker include Sidekiq::Worker def perform(table, user_id, year, times, start_dates, end_dates, values) connection = DataServices::CassandraConnection.instance - case table - when 'heart_rate' - connection.insert_heart_rate(user_id, year, times, start_dates, end_dates, values) - when 'sleep' - connection.insert_sleep(user_id, year, times, start_dates, end_dates, values) - when 'calories' - connection.insert_calories(user_id, year, times, start_dates, end_dates, values) - when 'distance' - connection.insert_distance(user_id, year, times, start_dates, end_dates, values) - when 'steps' - connection.insert_steps(user_id, year, times, start_dates, end_dates, values) - when 'activities' - connection.insert_activities(user_id, year, times, start_dates, end_dates, values) - end + connection.insert(table, user_id, year, times, start_dates, end_dates, values) end end end \ No newline at end of file diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index 277ff90..ad0fa33 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -125,7 +125,7 @@ def initialize - def insert_heart_rate(user_id, year, times, start_dates, end_dates, values) + def insert(table, user_id, year, times, start_dates, end_dates, values) times_slices = times.each_slice(100).to_a start_dates_slices = start_dates.each_slice(100).to_a end_dates_slices = end_dates.each_slice(100).to_a @@ -138,115 +138,26 @@ def insert_heart_rate(user_id, year, times, start_dates, end_dates, values) batch = @session.batch do |b| times_slice.each_with_index do |time, i| - b.add(@insert_heart_rate, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + case table + when 'heart_rate' + b.add(@insert_heart_rate, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + when 'sleep' + b.add(@insert_sleep, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + when 'calories' + b.add(@insert_calories, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + when 'distance' + b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + when 'steps' + b.add(@insert_steps, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) + when 'activities' + b.add(@insert_activities, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, values_slice[i], 10]) + end end end @session.execute(batch) end end - def insert_sleep(user_id, year, times, start_dates, end_dates, values) - times_slices = times.each_slice(100).to_a - start_dates_slices = start_dates.each_slice(100).to_a - end_dates_slices = end_dates.each_slice(100).to_a - values_slices = values.each_slice(100).to_a - - times_slices.each_with_index do |times_slice, i| - start_dates_slice = start_dates_slices[i] - end_dates_slice = end_dates_slices[i] - values_slice = values_slices[i] - - batch = @session.batch do |b| - times_slice.each_with_index do |time, i| - b.add(@insert_sleep, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - end - end - @session.execute(batch) - end - end - - def insert_calories(user_id, year, times, start_dates, end_dates, values) - times_slices = times.each_slice(100).to_a - start_dates_slices = start_dates.each_slice(100).to_a - end_dates_slices = end_dates.each_slice(100).to_a - values_slices = values.each_slice(100).to_a - - times_slices.each_with_index do |times_slice, i| - start_dates_slice = start_dates_slices[i] - end_dates_slice = end_dates_slices[i] - values_slice = values_slices[i] - - batch = @session.batch do |b| - times_slice.each_with_index do |time, i| - b.add(@insert_calories, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - end - end - @session.execute(batch) - end - end - - def insert_distance(user_id, year, times, start_dates, end_dates, values) - times_slices = times.each_slice(100).to_a - start_dates_slices = start_dates.each_slice(100).to_a - end_dates_slices = end_dates.each_slice(100).to_a - values_slices = values.each_slice(100).to_a - - times_slices.each_with_index do |times_slice, i| - start_dates_slice = start_dates_slices[i] - end_dates_slice = end_dates_slices[i] - values_slice = values_slices[i] - - batch = @session.batch do |b| - times_slice.each_with_index do |time, i| - b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - end - end - @session.execute(batch) - end - end - - def insert_steps(user_id, year, times, start_dates, end_dates, values) - times_slices = times.each_slice(100).to_a - start_dates_slices = start_dates.each_slice(100).to_a - end_dates_slices = end_dates.each_slice(100).to_a - values_slices = values.each_slice(100).to_a - - times_slices.each_with_index do |times_slice, i| - start_dates_slice = start_dates_slices[i] - end_dates_slice = end_dates_slices[i] - values_slice = values_slices[i] - - batch = @session.batch do |b| - times_slice.each_with_index do |time, i| - b.add(@insert_steps, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - end - end - @session.execute(batch) - end - end - - def insert_activities(user_id, year, times, start_dates, end_dates, values) - times_slices = times.each_slice(100).to_a - start_dates_slices = start_dates.each_slice(100).to_a - end_dates_slices = end_dates.each_slice(100).to_a - values_slices = values.each_slice(100).to_a - - times_slices.each_with_index do |times_slice, i| - start_dates_slice = start_dates_slices[i] - end_dates_slice = end_dates_slices[i] - values_slice = values_slices[i] - - batch = @session.batch do |b| - times_slice.each_with_index do |time, i| - b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, values_slice[i]]) - end - end - @session.execute(batch) - end - end - - - def query_heart_rate(user_id, year, from, to) @session.execute(@query_heart_rate, arguments: [user_id, year, from, to]) end @@ -271,8 +182,6 @@ def query_activities(user_id, year, from, to) @session.execute(@query_activities, arguments: [user_id, year, from, to]) end - - private def init_db diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 7289b97..fb5015e 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -12,156 +12,94 @@ def service_name end def heart_rate(from, to) - entries = [] - years(from, to) do |year, from_per_year, to_per_year| - entries += make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) - end - new_entries = [] - if entries.blank? - new_entries = data_service.heart_rate(from, to) - else - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service.heart_rate(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service.heart_rate(entries.last.end_date, to) - end - end - if new_entries.present? - cache('heart_rate', @user_id, new_entries) - end - entries + new_entries + get_data('heart_rate', from, to) end def sleep(from, to) - entries = [] - years(from, to) do |year, from_per_year, to_per_year| - entries += make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) - end - new_entries = [] - if entries.blank? - new_entries = data_service.sleep(from, to) - else - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service.sleep(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service.sleep(entries.last.end_date, to) - end - end - if new_entries.present? - cache('sleep', @user_id, new_entries) - end - entries + new_entries + get_data('sleep', from, to) end def calories(from, to) - entries = [] - years(from, to) do |year, from_per_year, to_per_year| - entries += make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) - end - new_entries = [] - if entries.blank? - new_entries = data_service.calories(from, to) - else - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service.calories(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service.calories(entries.last.end_date, to) - end - end - if new_entries.present? - cache('calories', @user_id, new_entries) - end - entries + new_entries + get_data('calories', from, to) end def distance(from, to) - entries = [] - years(from, to) do |year, from_per_year, to_per_year| - entries += make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) - end - new_entries = [] - if entries.blank? - new_entries = data_service.distance(from, to) - else - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service.distance(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service.distance(entries.last.end_date, to) - end - end - if new_entries.present? - cache('distance', @user_id, new_entries) - end - entries + new_entries + get_data('distance', from, to) end def steps(from, to) - entries = [] - years(from, to) do |year, from_per_year, to_per_year| - entries += make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) - end - new_entries = [] - if entries.blank? - new_entries = data_service.steps(from, to) - else - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service.steps(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service.steps(entries.last.end_date, to) - end - end - if new_entries.present? - cache('steps', @user_id, new_entries) - end - entries + new_entries + get_data('steps', from, to) end def activities(from, to) + get_data('activities', from, to) + end + + private + + def get_data(table, from, to) entries = [] years(from, to) do |year, from_per_year, to_per_year| - entries += make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) - end + entries += case table + when 'heart_rate' + make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + when 'sleep' + make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + when 'calories' + make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) + when 'distance' + make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) + when 'steps' + make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) + when 'activities' + make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) + end + end + data_service_function = case table + when 'heart_rate' + data_service.method(:heart_rate) + when 'sleep' + data_service.method(:sleep) + when 'calories' + data_service.method(:calories) + when 'distance' + data_service.method(:distance) + when 'steps' + data_service.method(:steps) + when 'activities' + data_service.method(:activities) + end new_entries = [] if entries.blank? - new_entries = data_service.activities(from, to) + new_entries = data_service_function.call(from, to) else - if entries.first.start_date > from - new_entries += data_service.activities(from, entries.last.start_date) - end find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service.activities(from_gap, to_gap) + new_entries += data_service_function.call(from_gap, to_gap) end if entries.last.end_date < to - new_entries += data_service.activities(entries.last.end_date, to) + new_entries += data_service_function.call(entries.last.end_date, to) end end if new_entries.present? - cache('activities', @user_id, new_entries) + cache(table, @user_id, new_entries) end entries + new_entries end - private - def years(from, to) from_year = from.strftime('%Y').to_i to_year = to.strftime('%Y').to_i (from_year..to_year).each do |year| - if from_year < year - from_per_year = Time.zone.local(year, 1, 1, 0, 0, 0) - else - from_per_year = from - end - if to_year > year - to_per_year = Time.zone.local(year, 12, 31, 23, 59, 59) - else - to_per_year = to - end + from_per_year = if from_year < year + Time.zone.local(year, 1, 1, 0, 0, 0) + else + from + end + to_per_year = if to_year > year + Time.zone.local(year, 12, 31, 23, 59, 59) + else + to + end yield(year, from_per_year, to_per_year) end end @@ -180,8 +118,8 @@ def make_data_entries(results) entries = [] results.each do |result| entries << DataEntry.new(start_date: result['start_date'].in_time_zone, end_date: result['end_date'].in_time_zone, - values: result['value'], - measurement_moment: result['time'].in_time_zone) + values: result['value'], + measurement_moment: result['time'].in_time_zone) end entries end From 50f0de434260b53e6675be8016d7b491bd0a7d14 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 16 Jun 2016 14:15:34 +0200 Subject: [PATCH 12/35] Some refactoring --- app/workers/physiqual/cache_worker.rb | 2 +- .../data_services/cassandra_connection.rb | 262 ++++++++---------- .../data_services/cassandra_data_service.rb | 60 ++-- 3 files changed, 144 insertions(+), 180 deletions(-) diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb index fa961fe..f2a70cc 100644 --- a/app/workers/physiqual/cache_worker.rb +++ b/app/workers/physiqual/cache_worker.rb @@ -8,4 +8,4 @@ def perform(table, user_id, year, times, start_dates, end_dates, values) connection.insert(table, user_id, year, times, start_dates, end_dates, values) end end -end \ No newline at end of file +end diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index ad0fa33..37ca6d0 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -8,7 +8,7 @@ class CassandraConnection def initialize @cluster = nil - if (!ENV['CASSANDRA_USERNAME']) || (!ENV['CASSANDRA_PASSWORD']) || ENV['CASSANDRA_USERNAME'] == '' then + if !ENV['CASSANDRA_USERNAME'] || !ENV['CASSANDRA_PASSWORD'] || ENV['CASSANDRA_USERNAME'] == '' @cluster = Cassandra.cluster( hosts: (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') ) @@ -22,109 +22,10 @@ def initialize @session = @cluster.connect('physiqual') init_db - - @insert_heart_rate = @session.prepare(' - INSERT INTO heart_rate ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_sleep = @session.prepare(' - INSERT INTO sleep ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_calories = @session.prepare(' - INSERT INTO calories ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_distance = @session.prepare(' - INSERT INTO distance ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_steps = @session.prepare(' - INSERT INTO steps ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_activities = @session.prepare(' - INSERT INTO activities ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - - @query_heart_rate = @session.prepare(' - SELECT time, start_date, end_date, value - FROM heart_rate - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_sleep = @session.prepare(' - SELECT time, start_date, end_date, value - FROM sleep - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_calories = @session.prepare(' - SELECT time, start_date, end_date, value - FROM calories - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_distance = @session.prepare(' - SELECT time, start_date, end_date, value - FROM distance - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_steps = @session.prepare(' - SELECT time, start_date, end_date, value - FROM steps - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_activities = @session.prepare(' - SELECT time, start_date, end_date, value - FROM activities - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - + init_insert + init_query end - - def insert(table, user_id, year, times, start_dates, end_dates, values) times_slices = times.each_slice(100).to_a start_dates_slices = start_dates.each_slice(100).to_a @@ -137,21 +38,26 @@ def insert(table, user_id, year, times, start_dates, end_dates, values) values_slice = values_slices[i] batch = @session.batch do |b| - times_slice.each_with_index do |time, i| + times_slice.each_with_index do |time, j| + insert_type = nil + value = BigDecimal(values_slice[j], 10) case table - when 'heart_rate' - b.add(@insert_heart_rate, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'sleep' - b.add(@insert_sleep, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'calories' - b.add(@insert_calories, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'distance' - b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'steps' - b.add(@insert_steps, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'activities' - b.add(@insert_activities, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, values_slice[i], 10]) + when 'heart_rate' + insert_type = @insert_heart_rate + when 'sleep' + insert_type = @insert_sleep + when 'calories' + insert_type = @insert_calories + when 'distance' + insert_type = @insert_distance + when 'steps' + insert_type = @insert_steps + when 'activities' + insert_type = @insert_activities + value = values_slice[j] end + b.add(insert_type, arguments: [user_id, year, time.to_time, start_dates_slice[j].to_time, + end_dates_slice[j].to_time, value]) end end @session.execute(batch) @@ -187,71 +93,125 @@ def query_activities(user_id, year, from, to) def init_db @session.execute(' CREATE TABLE IF NOT EXISTS heart_rate ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS sleep ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS calories ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS distance ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS steps ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS activities ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value varchar, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value varchar, PRIMARY KEY ((user_id, year), time) ) ') end + + def init_insert + @insert_heart_rate = @session.prepare(' + INSERT INTO heart_rate ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_sleep = @session.prepare(' + INSERT INTO sleep ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_calories = @session.prepare(' + INSERT INTO calories ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_distance = @session.prepare(' + INSERT INTO distance ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_steps = @session.prepare(' + INSERT INTO steps ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_activities = @session.prepare(' + INSERT INTO activities ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + end + + def init_query + @query_heart_rate = @session.prepare(' + SELECT time, start_date, end_date, value + FROM heart_rate + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_sleep = @session.prepare(' + SELECT time, start_date, end_date, value + FROM sleep + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_calories = @session.prepare(' + SELECT time, start_date, end_date, value + FROM calories + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_distance = @session.prepare(' + SELECT time, start_date, end_date, value + FROM distance + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_steps = @session.prepare(' + SELECT time, start_date, end_date, value + FROM steps + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_activities = @session.prepare(' + SELECT time, start_date, end_date, value + FROM activities + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + end end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index fb5015e..519d171 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -41,34 +41,21 @@ def get_data(table, from, to) entries = [] years(from, to) do |year, from_per_year, to_per_year| entries += case table - when 'heart_rate' - make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) - when 'sleep' - make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) - when 'calories' - make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) - when 'distance' - make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) - when 'steps' - make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) - when 'activities' - make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) + when 'heart_rate' + make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + when 'sleep' + make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + when 'calories' + make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) + when 'distance' + make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) + when 'steps' + make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) + when 'activities' + make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) end end - data_service_function = case table - when 'heart_rate' - data_service.method(:heart_rate) - when 'sleep' - data_service.method(:sleep) - when 'calories' - data_service.method(:calories) - when 'distance' - data_service.method(:distance) - when 'steps' - data_service.method(:steps) - when 'activities' - data_service.method(:activities) - end + data_service_function = get_data_function(table) new_entries = [] if entries.blank? new_entries = data_service_function.call(from, to) @@ -86,6 +73,23 @@ def get_data(table, from, to) entries + new_entries end + def get_data_function(table) + case table + when 'heart_rate' + data_service.method(:heart_rate) + when 'sleep' + data_service.method(:sleep) + when 'calories' + data_service.method(:calories) + when 'distance' + data_service.method(:distance) + when 'steps' + data_service.method(:steps) + when 'activities' + data_service.method(:activities) + end + end + def years(from, to) from_year = from.strftime('%Y').to_i to_year = to.strftime('%Y').to_i @@ -107,8 +111,8 @@ def years(from, to) def find_gaps(entries) entries.each_with_index do |entry, i| break if i == entries.length - 1 - if entry.end_date != entries[i+1].start_date - yield(entry.end_date, entries[i+1].start_date) + if entry.end_date != entries[i + 1].start_date + yield(entry.end_date, entries[i + 1].start_date) end end end From e7e9c8266b11c89a9ceb30ac57f581e8a38b5d3f Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 16 Jun 2016 14:15:34 +0200 Subject: [PATCH 13/35] Some refactoring --- app/models/physiqual/data_entry.rb | 2 - app/workers/physiqual/cache_worker.rb | 2 +- .../data_services/cassandra_connection.rb | 262 ++++++++---------- .../data_services/cassandra_data_service.rb | 64 ++--- 4 files changed, 145 insertions(+), 185 deletions(-) diff --git a/app/models/physiqual/data_entry.rb b/app/models/physiqual/data_entry.rb index 6cdff5f..3d526cb 100644 --- a/app/models/physiqual/data_entry.rb +++ b/app/models/physiqual/data_entry.rb @@ -7,12 +7,10 @@ class DataEntry # The start date property is the start of this dataentry object, meaning it # denotes the actual start of the data in this measurement attribute :start_date, DateTime - # The end date property denotes the end of this measurement (exclusing the time). # Meaning, data in this data entry is upto, but not including this time point attribute :end_date, DateTime attribute :values, Array, default: [] - # Measurement moment is used in the physiqual process. It is used to determine # the actual 'measurement time' of the current dataentry object. Often it's # precisely in the middle of the start and end date of this data entry object. diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb index fa961fe..f2a70cc 100644 --- a/app/workers/physiqual/cache_worker.rb +++ b/app/workers/physiqual/cache_worker.rb @@ -8,4 +8,4 @@ def perform(table, user_id, year, times, start_dates, end_dates, values) connection.insert(table, user_id, year, times, start_dates, end_dates, values) end end -end \ No newline at end of file +end diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index ad0fa33..37ca6d0 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -8,7 +8,7 @@ class CassandraConnection def initialize @cluster = nil - if (!ENV['CASSANDRA_USERNAME']) || (!ENV['CASSANDRA_PASSWORD']) || ENV['CASSANDRA_USERNAME'] == '' then + if !ENV['CASSANDRA_USERNAME'] || !ENV['CASSANDRA_PASSWORD'] || ENV['CASSANDRA_USERNAME'] == '' @cluster = Cassandra.cluster( hosts: (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') ) @@ -22,109 +22,10 @@ def initialize @session = @cluster.connect('physiqual') init_db - - @insert_heart_rate = @session.prepare(' - INSERT INTO heart_rate ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_sleep = @session.prepare(' - INSERT INTO sleep ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_calories = @session.prepare(' - INSERT INTO calories ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_distance = @session.prepare(' - INSERT INTO distance ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_steps = @session.prepare(' - INSERT INTO steps ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_activities = @session.prepare(' - INSERT INTO activities ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - - @query_heart_rate = @session.prepare(' - SELECT time, start_date, end_date, value - FROM heart_rate - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_sleep = @session.prepare(' - SELECT time, start_date, end_date, value - FROM sleep - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_calories = @session.prepare(' - SELECT time, start_date, end_date, value - FROM calories - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_distance = @session.prepare(' - SELECT time, start_date, end_date, value - FROM distance - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_steps = @session.prepare(' - SELECT time, start_date, end_date, value - FROM steps - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - @query_activities = @session.prepare(' - SELECT time, start_date, end_date, value - FROM activities - WHERE user_id = ? - AND year = ? - AND time >= ? - AND time < ? - ORDER BY time ASC - ') - + init_insert + init_query end - - def insert(table, user_id, year, times, start_dates, end_dates, values) times_slices = times.each_slice(100).to_a start_dates_slices = start_dates.each_slice(100).to_a @@ -137,21 +38,26 @@ def insert(table, user_id, year, times, start_dates, end_dates, values) values_slice = values_slices[i] batch = @session.batch do |b| - times_slice.each_with_index do |time, i| + times_slice.each_with_index do |time, j| + insert_type = nil + value = BigDecimal(values_slice[j], 10) case table - when 'heart_rate' - b.add(@insert_heart_rate, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'sleep' - b.add(@insert_sleep, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'calories' - b.add(@insert_calories, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'distance' - b.add(@insert_distance, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'steps' - b.add(@insert_steps, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, BigDecimal(values_slice[i], 10)]) - when 'activities' - b.add(@insert_activities, arguments: [user_id, year, time.to_time, start_dates_slice[i].to_time, end_dates_slice[i].to_time, values_slice[i], 10]) + when 'heart_rate' + insert_type = @insert_heart_rate + when 'sleep' + insert_type = @insert_sleep + when 'calories' + insert_type = @insert_calories + when 'distance' + insert_type = @insert_distance + when 'steps' + insert_type = @insert_steps + when 'activities' + insert_type = @insert_activities + value = values_slice[j] end + b.add(insert_type, arguments: [user_id, year, time.to_time, start_dates_slice[j].to_time, + end_dates_slice[j].to_time, value]) end end @session.execute(batch) @@ -187,71 +93,125 @@ def query_activities(user_id, year, from, to) def init_db @session.execute(' CREATE TABLE IF NOT EXISTS heart_rate ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS sleep ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS calories ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS distance ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS steps ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value decimal, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, PRIMARY KEY ((user_id, year), time) ) ') @session.execute(' CREATE TABLE IF NOT EXISTS activities ( - user_id text, - year int, - time timestamp, - start_date timestamp, - end_date timestamp, - value varchar, + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value varchar, PRIMARY KEY ((user_id, year), time) ) ') end + + def init_insert + @insert_heart_rate = @session.prepare(' + INSERT INTO heart_rate ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_sleep = @session.prepare(' + INSERT INTO sleep ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_calories = @session.prepare(' + INSERT INTO calories ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_distance = @session.prepare(' + INSERT INTO distance ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_steps = @session.prepare(' + INSERT INTO steps ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + @insert_activities = @session.prepare(' + INSERT INTO activities ( + user_id, year, time, start_date, end_date, value + ) VALUES ( + ?, ?, ?, ?, ?, ? + ) + ') + end + + def init_query + @query_heart_rate = @session.prepare(' + SELECT time, start_date, end_date, value + FROM heart_rate + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_sleep = @session.prepare(' + SELECT time, start_date, end_date, value + FROM sleep + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_calories = @session.prepare(' + SELECT time, start_date, end_date, value + FROM calories + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_distance = @session.prepare(' + SELECT time, start_date, end_date, value + FROM distance + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_steps = @session.prepare(' + SELECT time, start_date, end_date, value + FROM steps + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + @query_activities = @session.prepare(' + SELECT time, start_date, end_date, value + FROM activities + WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + ORDER BY time ASC + ') + end end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index fb5015e..5fe0347 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -41,34 +41,21 @@ def get_data(table, from, to) entries = [] years(from, to) do |year, from_per_year, to_per_year| entries += case table - when 'heart_rate' - make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) - when 'sleep' - make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) - when 'calories' - make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) - when 'distance' - make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) - when 'steps' - make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) - when 'activities' - make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) + when 'heart_rate' + make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + when 'sleep' + make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + when 'calories' + make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) + when 'distance' + make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) + when 'steps' + make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) + when 'activities' + make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) end end - data_service_function = case table - when 'heart_rate' - data_service.method(:heart_rate) - when 'sleep' - data_service.method(:sleep) - when 'calories' - data_service.method(:calories) - when 'distance' - data_service.method(:distance) - when 'steps' - data_service.method(:steps) - when 'activities' - data_service.method(:activities) - end + data_service_function = get_data_function(table) new_entries = [] if entries.blank? new_entries = data_service_function.call(from, to) @@ -80,12 +67,27 @@ def get_data(table, from, to) new_entries += data_service_function.call(entries.last.end_date, to) end end - if new_entries.present? - cache(table, @user_id, new_entries) - end + cache(table, @user_id, new_entries) if new_entries.present? entries + new_entries end + def get_data_function(table) + case table + when 'heart_rate' + data_service.method(:heart_rate) + when 'sleep' + data_service.method(:sleep) + when 'calories' + data_service.method(:calories) + when 'distance' + data_service.method(:distance) + when 'steps' + data_service.method(:steps) + when 'activities' + data_service.method(:activities) + end + end + def years(from, to) from_year = from.strftime('%Y').to_i to_year = to.strftime('%Y').to_i @@ -107,8 +109,8 @@ def years(from, to) def find_gaps(entries) entries.each_with_index do |entry, i| break if i == entries.length - 1 - if entry.end_date != entries[i+1].start_date - yield(entry.end_date, entries[i+1].start_date) + if entry.end_date != entries[i + 1].start_date + yield(entry.end_date, entries[i + 1].start_date) end end end From 848ffebab90feacaf33e708183444da3324a8282 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 16 Jun 2016 15:37:03 +0200 Subject: [PATCH 14/35] Fix bugs, pull candidate --- .../data_services/cached_data_service.rb | 2 +- .../data_services/cassandra_connection.rb | 14 ++++----- .../data_services/cassandra_data_service.rb | 30 ++++++++++++++----- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lib/physiqual/data_services/cached_data_service.rb b/lib/physiqual/data_services/cached_data_service.rb index 3fdccbc..68c3863 100644 --- a/lib/physiqual/data_services/cached_data_service.rb +++ b/lib/physiqual/data_services/cached_data_service.rb @@ -37,7 +37,7 @@ def distance(from, to) private def from_cache(type) - Rails.logger.warn("#{type} of #{service_name} not from cache.. ") unless @cache.include? type + Rails.logger.warn("#{type} of #{service_name} not duplicate...") unless @cache.include? type @cache[type] = yield unless @cache.include? type @cache[type] end diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index 37ca6d0..6512d48 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -40,7 +40,7 @@ def insert(table, user_id, year, times, start_dates, end_dates, values) batch = @session.batch do |b| times_slice.each_with_index do |time, j| insert_type = nil - value = BigDecimal(values_slice[j], 10) + value = BigDecimal(values_slice[j], Float::DIG + 1) case table when 'heart_rate' insert_type = @insert_heart_rate @@ -178,37 +178,37 @@ def init_query @query_heart_rate = @session.prepare(' SELECT time, start_date, end_date, value FROM heart_rate - WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC ') @query_sleep = @session.prepare(' SELECT time, start_date, end_date, value FROM sleep - WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC ') @query_calories = @session.prepare(' SELECT time, start_date, end_date, value FROM calories - WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC ') @query_distance = @session.prepare(' SELECT time, start_date, end_date, value FROM distance - WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC ') @query_steps = @session.prepare(' SELECT time, start_date, end_date, value FROM steps - WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC ') @query_activities = @session.prepare(' SELECT time, start_date, end_date, value FROM activities - WHERE user_id = ? AND year = ? AND time >= ? AND time < ? + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC ') end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 30eb9e3..6d91f1c 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -42,28 +42,35 @@ def get_data(table, from, to) years(from, to) do |year, from_per_year, to_per_year| entries += case table when 'heart_rate' - make_data_entries(@connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, @connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) when 'sleep' - make_data_entries(@connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, @connection.query_sleep(@user_id, year, from_per_year, to_per_year)) when 'calories' - make_data_entries(@connection.query_calories(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, @connection.query_calories(@user_id, year, from_per_year, to_per_year)) when 'distance' - make_data_entries(@connection.query_distance(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, @connection.query_distance(@user_id, year, from_per_year, to_per_year)) when 'steps' - make_data_entries(@connection.query_steps(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, @connection.query_steps(@user_id, year, from_per_year, to_per_year)) when 'activities' - make_data_entries(@connection.query_activities(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, @connection.query_activities(@user_id, year, from_per_year, to_per_year)) end end data_service_function = get_data_function(table) new_entries = [] if entries.blank? + Rails.logger.warn("#{table} call not in cache...") new_entries = data_service_function.call(from, to) else + if entries.first.start_date > from + Rails.logger.warn("#{table} data from #{from} to #{entries.first.start_date} not in cache...") + new_entries += data_service_function.call(from, entries.first.start_date) + end find_gaps(entries) do |from_gap, to_gap| + Rails.logger.warn("#{table} data from #{from_gap} to #{to_gap} not in cache...") new_entries += data_service_function.call(from_gap, to_gap) end if entries.last.end_date < to + Rails.logger.warn("#{table} data from #{entries.last.end_date} to #{to} not in cache...") new_entries += data_service_function.call(entries.last.end_date, to) end end @@ -115,13 +122,18 @@ def find_gaps(entries) end end - def make_data_entries(results) + def make_data_entries(table, results) return [] if results.blank? entries = [] results.each do |result| + value = if table == 'activities' + result['value'] + else + result['value'].to_f + end entries << DataEntry.new(start_date: result['start_date'].in_time_zone, end_date: result['end_date'].in_time_zone, - values: result['value'], + values: value, measurement_moment: result['time'].in_time_zone) end entries @@ -135,6 +147,7 @@ def cache(table, user_id, new_entries) values = [] new_entries.each do |entry| if year != entry.measurement_moment.strftime('%Y').to_i + Rails.logger.info("Caching from #{start_dates.first} to #{end_dates.last}...") Physiqual::CacheWorker.perform_async(table, user_id, year, times, start_dates, end_dates, values) year = entry.measurement_moment.strftime('%Y').to_i times = [] @@ -147,6 +160,7 @@ def cache(table, user_id, new_entries) end_dates << entry.end_date values << entry.values.first if entry == new_entries.last + Rails.logger.info("Caching from #{start_dates.first} to #{end_dates.last}...") Physiqual::CacheWorker.perform_async(table, user_id, year, times, start_dates, end_dates, values) end end From e99e3a474d6a02c4db0e1f398e7e63b97ed45209 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Thu, 16 Jun 2016 15:40:13 +0200 Subject: [PATCH 15/35] Rubocop --- .../data_services/cassandra_data_service.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 6d91f1c..34f6cc2 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -42,17 +42,23 @@ def get_data(table, from, to) years(from, to) do |year, from_per_year, to_per_year| entries += case table when 'heart_rate' - make_data_entries(table, @connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, + @connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) when 'sleep' - make_data_entries(table, @connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, + @connection.query_sleep(@user_id, year, from_per_year, to_per_year)) when 'calories' - make_data_entries(table, @connection.query_calories(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, + @connection.query_calories(@user_id, year, from_per_year, to_per_year)) when 'distance' - make_data_entries(table, @connection.query_distance(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, + @connection.query_distance(@user_id, year, from_per_year, to_per_year)) when 'steps' - make_data_entries(table, @connection.query_steps(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, + @connection.query_steps(@user_id, year, from_per_year, to_per_year)) when 'activities' - make_data_entries(table, @connection.query_activities(@user_id, year, from_per_year, to_per_year)) + make_data_entries(table, + @connection.query_activities(@user_id, year, from_per_year, to_per_year)) end end data_service_function = get_data_function(table) From e9135237c95dfd64e71cfe310e4ac7fab35671b0 Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Wed, 6 Jul 2016 23:11:35 +0200 Subject: [PATCH 16/35] Entirety of caching moved to Sidekiq. --- app/workers/physiqual/cache_worker.rb | 89 ++++++++++++- .../data_services/cassandra_data_service.rb | 118 +++++------------- physiqual.gemspec | 1 + spec/dummy/config/initializers/sidekiq.rb | 20 +++ 4 files changed, 138 insertions(+), 90 deletions(-) create mode 100644 spec/dummy/config/initializers/sidekiq.rb diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb index f2a70cc..aaec809 100644 --- a/app/workers/physiqual/cache_worker.rb +++ b/app/workers/physiqual/cache_worker.rb @@ -3,9 +3,94 @@ module Physiqual class CacheWorker include Sidekiq::Worker - def perform(table, user_id, year, times, start_dates, end_dates, values) + def perform(table, user_id, from, to) + token = User.find_by_user_id(user_id).physiqual_token + return [] unless token.complete? + session = Sessions::TokenAuthorizedSession.new(token) + @data_service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) + @user_id = user_id connection = DataServices::CassandraConnection.instance - connection.insert(table, user_id, year, times, start_dates, end_dates, values) + from = Time.zone.parse(from) + to = Time.zone.parse(to) + store_data(connection, table, from, to) + end + + private + + def store_data(connection, table, from, to) + begin + entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) + data_service_function = get_data_function(table) + new_entries = [] + if entries.blank? + new_entries = data_service_function.call(from, to) + else + if entries.first.start_date > from + new_entries += data_service_function.call(from, entries.first.start_date) + end + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service_function.call(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service_function.call(entries.last.end_date, to) + end + end + cache(connection, table, @user_id, new_entries) if new_entries.present? + rescue Errors::NotSupportedError => e + Rails.logger.warn e.message + end + + end + + def get_data_function(table) + case table + when 'heart_rate' + @data_service.method(:heart_rate) + when 'sleep' + @data_service.method(:sleep) + when 'calories' + @data_service.method(:calories) + when 'distance' + @data_service.method(:distance) + when 'steps' + @data_service.method(:steps) + when 'activities' + @data_service.method(:activities) + end + end + + def find_gaps(entries) + entries.each_with_index do |entry, i| + break if i == entries.length - 1 + if entry.end_date != entries[i + 1].start_date + yield(entry.end_date, entries[i + 1].start_date) + end + end + end + + def cache(connection, table, user_id, new_entries) + year = new_entries.first.measurement_moment.strftime('%Y').to_i + times = [] + start_dates = [] + end_dates = [] + values = [] + new_entries.each do |entry| + if year != entry.measurement_moment.strftime('%Y').to_i + connection.insert(table, user_id, year, times, start_dates, end_dates, values) + year = entry.measurement_moment.strftime('%Y').to_i + times = [] + start_dates = [] + end_dates = [] + values = [] + end + times << entry.measurement_moment + start_dates << entry.start_date + end_dates << entry.end_date + values << entry.values.first + if entry == new_entries.last + connection.insert(table, user_id, year, times, start_dates, end_dates, values) + end + end end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 34f6cc2..424a0ad 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -1,3 +1,5 @@ +require 'sidekiq-status' + module Physiqual module DataServices class CassandraDataService < DataServiceDecorator @@ -12,96 +14,72 @@ def service_name end def heart_rate(from, to) - get_data('heart_rate', from, to) + cache_data('heart_rate', @user_id, from, to) + CassandraDataService.get_data(@connection, @user_id, 'heart_rate', from, to) end def sleep(from, to) - get_data('sleep', from, to) + cache_data('sleep', @user_id, from, to) + CassandraDataService.get_data(@connection, @user_id, 'sleep', from, to) end def calories(from, to) - get_data('calories', from, to) + cache_data('calories', @user_id, from, to) + CassandraDataService.get_data(@connection, @user_id, 'calories', from, to) end def distance(from, to) - get_data('distance', from, to) + cache_data('distance', @user_id, from, to) + CassandraDataService.get_data(@connection, @user_id, 'distance', from, to) end def steps(from, to) - get_data('steps', from, to) + cache_data('steps', @user_id, from, to) + CassandraDataService.get_data(@connection, @user_id, 'steps', from, to) end def activities(from, to) - get_data('activities', from, to) + cache_data('activities', @user_id, from, to) + CassandraDataService.get_data(@connection, @user_id, 'activities', from, to) end - private - - def get_data(table, from, to) + def self.get_data(connection, user_id, table, from, to) entries = [] years(from, to) do |year, from_per_year, to_per_year| entries += case table when 'heart_rate' make_data_entries(table, - @connection.query_heart_rate(@user_id, year, from_per_year, to_per_year)) + connection.query_heart_rate(user_id, year, from_per_year, to_per_year)) when 'sleep' make_data_entries(table, - @connection.query_sleep(@user_id, year, from_per_year, to_per_year)) + connection.query_sleep(user_id, year, from_per_year, to_per_year)) when 'calories' make_data_entries(table, - @connection.query_calories(@user_id, year, from_per_year, to_per_year)) + connection.query_calories(user_id, year, from_per_year, to_per_year)) when 'distance' make_data_entries(table, - @connection.query_distance(@user_id, year, from_per_year, to_per_year)) + connection.query_distance(user_id, year, from_per_year, to_per_year)) when 'steps' make_data_entries(table, - @connection.query_steps(@user_id, year, from_per_year, to_per_year)) + connection.query_steps(user_id, year, from_per_year, to_per_year)) when 'activities' make_data_entries(table, - @connection.query_activities(@user_id, year, from_per_year, to_per_year)) + connection.query_activities(user_id, year, from_per_year, to_per_year)) end end - data_service_function = get_data_function(table) - new_entries = [] - if entries.blank? - Rails.logger.warn("#{table} call not in cache...") - new_entries = data_service_function.call(from, to) - else - if entries.first.start_date > from - Rails.logger.warn("#{table} data from #{from} to #{entries.first.start_date} not in cache...") - new_entries += data_service_function.call(from, entries.first.start_date) - end - find_gaps(entries) do |from_gap, to_gap| - Rails.logger.warn("#{table} data from #{from_gap} to #{to_gap} not in cache...") - new_entries += data_service_function.call(from_gap, to_gap) - end - if entries.last.end_date < to - Rails.logger.warn("#{table} data from #{entries.last.end_date} to #{to} not in cache...") - new_entries += data_service_function.call(entries.last.end_date, to) - end - end - cache(table, @user_id, new_entries) if new_entries.present? - entries + new_entries + entries end - def get_data_function(table) - case table - when 'heart_rate' - data_service.method(:heart_rate) - when 'sleep' - data_service.method(:sleep) - when 'calories' - data_service.method(:calories) - when 'distance' - data_service.method(:distance) - when 'steps' - data_service.method(:steps) - when 'activities' - data_service.method(:activities) + private + + def cache_data(table, user_id, from, to) + job = Physiqual::CacheWorker.perform_async(table, user_id, from, to) + while Sidekiq::Status::queued? job or Sidekiq::Status::working? job + Kernel::sleep(1) end end - def years(from, to) + def self.years(from, to) from_year = from.strftime('%Y').to_i to_year = to.strftime('%Y').to_i (from_year..to_year).each do |year| @@ -119,16 +97,7 @@ def years(from, to) end end - def find_gaps(entries) - entries.each_with_index do |entry, i| - break if i == entries.length - 1 - if entry.end_date != entries[i + 1].start_date - yield(entry.end_date, entries[i + 1].start_date) - end - end - end - - def make_data_entries(table, results) + def self.make_data_entries(table, results) return [] if results.blank? entries = [] results.each do |result| @@ -144,33 +113,6 @@ def make_data_entries(table, results) end entries end - - def cache(table, user_id, new_entries) - year = new_entries.first.measurement_moment.strftime('%Y').to_i - times = [] - start_dates = [] - end_dates = [] - values = [] - new_entries.each do |entry| - if year != entry.measurement_moment.strftime('%Y').to_i - Rails.logger.info("Caching from #{start_dates.first} to #{end_dates.last}...") - Physiqual::CacheWorker.perform_async(table, user_id, year, times, start_dates, end_dates, values) - year = entry.measurement_moment.strftime('%Y').to_i - times = [] - start_dates = [] - end_dates = [] - values = [] - end - times << entry.measurement_moment - start_dates << entry.start_date - end_dates << entry.end_date - values << entry.values.first - if entry == new_entries.last - Rails.logger.info("Caching from #{start_dates.first} to #{end_dates.last}...") - Physiqual::CacheWorker.perform_async(table, user_id, year, times, start_dates, end_dates, values) - end - end - end end end end diff --git a/physiqual.gemspec b/physiqual.gemspec index cd8588b..9eb0419 100644 --- a/physiqual.gemspec +++ b/physiqual.gemspec @@ -41,6 +41,7 @@ Gem::Specification.new do |s| s.add_dependency 'cassandra-driver' s.add_dependency 'sidekiq' + s.add_dependency 'sidekiq-status' s.add_development_dependency 'codeclimate-test-reporter' s.add_development_dependency 'sqlite3' diff --git a/spec/dummy/config/initializers/sidekiq.rb b/spec/dummy/config/initializers/sidekiq.rb new file mode 100644 index 0000000..3f77e7e --- /dev/null +++ b/spec/dummy/config/initializers/sidekiq.rb @@ -0,0 +1,20 @@ +require 'sidekiq' +require 'sidekiq-status' + +Sidekiq.configure_client do |config| + config.client_middleware do |chain| + # accepts :expiration (optional) + chain.add Sidekiq::Status::ClientMiddleware, expiration: 30.minutes # default + end +end + +Sidekiq.configure_server do |config| + config.server_middleware do |chain| + # accepts :expiration (optional) + chain.add Sidekiq::Status::ServerMiddleware, expiration: 30.minutes # default + end + config.client_middleware do |chain| + # accepts :expiration (optional) + chain.add Sidekiq::Status::ClientMiddleware, expiration: 30.minutes # default + end +end \ No newline at end of file From 1249f5ba89be3c04941a2522bf21838e519782fa Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Wed, 6 Jul 2016 23:31:42 +0200 Subject: [PATCH 17/35] Rubocop --- app/workers/physiqual/cache_worker.rb | 34 ++--- .../data_services/cassandra_connection.rb | 142 +++++------------- .../data_services/cassandra_data_service.rb | 64 ++++---- 3 files changed, 87 insertions(+), 153 deletions(-) diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb index aaec809..823fecf 100644 --- a/app/workers/physiqual/cache_worker.rb +++ b/app/workers/physiqual/cache_worker.rb @@ -18,27 +18,25 @@ def perform(table, user_id, from, to) private def store_data(connection, table, from, to) - begin - entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) - data_service_function = get_data_function(table) - new_entries = [] - if entries.blank? - new_entries = data_service_function.call(from, to) - else - if entries.first.start_date > from - new_entries += data_service_function.call(from, entries.first.start_date) - end - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service_function.call(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service_function.call(entries.last.end_date, to) - end + entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) + data_service_function = get_data_function(table) + new_entries = [] + if entries.blank? + new_entries = data_service_function.call(from, to) + else + if entries.first.start_date > from + new_entries += data_service_function.call(from, entries.first.start_date) end - cache(connection, table, @user_id, new_entries) if new_entries.present? + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service_function.call(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service_function.call(entries.last.end_date, to) + end + end + cache(connection, table, @user_id, new_entries) if new_entries.present? rescue Errors::NotSupportedError => e Rails.logger.warn e.message - end end diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index 6512d48..5bd459e 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -91,126 +91,58 @@ def query_activities(user_id, year, from, to) private def init_db - @session.execute(' - CREATE TABLE IF NOT EXISTS heart_rate ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS sleep ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS calories ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS distance ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS steps ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS activities ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value varchar, + create_table('heart_rate', 'decimal') + create_table('sleep', 'decimal') + create_table('calories', 'decimal') + create_table('distance', 'decimal') + create_table('steps', 'decimal') + create_table('activities', 'varchar') + end + + def create_table(name, value_type) + @session.execute(" + CREATE TABLE IF NOT EXISTS #{name} ( + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, PRIMARY KEY ((user_id, year), time) ) - ') + ") end def init_insert - @insert_heart_rate = @session.prepare(' - INSERT INTO heart_rate ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_sleep = @session.prepare(' - INSERT INTO sleep ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_calories = @session.prepare(' - INSERT INTO calories ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_distance = @session.prepare(' - INSERT INTO distance ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_steps = @session.prepare(' - INSERT INTO steps ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_activities = @session.prepare(' - INSERT INTO activities ( + @insert_heart_rate = prepare_insert('heart_rate') + @insert_sleep = prepare_insert('sleep') + @insert_calories = prepare_insert('calories') + @insert_distance = prepare_insert('distance') + @insert_steps = prepare_insert('steps') + @insert_activities = prepare_insert('activities') + end + + def prepare_insert(table_name) + @session.prepare(" + INSERT INTO #{table_name} ( user_id, year, time, start_date, end_date, value ) VALUES ( ?, ?, ?, ?, ?, ? ) - ') + ") end def init_query - @query_heart_rate = @session.prepare(' - SELECT time, start_date, end_date, value - FROM heart_rate - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_sleep = @session.prepare(' - SELECT time, start_date, end_date, value - FROM sleep - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_calories = @session.prepare(' - SELECT time, start_date, end_date, value - FROM calories - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_distance = @session.prepare(' - SELECT time, start_date, end_date, value - FROM distance - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_steps = @session.prepare(' - SELECT time, start_date, end_date, value - FROM steps - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_activities = @session.prepare(' + @query_heart_rate = prepare_query('heart_rate') + @query_sleep = prepare_query('sleep') + @query_calories = prepare_query('calories') + @query_distance = prepare_query('distance') + @query_steps = prepare_query('steps') + @query_activities = prepare_query('activities') + end + + def prepare_query(table_name) + @session.prepare(" SELECT time, start_date, end_date, value - FROM activities + FROM #{table_name} WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC - ') + ") end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 424a0ad..5c3c572 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -74,44 +74,48 @@ def self.get_data(connection, user_id, table, from, to) def cache_data(table, user_id, from, to) job = Physiqual::CacheWorker.perform_async(table, user_id, from, to) - while Sidekiq::Status::queued? job or Sidekiq::Status::working? job - Kernel::sleep(1) + while Sidekiq::Status.queued? job or Sidekiq::Status.working? job + Kernel.sleep(1) end end - def self.years(from, to) - from_year = from.strftime('%Y').to_i - to_year = to.strftime('%Y').to_i - (from_year..to_year).each do |year| - from_per_year = if from_year < year - Time.zone.local(year, 1, 1, 0, 0, 0) + class << self + private + + def years(from, to) + from_year = from.strftime('%Y').to_i + to_year = to.strftime('%Y').to_i + (from_year..to_year).each do |year| + from_per_year = if from_year < year + Time.zone.local(year, 1, 1, 0, 0, 0) + else + from + end + to_per_year = if to_year > year + Time.zone.local(year, 12, 31, 23, 59, 59) else - from + to end - to_per_year = if to_year > year - Time.zone.local(year, 12, 31, 23, 59, 59) - else - to - end - yield(year, from_per_year, to_per_year) + yield(year, from_per_year, to_per_year) + end end - end - def self.make_data_entries(table, results) - return [] if results.blank? - entries = [] - results.each do |result| - value = if table == 'activities' - result['value'] - else - result['value'].to_f - end - entries << DataEntry.new(start_date: result['start_date'].in_time_zone, - end_date: result['end_date'].in_time_zone, - values: value, - measurement_moment: result['time'].in_time_zone) + def make_data_entries(table, results) + return [] if results.blank? + entries = [] + results.each do |result| + value = if table == 'activities' + result['value'] + else + result['value'].to_f + end + entries << DataEntry.new(start_date: result['start_date'].in_time_zone, + end_date: result['end_date'].in_time_zone, + values: value, + measurement_moment: result['time'].in_time_zone) + end + entries end - entries end end end From 899b204c8c6213611e20704618f54db305c2db6e Mon Sep 17 00:00:00 2001 From: Marc Babtist Date: Wed, 6 Jul 2016 23:31:42 +0200 Subject: [PATCH 18/35] Rubocop --- app/workers/physiqual/cache_worker.rb | 37 ++--- .../data_services/cassandra_connection.rb | 157 ++++++------------ .../data_services/cassandra_data_service.rb | 64 +++---- 3 files changed, 98 insertions(+), 160 deletions(-) diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb index aaec809..7f5d895 100644 --- a/app/workers/physiqual/cache_worker.rb +++ b/app/workers/physiqual/cache_worker.rb @@ -18,28 +18,25 @@ def perform(table, user_id, from, to) private def store_data(connection, table, from, to) - begin - entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) - data_service_function = get_data_function(table) - new_entries = [] - if entries.blank? - new_entries = data_service_function.call(from, to) - else - if entries.first.start_date > from - new_entries += data_service_function.call(from, entries.first.start_date) - end - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service_function.call(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service_function.call(entries.last.end_date, to) - end + entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) + data_service_function = get_data_function(table) + new_entries = [] + if entries.blank? + new_entries = data_service_function.call(from, to) + else + if entries.first.start_date > from + new_entries += data_service_function.call(from, entries.first.start_date) + end + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service_function.call(from_gap, to_gap) + end + if entries.last.end_date < to + new_entries += data_service_function.call(entries.last.end_date, to) end - cache(connection, table, @user_id, new_entries) if new_entries.present? - rescue Errors::NotSupportedError => e - Rails.logger.warn e.message end - + cache(connection, table, @user_id, new_entries) if new_entries.present? + rescue Errors::NotSupportedError => e + Rails.logger.warn e.message end def get_data_function(table) diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index 6512d48..ae7e01b 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -5,6 +5,7 @@ module Physiqual module DataServices class CassandraConnection include Singleton + SLICE_SIZE = 100 def initialize @cluster = nil @@ -27,11 +28,8 @@ def initialize end def insert(table, user_id, year, times, start_dates, end_dates, values) - times_slices = times.each_slice(100).to_a - start_dates_slices = start_dates.each_slice(100).to_a - end_dates_slices = end_dates.each_slice(100).to_a - values_slices = values.each_slice(100).to_a - + times_slices, start_dates_slices, end_dates_slices, values_slices = + slice(times, start_dates, end_dates, values) times_slices.each_with_index do |times_slice, i| start_dates_slice = start_dates_slices[i] end_dates_slice = end_dates_slices[i] @@ -64,6 +62,13 @@ def insert(table, user_id, year, times, start_dates, end_dates, values) end end + def slice(times, start_dates, end_dates, values) + return times.each_slice(SLICE_SIZE).to_a, + start_dates.each_slice(SLICE_SIZE).to_a, + end_dates.each_slice(SLICE_SIZE).to_a, + values.each_slice(SLICE_SIZE).to_a + end + def query_heart_rate(user_id, year, from, to) @session.execute(@query_heart_rate, arguments: [user_id, year, from, to]) end @@ -91,126 +96,58 @@ def query_activities(user_id, year, from, to) private def init_db - @session.execute(' - CREATE TABLE IF NOT EXISTS heart_rate ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS sleep ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS calories ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS distance ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS steps ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value decimal, - PRIMARY KEY ((user_id, year), time) - ) - ') - @session.execute(' - CREATE TABLE IF NOT EXISTS activities ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value varchar, + create_table('heart_rate', 'decimal') + create_table('sleep', 'decimal') + create_table('calories', 'decimal') + create_table('distance', 'decimal') + create_table('steps', 'decimal') + create_table('activities', 'varchar') + end + + def create_table(name, value_type) + @session.execute(" + CREATE TABLE IF NOT EXISTS #{name} ( + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, PRIMARY KEY ((user_id, year), time) ) - ') + ") end def init_insert - @insert_heart_rate = @session.prepare(' - INSERT INTO heart_rate ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_sleep = @session.prepare(' - INSERT INTO sleep ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_calories = @session.prepare(' - INSERT INTO calories ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_distance = @session.prepare(' - INSERT INTO distance ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_steps = @session.prepare(' - INSERT INTO steps ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ') - @insert_activities = @session.prepare(' - INSERT INTO activities ( + @insert_heart_rate = prepare_insert('heart_rate') + @insert_sleep = prepare_insert('sleep') + @insert_calories = prepare_insert('calories') + @insert_distance = prepare_insert('distance') + @insert_steps = prepare_insert('steps') + @insert_activities = prepare_insert('activities') + end + + def prepare_insert(table_name) + @session.prepare(" + INSERT INTO #{table_name} ( user_id, year, time, start_date, end_date, value ) VALUES ( ?, ?, ?, ?, ?, ? ) - ') + ") end def init_query - @query_heart_rate = @session.prepare(' - SELECT time, start_date, end_date, value - FROM heart_rate - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_sleep = @session.prepare(' - SELECT time, start_date, end_date, value - FROM sleep - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_calories = @session.prepare(' - SELECT time, start_date, end_date, value - FROM calories - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_distance = @session.prepare(' - SELECT time, start_date, end_date, value - FROM distance - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_steps = @session.prepare(' - SELECT time, start_date, end_date, value - FROM steps - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ') - @query_activities = @session.prepare(' + @query_heart_rate = prepare_query('heart_rate') + @query_sleep = prepare_query('sleep') + @query_calories = prepare_query('calories') + @query_distance = prepare_query('distance') + @query_steps = prepare_query('steps') + @query_activities = prepare_query('activities') + end + + def prepare_query(table_name) + @session.prepare(" SELECT time, start_date, end_date, value - FROM activities + FROM #{table_name} WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC - ') + ") end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 424a0ad..5c3c572 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -74,44 +74,48 @@ def self.get_data(connection, user_id, table, from, to) def cache_data(table, user_id, from, to) job = Physiqual::CacheWorker.perform_async(table, user_id, from, to) - while Sidekiq::Status::queued? job or Sidekiq::Status::working? job - Kernel::sleep(1) + while Sidekiq::Status.queued? job or Sidekiq::Status.working? job + Kernel.sleep(1) end end - def self.years(from, to) - from_year = from.strftime('%Y').to_i - to_year = to.strftime('%Y').to_i - (from_year..to_year).each do |year| - from_per_year = if from_year < year - Time.zone.local(year, 1, 1, 0, 0, 0) + class << self + private + + def years(from, to) + from_year = from.strftime('%Y').to_i + to_year = to.strftime('%Y').to_i + (from_year..to_year).each do |year| + from_per_year = if from_year < year + Time.zone.local(year, 1, 1, 0, 0, 0) + else + from + end + to_per_year = if to_year > year + Time.zone.local(year, 12, 31, 23, 59, 59) else - from + to end - to_per_year = if to_year > year - Time.zone.local(year, 12, 31, 23, 59, 59) - else - to - end - yield(year, from_per_year, to_per_year) + yield(year, from_per_year, to_per_year) + end end - end - def self.make_data_entries(table, results) - return [] if results.blank? - entries = [] - results.each do |result| - value = if table == 'activities' - result['value'] - else - result['value'].to_f - end - entries << DataEntry.new(start_date: result['start_date'].in_time_zone, - end_date: result['end_date'].in_time_zone, - values: value, - measurement_moment: result['time'].in_time_zone) + def make_data_entries(table, results) + return [] if results.blank? + entries = [] + results.each do |result| + value = if table == 'activities' + result['value'] + else + result['value'].to_f + end + entries << DataEntry.new(start_date: result['start_date'].in_time_zone, + end_date: result['end_date'].in_time_zone, + values: value, + measurement_moment: result['time'].in_time_zone) + end + entries end - entries end end end From de0f43ad3c7848f50f3ce1cc0a7f86570f22d314 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Tue, 19 Jul 2016 12:35:00 +0200 Subject: [PATCH 19/35] Cleaned up the code a bit --- .../data_services/cassandra_connection.rb | 136 ++++++++---------- .../data_services/cassandra_data_service.rb | 18 ++- lib/physiqual/engine.rb | 12 +- lib/physiqual/exporters/exporter.rb | 10 +- spec/dummy/config/initializers/physiqual.rb | 1 + 5 files changed, 91 insertions(+), 86 deletions(-) diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb index 38a0b07..b50c263 100644 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ b/lib/physiqual/data_services/cassandra_connection.rb @@ -8,116 +8,98 @@ class CassandraConnection SLICE_SIZE = 100 def initialize - @cluster = nil - if !ENV['CASSANDRA_USERNAME'] || !ENV['CASSANDRA_PASSWORD'] || ENV['CASSANDRA_USERNAME'] == '' + + # Setup the connection to the cluster + if Physiqual.cassandra_username.blank? || Physiqual.cassandra_password.blank? @cluster = Cassandra.cluster( - hosts: (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + hosts: Physiqual.cassandra_host_urls ) else @cluster = Cassandra.cluster( username: ENV['CASSANDRA_USERNAME'], password: ENV['CASSANDRA_PASSWORD'], - hosts: (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + hosts: Physiqual.cassandra_host_urls ) end @session = @cluster.connect('physiqual') - init_db - init_insert - init_query + variables = {'heart_rate' => 'decimal', + 'sleep' => 'decimal', + 'calories' => 'decimal', + 'distance' => 'decimal', + 'steps' => 'decimal', + 'activities' => 'varchar'} + + initialize_database(variables) end def insert(table, user_id, year, times, start_dates, end_dates, values) - times_slices, start_dates_slices, end_dates_slices, values_slices = - slice(times, start_dates, end_dates, values) - times_slices.each_with_index do |times_slice, i| - start_dates_slice = start_dates_slices[i] - end_dates_slice = end_dates_slices[i] - values_slice = values_slices[i] + + # Slice the dates in chuncs of SLICE_SIZE + times_slices, start_dates_slices, end_dates_slices, values_slices = slice(times, start_dates, end_dates, values) + + # Merge all slices into one array + times_slices.zip!(start_dates_slices, end_dates_slices, values_slices) + + times_slices.each_with_index do |times_slice, start_dates_slice, end_dates_slice, values_slice| batch = @session.batch do |b| - times_slice.each_with_index do |time, j| - value = BigDecimal(values_slice[j], Float::DIG + 1) - case table - when 'heart_rate' - insert_type = @insert_heart_rate - when 'sleep' - insert_type = @insert_sleep - when 'calories' - insert_type = @insert_calories - when 'distance' - insert_type = @insert_distance - when 'steps' - insert_type = @insert_steps - when 'activities' - insert_type = @insert_activities - value = values_slice[j] - end - b.add(insert_type, arguments: [user_id, year, time.to_time, start_dates_slice[j].to_time, - end_dates_slice[j].to_time, value]) + zipped_slices = times_slice.zip(start_dates_slice, end_dates_slice, values_slice) + zipped_slices.each do |time, start_date, end_date, value| + + # If the table is 'activities', we should not convert the slice to a bigdecimal + value = BigDecimal(value, Float::DIG + 1) if table != 'activities' + + # Retrieve the prepared statement + insert_type = @insert_queries[table] + b.add(insert_type, arguments: [user_id, year, time.to_time, start_date.to_time, + end_date.to_time, value]) end end @session.execute(batch) end end - def slice(times, start_dates, end_dates, values) - return times.each_slice(SLICE_SIZE).to_a, - start_dates.each_slice(SLICE_SIZE).to_a, - end_dates.each_slice(SLICE_SIZE).to_a, - values.each_slice(SLICE_SIZE).to_a - end - def query_heart_rate(user_id, year, from, to) - @session.execute(@query_heart_rate, arguments: [user_id, year, from, to]) + @session.execute(@queries['heart_rate'], arguments: [user_id, year, from, to]) end def query_sleep(user_id, year, from, to) - @session.execute(@query_sleep, arguments: [user_id, year, from, to]) + @session.execute(@query['sleep'], arguments: [user_id, year, from, to]) end def query_calories(user_id, year, from, to) - @session.execute(@query_calories, arguments: [user_id, year, from, to]) + @session.execute(@query['calories'], arguments: [user_id, year, from, to]) end def query_distance(user_id, year, from, to) - @session.execute(@query_distance, arguments: [user_id, year, from, to]) + @session.execute(@query['distance'], arguments: [user_id, year, from, to]) end def query_steps(user_id, year, from, to) - @session.execute(@query_steps, arguments: [user_id, year, from, to]) + @session.execute(@query['steps'], arguments: [user_id, year, from, to]) end def query_activities(user_id, year, from, to) - @session.execute(@query_activities, arguments: [user_id, year, from, to]) + @session.execute(@query['activities'], arguments: [user_id, year, from, to]) end private - def init_db - create_table('heart_rate', 'decimal') - create_table('sleep', 'decimal') - create_table('calories', 'decimal') - create_table('distance', 'decimal') - create_table('steps', 'decimal') - create_table('activities', 'varchar') - end - - def create_table(name, value_type) - @session.execute(" - CREATE TABLE IF NOT EXISTS #{name} ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, - PRIMARY KEY ((user_id, year), time) - ) - ") + def slice(times, start_dates, end_dates, values) + return times.each_slice(SLICE_SIZE).to_a, + start_dates.each_slice(SLICE_SIZE).to_a, + end_dates.each_slice(SLICE_SIZE).to_a, + values.each_slice(SLICE_SIZE).to_a end - def init_insert - @insert_heart_rate = prepare_insert('heart_rate') - @insert_sleep = prepare_insert('sleep') - @insert_calories = prepare_insert('calories') - @insert_distance = prepare_insert('distance') - @insert_steps = prepare_insert('steps') - @insert_activities = prepare_insert('activities') + def initialize_database(variable_names) + @insert_queries = {} + @queries = {} + variable_names.each do |variable, type| + @insert_queries[variable] = prepare_insert(variable) + @queries[variable] = prepare_query(variable) + create_table(variable, type) + end end def prepare_insert(table_name) @@ -130,15 +112,6 @@ def prepare_insert(table_name) ") end - def init_query - @query_heart_rate = prepare_query('heart_rate') - @query_sleep = prepare_query('sleep') - @query_calories = prepare_query('calories') - @query_distance = prepare_query('distance') - @query_steps = prepare_query('steps') - @query_activities = prepare_query('activities') - end - def prepare_query(table_name) @session.prepare(" SELECT time, start_date, end_date, value @@ -147,6 +120,15 @@ def prepare_query(table_name) ORDER BY time ASC ") end + + def create_table(name, value_type) + @session.execute(" + CREATE TABLE IF NOT EXISTS #{name} ( + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, + PRIMARY KEY ((user_id, year), time) + ) + ") + end end end end diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 5c3c572..07a7338 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -6,7 +6,6 @@ class CassandraDataService < DataServiceDecorator def initialize(data_service, user_id) super(data_service) @user_id = user_id - @connection = CassandraConnection.instance end def service_name @@ -15,32 +14,32 @@ def service_name def heart_rate(from, to) cache_data('heart_rate', @user_id, from, to) - CassandraDataService.get_data(@connection, @user_id, 'heart_rate', from, to) + CassandraDataService.get_data(cassandra_connection, @user_id, 'heart_rate', from, to) end def sleep(from, to) cache_data('sleep', @user_id, from, to) - CassandraDataService.get_data(@connection, @user_id, 'sleep', from, to) + CassandraDataService.get_data(cassandra_connection, @user_id, 'sleep', from, to) end def calories(from, to) cache_data('calories', @user_id, from, to) - CassandraDataService.get_data(@connection, @user_id, 'calories', from, to) + CassandraDataService.get_data(cassandra_connection, @user_id, 'calories', from, to) end def distance(from, to) cache_data('distance', @user_id, from, to) - CassandraDataService.get_data(@connection, @user_id, 'distance', from, to) + CassandraDataService.get_data(cassandra_connection, @user_id, 'distance', from, to) end def steps(from, to) cache_data('steps', @user_id, from, to) - CassandraDataService.get_data(@connection, @user_id, 'steps', from, to) + CassandraDataService.get_data(cassandra_connection, @user_id, 'steps', from, to) end def activities(from, to) cache_data('activities', @user_id, from, to) - CassandraDataService.get_data(@connection, @user_id, 'activities', from, to) + CassandraDataService.get_data(cassandra_connection, @user_id, 'activities', from, to) end def self.get_data(connection, user_id, table, from, to) @@ -72,6 +71,11 @@ def self.get_data(connection, user_id, table, from, to) private + def cassandra_connection + @connection ||= CassandraConnection.instance + @connection + end + def cache_data(table, user_id, from, to) job = Physiqual::CacheWorker.perform_async(table, user_id, from, to) while Sidekiq::Status.queued? job or Sidekiq::Status.working? job diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index 1987386..ab50f0b 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -9,10 +9,19 @@ class << self mattr_accessor :google_client_secret mattr_accessor :fitbit_client_id mattr_accessor :fitbit_client_secret + + # Cassandra settings + mattr_accessor :enable_cassandra mattr_accessor :cassandra_username mattr_accessor :cassandra_password - mattr_accessor :cassandra_host_urls + mattr_writer :cassandra_host_urls + + def cassandra_host_urls + @cassandra_host_urls.split(' ') unless @cassandra_host_urls.blank? + end + mattr_accessor :cassandra_keyspace + mattr_accessor :host_url mattr_accessor :host_protocol mattr_accessor :measurements_per_day @@ -21,6 +30,7 @@ class << self mattr_accessor :imputers end + def self.configure(&_block) yield self end diff --git a/lib/physiqual/exporters/exporter.rb b/lib/physiqual/exporters/exporter.rb index 184bb26..770b465 100644 --- a/lib/physiqual/exporters/exporter.rb +++ b/lib/physiqual/exporters/exporter.rb @@ -34,9 +34,17 @@ def create_service(user, bucket_generator) token = user.physiqual_token return [] unless token.complete? session = Sessions::TokenAuthorizedSession.new(token) + + # Create the actual service accessing the service provider service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) - service = DataServices::CassandraDataService.new(service, user.user_id) + + # Enable caching with cassandra if it is enabled + service = DataServices::CassandraDataService.new(service, user.user_id) if Physiqual.enable_cassandra + + # Transform the raw data into buckets service = DataServices::BucketeerDataService.new(service, bucket_generator) + + # Generate a summary score per bucket service = DataServices::SummarizedDataService.new(service) DataServices::CachedDataService.new service end diff --git a/spec/dummy/config/initializers/physiqual.rb b/spec/dummy/config/initializers/physiqual.rb index bde1dbc..ef091c6 100644 --- a/spec/dummy/config/initializers/physiqual.rb +++ b/spec/dummy/config/initializers/physiqual.rb @@ -12,6 +12,7 @@ config.host_protocol = ENV['HOST_PROTOCOL'] || 'http' # Cassandra settings + config.enable_cassandra = ENV['ENABLE_CASSANDRA'] ? ENV['ENABLE_CASSANDRA'].downcase == 'true' : false config.cassandra_username = ENV['CASSANDRA_USERNAME'] || '' config.cassandra_password = ENV['CASSANDRA_PASSWORD'] || '' config.cassandra_host_urls = (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') From 4dba7d755640e1fcce6e1c1165ed4d91cd44eefe Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Tue, 19 Jul 2016 14:09:12 +0200 Subject: [PATCH 20/35] Added some specs --- lib/physiqual.rb | 1 + lib/physiqual/cassandra_connection.rb | 126 ++++++++++ lib/physiqual/data_services.rb | 1 - .../data_services/bucketeer_data_service.rb | 2 +- .../data_services/cassandra_connection.rb | 134 ----------- lib/physiqual/engine.rb | 1 - .../physiqual/cassandra_connection_spec.rb | 215 ++++++++++++++++++ .../cassandra_data_service_spec.rb | 0 8 files changed, 343 insertions(+), 137 deletions(-) create mode 100644 lib/physiqual/cassandra_connection.rb delete mode 100644 lib/physiqual/data_services/cassandra_connection.rb create mode 100644 spec/lib/physiqual/cassandra_connection_spec.rb create mode 100644 spec/lib/physiqual/data_services/cassandra_data_service_spec.rb diff --git a/lib/physiqual.rb b/lib/physiqual.rb index 12ad2d1..86f518c 100644 --- a/lib/physiqual.rb +++ b/lib/physiqual.rb @@ -9,6 +9,7 @@ require 'physiqual/sessions' require 'physiqual/users' require 'physiqual/version' +require 'physiqual/cassandra_connection' module Physiqual end diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb new file mode 100644 index 0000000..750a731 --- /dev/null +++ b/lib/physiqual/cassandra_connection.rb @@ -0,0 +1,126 @@ +require 'singleton' +require 'cassandra' + +module Physiqual + class CassandraConnection + include Singleton + SLICE_SIZE = 100 + + def initialize + # Setup the connection to the cluster + if Physiqual.cassandra_username.blank? || Physiqual.cassandra_password.blank? + cluster = Cassandra.cluster( + hosts: Physiqual.cassandra_host_urls + ) + else + cluster = Cassandra.cluster( + username: ENV['CASSANDRA_USERNAME'], + password: ENV['CASSANDRA_PASSWORD'], + hosts: Physiqual.cassandra_host_urls + ) + end + @session = cluster.connect('physiqual') + + variables = { 'heart_rate' => 'decimal', + 'sleep' => 'decimal', + 'calories' => 'decimal', + 'distance' => 'decimal', + 'steps' => 'decimal', + 'activities' => 'varchar' } + + initialize_database(variables) + end + + def insert(table, user_id, year, times, start_dates, end_dates, values) + # Slice the dates in chuncs of SLICE_SIZE + times_slices, start_dates_slices, end_dates_slices, values_slices = slice(times, start_dates, end_dates, values) + + # Merge all slices into one array + times_slices.zip!(start_dates_slices, end_dates_slices, values_slices) + + times_slices.each_with_index do |times_slice, start_dates_slice, end_dates_slice, values_slice| + batch = @session.batch do |b| + zipped_slices = times_slice.zip(start_dates_slice, end_dates_slice, values_slice) + zipped_slices.each do |time, start_date, end_date, value| + # If the table is 'activities', we should not convert the slice to a bigdecimal + value = BigDecimal(value, Float::DIG + 1) if table != 'activities' + + # Retrieve the prepared statement + insert_type = @insert_queries[table] + b.add(insert_type, arguments: [user_id, year, time.to_time, start_date.to_time, + end_date.to_time, value]) + end + end + @session.execute(batch) + end + end + + def query_heart_rate(user_id, year, from, to) + @session.execute(@queries['heart_rate'], arguments: [user_id, year, from, to]) + end + + def query_sleep(user_id, year, from, to) + @session.execute(@queries['sleep'], arguments: [user_id, year, from, to]) + end + + def query_calories(user_id, year, from, to) + @session.execute(@queries['calories'], arguments: [user_id, year, from, to]) + end + + def query_distance(user_id, year, from, to) + @session.execute(@queries['distance'], arguments: [user_id, year, from, to]) + end + + def query_steps(user_id, year, from, to) + @session.execute(@queries['steps'], arguments: [user_id, year, from, to]) + end + + def query_activities(user_id, year, from, to) + @session.execute(@queries['activities'], arguments: [user_id, year, from, to]) + end + + private + + def slice(times, start_dates, end_dates, values) + [times.each_slice(SLICE_SIZE).to_a, + start_dates.each_slice(SLICE_SIZE).to_a, + end_dates.each_slice(SLICE_SIZE).to_a, + values.each_slice(SLICE_SIZE).to_a] + end + + def initialize_database(variable_names) + @insert_queries = {} + @queries = {} + variable_names.each do |variable, type| + @insert_queries[variable] = prepare_insert(variable) + @queries[variable] = prepare_query(variable) + create_table(variable, type) + end + end + + def prepare_insert(table_name) + query = "INSERT INTO #{table_name} (user_id, year, time, start_date, end_date, value) VALUES (?, ?, ?, ?, ?, ?)" + @session.prepare(query) + end + + def prepare_query(table_name) + query = " + SELECT time, start_date, end_date, value + FROM #{table_name} + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? + ORDER BY time ASC + " + @session.prepare(query) + end + + def create_table(name, value_type) + query = " + CREATE TABLE IF NOT EXISTS #{name} ( + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, + PRIMARY KEY ((user_id, year), time) + ) + " + @session.execute(query) + end + end +end diff --git a/lib/physiqual/data_services.rb b/lib/physiqual/data_services.rb index ad58b70..53b17db 100644 --- a/lib/physiqual/data_services.rb +++ b/lib/physiqual/data_services.rb @@ -8,7 +8,6 @@ require 'physiqual/data_services/summarized_data_service' require 'physiqual/data_services/bucketeer_data_service' require 'physiqual/data_services/cassandra_data_service' -require 'physiqual/data_services/cassandra_connection' module Physiqual module DataServices diff --git a/lib/physiqual/data_services/bucketeer_data_service.rb b/lib/physiqual/data_services/bucketeer_data_service.rb index bdc127c..ea6a0d0 100644 --- a/lib/physiqual/data_services/bucketeer_data_service.rb +++ b/lib/physiqual/data_services/bucketeer_data_service.rb @@ -67,7 +67,7 @@ def create_buckets(buckets, data) next unless entry.measurement_moment > buckets[current_bucket].start_date values = entry.values - buckets[current_bucket].values.push(*values) + buckets[current_bucket].values.concat(values) end buckets end diff --git a/lib/physiqual/data_services/cassandra_connection.rb b/lib/physiqual/data_services/cassandra_connection.rb deleted file mode 100644 index b50c263..0000000 --- a/lib/physiqual/data_services/cassandra_connection.rb +++ /dev/null @@ -1,134 +0,0 @@ -require 'singleton' -require 'cassandra' - -module Physiqual - module DataServices - class CassandraConnection - include Singleton - SLICE_SIZE = 100 - - def initialize - - # Setup the connection to the cluster - if Physiqual.cassandra_username.blank? || Physiqual.cassandra_password.blank? - @cluster = Cassandra.cluster( - hosts: Physiqual.cassandra_host_urls - ) - else - @cluster = Cassandra.cluster( - username: ENV['CASSANDRA_USERNAME'], - password: ENV['CASSANDRA_PASSWORD'], - hosts: Physiqual.cassandra_host_urls - ) - end - @session = @cluster.connect('physiqual') - - variables = {'heart_rate' => 'decimal', - 'sleep' => 'decimal', - 'calories' => 'decimal', - 'distance' => 'decimal', - 'steps' => 'decimal', - 'activities' => 'varchar'} - - initialize_database(variables) - end - - def insert(table, user_id, year, times, start_dates, end_dates, values) - - # Slice the dates in chuncs of SLICE_SIZE - times_slices, start_dates_slices, end_dates_slices, values_slices = slice(times, start_dates, end_dates, values) - - # Merge all slices into one array - times_slices.zip!(start_dates_slices, end_dates_slices, values_slices) - - times_slices.each_with_index do |times_slice, start_dates_slice, end_dates_slice, values_slice| - batch = @session.batch do |b| - zipped_slices = times_slice.zip(start_dates_slice, end_dates_slice, values_slice) - zipped_slices.each do |time, start_date, end_date, value| - - # If the table is 'activities', we should not convert the slice to a bigdecimal - value = BigDecimal(value, Float::DIG + 1) if table != 'activities' - - # Retrieve the prepared statement - insert_type = @insert_queries[table] - b.add(insert_type, arguments: [user_id, year, time.to_time, start_date.to_time, - end_date.to_time, value]) - end - end - @session.execute(batch) - end - end - - def query_heart_rate(user_id, year, from, to) - @session.execute(@queries['heart_rate'], arguments: [user_id, year, from, to]) - end - - def query_sleep(user_id, year, from, to) - @session.execute(@query['sleep'], arguments: [user_id, year, from, to]) - end - - def query_calories(user_id, year, from, to) - @session.execute(@query['calories'], arguments: [user_id, year, from, to]) - end - - def query_distance(user_id, year, from, to) - @session.execute(@query['distance'], arguments: [user_id, year, from, to]) - end - - def query_steps(user_id, year, from, to) - @session.execute(@query['steps'], arguments: [user_id, year, from, to]) - end - - def query_activities(user_id, year, from, to) - @session.execute(@query['activities'], arguments: [user_id, year, from, to]) - end - - private - - def slice(times, start_dates, end_dates, values) - return times.each_slice(SLICE_SIZE).to_a, - start_dates.each_slice(SLICE_SIZE).to_a, - end_dates.each_slice(SLICE_SIZE).to_a, - values.each_slice(SLICE_SIZE).to_a - end - - def initialize_database(variable_names) - @insert_queries = {} - @queries = {} - variable_names.each do |variable, type| - @insert_queries[variable] = prepare_insert(variable) - @queries[variable] = prepare_query(variable) - create_table(variable, type) - end - end - - def prepare_insert(table_name) - @session.prepare(" - INSERT INTO #{table_name} ( - user_id, year, time, start_date, end_date, value - ) VALUES ( - ?, ?, ?, ?, ?, ? - ) - ") - end - - def prepare_query(table_name) - @session.prepare(" - SELECT time, start_date, end_date, value - FROM #{table_name} - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - ") - end - - def create_table(name, value_type) - @session.execute(" - CREATE TABLE IF NOT EXISTS #{name} ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, - PRIMARY KEY ((user_id, year), time) - ) - ") - end - end - end -end diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index ab50f0b..ab7c4ab 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -30,7 +30,6 @@ def cassandra_host_urls mattr_accessor :imputers end - def self.configure(&_block) yield self end diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb new file mode 100644 index 0000000..43b05c1 --- /dev/null +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -0,0 +1,215 @@ +module Physiqual + require 'rails_helper' + describe CassandraConnection do + describe 'initialize' do + end + + describe 'with mock connection' do + let(:user_id) { 0 } + let(:year) { 2016 } + let(:from) { 3.days.ago } + let(:to) { Time.now } + let(:arguments_array) { [user_id, year, from, to] } + + before(:each) do + @cluster = double('cluster') + @session = double('session') + query = double('query') + insert_query = double('insert_query') + + @queries = { 'heart_rate' => query, + 'sleep' => query, + 'calories' => query, + 'distance' => query, + 'steps' => query, + 'activities' => query } + + @insert_queries = { 'heart_rate' => insert_query, + 'sleep' => insert_query, + 'calories' => insert_query, + 'distance' => insert_query, + 'steps' => insert_query, + 'activities' => insert_query } + + allow(@cluster).to receive(:connect).and_return(@session) + allow(Cassandra).to receive(:cluster).with(any_args).and_return(@cluster) + allow_any_instance_of(described_class).to receive(:initialize_database).and_return(true) + + described_class.instance.instance_variable_set(:@queries, @queries) + described_class.instance.instance_variable_set(:@insert_queries, @insert_queries) + end + + describe 'insert' do + end + + describe 'query_heart_rate' do + it 'calls the execute method on the session object' do + var_name = 'heart_rate' + session = double('session') + expect(session).to receive(:execute).with(@queries[var_name], arguments: arguments_array) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.query_heart_rate(user_id, year, from, to) + end + end + + describe 'query_sleep' do + it 'calls the execute method on the session object' do + var_name = 'sleep' + session = double('session') + expect(session).to receive(:execute).with(@queries[var_name], arguments: arguments_array) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.query_sleep(user_id, year, from, to) + end + end + + describe 'query_calories' do + it 'calls the execute method on the session object' do + var_name = 'calories' + session = double('session') + expect(session).to receive(:execute).with(@queries[var_name], arguments: arguments_array) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.query_heart_rate(user_id, year, from, to) + end + end + + describe 'query_distance' do + it 'calls the execute method on the session object' do + var_name = 'distance' + session = double('session') + expect(session).to receive(:execute).with(@queries[var_name], arguments: arguments_array) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.query_distance(user_id, year, from, to) + end + end + + describe 'query_steps' do + it 'calls the execute method on the session object' do + var_name = 'steps' + session = double('session') + expect(session).to receive(:execute).with(@queries[var_name], arguments: arguments_array) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.query_steps(user_id, year, from, to) + end + end + + describe 'query_activities' do + it 'calls the execute method on the session object' do + var_name = 'activities' + session = double('session') + expect(session).to receive(:execute).with(@queries[var_name], arguments: arguments_array) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.query_activities(user_id, year, from, to) + end + end + + # private methods + + describe 'slice' do + let!(:times) { [1, 2, 3, 4, 5, 6] } + let(:start_dates) { [6, 5, 4, 3, 2, 1] } + let(:end_dates) { %w(a c d e f) } + let(:values) { %w(f e d c b a) } + let(:number_of_variables) { 4 } + let(:number_of_entries) { times.length } + it 'should slice the provided arrays in the slice_size' do + stub_const('Physiqual::CassandraConnection::SLICE_SIZE', 2) + expect(described_class::SLICE_SIZE).to eq(2) + res = described_class.instance.send(:slice, times, start_dates, end_dates, values) + res.each { |x| expect(x.length).to eq((number_of_entries.to_f / described_class::SLICE_SIZE.to_f).ceil) } + end + + it 'should be able to deal with SLICE_SIZE > then the number of entries' do + res = described_class.instance.send(:slice, times, start_dates, end_dates, values) + expect(res.length).to eq(number_of_variables) + res.each { |x| expect(x.length).to eq((number_of_entries.to_f / described_class::SLICE_SIZE.to_f).ceil) } + end + end + + describe 'initialize_database' do + before(:each) do + allow_any_instance_of(described_class).to receive(:initialize_database).and_call_original + end + + let(:variables) do + { 'heart_rate' => 'decimal', + 'sleep' => 'decimal', + 'calories' => 'decimal', + 'distance' => 'decimal', + 'steps' => 'decimal', + 'activities' => 'varchar' } + end + it 'should initialize the insert queries' do + variables.each do |variable, _type| + allow(described_class.instance).to receive(:prepare_query).with(any_args) + allow(described_class.instance).to receive(:create_table).with(any_args) + expect(described_class.instance).to receive(:prepare_insert).with(variable) + end + described_class.instance.send(:initialize_database, variables) + end + + it 'should initialize the retrieve queries' do + variables.each do |variable, _type| + expect(described_class.instance).to receive(:prepare_query).with(variable) + allow(described_class.instance).to receive(:create_table).with(any_args) + allow(described_class.instance).to receive(:prepare_insert).with(any_args) + end + described_class.instance.send(:initialize_database, variables) + end + + it 'should initialize the tables' do + variables.each do |variable, type| + allow(described_class.instance).to receive(:prepare_query).with(any_args) + expect(described_class.instance).to receive(:create_table).with(variable, type) + allow(described_class.instance).to receive(:prepare_insert).with(any_args) + end + described_class.instance.send(:initialize_database, variables) + end + end + + describe 'prepare_insert' do + it 'should use the provided name in the query' do + table_name = 'some random variable name' + qry = "INSERT INTO #{table_name} (user_id, year, time, start_date, end_date, value) VALUES (?, ?, ?, ?, ?, ?)" + + session = double('session') + expect(session).to receive(:prepare).with(qry) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.send(:prepare_insert, table_name) + end + end + + describe 'prepare_query' do + it 'should use the provided name in the query' do + table_name = 'some random variable name' + qry = " + SELECT time, start_date, end_date, value + FROM #{table_name} + WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? + ORDER BY time ASC + " + session = double('session') + expect(session).to receive(:prepare).with(qry) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.send(:prepare_query, table_name) + end + end + + describe 'create_table' do + it 'should use the provided name in the query' do + name = 'some random variable name' + value_type = 'some random variable type' + query = " + CREATE TABLE IF NOT EXISTS #{name} ( + user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, + PRIMARY KEY ((user_id, year), time) + ) + " + session = double('session') + expect(session).to receive(:execute).with(query) + described_class.instance.instance_variable_set(:@session, session) + described_class.instance.send(:create_table, name, value_type) + end + end + end + end +end diff --git a/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb b/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb new file mode 100644 index 0000000..e69de29 From 3b4ac058f75d83e292c7c8d7a854c949d6d7c78a Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Tue, 19 Jul 2016 15:06:47 +0200 Subject: [PATCH 21/35] Removed the time splitting construction, refactored the code to use entries --- app/workers/physiqual/cache_worker.rb | 93 ------------------ lib/physiqual/cassandra_connection.rb | 16 +-- .../data_services/cassandra_data_service.rb | 4 +- lib/physiqual/workers.rb | 7 ++ lib/physiqual/workers/cache_worker.rb | 98 +++++++++++++++++++ .../physiqual/cassandra_connection_spec.rb | 11 +++ .../cassandra_data_service_spec.rb | 6 ++ .../physiqual/workers/cache_worker_spec.rb | 8 ++ 8 files changed, 140 insertions(+), 103 deletions(-) delete mode 100644 app/workers/physiqual/cache_worker.rb create mode 100644 lib/physiqual/workers.rb create mode 100644 lib/physiqual/workers/cache_worker.rb create mode 100644 spec/lib/physiqual/workers/cache_worker_spec.rb diff --git a/app/workers/physiqual/cache_worker.rb b/app/workers/physiqual/cache_worker.rb deleted file mode 100644 index 7f5d895..0000000 --- a/app/workers/physiqual/cache_worker.rb +++ /dev/null @@ -1,93 +0,0 @@ -require 'sidekiq' - -module Physiqual - class CacheWorker - include Sidekiq::Worker - def perform(table, user_id, from, to) - token = User.find_by_user_id(user_id).physiqual_token - return [] unless token.complete? - session = Sessions::TokenAuthorizedSession.new(token) - @data_service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) - @user_id = user_id - connection = DataServices::CassandraConnection.instance - from = Time.zone.parse(from) - to = Time.zone.parse(to) - store_data(connection, table, from, to) - end - - private - - def store_data(connection, table, from, to) - entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) - data_service_function = get_data_function(table) - new_entries = [] - if entries.blank? - new_entries = data_service_function.call(from, to) - else - if entries.first.start_date > from - new_entries += data_service_function.call(from, entries.first.start_date) - end - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service_function.call(from_gap, to_gap) - end - if entries.last.end_date < to - new_entries += data_service_function.call(entries.last.end_date, to) - end - end - cache(connection, table, @user_id, new_entries) if new_entries.present? - rescue Errors::NotSupportedError => e - Rails.logger.warn e.message - end - - def get_data_function(table) - case table - when 'heart_rate' - @data_service.method(:heart_rate) - when 'sleep' - @data_service.method(:sleep) - when 'calories' - @data_service.method(:calories) - when 'distance' - @data_service.method(:distance) - when 'steps' - @data_service.method(:steps) - when 'activities' - @data_service.method(:activities) - end - end - - def find_gaps(entries) - entries.each_with_index do |entry, i| - break if i == entries.length - 1 - if entry.end_date != entries[i + 1].start_date - yield(entry.end_date, entries[i + 1].start_date) - end - end - end - - def cache(connection, table, user_id, new_entries) - year = new_entries.first.measurement_moment.strftime('%Y').to_i - times = [] - start_dates = [] - end_dates = [] - values = [] - new_entries.each do |entry| - if year != entry.measurement_moment.strftime('%Y').to_i - connection.insert(table, user_id, year, times, start_dates, end_dates, values) - year = entry.measurement_moment.strftime('%Y').to_i - times = [] - start_dates = [] - end_dates = [] - values = [] - end - times << entry.measurement_moment - start_dates << entry.start_date - end_dates << entry.end_date - values << entry.values.first - if entry == new_entries.last - connection.insert(table, user_id, year, times, start_dates, end_dates, values) - end - end - end - end -end diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb index 750a731..f2d8f93 100644 --- a/lib/physiqual/cassandra_connection.rb +++ b/lib/physiqual/cassandra_connection.rb @@ -31,17 +31,17 @@ def initialize initialize_database(variables) end - def insert(table, user_id, year, times, start_dates, end_dates, values) + def insert(table, user_id, year, entries) # Slice the dates in chuncs of SLICE_SIZE - times_slices, start_dates_slices, end_dates_slices, values_slices = slice(times, start_dates, end_dates, values) + entries = entries.each_slice(SLICE_SIZE).to_a - # Merge all slices into one array - times_slices.zip!(start_dates_slices, end_dates_slices, values_slices) - - times_slices.each_with_index do |times_slice, start_dates_slice, end_dates_slice, values_slice| + entries.each_with_index do |entry_slice| batch = @session.batch do |b| - zipped_slices = times_slice.zip(start_dates_slice, end_dates_slice, values_slice) - zipped_slices.each do |time, start_date, end_date, value| + entry_slice.each do |entry| + time = entry.measurement_moment + start_date = entry.start_date + end_date = entry.end_date + value = entry.values.first # If the table is 'activities', we should not convert the slice to a bigdecimal value = BigDecimal(value, Float::DIG + 1) if table != 'activities' diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 07a7338..d43974b 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -76,8 +76,8 @@ def cassandra_connection @connection end - def cache_data(table, user_id, from, to) - job = Physiqual::CacheWorker.perform_async(table, user_id, from, to) + def cache_data(variable, user_id, from, to) + job = Physiqual::Workers::CacheWorker.perform_async(data_service, variable, user_id, from, to) while Sidekiq::Status.queued? job or Sidekiq::Status.working? job Kernel.sleep(1) end diff --git a/lib/physiqual/workers.rb b/lib/physiqual/workers.rb new file mode 100644 index 0000000..2ffe898 --- /dev/null +++ b/lib/physiqual/workers.rb @@ -0,0 +1,7 @@ +require 'httparty' +require 'physiqual/workers/cache_worker' + +module Physiqual + module Workers + end +end diff --git a/lib/physiqual/workers/cache_worker.rb b/lib/physiqual/workers/cache_worker.rb new file mode 100644 index 0000000..e2e3380 --- /dev/null +++ b/lib/physiqual/workers/cache_worker.rb @@ -0,0 +1,98 @@ +require 'sidekiq' + +module Physiqual + module Workers + class CacheWorker + include Sidekiq::Worker + + def perform(data_service, variable, user_id, from, to) + @user_id = user_id + @data_service = data_service + + connection = DataServices::CassandraConnection.instance + from = Time.zone.parse(from) + to = Time.zone.parse(to) + store_data(connection, variable, from, to) + end + + private + + def store_data(connection, table, from, to) + entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) + + # Retrieve the function to use for getting the data + data_service_function = get_data_function(table) + + new_entries = [] + + # If there were no enties in the first place, just make the call to the dataservice + if entries.blank? + new_entries = data_service_function.call(from, to) + else + if entries.first.start_date > from + new_entries += data_service_function.call(from, entries.first.start_date) + end + + find_gaps(entries) do |from_gap, to_gap| + new_entries += data_service_function.call(from_gap, to_gap) + end + + if entries.last.end_date < to + new_entries += data_service_function.call(entries.last.end_date, to) + end + end + + # Cache the newly retrieved data + cache(connection, table, @user_id, new_entries) if new_entries.present? + rescue Errors::NotSupportedError => e + Rails.logger.warn e.message + end + + def get_data_function(table) + case table + when 'heart_rate' + @data_service.method(:heart_rate) + when 'sleep' + @data_service.method(:sleep) + when 'calories' + @data_service.method(:calories) + when 'distance' + @data_service.method(:distance) + when 'steps' + @data_service.method(:steps) + when 'activities' + @data_service.method(:activities) + end + end + + def find_gaps(entries) + entries.each_with_index do |entry, i| + break if i == entries.length - 1 + if entry.end_date != entries[i + 1].start_date + yield(entry.end_date, entries[i + 1].start_date) + end + end + end + + def cache(connection, table, user_id, new_entries) + year_of_old_entry = new_entries.first.measurement_moment.strftime('%Y').to_i + entries = [] + new_entries.each do |entry| + year_of_current_entry = entry.measurement_moment.strftime('%Y').to_i + + # If the previous year is not equal to the current year we insert a new row + if year_of_old_entry != year_of_current_entry + connection.insert(table, user_id, year_of_first_entry, entries) + year_of_old_entry = year_of_current_entry + + entries = [] + end + entries << entry + end + + # Insert the remaining data + connection.insert(table, user_id, year, entries) + end + end + end +end diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index 43b05c1..b7fb0db 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -40,6 +40,17 @@ module Physiqual end describe 'insert' do + it "should insert data in the database in batches" do + expect(true).to eq false + end + + it "should be able to work with nil values" do + expect(true).to eq false + end + + it 'should convert values to big decimal values if needed' do + + end end describe 'query_heart_rate' do diff --git a/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb b/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb index e69de29..a933f7f 100644 --- a/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb +++ b/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb @@ -0,0 +1,6 @@ +module Physiqual + require 'rails_helper' + describe CassandraDataService do + + end +end \ No newline at end of file diff --git a/spec/lib/physiqual/workers/cache_worker_spec.rb b/spec/lib/physiqual/workers/cache_worker_spec.rb new file mode 100644 index 0000000..a255e87 --- /dev/null +++ b/spec/lib/physiqual/workers/cache_worker_spec.rb @@ -0,0 +1,8 @@ +module Physiqual + module Workers + require 'rails_helper' + describe CacheWorker do + + end + end +end \ No newline at end of file From 20a5165f96b9db2ad8cb3b02e83a41b1efe26dd9 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Tue, 19 Jul 2016 17:40:22 +0200 Subject: [PATCH 22/35] WIP --- cassandra.sh | 5 + lib/physiqual.rb | 1 + lib/physiqual/cassandra_connection.rb | 23 +++- .../data_services/cassandra_data_service.rb | 115 +++++++----------- lib/physiqual/engine.rb | 12 +- lib/physiqual/workers/cache_worker.rb | 30 ++--- spec/dummy/config/initializers/physiqual.rb | 2 +- spec/dummy/config/initializers/sidekiq.rb | 2 + 8 files changed, 97 insertions(+), 93 deletions(-) create mode 100755 cassandra.sh diff --git a/cassandra.sh b/cassandra.sh new file mode 100755 index 0000000..8833cdc --- /dev/null +++ b/cassandra.sh @@ -0,0 +1,5 @@ +#!/bin/bash +line="CREATE KEYSPACE physiqual WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};" +echo $line | pbcopy + +docker run -it --rm --name cassmgmt --link cassandra1:cassmgmt cassandra cqlsh cassandra1 diff --git a/lib/physiqual.rb b/lib/physiqual.rb index 86f518c..a3dddad 100644 --- a/lib/physiqual.rb +++ b/lib/physiqual.rb @@ -10,6 +10,7 @@ require 'physiqual/users' require 'physiqual/version' require 'physiqual/cassandra_connection' +require 'physiqual/workers' module Physiqual end diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb index f2d8f93..6cadef0 100644 --- a/lib/physiqual/cassandra_connection.rb +++ b/lib/physiqual/cassandra_connection.rb @@ -8,18 +8,22 @@ class CassandraConnection def initialize # Setup the connection to the cluster + Rails.logger.info(Physiqual.cassandra_username) + Rails.logger.info(Physiqual.cassandra_password) + Rails.logger.info(Physiqual.cassandra_host_urls) if Physiqual.cassandra_username.blank? || Physiqual.cassandra_password.blank? cluster = Cassandra.cluster( - hosts: Physiqual.cassandra_host_urls + hosts: Physiqual.cassandra_urls ) else cluster = Cassandra.cluster( - username: ENV['CASSANDRA_USERNAME'], - password: ENV['CASSANDRA_PASSWORD'], - hosts: Physiqual.cassandra_host_urls + username: Physiqual.cassandra_username, + password: Physiqual.cassandra_password, + hosts: Physiqual.cassandra_urls ) end - @session = cluster.connect('physiqual') + + @session = cluster.connect(Physiqual.cassandra_keyspace) variables = { 'heart_rate' => 'decimal', 'sleep' => 'decimal', @@ -28,6 +32,7 @@ def initialize 'steps' => 'decimal', 'activities' => 'varchar' } + initialize_database(variables) end @@ -92,12 +97,18 @@ def initialize_database(variable_names) @insert_queries = {} @queries = {} variable_names.each do |variable, type| + create_table(variable, type) @insert_queries[variable] = prepare_insert(variable) @queries[variable] = prepare_query(variable) - create_table(variable, type) end end + def create_keyspace(keypsace_name) + query = "CREATE KEYSPACE #{keyspace_name} + WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};" + @session.execute(query) + end + def prepare_insert(table_name) query = "INSERT INTO #{table_name} (user_id, year, time, start_date, end_date, value) VALUES (?, ?, ?, ?, ?, ?)" @session.prepare(query) diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index d43974b..04bc315 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -6,6 +6,7 @@ class CassandraDataService < DataServiceDecorator def initialize(data_service, user_id) super(data_service) @user_id = user_id + @cassandra_connection ||= CassandraConnection.instance end def service_name @@ -14,112 +15,90 @@ def service_name def heart_rate(from, to) cache_data('heart_rate', @user_id, from, to) - CassandraDataService.get_data(cassandra_connection, @user_id, 'heart_rate', from, to) + get_data(@cassandra_connection, @user_id, 'heart_rate', from, to) end def sleep(from, to) cache_data('sleep', @user_id, from, to) - CassandraDataService.get_data(cassandra_connection, @user_id, 'sleep', from, to) + get_data(@cassandra_connection, @user_id, 'sleep', from, to) end def calories(from, to) cache_data('calories', @user_id, from, to) - CassandraDataService.get_data(cassandra_connection, @user_id, 'calories', from, to) + get_data(@cassandra_connection, @user_id, 'calories', from, to) end def distance(from, to) cache_data('distance', @user_id, from, to) - CassandraDataService.get_data(cassandra_connection, @user_id, 'distance', from, to) + get_data(@cassandra_connection, @user_id, 'distance', from, to) end def steps(from, to) cache_data('steps', @user_id, from, to) - CassandraDataService.get_data(cassandra_connection, @user_id, 'steps', from, to) + get_data(@cassandra_connection, @user_id, 'steps', from, to) end def activities(from, to) cache_data('activities', @user_id, from, to) - CassandraDataService.get_data(cassandra_connection, @user_id, 'activities', from, to) + get_data(@cassandra_connection, @user_id, 'activities', from, to) end - def self.get_data(connection, user_id, table, from, to) + def get_data(connection, user_id, variable, from, to) entries = [] years(from, to) do |year, from_per_year, to_per_year| - entries += case table - when 'heart_rate' - make_data_entries(table, - connection.query_heart_rate(user_id, year, from_per_year, to_per_year)) - when 'sleep' - make_data_entries(table, - connection.query_sleep(user_id, year, from_per_year, to_per_year)) - when 'calories' - make_data_entries(table, - connection.query_calories(user_id, year, from_per_year, to_per_year)) - when 'distance' - make_data_entries(table, - connection.query_distance(user_id, year, from_per_year, to_per_year)) - when 'steps' - make_data_entries(table, - connection.query_steps(user_id, year, from_per_year, to_per_year)) - when 'activities' - make_data_entries(table, - connection.query_activities(user_id, year, from_per_year, to_per_year)) - end + case variable + when 'heart_rate' + query_result = connection.query_heart_rate(user_id, year, from_per_year, to_per_year) + when 'sleep' + query_result = connection.query_sleep(user_id, year, from_per_year, to_per_year) + when 'calories' + query_result = connection.query_calories(user_id, year, from_per_year, to_per_year) + when 'distance' + query_result = connection.query_distance(user_id, year, from_per_year, to_per_year) + when 'steps' + query_result = connection.query_steps(user_id, year, from_per_year, to_per_year) + when 'activities' + query_result = connection.query_activities(user_id, year, from_per_year, to_per_year) + end + + entries += make_data_entries(variable, query_result) end entries end private - - def cassandra_connection - @connection ||= CassandraConnection.instance - @connection - end - + def cache_data(variable, user_id, from, to) - job = Physiqual::Workers::CacheWorker.perform_async(data_service, variable, user_id, from, to) + job = Physiqual::Workers::CacheWorker.perform_async(data_service, self, variable, user_id, from, to) + #job = Physiqual::Workers::CacheWorker.new.perform(data_service, self, variable, user_id, from, to) while Sidekiq::Status.queued? job or Sidekiq::Status.working? job + Rails.logger.info('Sleeping!') Kernel.sleep(1) end end - class << self - private - - def years(from, to) - from_year = from.strftime('%Y').to_i - to_year = to.strftime('%Y').to_i - (from_year..to_year).each do |year| - from_per_year = if from_year < year - Time.zone.local(year, 1, 1, 0, 0, 0) - else - from - end - to_per_year = if to_year > year - Time.zone.local(year, 12, 31, 23, 59, 59) - else - to - end - yield(year, from_per_year, to_per_year) - end + def years(from, to) + from_year = from.strftime('%Y').to_i + to_year = to.strftime('%Y').to_i + (from_year..to_year).each do |year| + from_per_year = from_year < year ? Time.zone.local(year, 1, 1, 0, 0, 0) : from + to_per_year = to_year > year ? Time.zone.local(year, 12, 31, 23, 59, 59) : to + yield(year, from_per_year, to_per_year) end + end - def make_data_entries(table, results) - return [] if results.blank? - entries = [] - results.each do |result| - value = if table == 'activities' - result['value'] - else - result['value'].to_f - end - entries << DataEntry.new(start_date: result['start_date'].in_time_zone, - end_date: result['end_date'].in_time_zone, - values: value, - measurement_moment: result['time'].in_time_zone) - end - entries + def make_data_entries(table, results) + return [] if results.blank? + entries = [] + results.each do |result| + # TODO Should this be to_i or to_f + value = table == 'activities' ? result['value'] : result['value'].to_i + entries << DataEntry.new(start_date: result['start_date'].in_time_zone, + end_date: result['end_date'].in_time_zone, + values: [value], + measurement_moment: result['time'].in_time_zone) end + entries end end end diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index ab7c4ab..d06b1e0 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -14,14 +14,18 @@ class << self mattr_accessor :enable_cassandra mattr_accessor :cassandra_username mattr_accessor :cassandra_password - mattr_writer :cassandra_host_urls + mattr_accessor :cassandra_host_urls + mattr_accessor :cassandra_keyspace - def cassandra_host_urls - @cassandra_host_urls.split(' ') unless @cassandra_host_urls.blank? + def cassandra_urls + urls = self.cassandra_host_urls + a = urls.split(' ') unless urls.blank? + Rails.logger.info(urls) + Rails.logger.info(a) + a end mattr_accessor :cassandra_keyspace - mattr_accessor :host_url mattr_accessor :host_protocol mattr_accessor :measurements_per_day diff --git a/lib/physiqual/workers/cache_worker.rb b/lib/physiqual/workers/cache_worker.rb index e2e3380..b17ec14 100644 --- a/lib/physiqual/workers/cache_worker.rb +++ b/lib/physiqual/workers/cache_worker.rb @@ -5,26 +5,27 @@ module Workers class CacheWorker include Sidekiq::Worker - def perform(data_service, variable, user_id, from, to) + def perform(data_service, cassandra_dataservice, variable, user_id, from, to) @user_id = user_id @data_service = data_service - connection = DataServices::CassandraConnection.instance - from = Time.zone.parse(from) - to = Time.zone.parse(to) - store_data(connection, variable, from, to) + connection = CassandraConnection.instance + + # Sidekiq converts all dates to strings, so we have to parse them back to dates, + from = Time.zone.parse(from) if from.is_a? String + to = Time.zone.parse(to) if to.is_a? String + store_data(connection, cassandra_dataservice, variable, from, to) end private - def store_data(connection, table, from, to) - entries = DataServices::CassandraDataService.get_data(connection, @user_id, table, from, to) - + def store_data(connection, cassandra_dataservice, table, from, to) + entries = cassandra_dataservice.get_data(connection, @user_id, table, from, to) # Retrieve the function to use for getting the data data_service_function = get_data_function(table) new_entries = [] - + Rails.logger.info('new entries') # If there were no enties in the first place, just make the call to the dataservice if entries.blank? new_entries = data_service_function.call(from, to) @@ -33,15 +34,15 @@ def store_data(connection, table, from, to) new_entries += data_service_function.call(from, entries.first.start_date) end - find_gaps(entries) do |from_gap, to_gap| - new_entries += data_service_function.call(from_gap, to_gap) - end + # find_gaps(entries) do |from_gap, to_gap| + # new_entries += data_service_function.call(from_gap, to_gap) + # end if entries.last.end_date < to new_entries += data_service_function.call(entries.last.end_date, to) end end - + Rails.logger.info('caching new entries') # Cache the newly retrieved data cache(connection, table, @user_id, new_entries) if new_entries.present? rescue Errors::NotSupportedError => e @@ -68,6 +69,7 @@ def get_data_function(table) def find_gaps(entries) entries.each_with_index do |entry, i| break if i == entries.length - 1 + # TODO: This is dramatically inefficient. if entry.end_date != entries[i + 1].start_date yield(entry.end_date, entries[i + 1].start_date) end @@ -91,7 +93,7 @@ def cache(connection, table, user_id, new_entries) end # Insert the remaining data - connection.insert(table, user_id, year, entries) + connection.insert(table, user_id, year_of_old_entry, entries) end end end diff --git a/spec/dummy/config/initializers/physiqual.rb b/spec/dummy/config/initializers/physiqual.rb index ef091c6..addfa00 100644 --- a/spec/dummy/config/initializers/physiqual.rb +++ b/spec/dummy/config/initializers/physiqual.rb @@ -15,7 +15,7 @@ config.enable_cassandra = ENV['ENABLE_CASSANDRA'] ? ENV['ENABLE_CASSANDRA'].downcase == 'true' : false config.cassandra_username = ENV['CASSANDRA_USERNAME'] || '' config.cassandra_password = ENV['CASSANDRA_PASSWORD'] || '' - config.cassandra_host_urls = (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + config.cassandra_host_urls = ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev' config.cassandra_keyspace = ENV['CASSANDRA_KEYSPACE'] # EMA Settings diff --git a/spec/dummy/config/initializers/sidekiq.rb b/spec/dummy/config/initializers/sidekiq.rb index 3f77e7e..15e05f2 100644 --- a/spec/dummy/config/initializers/sidekiq.rb +++ b/spec/dummy/config/initializers/sidekiq.rb @@ -2,6 +2,7 @@ require 'sidekiq-status' Sidekiq.configure_client do |config| + config.redis = { url: ENV['REDIS_URL'], password: ENV['REDIS_PASSWORD']} config.client_middleware do |chain| # accepts :expiration (optional) chain.add Sidekiq::Status::ClientMiddleware, expiration: 30.minutes # default @@ -9,6 +10,7 @@ end Sidekiq.configure_server do |config| + config.redis = { url: ENV['REDIS_URL'], password: ENV['REDIS_PASSWORD']} config.server_middleware do |chain| # accepts :expiration (optional) chain.add Sidekiq::Status::ServerMiddleware, expiration: 30.minutes # default From 7c21ba3771d1dc76a3cb19dc1820ebe135e0a021 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 00:04:37 +0200 Subject: [PATCH 23/35] More refactoring, added some specs --- lib/physiqual/cassandra_connection.rb | 3 +- .../data_services/cassandra_data_service.rb | 34 ++--- lib/physiqual/engine.rb | 2 +- lib/physiqual/exporters/exporter.rb | 4 +- lib/physiqual/workers/cache_worker.rb | 24 ++-- .../physiqual/cassandra_connection_spec.rb | 5 +- .../cassandra_data_service_spec.rb | 124 +++++++++++++++++- .../physiqual/workers/cache_worker_spec.rb | 3 +- 8 files changed, 158 insertions(+), 41 deletions(-) diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb index 6cadef0..33b3a7b 100644 --- a/lib/physiqual/cassandra_connection.rb +++ b/lib/physiqual/cassandra_connection.rb @@ -32,7 +32,6 @@ def initialize 'steps' => 'decimal', 'activities' => 'varchar' } - initialize_database(variables) end @@ -103,7 +102,7 @@ def initialize_database(variable_names) end end - def create_keyspace(keypsace_name) + def create_keyspace(_keypsace_name) query = "CREATE KEYSPACE #{keyspace_name} WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};" @session.execute(query) diff --git a/lib/physiqual/data_services/cassandra_data_service.rb b/lib/physiqual/data_services/cassandra_data_service.rb index 04bc315..ca367b9 100644 --- a/lib/physiqual/data_services/cassandra_data_service.rb +++ b/lib/physiqual/data_services/cassandra_data_service.rb @@ -3,10 +3,10 @@ module Physiqual module DataServices class CassandraDataService < DataServiceDecorator - def initialize(data_service, user_id) + def initialize(data_service, user_id, cassandra_connection) super(data_service) @user_id = user_id - @cassandra_connection ||= CassandraConnection.instance + @cassandra_connection ||= cassandra_connection end def service_name @@ -47,18 +47,18 @@ def get_data(connection, user_id, variable, from, to) entries = [] years(from, to) do |year, from_per_year, to_per_year| case variable - when 'heart_rate' - query_result = connection.query_heart_rate(user_id, year, from_per_year, to_per_year) - when 'sleep' - query_result = connection.query_sleep(user_id, year, from_per_year, to_per_year) - when 'calories' - query_result = connection.query_calories(user_id, year, from_per_year, to_per_year) - when 'distance' - query_result = connection.query_distance(user_id, year, from_per_year, to_per_year) - when 'steps' - query_result = connection.query_steps(user_id, year, from_per_year, to_per_year) - when 'activities' - query_result = connection.query_activities(user_id, year, from_per_year, to_per_year) + when 'heart_rate' + query_result = connection.query_heart_rate(user_id, year, from_per_year, to_per_year) + when 'sleep' + query_result = connection.query_sleep(user_id, year, from_per_year, to_per_year) + when 'calories' + query_result = connection.query_calories(user_id, year, from_per_year, to_per_year) + when 'distance' + query_result = connection.query_distance(user_id, year, from_per_year, to_per_year) + when 'steps' + query_result = connection.query_steps(user_id, year, from_per_year, to_per_year) + when 'activities' + query_result = connection.query_activities(user_id, year, from_per_year, to_per_year) end entries += make_data_entries(variable, query_result) @@ -67,10 +67,10 @@ def get_data(connection, user_id, variable, from, to) end private - + def cache_data(variable, user_id, from, to) job = Physiqual::Workers::CacheWorker.perform_async(data_service, self, variable, user_id, from, to) - #job = Physiqual::Workers::CacheWorker.new.perform(data_service, self, variable, user_id, from, to) + # job = Physiqual::Workers::CacheWorker.new.perform(data_service, self, variable, user_id, from, to) while Sidekiq::Status.queued? job or Sidekiq::Status.working? job Rails.logger.info('Sleeping!') Kernel.sleep(1) @@ -91,7 +91,7 @@ def make_data_entries(table, results) return [] if results.blank? entries = [] results.each do |result| - # TODO Should this be to_i or to_f + # TODO: Should this be to_i or to_f value = table == 'activities' ? result['value'] : result['value'].to_i entries << DataEntry.new(start_date: result['start_date'].in_time_zone, end_date: result['end_date'].in_time_zone, diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index d06b1e0..54a8044 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -18,7 +18,7 @@ class << self mattr_accessor :cassandra_keyspace def cassandra_urls - urls = self.cassandra_host_urls + urls = cassandra_host_urls a = urls.split(' ') unless urls.blank? Rails.logger.info(urls) Rails.logger.info(a) diff --git a/lib/physiqual/exporters/exporter.rb b/lib/physiqual/exporters/exporter.rb index 770b465..4352fb6 100644 --- a/lib/physiqual/exporters/exporter.rb +++ b/lib/physiqual/exporters/exporter.rb @@ -39,7 +39,9 @@ def create_service(user, bucket_generator) service = DataServices::DataServiceFactory.fabricate!(token.class.csrf_token, session) # Enable caching with cassandra if it is enabled - service = DataServices::CassandraDataService.new(service, user.user_id) if Physiqual.enable_cassandra + if Physiqual.enable_cassandra + service = DataServices::CassandraDataService.new(service, user.user_id, CassandraConnection.instance) + end # Transform the raw data into buckets service = DataServices::BucketeerDataService.new(service, bucket_generator) diff --git a/lib/physiqual/workers/cache_worker.rb b/lib/physiqual/workers/cache_worker.rb index b17ec14..d7659ef 100644 --- a/lib/physiqual/workers/cache_worker.rb +++ b/lib/physiqual/workers/cache_worker.rb @@ -51,18 +51,18 @@ def store_data(connection, cassandra_dataservice, table, from, to) def get_data_function(table) case table - when 'heart_rate' - @data_service.method(:heart_rate) - when 'sleep' - @data_service.method(:sleep) - when 'calories' - @data_service.method(:calories) - when 'distance' - @data_service.method(:distance) - when 'steps' - @data_service.method(:steps) - when 'activities' - @data_service.method(:activities) + when 'heart_rate' + @data_service.method(:heart_rate) + when 'sleep' + @data_service.method(:sleep) + when 'calories' + @data_service.method(:calories) + when 'distance' + @data_service.method(:distance) + when 'steps' + @data_service.method(:steps) + when 'activities' + @data_service.method(:activities) end end diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index b7fb0db..1196a1c 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -40,16 +40,15 @@ module Physiqual end describe 'insert' do - it "should insert data in the database in batches" do + it 'should insert data in the database in batches' do expect(true).to eq false end - it "should be able to work with nil values" do + it 'should be able to work with nil values' do expect(true).to eq false end it 'should convert values to big decimal values if needed' do - end end diff --git a/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb b/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb index a933f7f..0322a0a 100644 --- a/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb +++ b/spec/lib/physiqual/data_services/cassandra_data_service_spec.rb @@ -1,6 +1,124 @@ module Physiqual - require 'rails_helper' - describe CassandraDataService do + module DataServices + require 'rails_helper' + describe CassandraDataService do + let(:subject) { described_class.new(nil, 1, nil) } + describe 'years' do + it 'should return blocks of years' do + Timecop.freeze(Date.new(2016, 0o1, 0o5)) + endd = Time.now + start = 10.days.ago - 1.year + expected_years = [2014, 2015, 2016] + number_of_years = expected_years.length + froms = [] + tos = [] + subject.send(:years, start, endd) do |year, from, to| + expect(year).to eq expected_years.shift + froms << from + tos << to + end + expect(froms.length).to eq number_of_years + expect(tos.length).to eq number_of_years + expect(froms.first).to eq start + expect(froms.second).to eq Date.new(2015, 0o1, 0o1).to_time + expect(froms.last).to eq Date.new(2016, 0o1, 0o1).to_time + + expect(tos.first).to eq Time.new(2014, 12, 31, 23, 59, 59).to_time + expect(tos.second).to eq Time.new(2015, 12, 31, 23, 59, 59).to_time + expect(tos.last).to eq endd + Timecop.return + end + + it 'should only return one year if there is no year shift in between' do + Timecop.freeze(Date.new(2016, 0o1, 25)) + endd = Time.now + start = 10.days.ago + expected_years = [2016] + number_of_years = expected_years.length + froms = [] + tos = [] + subject.send(:years, start, endd) do |year, from, to| + expect(year).to eq expected_years.shift + froms << from + tos << to + end + expect(froms.length).to eq number_of_years + expect(tos.length).to eq number_of_years + + expect(froms.first).to eq start + expect(tos.last).to eq endd + end + end + + describe 'cache_data' do + let(:start_date) { Time.new(2014, 12, 31, 23, 56, 0) } + let(:end_date) { Time.new(2014, 12, 31, 23, 57, 0) } + let(:user_id) { 1 } + let(:variable) { 'some_variable' } + it 'should call sidekick asynchronously' do + expect(Sidekiq::Status).to receive(:queued?).and_return(false) + expect(Sidekiq::Status).to receive(:working?).and_return(false) + + expect(Physiqual::Workers::CacheWorker).to receive(:perform_async) + .with(subject.data_service, subject, variable, user_id, start_date, end_date) + .and_return(true) + subject.send(:cache_data, variable, user_id, start_date, end_date) + end + end + + describe 'make_data_enries' do + let(:results) do + [ + { + 'value' => '10', + 'start_date' => Time.new(2014, 12, 31, 23, 56, 0), + 'end_date' => Time.new(2014, 12, 31, 23, 57, 0), + 'time' => Time.new(2014, 12, 31, 23, 56, 30) + }, + { + 'value' => '42', + 'start_date' => Time.new(2014, 12, 31, 23, 57, 0), + 'end_date' => Time.new(2014, 12, 31, 23, 58, 0), + 'time' => Time.new(2014, 12, 31, 23, 57, 30) + }, + { + 'value' => '1', + 'start_date' => Time.new(2014, 12, 31, 23, 58, 0), + 'end_date' => Time.new(2014, 12, 31, 23, 59, 0), + 'time' => Time.new(2014, 12, 31, 23, 58, 30) + } + ] + end + it 'should return [] if there are no results' do + expect(subject.send(:make_data_entries, 'variable', nil)).to eq [] + expect(subject.send(:make_data_entries, 'variable', [])).to eq [] + end + + it 'should transform the value of a result to int if it is not activity' do + result = subject.send(:make_data_entries, 'variable', results) + expect(result.length).to eq 3 + zipped = result.zip(results) + zipped.each do |res, expected| + expect(res.start_date).to eq expected['start_date'] + expect(res.end_date).to eq expected['end_date'] + expect(res.measurement_moment).to eq expected['time'] + expect(res.values).to be_an Array + expect(res.values.first).to be_an Integer + expect(res.values.first).to eq expected['value'].to_i + end + end + + it 'should not transform the value of a result to int if it is not activity' do + result = subject.send(:make_data_entries, 'activities', results) + zipped = result.zip(results) + zipped.each do |res, expected| + expect(res.values).to be_an Array + expect(res.values.first).to be_a String + expect(res.values.first).to eq expected['value'] + end + end + end + end end -end \ No newline at end of file +end diff --git a/spec/lib/physiqual/workers/cache_worker_spec.rb b/spec/lib/physiqual/workers/cache_worker_spec.rb index a255e87..ae82dc2 100644 --- a/spec/lib/physiqual/workers/cache_worker_spec.rb +++ b/spec/lib/physiqual/workers/cache_worker_spec.rb @@ -2,7 +2,6 @@ module Physiqual module Workers require 'rails_helper' describe CacheWorker do - end end -end \ No newline at end of file +end From cfb95f02ff7d2c4df2f548431e5dbef3a386494b Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 00:24:13 +0200 Subject: [PATCH 24/35] Updated specs to expect a user instead of just a token --- spec/lib/physiqual/exporters/exporter_spec.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spec/lib/physiqual/exporters/exporter_spec.rb b/spec/lib/physiqual/exporters/exporter_spec.rb index a8243b8..6c60a3a 100644 --- a/spec/lib/physiqual/exporters/exporter_spec.rb +++ b/spec/lib/physiqual/exporters/exporter_spec.rb @@ -21,25 +21,27 @@ module Exporters it 'should create a fitbit service for a correct fitbittoken provided' do token = FactoryGirl.build(:fitbit_token) + user = FactoryGirl.build(:physiqual_user, physiqual_token: token) expect(token.complete?).to be_truthy bucket_generator = BucketGenerators::EquidistantBucketGenerator.new( Physiqual.measurements_per_day, Physiqual.interval, Physiqual.hours_before_first_measurement ) - result = subject.send(:create_service, token, bucket_generator) + result = subject.send(:create_service, user, bucket_generator) expect(result.service_name).to end_with('fitbit_oauth2') end it 'should create a fitbit service for a correct fitbittoken provided' do token = FactoryGirl.build(:google_token) + user = FactoryGirl.build(:physiqual_user, physiqual_token: token) expect(token.complete?).to be_truthy bucket_generator = BucketGenerators::EquidistantBucketGenerator.new( Physiqual.measurements_per_day, Physiqual.interval, Physiqual.hours_before_first_measurement ) - result = subject.send(:create_service, token, bucket_generator) + result = subject.send(:create_service, user, bucket_generator) expect(result.service_name).to end_with('google_oauth2') end @@ -47,13 +49,14 @@ module Exporters let(:token) { FactoryGirl.build(:fitbit_token) } it 'should not create a service for a token which is not complete, but should return an empty array' do allow(token).to receive(:complete?).and_return(false) + user = FactoryGirl.build(:physiqual_user, physiqual_token: token) expect(token.complete?).to be_falsey bucket_generator = BucketGenerators::EquidistantBucketGenerator.new( Physiqual.measurements_per_day, Physiqual.interval, Physiqual.hours_before_first_measurement ) - result = subject.send(:create_service, token, bucket_generator) + result = subject.send(:create_service, user, bucket_generator) expect(result).to eq([]) end end From e6f9b5c2c91d8ca9e9404c1acc01162854e4d9f9 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 00:28:51 +0200 Subject: [PATCH 25/35] remove failing specs and rubocop' --- lib/physiqual/workers/cache_worker.rb | 2 ++ spec/lib/physiqual/cassandra_connection_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/physiqual/workers/cache_worker.rb b/lib/physiqual/workers/cache_worker.rb index d7659ef..9e57a34 100644 --- a/lib/physiqual/workers/cache_worker.rb +++ b/lib/physiqual/workers/cache_worker.rb @@ -5,6 +5,7 @@ module Workers class CacheWorker include Sidekiq::Worker + # rubocop:disable Metrics/ParameterLists def perform(data_service, cassandra_dataservice, variable, user_id, from, to) @user_id = user_id @data_service = data_service @@ -16,6 +17,7 @@ def perform(data_service, cassandra_dataservice, variable, user_id, from, to) to = Time.zone.parse(to) if to.is_a? String store_data(connection, cassandra_dataservice, variable, from, to) end + # rubocop:enable Metrics/ParameterLists private diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index 1196a1c..a523e80 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -41,11 +41,11 @@ module Physiqual describe 'insert' do it 'should insert data in the database in batches' do - expect(true).to eq false + # expect(true).to eq false end it 'should be able to work with nil values' do - expect(true).to eq false + # expect(true).to eq false end it 'should convert values to big decimal values if needed' do From 6da8226f82273ac88ab23de9eaf601f9ea822ec0 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 16:58:17 +0200 Subject: [PATCH 26/35] Added some more specs --- .rubocop.yml | 3 + lib/physiqual/cassandra_connection.rb | 43 +++---- .../physiqual/cassandra_connection_spec.rb | 117 ++++++++++++++---- .../summarized_data_service_spec.rb | 1 - 4 files changed, 113 insertions(+), 51 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index cf17c13..d47d95d 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -30,6 +30,9 @@ MethodLength: # Limit modules to have a length of 200 Metrics/ModuleLength: Max: 200 + Exclude: + - 'spec/**/*' + # Limit classes to have a length of 200 Metrics/ClassLength: diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb index 33b3a7b..e0d6de5 100644 --- a/lib/physiqual/cassandra_connection.rb +++ b/lib/physiqual/cassandra_connection.rb @@ -35,27 +35,31 @@ def initialize initialize_database(variables) end - def insert(table, user_id, year, entries) + def insert(variable, user_id, year, entries) # Slice the dates in chuncs of SLICE_SIZE entries = entries.each_slice(SLICE_SIZE).to_a entries.each_with_index do |entry_slice| - batch = @session.batch do |b| - entry_slice.each do |entry| - time = entry.measurement_moment - start_date = entry.start_date - end_date = entry.end_date - value = entry.values.first - # If the table is 'activities', we should not convert the slice to a bigdecimal - value = BigDecimal(value, Float::DIG + 1) if table != 'activities' - - # Retrieve the prepared statement - insert_type = @insert_queries[table] - b.add(insert_type, arguments: [user_id, year, time.to_time, start_date.to_time, - end_date.to_time, value]) - end + current_batch = create_batches(entry_slice, variable, user_id, year) + @session.execute(current_batch) + end + end + + def create_batches(entry_slice, variable, user_id, year) + @session.batch do |batch| + entry_slice.each do |entry| + value = entry.values.first + # If the table is 'activities', we should not convert the slice to a bigdecimal + value = BigDecimal(value, Float::DIG + 1) if variable != 'activities' + + # Retrieve the prepared statement + insert_type = @insert_queries[variable] + batch.add(insert_type, + arguments: [user_id, year, + entry.measurement_moment.to_time, + entry.start_date.to_time, + entry.end_date.to_time, value]) end - @session.execute(batch) end end @@ -85,13 +89,6 @@ def query_activities(user_id, year, from, to) private - def slice(times, start_dates, end_dates, values) - [times.each_slice(SLICE_SIZE).to_a, - start_dates.each_slice(SLICE_SIZE).to_a, - end_dates.each_slice(SLICE_SIZE).to_a, - values.each_slice(SLICE_SIZE).to_a] - end - def initialize_database(variable_names) @insert_queries = {} @queries = {} diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index a523e80..e42f993 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -40,15 +40,99 @@ module Physiqual end describe 'insert' do - it 'should insert data in the database in batches' do + it 'should call execute on the session object x nr of times' do # expect(true).to eq false + slice_size = 3 + entries = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18] + stub_const('Physiqual::CassandraConnection::SLICE_SIZE', slice_size) + expect(described_class::SLICE_SIZE).to eq(slice_size) + + (entries.length / slice_size).times do + rand = SecureRandom.hex(10) + expect(@session).to receive(:execute).with(rand) + expect(described_class.instance).to receive(:create_batches).with(any_args, user_id, year).and_return(rand) + end + + described_class.instance.instance_variable_set(:@session, @session) + described_class.instance.insert(table, user_id, year, entries) end + end - it 'should be able to work with nil values' do - # expect(true).to eq false + fdescribe 'create_batches' do + let(:table) { 'activities' } + let(:year) { 2013 } + let(:user_id) { 10 } + + it 'should call the correct cassandra batch functions' do + entry_slice = [FactoryGirl.build(:data_entry)] + entry = entry_slice.first + table = 'heart_rate' + arguments = [user_id, year, entry.measurement_moment.to_time, + entry.start_date.to_time, + entry.end_date.to_time, entry.values.first] + + batch = double(:batch) + expect(batch).to receive(:add).with(@insert_queries[table], arguments: arguments) + expect(@session).to receive(:batch).and_yield(batch) + described_class.instance.instance_variable_set(:@session, @session) + + described_class.instance.create_batches(entry_slice, table, user_id, year) end - it 'should convert values to big decimal values if needed' do + it 'should add each of the entry slices to the batch' do + entry_slice = [ + FactoryGirl.build(:data_entry), + FactoryGirl.build(:data_entry), + FactoryGirl.build(:data_entry), + FactoryGirl.build(:data_entry), + FactoryGirl.build(:data_entry) + ] + entry = entry_slice.first + table = 'heart_rate' + arguments = [user_id, year, entry.measurement_moment.to_time, + entry.start_date.to_time, + entry.end_date.to_time, entry.values.first] + + batch = double(:batch) + expect(batch).to receive(:add).with(@insert_queries[table], arguments: arguments) + .exactly(entry_slice.length).times + expect(@session).to receive(:batch).and_yield(batch) + described_class.instance.instance_variable_set(:@session, @session) + + described_class.instance.create_batches(entry_slice, table, user_id, year) + end + + it 'should not transform the values of a variable to big float if the variable type is activities' do + entry_slice = [FactoryGirl.build(:data_entry, values: ['running'])] + entry = entry_slice.first + arguments = [user_id, year, entry.measurement_moment.to_time, + entry.start_date.to_time, + entry.end_date.to_time, entry.values.first] + + batch = double(:batch) + expect(batch).to receive(:add).with(@insert_queries[table], arguments: arguments) + expect(@session).to receive(:batch).and_yield(batch) + described_class.instance.instance_variable_set(:@session, @session) + described_class.instance.create_batches(entry_slice, table, user_id, year) + end + + it 'should transform the float data to a big float whenever the data is not activity data' do + entry_slice = [FactoryGirl.build(:data_entry, values: [1.21312])] + table = 'heart_rate' + entry = entry_slice.first + value = BigDecimal(entry.values.first, Float::DIG + 1) + arguments = [user_id, year, entry.measurement_moment.to_time, + entry.start_date.to_time, + entry.end_date.to_time, value] + + batch = double(:batch) + expect(batch).to receive(:add).with(@insert_queries[table], arguments: arguments) + expect(@session).to receive(:batch).and_yield(batch) + described_class.instance.instance_variable_set(:@session, @session) + described_class.instance.create_batches(entry_slice, table, user_id, year) + end + + xit 'should return batches' do end end @@ -114,27 +198,6 @@ module Physiqual # private methods - describe 'slice' do - let!(:times) { [1, 2, 3, 4, 5, 6] } - let(:start_dates) { [6, 5, 4, 3, 2, 1] } - let(:end_dates) { %w(a c d e f) } - let(:values) { %w(f e d c b a) } - let(:number_of_variables) { 4 } - let(:number_of_entries) { times.length } - it 'should slice the provided arrays in the slice_size' do - stub_const('Physiqual::CassandraConnection::SLICE_SIZE', 2) - expect(described_class::SLICE_SIZE).to eq(2) - res = described_class.instance.send(:slice, times, start_dates, end_dates, values) - res.each { |x| expect(x.length).to eq((number_of_entries.to_f / described_class::SLICE_SIZE.to_f).ceil) } - end - - it 'should be able to deal with SLICE_SIZE > then the number of entries' do - res = described_class.instance.send(:slice, times, start_dates, end_dates, values) - expect(res.length).to eq(number_of_variables) - res.each { |x| expect(x.length).to eq((number_of_entries.to_f / described_class::SLICE_SIZE.to_f).ceil) } - end - end - describe 'initialize_database' do before(:each) do allow_any_instance_of(described_class).to receive(:initialize_database).and_call_original @@ -196,7 +259,7 @@ module Physiqual FROM #{table_name} WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? ORDER BY time ASC - " + " session = double('session') expect(session).to receive(:prepare).with(qry) described_class.instance.instance_variable_set(:@session, session) @@ -213,7 +276,7 @@ module Physiqual user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, PRIMARY KEY ((user_id, year), time) ) - " + " session = double('session') expect(session).to receive(:execute).with(query) described_class.instance.instance_variable_set(:@session, session) diff --git a/spec/lib/physiqual/data_services/summarized_data_service_spec.rb b/spec/lib/physiqual/data_services/summarized_data_service_spec.rb index 3399adf..cbdaeb2 100644 --- a/spec/lib/physiqual/data_services/summarized_data_service_spec.rb +++ b/spec/lib/physiqual/data_services/summarized_data_service_spec.rb @@ -1,4 +1,3 @@ -# rubocop:disable Metrics/ModuleLength module Physiqual require 'rails_helper' From 2febdcd863a2de3a48da4db0e719e6201debfff0 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 16:58:39 +0200 Subject: [PATCH 27/35] Remove focus --- spec/lib/physiqual/cassandra_connection_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index e42f993..29862fc 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -58,7 +58,7 @@ module Physiqual end end - fdescribe 'create_batches' do + describe 'create_batches' do let(:table) { 'activities' } let(:year) { 2013 } let(:user_id) { 10 } From 3aa34d16f4d5f016f29aefa05fc600c5b6fa9b93 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 17:15:23 +0200 Subject: [PATCH 28/35] Reduced complexity of some methods --- lib/physiqual/cassandra_connection.rb | 25 +++++------ lib/physiqual/workers/cache_worker.rb | 41 ++++++++++--------- .../physiqual/cassandra_connection_spec.rb | 1 - 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb index e0d6de5..1a317c6 100644 --- a/lib/physiqual/cassandra_connection.rb +++ b/lib/physiqual/cassandra_connection.rb @@ -11,18 +11,7 @@ def initialize Rails.logger.info(Physiqual.cassandra_username) Rails.logger.info(Physiqual.cassandra_password) Rails.logger.info(Physiqual.cassandra_host_urls) - if Physiqual.cassandra_username.blank? || Physiqual.cassandra_password.blank? - cluster = Cassandra.cluster( - hosts: Physiqual.cassandra_urls - ) - else - cluster = Cassandra.cluster( - username: Physiqual.cassandra_username, - password: Physiqual.cassandra_password, - hosts: Physiqual.cassandra_urls - ) - end - + cluster = initialize_cassandra_cluster @session = cluster.connect(Physiqual.cassandra_keyspace) variables = { 'heart_rate' => 'decimal', @@ -89,6 +78,18 @@ def query_activities(user_id, year, from, to) private + def initialize_cassandra_cluster + if Physiqual.cassandra_username.blank? || Physiqual.cassandra_password.blank? + return Cassandra.cluster(hosts: Physiqual.cassandra_urls) + end + + Cassandra.cluster( + username: Physiqual.cassandra_username, + password: Physiqual.cassandra_password, + hosts: Physiqual.cassandra_urls + ) + end + def initialize_database(variable_names) @insert_queries = {} @queries = {} diff --git a/lib/physiqual/workers/cache_worker.rb b/lib/physiqual/workers/cache_worker.rb index 9e57a34..18e2378 100644 --- a/lib/physiqual/workers/cache_worker.rb +++ b/lib/physiqual/workers/cache_worker.rb @@ -15,35 +15,23 @@ def perform(data_service, cassandra_dataservice, variable, user_id, from, to) # Sidekiq converts all dates to strings, so we have to parse them back to dates, from = Time.zone.parse(from) if from.is_a? String to = Time.zone.parse(to) if to.is_a? String - store_data(connection, cassandra_dataservice, variable, from, to) + entries = cassandra_dataservice.get_data(connection, @user_id, table, from, to) + store_data(connection, entries, variable, from, to) end # rubocop:enable Metrics/ParameterLists private - def store_data(connection, cassandra_dataservice, table, from, to) - entries = cassandra_dataservice.get_data(connection, @user_id, table, from, to) + def store_data(connection, entries, table, from, to) # Retrieve the function to use for getting the data data_service_function = get_data_function(table) - new_entries = [] - Rails.logger.info('new entries') # If there were no enties in the first place, just make the call to the dataservice - if entries.blank? - new_entries = data_service_function.call(from, to) - else - if entries.first.start_date > from - new_entries += data_service_function.call(from, entries.first.start_date) - end - - # find_gaps(entries) do |from_gap, to_gap| - # new_entries += data_service_function.call(from_gap, to_gap) - # end - - if entries.last.end_date < to - new_entries += data_service_function.call(entries.last.end_date, to) - end - end + new_entries = if entries.blank? + data_service_function.call(from, to) + else + partial_entries(data_service_function, from, to, entries, false) + end Rails.logger.info('caching new entries') # Cache the newly retrieved data cache(connection, table, @user_id, new_entries) if new_entries.present? @@ -51,6 +39,19 @@ def store_data(connection, cassandra_dataservice, table, from, to) Rails.logger.warn e.message end + def partial_entries(data_service_function, from, to, entries, fill_gaps) + new_entries = [] + + new_entries << data_service_function.call(from, entries.first.start_date) if entries.first.start_date > from + + find_gaps(entries) do |from_gap, to_gap| + new_entries << data_service_function.call(from_gap, to_gap) + end if fill_gaps + + new_entries << data_service_function.call(entries.last.end_date, to) if entries.last.end_date < to + new_entries + end + def get_data_function(table) case table when 'heart_rate' diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index 29862fc..42a1dd1 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -41,7 +41,6 @@ module Physiqual describe 'insert' do it 'should call execute on the session object x nr of times' do - # expect(true).to eq false slice_size = 3 entries = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18] stub_const('Physiqual::CassandraConnection::SLICE_SIZE', slice_size) From dee8a79acd1db95f26ba8222e538d9989af01479 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 19:55:43 +0200 Subject: [PATCH 29/35] Fixed some specs --- lib/physiqual/cassandra_connection.rb | 22 ++++------- spec/factories/physiqual/data_entry.rb | 8 ++++ .../physiqual/cassandra_connection_spec.rb | 38 +++++++++++++------ 3 files changed, 41 insertions(+), 27 deletions(-) create mode 100644 spec/factories/physiqual/data_entry.rb diff --git a/lib/physiqual/cassandra_connection.rb b/lib/physiqual/cassandra_connection.rb index 1a317c6..847eb99 100644 --- a/lib/physiqual/cassandra_connection.rb +++ b/lib/physiqual/cassandra_connection.rb @@ -8,9 +8,6 @@ class CassandraConnection def initialize # Setup the connection to the cluster - Rails.logger.info(Physiqual.cassandra_username) - Rails.logger.info(Physiqual.cassandra_password) - Rails.logger.info(Physiqual.cassandra_host_urls) cluster = initialize_cassandra_cluster @session = cluster.connect(Physiqual.cassandra_keyspace) @@ -112,22 +109,17 @@ def prepare_insert(table_name) end def prepare_query(table_name) - query = " - SELECT time, start_date, end_date, value - FROM #{table_name} - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - " + query = 'SELECT time, start_date, end_date, value'\ + "FROM #{table_name}"\ + 'WHERE user_id = ? AND year = ? AND time >= ? AND time <= ?'\ + 'ORDER BY time ASC' @session.prepare(query) end def create_table(name, value_type) - query = " - CREATE TABLE IF NOT EXISTS #{name} ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, - PRIMARY KEY ((user_id, year), time) - ) - " + query = "CREATE TABLE IF NOT EXISTS #{name} ("\ + "user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type},"\ + 'PRIMARY KEY ((user_id, year), time))' @session.execute(query) end end diff --git a/spec/factories/physiqual/data_entry.rb b/spec/factories/physiqual/data_entry.rb new file mode 100644 index 0000000..df05bb5 --- /dev/null +++ b/spec/factories/physiqual/data_entry.rb @@ -0,0 +1,8 @@ +FactoryGirl.define do + factory :data_entry, class: Physiqual::DataEntry do + measurement_moment Date.new(2015, 0o6, 0o1).to_time + start_date Date.new(2015, 0o6, 0o1).to_time + end_date Date.new(2015, 0o6, 21).to_time + values [1, 2, 3, 4, 5] + end +end diff --git a/spec/lib/physiqual/cassandra_connection_spec.rb b/spec/lib/physiqual/cassandra_connection_spec.rb index 42a1dd1..cdb68c2 100644 --- a/spec/lib/physiqual/cassandra_connection_spec.rb +++ b/spec/lib/physiqual/cassandra_connection_spec.rb @@ -2,6 +2,22 @@ module Physiqual require 'rails_helper' describe CassandraConnection do describe 'initialize' do + it 'should setup the cassandracluster' do + expect_any_instance_of(described_class) + .to receive(:initialize_cassandra_cluster) { raise(StandardError, 'stop_execution') } + expect { described_class.instance }.to raise_error('stop_execution') + end + + it 'should initialize the database' do + session = double('session') + cluster = double('cluster') + allow(cluster).to receive(:connect).and_return(session) + expect_any_instance_of(described_class) + .to receive(:initialize_database) { raise(StandardError, 'stop_execution') } + allow_any_instance_of(described_class) + .to receive(:initialize_cassandra_cluster).and_return(cluster) + expect { described_class.instance }.to raise_error('stop_execution') + end end describe 'with mock connection' do @@ -40,6 +56,9 @@ module Physiqual end describe 'insert' do + let(:table) { 'activities' } + let(:year) { 2013 } + let(:user_id) { 10 } it 'should call execute on the session object x nr of times' do slice_size = 3 entries = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18] @@ -253,12 +272,10 @@ module Physiqual describe 'prepare_query' do it 'should use the provided name in the query' do table_name = 'some random variable name' - qry = " - SELECT time, start_date, end_date, value - FROM #{table_name} - WHERE user_id = ? AND year = ? AND time >= ? AND time <= ? - ORDER BY time ASC - " + qry = 'SELECT time, start_date, end_date, value'\ + "FROM #{table_name}"\ + 'WHERE user_id = ? AND year = ? AND time >= ? AND time <= ?'\ + 'ORDER BY time ASC' session = double('session') expect(session).to receive(:prepare).with(qry) described_class.instance.instance_variable_set(:@session, session) @@ -270,12 +287,9 @@ module Physiqual it 'should use the provided name in the query' do name = 'some random variable name' value_type = 'some random variable type' - query = " - CREATE TABLE IF NOT EXISTS #{name} ( - user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type}, - PRIMARY KEY ((user_id, year), time) - ) - " + query = "CREATE TABLE IF NOT EXISTS #{name} ("\ + "user_id text, year int, time timestamp, start_date timestamp, end_date timestamp, value #{value_type},"\ + 'PRIMARY KEY ((user_id, year), time))' session = double('session') expect(session).to receive(:execute).with(query) described_class.instance.instance_variable_set(:@session, session) From b1310623fcf171bef15f196e88203515702e40c3 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 22:32:13 +0200 Subject: [PATCH 30/35] Hoped to fix the codeclimate issues --- lib/physiqual/engine.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index 54a8044..ee5961b 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -19,10 +19,7 @@ class << self def cassandra_urls urls = cassandra_host_urls - a = urls.split(' ') unless urls.blank? - Rails.logger.info(urls) - Rails.logger.info(a) - a + urls.split(' ') unless urls.blank? end mattr_accessor :cassandra_keyspace From 8c54436df44d5030fc36610250666f39a8908172 Mon Sep 17 00:00:00 2001 From: Frank Blaauw Date: Wed, 20 Jul 2016 22:46:42 +0200 Subject: [PATCH 31/35] Moved function in class --- lib/physiqual/engine.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index ee5961b..a09774b 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -17,11 +17,6 @@ class << self mattr_accessor :cassandra_host_urls mattr_accessor :cassandra_keyspace - def cassandra_urls - urls = cassandra_host_urls - urls.split(' ') unless urls.blank? - end - mattr_accessor :cassandra_keyspace mattr_accessor :host_url mattr_accessor :host_protocol @@ -31,6 +26,11 @@ def cassandra_urls mattr_accessor :imputers end + def self.cassandra_urls + urls = cassandra_host_urls + urls.split(' ') unless urls.blank? + end + def self.configure(&_block) yield self end From 7686025b63f905b3f8873ad5ca2244f6127b0670 Mon Sep 17 00:00:00 2001 From: Babbie Date: Sun, 28 Aug 2016 21:25:53 +0200 Subject: [PATCH 32/35] Include information required since caching update --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index e6e7ba5..1288622 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ Ruby Engine for merging multiple data sources with diary questionnaire data [![Code Climate](https://codeclimate.com/github/roqua/physiqual/badges/gpa.svg)](https://codeclimate.com/github/roqua/physiqual) [![Test Coverage](https://codeclimate.com/github/roqua/physiqual/badges/coverage.svg)](https://codeclimate.com/github/roqua/physiqual/coverage) [![Dependency Status](https://gemnasium.com/roqua/physiqual.svg)](https://gemnasium.com/roqua/physiqual) [![Circle CI](https://circleci.com/gh/roqua/physiqual/tree/master.svg?style=svg)](https://circleci.com/gh/roqua/physiqual/tree/master) +## Requirements +Physiqual requires the following additional software for caching: +* A Redis database +* A Cassandra database + +Make sure the Cassandra database has a keyspace set up for use in Physiqual. ## Installation Add Physiqual to your Gemfile. Currently Physiqual is not yet on RubyGems, this will happen after Physiqual is in a more stable beta state. @@ -42,6 +48,15 @@ Physiqual.configure do |config| config.host_url = ENV['HOST_URL'] || 'physiqual.dev' config.host_protocol = ENV['HOST_PROTOCOL'] || 'http' + # Cassandra settings + config.cassandra_username = ENV['CASSANDRA_USERNAME'] || '' + config.cassandra_password = ENV['CASSANDRA_PASSWORD'] || '' + config.cassandra_host_urls = (ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev').split(' ') + config.cassandra_keyspace = ENV['CASSANDRA_KEYSPACE'] + + # Redis settings + config.redis_url = ENV['REDIS_URL'] + # EMA Settings config.measurements_per_day = 3 # Number of measurements per day, from the end of day downwards config.interval = 6 # Number of hours between measurements @@ -59,6 +74,12 @@ Physiqual.configure do |config| end ``` +On the machine(s) that will handle caching, install Physiqual as well. Then run the following: +```bash + bundle exec sidekiq +``` +This will allow Physiqual to cache data asynchronously to your Cassandra database. + Now you should be able to start your server. ## Dummy From 2fb84e834358738feb0d965d0e81dcffbb96805c Mon Sep 17 00:00:00 2001 From: Babbie Date: Sun, 28 Aug 2016 21:32:43 +0200 Subject: [PATCH 33/35] Add redis url config --- spec/dummy/config/initializers/physiqual.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/dummy/config/initializers/physiqual.rb b/spec/dummy/config/initializers/physiqual.rb index addfa00..722a5b8 100644 --- a/spec/dummy/config/initializers/physiqual.rb +++ b/spec/dummy/config/initializers/physiqual.rb @@ -17,6 +17,9 @@ config.cassandra_password = ENV['CASSANDRA_PASSWORD'] || '' config.cassandra_host_urls = ENV['CASSANDRA_HOST_URLS'] || 'physiqual.dev' config.cassandra_keyspace = ENV['CASSANDRA_KEYSPACE'] + + # Redis settings + config.redis_url = ENV['REDIS_URL'] || 'physiqual.dev' # EMA Settings config.measurements_per_day = 1 # Number of measurements per day From 056541fe7a3a0cccb6d71670d69a1b127180bd2b Mon Sep 17 00:00:00 2001 From: Babbie Date: Sun, 28 Aug 2016 21:33:29 +0200 Subject: [PATCH 34/35] Add redis url config --- spec/dummy/config/initializers/sidekiq.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/dummy/config/initializers/sidekiq.rb b/spec/dummy/config/initializers/sidekiq.rb index 15e05f2..c65ea36 100644 --- a/spec/dummy/config/initializers/sidekiq.rb +++ b/spec/dummy/config/initializers/sidekiq.rb @@ -7,6 +7,7 @@ # accepts :expiration (optional) chain.add Sidekiq::Status::ClientMiddleware, expiration: 30.minutes # default end + config.redis = { url: ENV['REDIS_URL'] } end Sidekiq.configure_server do |config| @@ -19,4 +20,5 @@ # accepts :expiration (optional) chain.add Sidekiq::Status::ClientMiddleware, expiration: 30.minutes # default end -end \ No newline at end of file + config.redis = { url: ENV['REDIS_URL'] } +end From 5019e3a848e6f6a189aa6a8c924de93f3ceeddc5 Mon Sep 17 00:00:00 2001 From: Babbie Date: Sun, 28 Aug 2016 21:34:29 +0200 Subject: [PATCH 35/35] Add redis url config --- lib/physiqual/engine.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/physiqual/engine.rb b/lib/physiqual/engine.rb index a09774b..a64210e 100644 --- a/lib/physiqual/engine.rb +++ b/lib/physiqual/engine.rb @@ -16,6 +16,9 @@ class << self mattr_accessor :cassandra_password mattr_accessor :cassandra_host_urls mattr_accessor :cassandra_keyspace + + # Redis settings + mattr_accessor :redis_url mattr_accessor :cassandra_keyspace mattr_accessor :host_url