diff --git a/Dockerfile.prod b/Dockerfile.prod
new file mode 100644
index 00000000..841f358d
--- /dev/null
+++ b/Dockerfile.prod
@@ -0,0 +1,37 @@
+FROM ruby:2.6
+ARG UNAME=app
+ARG UID=1000
+ARG GID=1000
+ARG URL_ROOT=/
+
+RUN curl https://deb.nodesource.com/setup_18.x | bash
+RUN curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends nodejs yarn
+RUN gem install bundler
+
+RUN groupadd -g $GID -o $UNAME
+RUN useradd -m -d /usr/src/app -u $UID -g $GID -o -s /bin/bash $UNAME
+RUN mkdir -p /gems && chown $UID:$GID /gems
+
+COPY --chown=$UID:$GID Gemfile* /usr/src/app/
+
+ENV RAILS_SERVE_STATIC_FILES true
+ENV RAILS_LOG_TO_STDOUT true
+ENV RAILS_ENV production
+ENV BUNDLE_PATH /gems
+
+# This can be anything but must be set.
+ENV SECRET_KEY_BASE 121222bccca
+
+WORKDIR /usr/src/app
+RUN bundle install --without development test
+
+COPY --chown=$UID:$GID . /usr/src/app
+RUN RAILS_ENV=production bin/rails assets:precompile
+RUN chown -R $UID:$GID /usr/src/app
+USER $UNAME
+
+
+CMD ["bin/rails", "s", "-b", "0.0.0.0"]
diff --git a/Gemfile.lock b/Gemfile.lock
index 33c4de18..3ebcb0c9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -204,7 +204,7 @@ GEM
parser (3.2.1.1)
ast (~> 2.4.1)
pg (1.4.6)
- prometheus-client (4.0.0)
+ prometheus-client (4.1.0)
pry (0.14.2)
coderay (~> 1.1)
method_source (~> 1.0)
@@ -346,10 +346,10 @@ GEM
activesupport (>= 5.2)
sprockets (>= 3.0.0)
sqlite3 (1.3.13)
- standard (1.25.0)
+ standard (1.25.3)
language_server-protocol (~> 3.17.0.2)
- rubocop (= 1.48.1)
- rubocop-performance (= 1.16.0)
+ rubocop (~> 1.48.1)
+ rubocop-performance (~> 1.16.0)
thor (1.2.1)
thread_safe (0.3.6)
tilt (2.1.0)
diff --git a/Rakefile b/Rakefile
index 9ca41066..348297fc 100644
--- a/Rakefile
+++ b/Rakefile
@@ -3,5 +3,43 @@
require_relative "config/application"
require "standard/rake"
+require_relative "app/jobs/job_index"
+require_relative "lib/med_installer/job_monitoring"
Rails.application.load_tasks
+
+desc "sanity-check sidekiq"
+task :poke_sidekiq do
+ Dromedary::PokeSidekiqJob.perform_async
+end
+
+desc "Check for updated data file"
+task :check_data do
+ # metrics = MiddleEnglishIndexMetrics.new({type: "check_data"})
+
+ # standard:disable Lint/ConstantDefinitionInBlock
+ UPDATE_WINDOW_SECONDS = 7 * 24 * 60 * 60 # should update once weekly
+ # standard:enable Lint/ConstantDefinitionInBlock
+ begin
+ puts ENV["DATA_FILE"]
+ last_modified = File.mtime(ENV["DATA_FILE"])
+ puts last_modified
+ if (Time.now - last_modified) < UPDATE_WINDOW_SECONDS
+ # metrics.log_success
+ puts "kicking off data job"
+ Dromedary::IndexDataJob.perform_async(ENV["DATA_FILE"])
+ end
+ rescue => _e
+ # metrics.log_error(e)
+ end
+end
+
+desc "add indexing job to sidekiq queue"
+task :queue_indexing do
+ Dromedary::IndexDataJob.perform_async(ENV["DATA_FILE"])
+end
+
+desc "do indexing job now"
+task :perform_indexing do
+ Dromedary::IndexDataJob.perform(ENV["DATA_FILE"])
+end
diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
deleted file mode 100644
index a009ace5..00000000
--- a/app/jobs/application_job.rb
+++ /dev/null
@@ -1,2 +0,0 @@
-class ApplicationJob < ActiveJob::Base
-end
diff --git a/app/jobs/index_data_job.rb b/app/jobs/index_data_job.rb
new file mode 100644
index 00000000..7bd4e8ca
--- /dev/null
+++ b/app/jobs/index_data_job.rb
@@ -0,0 +1,15 @@
+require "sidekiq"
+
+module Dromedary
+ class IndexDataJob
+ include Sidekiq::Job
+
+ queue_as :default
+
+ def perform(filename)
+ puts "indexing data"
+ exit_code = `bin/dromedary extract_convert_index #{filename}`
+ puts "finished preparing with exit code #{exit_code}"
+ end
+ end
+end
diff --git a/app/jobs/job_index.rb b/app/jobs/job_index.rb
new file mode 100644
index 00000000..9f6db10e
--- /dev/null
+++ b/app/jobs/job_index.rb
@@ -0,0 +1,2 @@
+require_relative "./poke_sidekiq"
+require_relative "./index_data_job"
diff --git a/app/jobs/poke_sidekiq.rb b/app/jobs/poke_sidekiq.rb
new file mode 100644
index 00000000..b7373dac
--- /dev/null
+++ b/app/jobs/poke_sidekiq.rb
@@ -0,0 +1,11 @@
+require "sidekiq"
+
+module Dromedary
+ class PokeSidekiqJob
+ include Sidekiq::Job
+
+ def perform
+ puts "hiiiiii"
+ end
+ end
+end
diff --git a/app/views/catalog/home.html.erb b/app/views/catalog/home.html.erb
index 5d50c4ab..a2a8aff6 100644
--- a/app/views/catalog/home.html.erb
+++ b/app/views/catalog/home.html.erb
@@ -9,10 +9,11 @@
- <%= render_search_bar %>
+ <%= render_search_bar %>
- Image: The Ellesmer Manuscript of Geoffrey Chaucer's The Canterbury Tales, c. 1400-1405. Shown is the introduction to "The Knight's Tale." Available online at the Huntington Library <%= image_tag('external-link-white.svg', alt: 'Opens in a new window') %>
+
+ Image: The Ellesmer Manuscript of Geoffrey Chaucer's The Canterbury Tales, c. 1400-1405. Shown is the introduction to "The Knight's Tale." Available online at the Huntington Library <%= image_tag(asset_path('external-link-white.svg'), alt: 'Opens in a new window') %>
diff --git a/app/views/catalog/splash.html.erb b/app/views/catalog/splash.html.erb
index 224dd756..1b3a7995 100644
--- a/app/views/catalog/splash.html.erb
+++ b/app/views/catalog/splash.html.erb
@@ -42,7 +42,8 @@
Shown is the introduction to "The Knight's Tale."
Available online at the Huntington
- Library <%= image_tag('external-link-white.svg', alt: 'Opens in a new window') %>
+
+ Library <%= image_tag(asset_path('external-link-white.svg'), alt: 'Opens in a new window') %>
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index 6ac074dd..ffc5a1f9 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -35,6 +35,10 @@
<%= render :partial => 'shared/header_navbar' %>
<% end %>
+ <% if ENV['DEBUG_ENV'] %>
+ x-forwarded host: <%= ENV["HTTP_X_FORWARDED_HOST"].inspect %>
+ <% end %>
+
<%= area :main do %>
Main content goes here.
<% end %>
diff --git a/app/views/shared/_footer.html.erb b/app/views/shared/_footer.html.erb
index 2d5387bb..5e0ee4cf 100644
--- a/app/views/shared/_footer.html.erb
+++ b/app/views/shared/_footer.html.erb
@@ -5,11 +5,11 @@
Middle English Compendium
Middle English Dictionary
Bibliography
- Corpus <%= image_tag('external-link-white.svg', alt:'Opens in a new window') %>
+
+ Corpus <%= image_tag(asset_path('external-link-white.svg'), alt:'Opens in a new window') %>
-
- Help and information
- <%= link_to "Search Help", help_root_path %>
diff --git a/bin/dromedary b/bin/dromedary
index 5fd8b74d..cd882aaf 100755
--- a/bin/dromedary
+++ b/bin/dromedary
@@ -94,6 +94,7 @@ MedInstaller::CLI.register 'extract_convert_index', MedInstaller::ExtractConvert
MedInstaller::CLI.register 'newdata prepare', MedInstaller::PrepareNewData
MedInstaller::CLI.register 'newdata index', MedInstaller::IndexNewData
+MedInstaller::CLI.register 'ping_prometheus', MedInstaller::PingPrometheus
Hanami::CLI.new(MedInstaller::CLI).call
diff --git a/config.ru b/config.ru
index 441e6ff0..469175fd 100644
--- a/config.ru
+++ b/config.ru
@@ -3,3 +3,6 @@
require_relative "config/environment"
run Rails.application
+# map Dromedary.config.relative_url_root || "/" do
+# run Rails.application
+# end
diff --git a/config/application.rb b/config/application.rb
index 5983afd5..fb53190c 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -31,6 +31,8 @@ class Application < Rails::Application
config.relative_url_root = Dromedary.config.relative_url_root
config.action_controller.relative_url_root = config.relative_url_root
+ # config.relative_url_root = '/'
+ # config.action_controller.relative_url_root = '/'
config.log_level = :info
@@ -50,6 +52,7 @@ class Application < Rails::Application
end
config.lograge.formatter = Lograge::Formatters::Json.new
+ config.active_job.queue_adapter = :sidekiq
# config.log_tags = {
# ip: :remote_ip,
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 2c33e77d..d5494c1e 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -80,11 +80,11 @@
# require 'syslog/logger'
# config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
#
- # if ENV["RAILS_LOG_TO_STDOUT"].present?
- # logger = ActiveSupport::Logger.new(STDOUT)
- # logger.formatter = config.log_formatter
- # config.logger = ActiveSupport::TaggedLogging.new(logger)
- # end
+ if ENV["RAILS_LOG_TO_STDOUT"].present?
+ logger = ActiveSupport::Logger.new($stdout)
+ logger.formatter = config.log_formatter
+ config.logger = ActiveSupport::TaggedLogging.new(logger)
+ end
# Do not dump schema after migrations.
config.active_record.dump_schema_after_migration = false
diff --git a/config/routes.rb b/config/routes.rb
index a974c52e..0bd7fdc8 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -3,6 +3,8 @@
Rails.application.routes.draw do
scope Dromedary.config.relative_url_root do
mount Blacklight::Engine => Dromedary.config.relative_url_root
+ # scope '/' do
+ # mount Blacklight::Engine => '/'
# Shunt it all to the maintenace page if we need to
match "*path" => "static#maintenance_mode", :status => 302, :via => [:get, :post],
diff --git a/config/settings/development.yml b/config/settings/development.yml
index f8213fd6..55c5afec 100644
--- a/config/settings/development.yml
+++ b/config/settings/development.yml
@@ -21,6 +21,9 @@ cable:
relative_url_root: /
data_dir: ./data
+build_dir: ./data/build
log_dir: ./log
+#data_dir: /home/esty/github/mlibrary/med/data
+#build_dir: /home/esty/github/mlibrary/med/data/build
diff --git a/config/settings/local.yml b/config/settings/local.yml
new file mode 100644
index 00000000..f9d26267
--- /dev/null
+++ b/config/settings/local.yml
@@ -0,0 +1,5 @@
+data_dir: /home/esty/github/mlibrary/med/data
+build_dir: /home/esty/github/mlibrary/med/data/build
+
+blacklight:
+ url: http://localhost:9639/solr/med
diff --git a/config/settings/production.yml b/config/settings/production.yml
new file mode 100644
index 00000000..27771951
--- /dev/null
+++ b/config/settings/production.yml
@@ -0,0 +1,29 @@
+blacklight:
+ adapter: solr
+ url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:9639/solr/med" %>
+
+solr:
+ url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:9639/solr/med" %>
+
+secret_key_base: <%= ENV["SECRET_KEY_BASE"] || "somesecretkey" %>
+
+
+
+db:
+ url: unused
+
+cable:
+ adapter: async
+ url: unused
+ channel_prefix: dromedary
+
+
+relative_url_root: <%= ENV["RAILS_RELATIVE_URL_ROOT"] || "/" %>
+
+data_dir: <%= ENV["DATA_DIR"] || "./data" %>
+build_dir: <%= ENV["BUILD_DIR"] || "./data/build" %>
+
+log_dir: <%= ENV["LOG_DIR"] || "./log" %>
+
+#data_dir: /home/esty/github/mlibrary/med/data
+#build_dir: /home/esty/github/mlibrary/med/data/build
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
new file mode 100644
index 00000000..a7bb8570
--- /dev/null
+++ b/config/sidekiq.yml
@@ -0,0 +1,8 @@
+development:
+ :concurrency: 1
+
+production:
+ :concurrency: 1
+
+:queues:
+ - default
diff --git a/data.Dockerfile b/data.Dockerfile
new file mode 100644
index 00000000..8522ed3f
--- /dev/null
+++ b/data.Dockerfile
@@ -0,0 +1,32 @@
+FROM ruby:2.6
+
+ENV DATA_FILE=/opt/app-data/All_MED_and_BIB_files.zip
+ENV data_dir=/usr/src/app/data
+ENV build_dir=/usr/src/app/data/build
+ENV SOLR_URL=http://localhost:9639/solr
+ARG UNAME=app
+ARG UID=1000
+ARG GID=1000
+
+RUN apt-get update -yqq && \
+ apt-get install -yqq --no-install-recommends \
+ apt-transport-https nodejs
+
+RUN groupadd -g $GID -o $UNAME
+RUN useradd -m -d /usr/src/app -u $UID -g $GID -o -s /bin/bash $UNAME
+RUN mkdir -p /gems && chown $UID:$GID /gems
+
+USER $UNAME
+COPY --chown=$UID:$GID Gemfile* /usr/src/app/
+
+ENV RAILS_LOG_TO_STDOUT true
+ENV BUNDLE_PATH /gems
+
+WORKDIR /usr/src/app
+RUN gem install 'bundler:2.1.4'
+RUN bundle install
+
+COPY --chown=$UID:$GID . /usr/src/app
+
+USER $UNAME
+RUN mkdir -p /usr/src/app/data/build
diff --git a/docker-compose.yml b/docker-compose.yml
index 418834f0..6eff2b65 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,18 @@
---
services:
+ # web:
+ # image: dromedary-web
+ # build:
+ # context: .
+ # dockerfile: Dockerfile.prod
+ # ports:
+ # - "3000:3000"
+ # environment:
+ # - DATA_FILE=/usr/src/app/All_MED_and_Bib_files.zip
+ # - PROMETHEUS_PUSH_GATEWAY=http://pushgateway:9091
+ # - SOLR_URL=http://nectar.umdl.umich.edu:8081/solr/dromedary_testing/
+ # - REDIS_URL=redis://redis:6379
+ # - DEBUG_ENV=true
app:
depends_on:
- db
@@ -69,6 +82,44 @@ services:
# - sidekiq
# - -r
# - ./app/jobs/job_index.rb
+ - solr_core:/opt/solr/server/solr/dromedary
+# data:
+# build:
+# dockerfile: data.Dockerfile
+# context: .
+# volumes:
+# - ../data:/opt/app-data
+# - gem_cache:/gems
+# redis:
+# image: redis:6-buster
+# ports:
+# - 6380:6379
+#
+# sidekiq:
+# image: dromedary-web
+# ports:
+# - 3001:3001
+# command:
+# - bundle
+# - exec
+# - sidekiq
+# - -r
+# - ./app/jobs/job_index.rb
+# environment:
+# - REDIS_URL=redis://redis:6379
+# - SOLR_URL=http://nectar.umdl.umich.edu:8081/solr/dromedary_testing/
+# - DATA_FILE=/opt/app-data/All_MED_and_Bib_files.zip
+# - PROMETHEUS_PUSH_GATEWAY=http://pushgateway:9091
+ prometheus:
+ image: prom/prometheus
+ ports:
+ - 9090:9090
+ volumes:
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
+ pushgateway:
+ image: prom/pushgateway
+ ports:
+ - 9091:9091
volumes:
data:
driver_opts:
diff --git a/index_data.sh b/index_data.sh
new file mode 100755
index 00000000..da2063b5
--- /dev/null
+++ b/index_data.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+bin/dromedary newdata prepare /opt/app-data/All_MED_and_BIB_files.zip
+bin/dromedary newdata index
diff --git a/indexer/main_indexing_rules.rb b/indexer/main_indexing_rules.rb
index 92d7dc8f..88f80898 100644
--- a/indexer/main_indexing_rules.rb
+++ b/indexer/main_indexing_rules.rb
@@ -5,6 +5,7 @@
require "middle_english_dictionary"
require "json"
require_relative "../config/load_local_config"
+require_relative "../lib/med_installer/job_monitoring"
require "quote/quote_indexer"
require "serialization/indexable_quote"
@@ -181,7 +182,9 @@ def entry_method(name)
q.bib_id = bid
q.author = bibset[q.bib_id].author
else
- logger.warn "RID #{rid} in #{entry.source} not found in bib_all.xml"
+ warning = "RID #{rid} in #{entry.source} not found in bib_all.xml"
+ logger.warn warning
+ # MiddleEnglishIndexMetrics.new({type: "index_quotes"}).log_warning(warning)
end
rescue => e
logger.error e.full_message
diff --git a/prometheus.yml b/prometheus.yml
new file mode 100644
index 00000000..deaf567e
--- /dev/null
+++ b/prometheus.yml
@@ -0,0 +1,6 @@
+scrape_configs:
+- job_name: pushgateway
+ honor_labels: true
+ honor_timestamps: true
+ static_configs:
+ - targets: ['pushgateway:9091']
diff --git a/solr-helm-values.yml b/solr-helm-values.yml
new file mode 100644
index 00000000..c5e0fc25
--- /dev/null
+++ b/solr-helm-values.yml
@@ -0,0 +1,9 @@
+# helm upgrade middle-english-solr-cloud apache-solr/solr --version 0.6 --namespace middle-english-testing -f solr-helm-values.yml
+image:
+ repository: ghcr.io/mlibrary/dromedary/dromedary_data
+ tag: "20221003.3"
+solrOptions:
+ javaMemory: "-Xms300m -Xmx300m"
+ javaOpts: "-Dsolr.sharedLib=/var/solr/um_plugins/"
+ security:
+ authenticationType: "Basic"
diff --git a/solr.Dockerfile b/solr.Dockerfile
new file mode 100644
index 00000000..f2385b66
--- /dev/null
+++ b/solr.Dockerfile
@@ -0,0 +1,8 @@
+FROM solr:8-slim
+
+COPY ./solr/med/conf /solr_config
+COPY ./solr/lib/*.jar /var/solr/um_plugins/
+#RUN ls /var/solr/um_plugins/
+
+#ENTRYPOINT ["docker-entrypoint.sh", "solr-precreate", "dromedary-testing", "/solr_config"]
+
diff --git a/solr/solr.xml b/solr/solr.xml
new file mode 100644
index 00000000..03d1f393
--- /dev/null
+++ b/solr/solr.xml
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
diff --git a/spec/jobs/index_data_job_spec.rb b/spec/jobs/index_data_job_spec.rb
new file mode 100644
index 00000000..57d536bb
--- /dev/null
+++ b/spec/jobs/index_data_job_spec.rb
@@ -0,0 +1,5 @@
+require "rails_helper"
+
+RSpec.describe IndexDataJob, pending: "review", type: :job do
+ pending "add some examples to (or delete) #{__FILE__}"
+end