merge squash delta

Dockerfile.prod

@@ -0,0 +1,37 @@
+FROM ruby:2.6
+ARG UNAME=app
+ARG UID=1000
+ARG GID=1000
+ARG URL_ROOT=/
+
+RUN curl https://deb.nodesource.com/setup_18.x | bash
+RUN curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends nodejs yarn
+RUN gem install bundler
+
+RUN groupadd -g $GID -o $UNAME
+RUN useradd -m -d /usr/src/app -u $UID -g $GID -o -s /bin/bash $UNAME
+RUN mkdir -p /gems && chown $UID:$GID /gems
+
+COPY --chown=$UID:$GID Gemfile* /usr/src/app/
+
+ENV RAILS_SERVE_STATIC_FILES true
+ENV RAILS_LOG_TO_STDOUT true
+ENV RAILS_ENV production
+ENV BUNDLE_PATH /gems
+
+# This can be anything but must be set.
+ENV SECRET_KEY_BASE 121222bccca
+
+WORKDIR /usr/src/app
+RUN bundle install --without development test
+
+COPY --chown=$UID:$GID . /usr/src/app
+RUN RAILS_ENV=production bin/rails assets:precompile
+RUN chown -R $UID:$GID /usr/src/app
+USER $UNAME
+
+
+CMD ["bin/rails", "s", "-b", "0.0.0.0"]

Gemfile.lock

@@ -204,7 +204,7 @@ GEM
     parser (3.2.1.1)
       ast (~> 2.4.1)
     pg (1.4.6)
-    prometheus-client (4.0.0)
+    prometheus-client (4.1.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -346,10 +346,10 @@ GEM
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
     sqlite3 (1.3.13)
-    standard (1.25.0)
+    standard (1.25.3)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (= 1.48.1)
-      rubocop-performance (= 1.16.0)
+      rubocop (~> 1.48.1)
+      rubocop-performance (~> 1.16.0)
     thor (1.2.1)
     thread_safe (0.3.6)
     tilt (2.1.0)

Rakefile

@@ -3,5 +3,43 @@
 
 require_relative "config/application"
 require "standard/rake"
+require_relative "app/jobs/job_index"
+require_relative "lib/med_installer/job_monitoring"
 
 Rails.application.load_tasks
+
+desc "sanity-check sidekiq"
+task :poke_sidekiq do
+  Dromedary::PokeSidekiqJob.perform_async
+end
+
+desc "Check for updated data file"
+task :check_data do
+  # metrics = MiddleEnglishIndexMetrics.new({type: "check_data"})
+
+  # standard:disable Lint/ConstantDefinitionInBlock
+  UPDATE_WINDOW_SECONDS = 7 * 24 * 60 * 60 # should update once weekly
+  # standard:enable Lint/ConstantDefinitionInBlock
+  begin
+    puts ENV["DATA_FILE"]
+    last_modified = File.mtime(ENV["DATA_FILE"])
+    puts last_modified
+    if (Time.now - last_modified) < UPDATE_WINDOW_SECONDS
+      # metrics.log_success
+      puts "kicking off data job"
+      Dromedary::IndexDataJob.perform_async(ENV["DATA_FILE"])
+    end
+  rescue => _e
+    # metrics.log_error(e)
+  end
+end
+
+desc "add indexing job to sidekiq queue"
+task :queue_indexing do
+  Dromedary::IndexDataJob.perform_async(ENV["DATA_FILE"])
+end
+
+desc "do indexing job now"
+task :perform_indexing do
+  Dromedary::IndexDataJob.perform(ENV["DATA_FILE"])
+end

app/jobs/index_data_job.rb

@@ -0,0 +1,15 @@
+require "sidekiq"
+
+module Dromedary
+  class IndexDataJob
+    include Sidekiq::Job
+
+    queue_as :default
+
+    def perform(filename)
+      puts "indexing data"
+      exit_code = `bin/dromedary extract_convert_index #{filename}`
+      puts "finished preparing with exit code #{exit_code}"
+    end
+  end
+end

app/jobs/job_index.rb

@@ -0,0 +1,2 @@
+require_relative "./poke_sidekiq"
+require_relative "./index_data_job"

app/jobs/poke_sidekiq.rb

@@ -0,0 +1,11 @@
+require "sidekiq"
+
+module Dromedary
+  class PokeSidekiqJob
+    include Sidekiq::Job
+
+    def perform
+      puts "hiiiiii"
+    end
+  end
+end

app/views/catalog/home.html.erb

@@ -9,10 +9,11 @@
 			  	</div>
 
 			    <div class="search-bar--home">
-			     <%= render_search_bar  %>  
+			     <%= render_search_bar  %>
 			    </div>
 
-			    <p class="banner-text--caption">Image: The Ellesmer Manuscript of Geoffrey Chaucer's The Canterbury Tales, c. 1400-1405. Shown is the introduction to "The Knight's Tale." <a href="https://hdl.huntington.org/digital/collection/p15150coll7/id/2385" target="blank"> Available online at the Huntington Library <%= image_tag('external-link-white.svg', alt: 'Opens in a new window') %></a></p>
+<!--			    <p class="banner-text--caption">Image: The Ellesmer Manuscript of Geoffrey Chaucer's The Canterbury Tales, c. 1400-1405. Shown is the introduction to "The Knight's Tale." <a href="https://hdl.huntington.org/digital/collection/p15150coll7/id/2385" target="blank"> Available online at the Huntington Library <%#= image_tag('external-link-white.svg', alt: 'Opens in a new window') %></a></p>-->
+			    <p class="banner-text--caption">Image: The Ellesmer Manuscript of Geoffrey Chaucer's The Canterbury Tales, c. 1400-1405. Shown is the introduction to "The Knight's Tale." <a href="https://hdl.huntington.org/digital/collection/p15150coll7/id/2385" target="blank"> Available online at the Huntington Library <%= image_tag(asset_path('external-link-white.svg'), alt: 'Opens in a new window') %></a></p>
 			   </div>
 			</div>
 	</div>

app/views/catalog/splash.html.erb

@@ -42,7 +42,8 @@
         Shown is the introduction to "The Knight's Tale."
         <a href="https://hdl.huntington.org/digital/collection/p15150coll7/id/2385" target="_blank">
           Available online at the Huntington
-          Library <%= image_tag('external-link-white.svg', alt: 'Opens in a new window') %></a>
+<!--          Library <%#= image_tag('external-link-white.svg', alt: 'Opens in a new window') %></a>-->
+          Library <%= image_tag(asset_path('external-link-white.svg'), alt: 'Opens in a new window') %></a>
       </div>
     </div>
   </div>

app/views/layouts/application.html.erb

@@ -35,6 +35,10 @@
         <%= render :partial => 'shared/header_navbar' %>
     <% end %>
 
+    <% if ENV['DEBUG_ENV'] %>
+      x-forwarded host: <%= ENV["HTTP_X_FORWARDED_HOST"].inspect %>
+    <% end %>
+
     <%= area :main do %>
       <p>Main content goes here.</p>
     <% end %>

app/views/shared/_footer.html.erb

@@ -5,11 +5,11 @@
         <li class="heading-medium">Middle English Compendium</li>
         <li><a href="<%= dictionary_home_path%>">Middle English Dictionary</a></li>
         <li><a href="<%= bib_home_path%>">Bibliography</a></li>
-        <li><a href="https://quod.lib.umich.edu/c/cme/" target="_blank">Corpus <%= image_tag('external-link-white.svg', alt:'Opens in a new window') %></a></li>
+<!--        <li><a href="https://quod.lib.umich.edu/c/cme/" target="_blank">Corpus <%#= image_tag('external-link-white.svg', alt:'Opens in a new window') %></a></li>-->
+        <li><a href="https://quod.lib.umich.edu/c/cme/" target="_blank">Corpus <%= image_tag(asset_path('external-link-white.svg'), alt:'Opens in a new window') %></a></li>
       </ul>
      </section>
     <section>
-
       <ul>
         <li class="heading-medium">Help and information</li>
         <li><%= link_to "Search Help", help_root_path %></li>

bin/dromedary

@@ -94,6 +94,7 @@ MedInstaller::CLI.register 'extract_convert_index', MedInstaller::ExtractConvert
 
 MedInstaller::CLI.register 'newdata prepare', MedInstaller::PrepareNewData
 MedInstaller::CLI.register 'newdata index', MedInstaller::IndexNewData
+MedInstaller::CLI.register 'ping_prometheus', MedInstaller::PingPrometheus
 
 Hanami::CLI.new(MedInstaller::CLI).call
 

config.ru

@@ -3,3 +3,6 @@
 require_relative "config/environment"
 
 run Rails.application
+# map Dromedary.config.relative_url_root || "/" do
+#   run Rails.application
+# end

config/application.rb

@@ -31,6 +31,8 @@ class Application < Rails::Application
 
     config.relative_url_root = Dromedary.config.relative_url_root
     config.action_controller.relative_url_root = config.relative_url_root
+    # config.relative_url_root                   = '/'
+    # config.action_controller.relative_url_root = '/'
 
     config.log_level = :info
 
@@ -50,6 +52,7 @@ class Application < Rails::Application
     end
 
     config.lograge.formatter = Lograge::Formatters::Json.new
+    config.active_job.queue_adapter = :sidekiq
 
     # config.log_tags = {
     #   ip:         :remote_ip,

config/environments/production.rb

@@ -80,11 +80,11 @@
   # require 'syslog/logger'
   # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
   #
-  # if ENV["RAILS_LOG_TO_STDOUT"].present?
-  #   logger           = ActiveSupport::Logger.new(STDOUT)
-  #   logger.formatter = config.log_formatter
-  #   config.logger    = ActiveSupport::TaggedLogging.new(logger)
-  # end
+  if ENV["RAILS_LOG_TO_STDOUT"].present?
+    logger = ActiveSupport::Logger.new($stdout)
+    logger.formatter = config.log_formatter
+    config.logger = ActiveSupport::TaggedLogging.new(logger)
+  end
 
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false

config/routes.rb

@@ -3,6 +3,8 @@
 Rails.application.routes.draw do
   scope Dromedary.config.relative_url_root do
     mount Blacklight::Engine => Dromedary.config.relative_url_root
+    # scope '/' do
+    #   mount Blacklight::Engine => '/'
 
     # Shunt it all to the maintenace page if we need to
     match "*path" => "static#maintenance_mode", :status => 302, :via => [:get, :post],

config/settings/development.yml

@@ -21,6 +21,9 @@ cable:
 relative_url_root: /
 
 data_dir: ./data
+build_dir: ./data/build
 
 log_dir: ./log
 
+#data_dir: /home/esty/github/mlibrary/med/data
+#build_dir: /home/esty/github/mlibrary/med/data/build

config/settings/local.yml

@@ -0,0 +1,5 @@
+data_dir: /home/esty/github/mlibrary/med/data
+build_dir: /home/esty/github/mlibrary/med/data/build
+
+blacklight:
+  url: http://localhost:9639/solr/med

config/settings/production.yml

@@ -0,0 +1,29 @@
+blacklight:
+  adapter: solr
+  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:9639/solr/med" %>
+
+solr:
+  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:9639/solr/med" %>
+
+secret_key_base: <%= ENV["SECRET_KEY_BASE"] || "somesecretkey" %>
+
+
+
+db:
+  url: unused
+
+cable:
+  adapter: async
+  url: unused
+  channel_prefix: dromedary
+
+
+relative_url_root: <%= ENV["RAILS_RELATIVE_URL_ROOT"] || "/" %>
+
+data_dir: <%= ENV["DATA_DIR"] || "./data" %>
+build_dir: <%= ENV["BUILD_DIR"] || "./data/build" %>
+
+log_dir: <%= ENV["LOG_DIR"] || "./log" %>
+
+#data_dir: /home/esty/github/mlibrary/med/data
+#build_dir: /home/esty/github/mlibrary/med/data/build

config/sidekiq.yml

@@ -0,0 +1,8 @@
+development:
+  :concurrency: 1
+
+production:
+  :concurrency: 1
+
+:queues:
+  - default

data.Dockerfile

@@ -0,0 +1,32 @@
+FROM ruby:2.6
+
+ENV DATA_FILE=/opt/app-data/All_MED_and_BIB_files.zip
+ENV data_dir=/usr/src/app/data
+ENV build_dir=/usr/src/app/data/build
+ENV SOLR_URL=http://localhost:9639/solr
+ARG UNAME=app
+ARG UID=1000
+ARG GID=1000
+
+RUN apt-get update -yqq && \
+    apt-get install -yqq --no-install-recommends \
+    apt-transport-https nodejs
+
+RUN groupadd -g $GID -o $UNAME
+RUN useradd -m -d /usr/src/app -u $UID -g $GID -o -s /bin/bash $UNAME
+RUN mkdir -p /gems && chown $UID:$GID /gems
+
+USER $UNAME
+COPY --chown=$UID:$GID Gemfile* /usr/src/app/
+
+ENV RAILS_LOG_TO_STDOUT true
+ENV BUNDLE_PATH /gems
+
+WORKDIR /usr/src/app
+RUN gem install 'bundler:2.1.4'
+RUN bundle install
+
+COPY --chown=$UID:$GID . /usr/src/app
+
+USER $UNAME
+RUN mkdir -p /usr/src/app/data/build

docker-compose.yml

@@ -1,5 +1,18 @@
 ---
 services:
+  #  web:
+  #    image: dromedary-web
+  #    build:
+  #      context: .
+  #      dockerfile: Dockerfile.prod
+  #    ports:
+  #      - "3000:3000"
+  #    environment:
+  #      - DATA_FILE=/usr/src/app/All_MED_and_Bib_files.zip
+  #      - PROMETHEUS_PUSH_GATEWAY=http://pushgateway:9091
+  #      - SOLR_URL=http://nectar.umdl.umich.edu:8081/solr/dromedary_testing/
+  #      - REDIS_URL=redis://redis:6379
+  #      - DEBUG_ENV=true
   app:
     depends_on:
       - db
@@ -69,6 +82,44 @@ services:
 #      - sidekiq
 #      - -r
 #      - ./app/jobs/job_index.rb
+      - solr_core:/opt/solr/server/solr/dromedary
+#  data:
+#    build:
+#      dockerfile: data.Dockerfile
+#      context: .
+#    volumes:
+#      - ../data:/opt/app-data
+#      - gem_cache:/gems
+#  redis:
+#    image: redis:6-buster
+#    ports:
+#      - 6380:6379
+#
+#  sidekiq:
+#    image: dromedary-web
+#    ports:
+#      - 3001:3001
+#    command:
+#      - bundle
+#      - exec
+#      - sidekiq
+#      - -r
+#      - ./app/jobs/job_index.rb
+#    environment:
+#      - REDIS_URL=redis://redis:6379
+#      - SOLR_URL=http://nectar.umdl.umich.edu:8081/solr/dromedary_testing/
+#      - DATA_FILE=/opt/app-data/All_MED_and_Bib_files.zip
+#      - PROMETHEUS_PUSH_GATEWAY=http://pushgateway:9091
+  prometheus:
+    image: prom/prometheus
+    ports:
+      - 9090:9090
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+  pushgateway:
+    image: prom/pushgateway
+    ports:
+      - 9091:9091
 volumes:
   data:
     driver_opts: