diff --git a/bes/.gitignore b/bes/.gitignore new file mode 100644 index 0000000..8fb89be --- /dev/null +++ b/bes/.gitignore @@ -0,0 +1 @@ +creds.conf \ No newline at end of file diff --git a/bes/cores/.gitignore b/bes/cores/.gitignore new file mode 100644 index 0000000..a780935 --- /dev/null +++ b/bes/cores/.gitignore @@ -0,0 +1 @@ +core* \ No newline at end of file diff --git a/bes/cores/SmGrp.dmr.bescmd b/bes/cores/SmGrp.dmr.bescmd new file mode 100644 index 0000000..af4da3a --- /dev/null +++ b/bes/cores/SmGrp.dmr.bescmd @@ -0,0 +1,18 @@ + + + 1800 + no + xml + http://ngap-west.opendap.org:8080/opendap/dap4/SimpleGroup.nc4.h5 + 0 + 0 + not_logged_in + false + /dap4/SimpleGroup.nc4.h5 + + + + + + + diff --git a/bes/site.conf b/bes/site.conf index b33afba..7612f85 100644 --- a/bes/site.conf +++ b/bes/site.conf @@ -20,6 +20,7 @@ FONc.ClassicModel=false # True causes the DAP4 DMR to be generated directly rather than via DDS and DAS. H5.EnableCFDMR=true +#DMRPP.UseBufferChunk=false # I turned this off on 6/23/24 at 20:52 MDT to see if this was causing the large # number of alarms from Better Uptime. jhrg 6/23/24 # @@ -40,7 +41,7 @@ BES.Catalog.catalog.TypeMatch+=h5:.*\.(HDF5|h5|he5|H5)(\.bz2|\.gz|\.Z)?$; BES.Catalog.catalog.TypeMatch+=nc:.*\.nc(4)?(\.bz2|\.gz|\.Z)?$; BES.Catalog.catalog.TypeMatch+=ncml:.*\.ncml(\.bz2|\.gz|\.Z)?$; - +AllowedHosts += ^https:\/\/s3\.us-east-1\.amazonaws\.com\/opendap\.scratch\/merra2\/.*$ DMRPP.UseParallelTransfers = false DMRPP.MaxParallelTransfers = 16 @@ -49,4 +50,14 @@ DMRPP.MaxComputeThreads = 512 +# This is capped below 2.0GB because of https://bugs.earthdata.nasa.gov/browse/HYRAX-1313 +#BES.MaxVariableSize.bytes = 2104533975 +BES.MaxVariableSize.bytes = 12104533975 + +# It looks like an r6a.4xlarge can build the nc4 response for 31 days of Daymet, +# 8329006592 bytes (uncompressed), in under 165 seconds. Use that as the response +# size limit since it is within the AWS 180 seconds TTFB limit +#BES.MaxResponseSize.bytes = 8329006592 +BES.MaxResponseSize.bytes = 18329006592 +CredentialsManager.config=/etc/bes/credentials.conf diff --git a/hyraxctl b/hyraxctl index 6f6c80c..28c8081 100755 --- a/hyraxctl +++ b/hyraxctl @@ -9,17 +9,26 @@ # # -########################################################################## + + +######################################################################### # loggy() # +# A handy logging function. +# function loggy() { - echo "# $*" >&2 + if test -n "${log_file}"; then + echo "$@" | awk '{ print "# "$0;}' | tee -a "${log_file}" >&2 + else + echo "$@" | awk '{ print "# "$0;}' >&2 + fi } export H1="######################################################################" export H2="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" export H3="- - - - - - - - - - - - - - - - - - - - - - -" + export HYRAX_NAME=${HYRAX_NAME:-"hyrax"} export HTTPD_NAME=${HTTPD_NAME:-"httpd"} export HYRAX_HOME=${HYRAX_HOME:-"/home/ubuntu/hyrax"} @@ -34,12 +43,18 @@ export HTTPD_IMAGE_DEFAULT="httpd:2.4.56-bullseye" export HTTPD_IMAGE=${HTTPD_IMAGE:-"$HTTPD_IMAGE_DEFAULT"} export APACHE_HTTPD_HOME=${APACHE_HTTPD_HOME:-"/usr/local/apache2"} -export HYRAX_NETWORK=${HYRAX_NETWORK:-"hyrax-net"} +export HYRAX_NETWORK="${HYRAX_NETWORK:-"hyrax-net"}" export HYRAX_DEPLOYMENT="http://test.opendap.org" export THREDDS_CATALOG_ROOT=${THREDDS_CATALOG_ROOT:-""} export HYRAX_MEMORY=${HYRAX_MEMORY:-"24g"} +export RPM_BUILD_REPO="s3://opendap.travis.build" + +export bes_cores_dir +export tomcat_server_xml +export run_privileged +export log_file ################################################################################ # @@ -67,7 +82,9 @@ function show_config() { loggy "${prolog} log_dir: ${log_dir}" loggy "${prolog} data_dir: ${data_dir}" loggy "${prolog} bes_site_conf_file: ${bes_site_conf_file}" + loggy "${prolog} bes_creds_conf_file: ${bes_creds_conf_file}" loggy "${prolog} bes_log_dir: ${bes_log_dir}" + loggy "${prolog} bes_cores_dir: ${bes_cores_dir}" loggy "${prolog} olfs_conf_dir: ${olfs_conf_dir}" loggy "${prolog} tomcat_server_xml: ${tomcat_server_xml}" loggy "${prolog} tomcat_server_xml_volume: ${tomcat_server_xml_volume}" @@ -77,13 +94,29 @@ function show_config() { loggy "${prolog} ncwms: ${ncwms}" loggy "${prolog}" } +################################################################################ +# +# config_tomcat_server() +# +function config_tomcat_server() { + local hyrax_home="$1" + # A Tomcat server.xml file that enables AJP pn 8009 + tomcat_server_xml="${hyrax_home}/tomcat/server.xml" + + # Creates the tomcat server.xml volume mount key pair for the Docker run command. + export tomcat_server_xml_volume="" + if test -n "${tomcat_server_xml}"; then + export tomcat_server_xml_volume="--volume ${tomcat_server_xml}:/tomcat/conf/server.xml" + fi + +} ################################################################################ # # config_test_opendap_org() # # shellcheck disable=SC2120 -function config_test_opendap_org(){ +function config_test_opendap_org() { local prolog="config_test_opendap_org() -" local hyrax_home=${1:-"${HYRAX_HOME}"} local data_dir_in=${2:-"${DATA_MOUNT}"} @@ -97,49 +130,44 @@ function config_test_opendap_org(){ export log_dir="${hyrax_home}/log" # Set up the location of the data directory for Hyrax. - # If one is not provided the default ${hyrax_home}/data + # If one is not provided the default ${hyrax_home}/data # will be used. - if test -n "${data_dir_in}" - then + if test -n "${data_dir_in}"; then export data_dir="${data_dir_in}" else export data_dir="${hyrax_home}/data" loggy "${prolog} Using default data_dir value of: ${data_dir}" fi - + # The location of the OLFS persistent configuration # to be injected into the Hyrax container at /etc/olfs # # **If this is used then the OLFS logs will be written - # to ${hyrax_home}/olfs and the log directory + # to ${hyrax_home}/olfs and the log directory # ${hyrax_home}/log/olfs will be ignored. export olfs_conf_dir="${hyrax_home}/olfs" export bes_site_conf_file="${hyrax_home}/bes/site.conf" - export bes_log_dir="${log_dir}/bes" - + # Credentials for service-chain + export bes_creds_conf_file="${hyrax_home}/bes/creds.conf" - # A Tomcat server.xml file that enables AJP pn 8009 - export tomcat_server_xml="${hyrax_home}/tomcat/server.xml" + export bes_log_dir="${log_dir}/bes" + bes_cores_dir="${hyrax_home}/bes/cores" - # Creates the tomcat server.xml volume mount key pair for the Docker run command. - export tomcat_server_xml_volume="" - if test -n "${tomcat_server_xml}"; then - export tomcat_server_xml_volume="--volume ${tomcat_server_xml}:/tomcat/conf/server.xml" - fi + config_tomcat_server "${hyrax_home}" # # Variables for configuring Docker launch of httpd # - + # We inject a modified httpd.conf file export httpd_conf="${hyrax_home}/httpd/httpd.conf" - - # Our own collection of things that need to be served via http, + + # Our own collection of things that need to be served via http, # possibly utilizing range get capabilities export htdocs_content="${hyrax_home}/httpd/htdocs" - + # Q: Is expires.sh still in use? Can we drop it? # A: Yes it's still in use by libdap4 tests. # Q: Can we drop it? @@ -148,34 +176,95 @@ function config_test_opendap_org(){ # ncWMS host name to be injected into the ncWMS configuration. export ncwms="${HYRAX_DEPLOYMENT}" - + show_config loggy "${prolog} END" } - ################################################################################ # # config_ndp_local() # -function config_ndp_local(){ - local prolog="make_network_as_needed() -" +function config_ndp_local() { + local prolog="config_ndp_local() -" + export HYRAX_HOME="/Users/ndp/OPeNDAP/hyrax/test.opendap.org" + config_ndp "$HYRAX_HOME" +} + +function config_ndp(){ + + local hyrax_home= + hyrax_home="${1:-$HYRAX_HOME}" + loggy "hyrax_home: $hyrax_home" + local pfix + pfix=${prefix:-"/Users/ndp/OPeNDAP/hyrax/build"} + export data_dir="${pfix}/share/hyrax" + loggy "${prolog} ${H1}" loggy "${prolog} BEGIN" loggy "${prolog} " prefix=${prefix:-"/Users/ndp/OPeNDAP/hyrax/build"} - export hyrax_home="/Users/ndp/OPeNDAP/test.opendap.org" export ncwms="http://localhost:8080" export log_dir="${hyrax_home}/log" export bes_log_dir="${log_dir}/bes" - export data_dir="${prefix}/share/hyrax" + export bes_cores_dir="${hyrax_home}/bes/cores" export olfs_conf_dir="${hyrax_home}/etc/olfs" export bes_site_conf_file="${hyrax_home}/bes/site.conf" - # export tomcat_server_xml="${hyrax_home}/tomcat/server.xml" - # export tomcat_server_xml_volume="--volume ${tomcat_server_xml}:/tomcat/conf/server.xml" - + + # Credentials for service-chain + export bes_creds_conf_file="${hyrax_home}/bes/creds.conf" + + export httpd_conf="${hyrax_home}/httpd/httpd.conf" + export htdocs_content="${hyrax_home}/httpd/htdocs" + export expires_sh="${hyrax_home}/httpd/cgi-bin/expires.sh" + + config_tomcat_server "${hyrax_home}" + + show_config + + loggy "${prolog} END" + +} +################################################################################ +# +# config_rocky_8_local() +# +function config_rocky_8_local() { + local prolog="config_rocky_8_local() -" + export HYRAX_HOME="/home/rocky/docker_deploy" + config_rocky_8 $HYRAX_HOME +} + +function config_rocky_8() { + local prolog="config_rocky_8() -" + + loggy "${prolog} ${H1}" + loggy "${prolog} BEGIN" + loggy "${prolog} " + + local hyrax_home + hyrax_home="${1:-$HYRAX_HOME}" + + local pfix + pfix=${prefix:-"/home/rocky/hyrax/build"} + export data_dir="${prefix}/share/hyrax" + + + export ncwms="http://localhost:8080" + export log_dir="${hyrax_home}/log" + export bes_log_dir="${log_dir}/bes" + export bes_cores_dir="${hyrax_home}/bes/cores" + export olfs_conf_dir="/etc/olfs" + export bes_site_conf_file="${hyrax_home}/bes/site.conf" + + export httpd_conf="${hyrax_home}/httpd/httpd.conf" + export htdocs_content="${hyrax_home}/httpd/htdocs" + export expires_sh="${hyrax_home}/httpd/cgi-bin/expires.sh" + + config_tomcat_server "${hyrax_home}" + show_config loggy "${prolog} END" @@ -187,19 +276,18 @@ function config_ndp_local(){ # # make_network_as_needed() # -function make_network_as_needed(){ +function make_network_as_needed() { local prolog="make_network_as_needed() -" # # Check for the network that will be shared with Apache httpd # If it's not found, create it. # local status= - docker network inspect ${HYRAX_NETWORK} > /dev/null 2>&1 + docker network inspect "${HYRAX_NETWORK}" >/dev/null 2>&1 status=$? - if test $status -ne 0 ; - then + if test $status -ne 0; then loggy "${prolog} No Docker network by the name of ${HYRAX_NETWORK} was found. Creating..." - docker network create hyrax-net + docker network create "${HYRAX_NETWORK}" loggy "${prolog} Created Docker network '${HYRAX_NETWORK}' for Hyrax and Apache httpd." loggy "${prolog} " else @@ -209,23 +297,24 @@ function make_network_as_needed(){ } - ################################################################################ # # delete_network() # -function delete_network(){ +function delete_network() { # Dump Hyrax Docker Network - docker network rm ${HYRAX_NETWORK} + docker network rm "${HYRAX_NETWORK}" } - ################################################################################ # # start_httpd() # -function start_httpd(){ +function start_httpd() { local prolog="start_httpd() -" + + make_network_as_needed + loggy "${prolog} Starting Docker container for Apache httpd: ${HTTPD_IMAGE}" docker run -d \ --name ${HTTPD_NAME} \ @@ -247,7 +336,7 @@ function start_httpd(){ # # stop_httpd() # -function stop_httpd(){ +function stop_httpd() { # Dump httpd container docker rm -f "${HTTPD_NAME}" } @@ -262,7 +351,11 @@ function update_httpd() { start_httpd } -function get_platform(){ +################################################################################ +# +# get_platform() +# +function get_platform() { local prolog="get_platform() -" local sys_arch= sys_arch=$(arch) @@ -272,58 +365,64 @@ function get_platform(){ echo "$platform" } +################################################################################ +# +# set_file_permissions() +# function set_file_permissions() { local prolog="set_file_permissions() -" loggy "${prolog} BEGIN" local vers_doc - vers_doc=$( which sw_vers ) - if test $? -ne 0 - then - local name="testy_besty" - - local platform="" - platform=$( get_platform ) - loggy "${prolog} platform: $platform" - - local use_disconnect="-d" # ${disconnect:-"-d"} - loggy "${prolog} use_disconnect: $use_disconnect" - - loggy "${prolog} Checking bes user info in Hyrax container: ${HYRAX_IMAGE}" - docker run $use_disconnect $platform --name "${name}" ${HYRAX_IMAGE} - - sleep_time=10 - loggy "${prolog} Waiting $sleep_time seconds for hyrax to get going..." - sleep $sleep_time - - local bes_uid - bes_uid=$(docker logs "${name}" 2>&1 | grep bes_uid | awk '{print $2;}') - loggy "${prolog} bes_uid: ${bes_uid}" - - local bes_gid - bes_gid=$(docker logs "${name}" 2>&1 | grep bes_gid | awk '{print $2;}') - loggy "${prolog} bes_gid: ${bes_gid}" - - loggy "${prolog} Stopping Hyrax container: ${HYRAX_IMAGE}" - docker container rm -f "${name}" - - if test -n "$bes_uid" && test -n "$bes_gid" - then - loggy "${prolog} Setting ownership of ${bes_log_dir}" - sudo chown -R ${bes_uid}:${bes_gid} "${bes_log_dir}" - else - loggy "${prolog} ERROR - Failed to determine one or both of bes user uid or gid." - return 1 - fi - else - loggy "${prolog} Detected OSX, no permissions were changed." - loggy $(sw_vers) - fi - loggy "${prolog} END" - return 0 + vers_doc=$(which sw_vers) + if test $? -ne 0; then + local name="testy_besty" + + local platform="" + platform="$(get_platform)" + loggy "${prolog} platform: $platform" + + local use_disconnect="-d" # ${disconnect:-"-d"} + loggy "${prolog} use_disconnect: $use_disconnect" + + loggy "${prolog} Checking bes user info in Hyrax container: ${HYRAX_IMAGE}" + docker run $use_disconnect $platform --name "${name}" ${HYRAX_IMAGE} + + sleep_time=10 + loggy "${prolog} Waiting $sleep_time seconds for hyrax to get going..." + sleep $sleep_time + + local bes_uid + bes_uid=$(docker logs "${name}" 2>&1 | grep bes_uid | awk '{print $2;}') + loggy "${prolog} bes_uid: ${bes_uid}" + + local bes_gid + bes_gid=$(docker logs "${name}" 2>&1 | grep bes_gid | awk '{print $2;}') + loggy "${prolog} bes_gid: ${bes_gid}" + + loggy "${prolog} Stopping Hyrax container: ${HYRAX_IMAGE}" + docker container rm -f "${name}" + + if test -n "$bes_uid" && test -n "$bes_gid"; then + loggy "${prolog} Setting ownership of ${bes_log_dir}" + sudo chown -R ${bes_uid}:${bes_gid} "${bes_log_dir}" + else + loggy "${prolog} ERROR - Failed to determine one or both of bes user uid or gid." + return 1 + fi + else + loggy "${prolog} Detected OSX, no permissions were changed." + loggy $(sw_vers) + fi + loggy "${prolog} END" + return 0 } -function clean_hyrax_logs(){ +################################################################################ +# +# clean_hyrax_logs() +# +function clean_hyrax_logs() { local prolog="clean_hyrax_logs() -" loggy "${prolog} BEGIN" loggy $(rm -vrf ${HYRAX_HOME}/log/bes/*) @@ -332,7 +431,6 @@ function clean_hyrax_logs(){ loggy "${prolog} END" } - ################################################################################ # save_hyrax_logs() # Saves the bes log file. The tomcat logs and OLFS logs are already being @@ -341,7 +439,6 @@ function save_hyrax_logs() { local prolog="save_hyrax_logs() -" loggy "${prolog} BEGIN" - local date_tag= date_tag=$(date +"%Y-%m-%d-%H:%M:%S") loggy "${prolog} date_tag: $date_tag" @@ -355,50 +452,81 @@ function save_hyrax_logs() { local source_file= source_file="${HYRAX_HOME}/log/bes/bes.log" loggy "${prolog} Processing: $source_file" - if test -f "${source_file}" - then - cp -v "${source_file}" "${target_dir}/bes.log" + if test -f "${source_file}"; then + cp -v "${source_file}" "${target_dir}/bes.log" else - loggy "$prolog Failed to locate: $source_file" + loggy "$prolog Failed to locate: $source_file" fi loggy "${prolog} Save the various OLFS logs..." - for log in olfs-log.json HyraxAccess.log HyraxErrors.log cloud_watch_request.log cloud_watch_response.log BESCommands.log - do - source_file="${HYRAX_HOME}/olfs/logs/$log" - loggy "${prolog} Processing: $source_file" - if test -f "${source_file}" - then - cp -v "${source_file}" "${target_dir}/${log}" - else - loggy "$prolog Failed to locate: $source_file" - fi + for log in olfs-log.json HyraxAccess.log HyraxErrors.log cloud_watch_request.log cloud_watch_response.log BESCommands.log; do + source_file="${HYRAX_HOME}/olfs/logs/$log" + loggy "${prolog} Processing: $source_file" + if test -f "${source_file}"; then + cp -v "${source_file}" "${target_dir}/${log}" + else + loggy "$prolog Failed to locate: $source_file" + fi done loggy "${prolog} Save the Tomcat logs..." local local_host_log= local_host_log=$(basename $(ls -1t "${HYRAX_HOME}/log/tomcat/localhost".* | head -1)) loggy "${prolog} local_host_log: $local_host_log" - for log in console.log "${local_host_log}" catalina.out - do - source_file="${HYRAX_HOME}/log/tomcat/$log" - loggy "${prolog} Processing: $source_file" - - if test -f "${source_file}"; then - cp -v "${source_file}" "${target_dir}/tomcat-${log}" - else - loggy "$prolog Failed to locate file: $source_file" - fi + for log in console.log "${local_host_log}" catalina.out; do + source_file="${HYRAX_HOME}/log/tomcat/$log" + loggy "${prolog} Processing: $source_file" + + if test -f "${source_file}"; then + cp -v "${source_file}" "${target_dir}/tomcat-${log}" + else + loggy "$prolog Failed to locate file: $source_file" + fi done loggy "${prolog} END" } +################################################################################ +# +# start_simple() +# +function start_debug() { + local prolog="start_hyrax() -" + loggy "${prolog} BEGIN" + + save_hyrax_logs + clean_hyrax_logs + + set_file_permissions + if test $? -ne 0; then + loggy "${prolog} ERROR - Unable to set host file permissions. Hyrax was not started." + return 1 + fi + + local use_disconnect=${disconnect:-"-d"} + loggy "${prolog} use_disconnect: $use_disconnect" + local platform="" + platform="$(get_platform)" + loggy "${prolog} platform: $platform" + + loggy "${prolog} run_privileged: $run_privileged" + + loggy "${prolog} Starting Hyrax container: ${HYRAX_IMAGE}" + docker run -d --name "$HYRAX_NAME" -p 8080:8080 -v /usr/share/hyrax:/usr/share/hyrax \ + -v /home/rocky/test.opendap.org/debug:/debug \ + "$HYRAX_IMAGE" + loggy "${prolog}" + loggy "${prolog} Docker container: ${HYRAX_IMAGE} STARTED." + loggy "${prolog}" + loggy "${prolog} END" + +} ################################################################################ # # start_hyrax() # -function start_hyrax(){ +function start_hyrax() { local prolog="start_hyrax() -" loggy "${prolog} BEGIN" @@ -406,21 +534,24 @@ function start_hyrax(){ clean_hyrax_logs set_file_permissions - if test $? -ne 0 - then + if test $? -ne 0; then loggy "${prolog} ERROR - Unable to set host file permissions. Hyrax was not started." - return 1; + return 1 fi + make_network_as_needed + local use_disconnect=${disconnect:-"-d"} loggy "${prolog} use_disconnect: $use_disconnect" local platform="" - platform=$( get_platform ) + platform="$(get_platform)" loggy "${prolog} platform: $platform" + loggy "${prolog} run_privileged: $run_privileged" + loggy "${prolog} Starting Hyrax container: ${HYRAX_IMAGE}" - docker run $use_disconnect $platform \ + docker run $use_disconnect $run_privileged $platform \ --name ${HYRAX_NAME} \ --net=${HYRAX_NETWORK} \ --publish 8009:8009 \ @@ -428,9 +559,11 @@ function start_hyrax(){ --publish 8443:8443 \ --volume ${data_dir}:/usr/share/hyrax:ro \ --volume ${bes_log_dir}:/var/log/bes \ + --volume ${bes_cores_dir}:/cores \ --volume ${log_dir}/tomcat:/usr/share/tomcat/logs \ --volume ${olfs_conf_dir}:/etc/olfs \ --volume ${bes_site_conf_file}:/etc/bes/site.conf \ + --volume ${bes_creds_conf_file}:/etc/bes/credentials.conf \ --memory=${HYRAX_MEMORY} \ ${tomcat_server_xml_volume} \ "${HYRAX_IMAGE}" \ @@ -441,17 +574,204 @@ function start_hyrax(){ loggy "${prolog} Docker container: ${HYRAX_IMAGE} STARTED." loggy "${prolog}" loggy "${prolog} END" + +} + +################################################################################ +# enable_core_dump() +# +# Restarts the container with privileged enabled. Then it uses docker exec and +# bash to configure the running container to create core dump files in /cores +# on container's disk. +# +function enable_core_dump() { + local prolog="enable_core_dump() -" + loggy "${H2}" + loggy "${prolog} BEGIN" + run_privileged="--ulimit core=-1 --privileged --security-opt seccomp=unconfined " + loggy "${prolog} run_privileged: $run_privileged" + stop_hyrax + start_hyrax + local naptime=10 + loggy "$prolog Waiting $naptime seconds for Hyrax to arrive..." + sleep 10 + + loggy "" + loggy "${prolog} Setting kernel.core.pattern in docker image:" + loggy "$(docker exec -it ${HYRAX_NAME} bash -c "sysctl -w kernel.core_pattern=/cores/core-%e.%p.%h.%t; echo \"status: $?\"")" + + loggy "" + loggy "${prolog} docker ulimit: " + loggy "$(docker exec -it ${HYRAX_NAME} bash -c "ulimit -a")" + + loggy "" + loggy "${prolog} host ulimit:" + loggy "$(ulimit -a)" + loggy "" + + loggy "" + loggy "${prolog} In docker container modify /etc/systemd/system.conf to all core dumps:" + loggy "$(docker exec -it ${HYRAX_NAME} bash -c "echo \"DumpCore=yes\" >> /etc/systemd/system.conf; cat /etc/systemd/system.conf; ")" + loggy "" + +} + + +################################################################################ +# check_for_aws_config() +# +function check_for_aws_config() { + local prolog="check_for_aws_config() -" + local status + local msg + + msg="$(aws sts get-caller-identity 2>&1 )" + status=$? + if test $status -ne 0 + then + loggy "$prolog ERROR! The AWS CLI is not configured unable to use AWS CLI." + loggy "$prolog AWS status: $status" + loggy "$prolog AWS message: "$msg + else + loggy "$prolog AWS CLI is configured! user: " + loggy "$msg" + fi + + return $status +} + + +################################################################################ +# install_linux_debug_packages() +# +# This is probably brittle - I think I got this list from gdb when +# I tried to look at a core dump with only the hyrax debuginfo RPMs +# installed. Given that the version numbers are "fully qualified" I +# suspect that when any of these packages gets updated the reference +# below will be out of date. For the core dump inspection the binary +# stuff has to match, or it's a debug train wreck. +# +function install_linux_debug_packages() { + local prolog="install_linux_debug_packages() -" + + loggy "$prolog Installing various debug info..." + docker exec -it hyrax bash -c "yum debuginfo-install -y \ + bzip2-libs-1.0.6-28.el8_10.x86_64 \ + glibc-2.28-251.el8_10.25.x86_64 \ + keyutils-libs-1.5.10-9.el8.x86_64 \ + krb5-libs-1.18.2-32.el8_10.x86_64 \ + libcom_err-1.45.6-6.el8_10.x86_64 \ + libcurl-minimal-7.61.1-34.el8_10.3.x86_64 \ + libgcc-8.5.0-28.el8_10.x86_64 \ + libicu-60.3-2.el8_1.x86_64 \ + libnghttp2-1.33.0-6.el8_10.1.x86_64 \ + libselinux-2.9-10.el8_10.x86_64 \ + libstdc++-8.5.0-28.el8_10.x86_64 \ + libtirpc-1.1.4-12.el8_10.x86_64 \ + libuuid-2.32.1-46.el8.x86_64 \ + libxml2-2.9.7-21.el8_10.3.x86_64 \ + libzstd-1.4.4-1.el8.x86_64 \ + openssl-libs-1.1.1k-14.el8_10.x86_64 \ + pcre2-10.32-3.el8_6.x86_64 \ + xz-libs-5.2.4-4.el8_6.x86_64 \ + zlib-1.2.11-26.el8.x86_64" +} + +################################################################################ +# enable_debugging() +# +# This will call enable_core_dump() which stops and then starts the server +# so the server should available for interrogation and configuration. +# +function enable_debugging() { + local prolog="enable_debugging() -" + local status + loggy "$H1" + loggy "$prolog BEGIN" + loggy "" + + #----------------------------------------------------------------- + # Make sure we have AWS CLI so we can get rpm files from our s3 bucket + check_for_aws_config + status=$? + if test $status -ne 0 + then + loggy "$prolog ERROR! AWS CLI Not configured." + # loggy "$prolog status: $status" + return $status + fi + loggy "" + + #----------------------------------------------------------------- + # Enable core dumps + enable_core_dump + + #----------------------------------------------------------------- + # We grab the version response from the running server + local version_response + local version_url="http://localhost:8080/opendap/version" + locl version_file="./hyrax-version.xml" + http_code=$(curl -o "%{http_code}" -o "$version_file" "$version_url") + if test $http_code -ne 200 + then + loggy "ERROR! Failed to get Hyrax version response from: $version_url" + loggy "ERROR! Failed to get Hyrax version response from: $version_url" + else + loggy "Hyrax Version Response: $version_response" + fi + + #----------------------------------------------------------------- + # Using the version response document, get and install the + # libdap4 debuginfo rpm + local libdap4_version + libdap4_version=$(echo "$version_response" | grep "library name=\"libdap\"" | awk '{ split($3,v,"="); gsub("\"","",v[2]); print v[2] ;}') + echo "libdap_version: $libdap4_version" + + local libdap_debug_rpm="libdap-debuginfo-$libdap4_version.el8.x86_64.rpm" + loggy "$prolog libdap_debug_rpm: $libdap_debug_rpm" + + loggy "$prolog Retrieving: $libdap_debug_rpm" + aws s3 cp "$RPM_BUILD_REPO/$libdap_debug_rpm" "$bes_cores_dir" + + loggy "$prolog Installing: $libdap_debug_rpm" + docker exec -it hyrax bash -c "yum install -y /cores/$libdap_debug_rpm" + + #----------------------------------------------------------------- + # Using the version response document, get and install the + # BES debuginfo rpm + local bes_version + bes_version=$(echo "$version_response" | grep "library name=\"bes\"" | awk '{ split($3,v,"="); gsub("\"","",v[2]); print v[2]; }') + echo "bes_version: $bes_version" + + local bes_debug_rpm="bes-debuginfo-$bes_version.static.el8.x86_64.rpm" + loggy "$prolog bes_debug_rpm: $bes_debug_rpm" + + loggy "$prolog Retrieving: $bes_debug_rpm" + aws s3 cp "$RPM_BUILD_REPO/$bes_debug_rpm" "$bes_cores_dir" + + loggy "$prolog Installing: $bes_debug_rpm" + docker exec -it hyrax bash -c "yum install -y /cores/$bes_debug_rpm" + + #----------------------------------------------------------------- + # Get the ancillary debugging symbols. + install_linux_debug_packages + + loggy "$prolog END" + + return $status } + + ################################################################################ # # stop_hyrax() # -function stop_hyrax(){ +function stop_hyrax() { local prolog="stop_hyrax() -" loggy "${prolog} BEGIN" # Dump Hyrax container loggy "${prolog} Forcibly removing the container: ${HYRAX_NAME}" - loggy "${prolog} Removed: "$( docker rm -f "${HYRAX_NAME}" ) + loggy "${prolog} Removed: "$(docker rm -f "${HYRAX_NAME}") loggy "${prolog} END" } ################################################################################ @@ -459,7 +779,7 @@ function stop_hyrax(){ # update_hyrax() # function update_hyrax() { - local prolog="stop_hyrax() -" + local prolog="update_hyrax() -" loggy "${prolog} BEGIN" docker pull "${HYRAX_IMAGE}" stop_hyrax @@ -467,27 +787,23 @@ function update_hyrax() { loggy "${prolog} END" } - - ################################################################################ # # start_all() # -function start_all(){ +function start_all() { local prolog="start_all() -" loggy "${prolog} BEGIN" - make_network_as_needed start_hyrax start_httpd loggy "${prolog} END" } - ################################################################################ # # stop_all() # -function stop_all(){ +function stop_all() { loggy "stop_all() - BEGIN" # Dump all containers docker rm -f $(docker ps -aq) @@ -500,7 +816,7 @@ function stop_all(){ # # clean_and_update() # -function clean_and_update_all(){ +function clean_and_update_all() { loggy "clean_and_update_all() - BEGIN" stop_all # Dump ALL the images @@ -511,12 +827,17 @@ function clean_and_update_all(){ loggy "clean_and_update_all() - END" } +################################################################################ +# Opens an interactive bash shell on the Hyrax container. +function dshell() { + docker exec -it ${HYRAX_NAME} /bin/bash +} ################################################################################ # # usage() # -function usage(){ +function usage() { loggy "${H1}" loggy "NAME" loggy " hyraxctl - hyrax control" @@ -559,6 +880,11 @@ function usage(){ loggy " test.opendap.org (ubuntu Docker host)" loggy " config_ndp_local - Configures the shell for dev deployment" loggy " on ndp's system." + loggy " config_rocky_8_local - Configures the shell for aam AWS rocky8 system" + loggy " and user rocky." + loggy "" + loggy " enable_debugging - Launches an instance set up with debuginfo symbols" + loggy " and core dumps enabled. AWS CLI must be conifgured!" loggy "" loggy " start_all - Makes the docker network, starts Hyrax," loggy " starts httpd." @@ -587,24 +913,46 @@ function usage(){ loggy " delete_network - Deletes the docker network for Hyrax" loggy " and httpd." loggy "" + if test "$USER" != "root" + then + loggy "" + loggy "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + loggy "WARNING: Your current user-id is: '$USER'" + loggy "WARNING: You must be the root user in order to use the functions in this script." + loggy "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + loggy "" + fi loggy "${H3}" } -function run_cmd(){ + +function run_cmd() { local cmd="${1}" - if test "${cmd}" == "show_config"; then show_config; - elif test "${cmd}" == "config_test_opendap_org"; then config_test_opendap_org; - elif test "${cmd}" == "config_ndp_local"; then config_ndp_local; - elif test "${cmd}" == "start_all"; then start_all; - elif test "${cmd}" == "stop_all"; then stop_all; - elif test "${cmd}" == "start_hyrax"; then start_hyrax; - elif test "${cmd}" == "stop_hyrax"; then stop_hyrax; - elif test "${cmd}" == "start_httpd"; then start_httpd; - elif test "${cmd}" == "stop_httpd"; then stop_httpd; - elif test "${cmd}" == "clean_and_update"; then clean_and_update; - elif test "${cmd}" == "make_network_as_needed"; then make_network_as_needed; - elif test "${cmd}" == "delete_network"; then delete_network; + if test "${cmd}" == "show_config"; then + show_config + elif test "${cmd}" == "config_test_opendap_org"; then + config_test_opendap_org + elif test "${cmd}" == "config_ndp_local"; then + config_ndp_local + elif test "${cmd}" == "start_all"; then + start_all + elif test "${cmd}" == "stop_all"; then + stop_all + elif test "${cmd}" == "start_hyrax"; then + start_hyrax + elif test "${cmd}" == "stop_hyrax"; then + stop_hyrax + elif test "${cmd}" == "start_httpd"; then + start_httpd + elif test "${cmd}" == "stop_httpd"; then + stop_httpd + elif test "${cmd}" == "clean_and_update"; then + clean_and_update + elif test "${cmd}" == "make_network_as_needed"; then + make_network_as_needed + elif test "${cmd}" == "delete_network"; then + delete_network else echo "ERROR Unrecognized command: ${cmd}" >&2 usage @@ -612,10 +960,8 @@ function run_cmd(){ } echo "ArgCount: $#" >&2 -if test "$#" -gt 0 -then - for a_cmd in "$@" - do +if test "$#" -gt 0; then + for a_cmd in "$@"; do echo "Running command: ${a_cmd}" >&2 run_cmd "${a_cmd}" done @@ -624,5 +970,5 @@ else usage fi - - +# libdap: 3.21.1-273 +# bes: 3.21.1-653 diff --git a/log/tomcat/console.log b/log/tomcat/console.log deleted file mode 100644 index e69de29..0000000 diff --git a/tomcat/server.xml b/tomcat/server.xml index 7ed5004..9e92b07 100644 --- a/tomcat/server.xml +++ b/tomcat/server.xml @@ -124,6 +124,7 @@ --> + + compression="force" + compressionMinSize="2048" + compressableMimeType="text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml,application/octet-stream,application/vnd.opendap.dap4.dataset-metadata+xml,application/vnd.opendap.dap4.data,application/vnd.opendap.dap4.error+xml,application/prs.coverage+json,application/rdf+xml,application/x-netcdf,image/tiff;application=geotiff" + />