From b519098b63b397f0ee30f43ae5e85b7dc87db549 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Wed, 10 Jan 2024 09:59:53 -0500 Subject: [PATCH 01/37] HubSpot Edit: Add HubSpot build setup --- .blazar.yaml | 25 + .build-jdk17 | 0 build-scripts/prepare_environment.sh | 97 ++++ hbase-rpm/.blazar.yaml | 30 ++ hbase-rpm/build.sh | 51 ++ hbase-rpm/hbase.spec | 131 +++++ hbase-rpm/sources/hbase.1 | 88 ++++ hbase-rpm/sources/install_hbase.sh | 180 +++++++ hubspot-client-bundles/.blazar.yaml | 24 + hubspot-client-bundles/.build-jdk17 | 0 hubspot-client-bundles/README.md | 59 +++ .../hbase-backup-restore-bundle/.blazar.yaml | 26 + .../hbase-backup-restore-bundle/.build-jdk17 | 0 .../hbase-backup-restore-bundle/pom.xml | 119 +++++ .../hbase-client-bundle/.blazar.yaml | 25 + .../hbase-client-bundle/.build-jdk17 | 0 .../hbase-client-bundle/pom.xml | 127 +++++ .../hbase-mapreduce-bundle/.blazar.yaml | 25 + .../hbase-mapreduce-bundle/.build-jdk17 | 0 .../hbase-mapreduce-bundle/pom.xml | 251 ++++++++++ .../hbase-server-it-bundle/.blazar.yaml | 26 + .../hbase-server-it-bundle/.build-jdk17 | 0 .../hbase-server-it-bundle/pom.xml | 168 +++++++ hubspot-client-bundles/pom.xml | 458 ++++++++++++++++++ 24 files changed, 1910 insertions(+) create mode 100644 .blazar.yaml create mode 100644 .build-jdk17 create mode 100755 build-scripts/prepare_environment.sh create mode 100644 hbase-rpm/.blazar.yaml create mode 100755 hbase-rpm/build.sh create mode 100644 hbase-rpm/hbase.spec create mode 100644 hbase-rpm/sources/hbase.1 create mode 100755 hbase-rpm/sources/install_hbase.sh create mode 100644 hubspot-client-bundles/.blazar.yaml create mode 100644 hubspot-client-bundles/.build-jdk17 create mode 100644 hubspot-client-bundles/README.md create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml create mode 100644 hubspot-client-bundles/hbase-client-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-client-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-client-bundle/pom.xml create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/pom.xml create mode 100644 hubspot-client-bundles/pom.xml diff --git a/.blazar.yaml b/.blazar.yaml new file mode 100644 index 000000000000..e034ada7508d --- /dev/null +++ b/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java-single-module + +env: + MAVEN_PHASE: "package assembly:single deploy" + HADOOP_DEP_VERSION: "3.3.6-hubspot-SNAPSHOT" + MAVEN_BUILD_ARGS: "-Phadoop-3.0 -Dhadoop.profile=3.0 -Dhadoop-three.version=$HADOOP_DEP_VERSION -Dgpg.skip=true -DskipTests -DdeployAtEnd -pl hbase-assembly -am -T1C" + + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +provides: + - hbase diff --git a/.build-jdk17 b/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build-scripts/prepare_environment.sh b/build-scripts/prepare_environment.sh new file mode 100755 index 000000000000..65842dcd4d17 --- /dev/null +++ b/build-scripts/prepare_environment.sh @@ -0,0 +1,97 @@ +# +# Generates the appropriate environment vars so that we: +# - build against the right version of hadoop, and properly set up maven +# - generate the correct maven version based on the branches +# - upload RPMs with the correct release based on the branch, and to the right yum repo +# +# Since we need to distribute .blazar.yaml to all sub-modules of the project, we define our constants once +# in this script which can be re-used by every .blazar.yaml. +# +set -ex +printenv + +# We base the expected main branch and resulting maven version for clients on the hbase minor version +# The reason for this is hbase re-branches for each minor release (2.4, 2.5, 2.6, etc). At each re-branch +# the histories diverge. So we'll need to create our own fork of each new minor release branch. +# The convention is a fork named "hubspot-$minorVersion", and the maven coordinates "$minorVersion-hubspot-SNAPSHOT" +MINOR_VERSION="2.6" +MAIN_BRANCH="hubspot-${MINOR_VERSION}" + +# +# Validate inputs from blazar +# + +if [ -z "$WORKSPACE" ]; then + echo "Missing env var \$WORKSPACE" + exit 1 +fi +if [ -z "$GIT_BRANCH" ]; then + echo "Missing env var \$GIT_BRANCH" + exit 1 +fi +if [ -z "$BUILD_COMMAND_RC_FILE" ]; then + echo "Missing env var \$BUILD_COMMAND_RC_FILE" + exit 1 +fi + +# +# Extract current hbase version from root pom.xml +# + +# the pom.xml has an invalid xml namespace, so just remove that so xmllint can parse it. +cat $WORKSPACE/pom.xml | sed '2 s/xmlns=".*"//g' > pom.xml.tmp +HBASE_VERSION=$(echo "cat /project/properties/revision/text()" | xmllint --nocdata --shell pom.xml.tmp | sed '1d;$d') +rm pom.xml.tmp + +# sanity check that we've got some that looks right. it wouldn't be the end of the world if we got it wrong, but +# will help avoid confusion. +if [[ ! "$HBASE_VERSION" =~ 2\.[0-9]+\.[0-9]+ ]]; then + echo "Unexpected HBASE_Version extracted from pom.xml. Got $HBASE_VERSION but expected a string like '2.4.3', with 3 numbers separated by decimals, the first number being 2." + exit 1 +fi + +# +# Generate branch-specific env vars +# We are going to generate the maven version and the RPM release here: +# - For the maven version, we need to special case our main branch +# - For RPM, we want our final version to be: +# main branch: {hbase_version}-hs.{build_number}.el6 +# other branches: {hbase_version}-hs~{branch_name}.{build_number}.el6, where branch_name substitutes underscore for non-alpha-numeric characters +# + +echo "Git branch $GIT_BRANCH. Detecting appropriate version override and RPM release." + +RELEASE="hs" + +if [[ "$GIT_BRANCH" = "$MAIN_BRANCH" ]]; then + SET_VERSION="${MINOR_VERSION}-hubspot-SNAPSHOT" + REPO_NAME="AnyLinuxVersion_hs-hbase" +elif [[ "$GIT_BRANCH" != "hubspot" ]]; then + SET_VERSION="${MINOR_VERSION}-${GIT_BRANCH}-SNAPSHOT" + RELEASE="${RELEASE}~${GIT_BRANCH//[^[:alnum:]]/_}" + REPO_NAME="AnyLinuxVersion_hs-hbase-develop" +else + echo "Invalid git branch $GIT_BRANCH" + exit 1 +fi + +RELEASE="${RELEASE}.${BUILD_NUMBER}" +FULL_BUILD_VERSION="${HBASE_VERSION}-${RELEASE}" + +# SET_VERSION is not the most intuitive name, but it's required for set-maven-versions script +write-build-env-var SET_VERSION "$SET_VERSION" +write-build-env-var HBASE_VERSION "$HBASE_VERSION" +write-build-env-var PKG_RELEASE "$RELEASE" +write-build-env-var FULL_BUILD_VERSION "$FULL_BUILD_VERSION" +write-build-env-var REPO_NAME "$REPO_NAME" +# Adding this value as versioninfo.version ensures we have the same value as would normally +# show up in a non-hubspot hbase build. Otherwise due to set-maven-versions we'd end up +# with 2.6-hubspot-SNAPSHOT which is not very useful as a point of reference. +# Another option would be to pass in our FULL_BUILD_VERSION but that might cause some funniness +# with the expectations in VersionInfo.compareVersion(). +write-build-env-var MAVEN_BUILD_ARGS "$MAVEN_BUILD_ARGS -Dversioninfo.version=$HBASE_VERSION" + +echo "Building HBase version $HBASE_VERSION" +echo "Will deploy to nexus with version $SET_VERSION" +echo "Will create rpm with version $FULL_BUILD_VERSION" +echo "Will run maven with extra args $MAVEN_BUILD_ARGS" diff --git a/hbase-rpm/.blazar.yaml b/hbase-rpm/.blazar.yaml new file mode 100644 index 000000000000..a1bfcb2ae17b --- /dev/null +++ b/hbase-rpm/.blazar.yaml @@ -0,0 +1,30 @@ +buildpack: + name: Buildpack-RPMs + +env: + RPM_BUILD_COMMAND: ./build.sh + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +enableBuildTargets: + - almalinux9_amd64 + +depends: + - hbase + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +stepActivation: + uploadRpms: + branchRegexes: ['.*'] diff --git a/hbase-rpm/build.sh b/hbase-rpm/build.sh new file mode 100755 index 000000000000..b527ca732913 --- /dev/null +++ b/hbase-rpm/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e +set -x + +ROOT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +for iv in HBASE_VERSION SET_VERSION PKG_RELEASE; do + if [[ "X${!iv}" = "X" ]]; then + echo "Must specifiy $iv" + exit 1 + fi +done + +# Setup build dir +BUILD_DIR="${ROOT_DIR}/build" +rm -rf $BUILD_DIR +mkdir -p ${BUILD_DIR}/{SOURCES,SPECS,RPMS} +cp -a $ROOT_DIR/sources/* ${BUILD_DIR}/SOURCES/ +cp $ROOT_DIR/hbase.spec ${BUILD_DIR}/SPECS/ + +# Download bin tar built by hbase-assembly +SOURCES_DIR=$BUILD_DIR/SOURCES +mvn dependency:copy \ + -Dartifact=org.apache.hbase:hbase-assembly:${SET_VERSION}:tar.gz:bin \ + -DoutputDirectory=$SOURCES_DIR \ + -DlocalRepositoryDirectory=$SOURCES_DIR \ + -Dtransitive=false +INPUT_TAR=`ls -d $SOURCES_DIR/hbase-assembly-*.tar.gz` + +if [[ $HBASE_VERSION == *"-SNAPSHOT" ]]; then + # unreleased verion. do i want to denote that in the rpm release somehow? + # it can't be in the version, so strip here + HBASE_VERSION=${HBASE_VERSION//-SNAPSHOT/} +fi + +rpmbuild \ + --define "_topdir $BUILD_DIR" \ + --define "input_tar $INPUT_TAR" \ + --define "hbase_version ${HBASE_VERSION}" \ + --define "maven_version ${SET_VERSION}" \ + --define "release ${PKG_RELEASE}%{?dist}" \ + -bb \ + $BUILD_DIR/SPECS/hbase.spec + +if [[ -d $RPMS_OUTPUT_DIR ]]; then + mkdir -p $RPMS_OUTPUT_DIR + + # Move rpms to output dir for upload + + find ${BUILD_DIR}/RPMS -name "*.rpm" -exec mv {} $RPMS_OUTPUT_DIR/ \; +fi diff --git a/hbase-rpm/hbase.spec b/hbase-rpm/hbase.spec new file mode 100644 index 000000000000..107c92636f06 --- /dev/null +++ b/hbase-rpm/hbase.spec @@ -0,0 +1,131 @@ +# taken from hbase.spec in https://github.com/apache/bigtop/ +# greatly modified to simplify and fix dependencies to work in the hubspot environment + +%define hadoop_major_version 3.2 +%define hbase_major_version 2.4 +%define etc_hbase_conf %{_sysconfdir}/hbase/conf +%define etc_hbase_conf_dist %{etc_hbase_conf}.dist +%define hbase_home /usr/lib/hbase +%define bin_hbase %{hbase_home}/bin +%define lib_hbase %{hbase_home}/lib +%define conf_hbase %{hbase_home}/conf +%define logs_hbase %{hbase_home}/logs +%define pids_hbase %{hbase_home}/pids +%define man_dir %{_mandir} +%define hbase_username hbase +%define hadoop_home /usr/lib/hadoop +%define zookeeper_home /usr/lib/zookeeper + +# FIXME: brp-repack-jars uses unzip to expand jar files +# Unfortunately guice-2.0.jar pulled by ivy contains some files and directories without any read permission +# and make whole process to fail. +# So for now brp-repack-jars is being deactivated until this is fixed. +# See BIGTOP-294 +%define __os_install_post \ + %{_rpmconfigdir}/brp-compress ; \ + %{_rpmconfigdir}/brp-strip-static-archive %{__strip} ; \ + %{_rpmconfigdir}/brp-strip-comment-note %{__strip} %{__objdump} ; \ + /usr/lib/rpm/brp-python-bytecompile ; \ + %{nil} + +%define doc_hbase %{_docdir}/hbase-%{hbase_version} +%global initd_dir %{_sysconfdir}/rc.d/init.d +%define alternatives_cmd alternatives + +# Disable debuginfo package +%define debug_package %{nil} + +# HubSpot: use zstd because it decompresses much faster +%define _binary_payload w19.zstdio +%define _source_payload w19.zstdio + +Name: hbase +Version: %{hbase_version} +Release: %{release} +BuildArch: noarch +Summary: HBase is the Hadoop database. Use it when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. +URL: http://hbase.apache.org/ +Group: Systems/Daemons +Buildroot: %{_topdir}/INSTALL/hbase-%{maven_version} +License: ASL 2.0 +Source0: %{input_tar} +Source1: install_hbase.sh + +Requires: coreutils, /usr/sbin/useradd, /sbin/chkconfig, /sbin/service +Requires: hadoop >= %{hadoop_major_version} + +AutoReq: no + +%description +HBase is an open-source, distributed, column-oriented store modeled after Google' Bigtable: A Distributed Storage System for Structured Data by Chang et al. Just as Bigtable leverages the distributed data storage provided by the Google File System, HBase provides Bigtable-like capabilities on top of Hadoop. HBase includes: + + * Convenient base classes for backing Hadoop MapReduce jobs with HBase tables + * Query predicate push down via server side scan and get filters + * Optimizations for real time queries + * A high performance Thrift gateway + * A REST-ful Web service gateway that supports XML, Protobuf, and binary data encoding options + * Cascading source and sink modules + * Extensible jruby-based (JIRB) shell + * Support for exporting metrics via the Hadoop metrics subsystem to files or Ganglia; or via JMX + +%prep +%setup -n hbase-%{maven_version} + +%install +%__rm -rf $RPM_BUILD_ROOT +bash %{SOURCE1} \ + --input-tar=%{SOURCE0} \ + --doc-dir=%{doc_hbase} \ + --conf-dir=%{etc_hbase_conf_dist} \ + --prefix=$RPM_BUILD_ROOT + +%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/hbase +ln -s %{_localstatedir}/log/hbase %{buildroot}/%{logs_hbase} + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/hbase +ln -s %{_localstatedir}/run/hbase %{buildroot}/%{pids_hbase} + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/lib/hbase + +%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin + +# Pull hadoop from its packages +rm -f $RPM_BUILD_ROOT/%{lib_hbase}/{hadoop,slf4j-log4j12-}*.jar + +ln -f -s %{hadoop_home}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-hdfs-client.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_hbase} + +%pre +getent group hbase 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hbase +getent passwd hbase 2>&1 > /dev/null || /usr/sbin/useradd -c "HBase" -s /sbin/nologin -g hbase -r -d /var/lib/hbase hbase 2> /dev/null || : + +%post +%{alternatives_cmd} --install %{etc_hbase_conf} %{name}-conf %{etc_hbase_conf_dist} 30 + +%files +%defattr(-,hbase,hbase) +%{logs_hbase} +%{pids_hbase} +%dir %{_localstatedir}/log/hbase +%dir %{_localstatedir}/run/hbase +%dir %{_localstatedir}/lib/hbase + +%defattr(-,root,root) +%{hbase_home} +%{hbase_home}/hbase-*.jar +/usr/bin/hbase +%config(noreplace) %{etc_hbase_conf_dist} + +# files from doc package +%defattr(-,root,root) +%doc %{doc_hbase}/ diff --git a/hbase-rpm/sources/hbase.1 b/hbase-rpm/sources/hbase.1 new file mode 100644 index 000000000000..349218fe1d87 --- /dev/null +++ b/hbase-rpm/sources/hbase.1 @@ -0,0 +1,88 @@ +.\" Licensed to the Apache Software Foundation (ASF) under one or more +.\" contributor license agreements. See the NOTICE file distributed with +.\" this work for additional information regarding copyright ownership. +.\" The ASF licenses this file to You under the Apache License, Version 2.0 +.\" (the "License"); you may not use this file except in compliance with +.\" the License. You may obtain a copy of the License at +.\" +.\" http://www.apache.org/licenses/LICENSE-2.0 +.\" +.\" Unless required by applicable law or agreed to in writing, software +.\" distributed under the License is distributed on an "AS IS" BASIS, +.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.\" See the License for the specific language governing permissions and +.\" limitations under the License. +.\" +.\" Process this file with +.\" groff -man -Tascii hbase.1 +.\" +.TH hbase 1 "October 2010 " Linux "User Manuals" + +.SH NAME +HBase \- HBase is the Hadoop database. + +.SH SYNOPSIS + +.B hbase +\fICOMMAND\fR + +.SH DESCRIPTION + +HBase is the Hadoop database. Use it when you need random, realtime +read/write access to your Big Data. This project's goal is the hosting +of very large tables -- billions of rows X millions of columns -- atop +clusters of commodity hardware. + +HBase is an open-source, distributed, versioned, column-oriented store +modeled after Google's Bigtable: A Distributed Storage System for +Structured Data by Chang et al. Just as Bigtable leverages the +distributed data storage provided by the Google File System, HBase +provides Bigtable-like capabilities on top of Hadoop. + +For more information about HBase, see http://hbase.apache.org. + +\fICOMMAND\fR may be one of the following: + shell run the HBase shell + shell-tests run the HBase shell tests + zkcli run the ZooKeeper shell + master run an HBase HMaster node + regionserver run an HBase HRegionServer node + zookeeper run a Zookeeper server + rest run an HBase REST server + thrift run an HBase Thrift server + avro run an HBase Avro server + migrate upgrade an hbase.rootdir + hbck run the hbase 'fsck' tool + or + CLASSNAME run the class named CLASSNAME + +Most commands print help when invoked w/o parameters or with --help. + +.SH ENVIRONMENT + +.IP JAVA_HOME +The java implementation to use. Overrides JAVA_HOME. + +.IP HBASE_CLASSPATH +Extra Java CLASSPATH entries. + +.IP HBASE_HEAPSIZE +The maximum amount of heap to use, in MB. Default is 1000. + +.IP HBASE_OPTS +Extra Java runtime options. + +.IP HBASE_CONF_DIR +Alternate conf dir. Default is ${HBASE_HOME}/conf. + +.IP HBASE_ROOT_LOGGER +The root appender. Default is INFO,console + +.IP HIVE_OPT +Extra Java runtime options. + +.IP HADOOP_HOME +Optionally, the Hadoop home to run with. + +.SH COPYRIGHT +Copyright (C) 2010 The Apache Software Foundation. All rights reserved. diff --git a/hbase-rpm/sources/install_hbase.sh b/hbase-rpm/sources/install_hbase.sh new file mode 100755 index 000000000000..95265d2100c8 --- /dev/null +++ b/hbase-rpm/sources/install_hbase.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() { + echo " +usage: $0 + Required not-so-options: + --mvn-target-dir=DIR path to the output of the mvn assembly + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/hbase] + --lib-dir=DIR path to install hbase home [/usr/lib/hbase] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + --examples-dir=DIR path to install examples [doc-dir/examples] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'examples-dir:' \ + -l 'conf-dir:' \ + -l 'input-tar:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --input-tar) + INPUT_TAR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --conf-dir) + CONF_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX INPUT_TAR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/hbase} +LIB_DIR=${LIB_DIR:-/usr/lib/hbase} + +BIN_DIR=${BIN_DIR:-/usr/lib/hbase/bin} +ETC_DIR=${ETC_DIR:-/etc/hbase} +CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist} +THRIFT_DIR=${THRIFT_DIR:-${LIB_DIR}/include/thrift} + +EXTRACT_DIR=extracted +rm -rf $EXTRACT_DIR +mkdir $EXTRACT_DIR + +version_part=$SET_VERSION +if [ -z "$version_part" ]; then + version_part=$HBASE_VERSION +fi + +tar -C $EXTRACT_DIR --strip-components=1 -xzf $INPUT_TAR + +# we do not need the shaded clients in our rpm. they bloat the size and cause classpath issues for hbck2. +rm -rf $EXTRACT_DIR/lib/shaded-clients + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$DOC_DIR +install -d -m 0755 $PREFIX/$BIN_DIR +install -d -m 0755 $PREFIX/$ETC_DIR +install -d -m 0755 $PREFIX/$MAN_DIR +install -d -m 0755 $PREFIX/$THRIFT_DIR + +cp -ra $EXTRACT_DIR/lib/* ${PREFIX}/${LIB_DIR}/lib/ +cp $EXTRACT_DIR/lib/hbase*.jar $PREFIX/$LIB_DIR + +# We do not currently run "mvn site", so do not have a docs dir. +# Only copy contents if dir exists +if [ -n "$(ls -A $EXTRACT_DIR/docs 2>/dev/null)" ]; then + cp -a $EXTRACT_DIR/docs/* $PREFIX/$DOC_DIR + cp $EXTRACT_DIR/*.txt $PREFIX/$DOC_DIR/ +else + echo "Doc generation is currently disabled in our RPM build. If this is an issue, it should be possible to enable them with some work. See https://git.hubteam.com/HubSpot/apache-hbase/blob/hubspot-2/rpm/sources/do-component-build#L17-L24 for details." > $PREFIX/$DOC_DIR/README.txt +fi + +cp -a $EXTRACT_DIR/conf $PREFIX/$CONF_DIR +cp -a $EXTRACT_DIR/bin/* $PREFIX/$BIN_DIR + +# Purge scripts that don't work with packages +for file in rolling-restart.sh graceful_stop.sh local-regionservers.sh \ + master-backup.sh regionservers.sh zookeepers.sh hbase-daemons.sh \ + start-hbase.sh stop-hbase.sh local-master-backup.sh ; do + rm -f $PREFIX/$BIN_DIR/$file +done + + +ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/conf + +# Make a symlink of hbase.jar to hbase-version.jar +pushd `pwd` +cd $PREFIX/$LIB_DIR +for i in `ls hbase*jar | grep -v tests.jar` +do + ln -s $i `echo $i | sed -n 's/\(.*\)\(-[0-9].*\)\(.jar\)/\1\3/p'` +done +popd + +wrapper=$PREFIX/usr/bin/hbase +mkdir -p `dirname $wrapper` +cat > $wrapper < dependencies.sorted` to get a file that can be compared with another such-processed file +4. Make the change you want in the bundle, then `mvn clean install` +5. Re-run steps 2 and 3, outputting to a new file +6. Run `comm -13 first second` to see what might be newly added after your change, or `comm -23` to see what might have been removed +7. If trying to track a specific dependency from the list, go back here and run `mvn dependency:tree -Dincludes=`. This might show you what dependency you need to add an exclusion to + +This ends up being pretty iterative and trial/error, but can eventually get to a jar which has what you want (and doesn't what you don't). diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml new file mode 100644 index 000000000000..9399e5dc0aa4 --- /dev/null +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles + - hbase-client-bundle + - hbase-mapreduce-bundle +provides: + - hbase-backup-restore-bundle diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-backup-restore-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml b/hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml new file mode 100644 index 000000000000..9707d9d8118d --- /dev/null +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-backup-restore-bundle + + + + com.hubspot.hbase + hbase-client-bundle + + + + commons-io + commons-io + + + + + com.hubspot.hbase + hbase-mapreduce-bundle + + + + commons-io + commons-io + + + + + org.apache.hbase + hbase-backup + + + + commons-io + commons-io + + + + org.apache.hbase + * + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + org.apache.hbase:* + + io.opentelemetry:opentelemetry-api + io.opentelemetry:opentelemetry-context + com.google.protobuf:protobuf-java + io.dropwizard.metrics:metrics-core + + + + + org.apache.kerby:* + + krb5-template.conf + krb5_udp-template.conf + ccache.txt + keytab.txt + + + + + + + + + + diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml new file mode 100644 index 000000000000..300be28892e8 --- /dev/null +++ b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles +provides: + - hbase-client-bundle + diff --git a/hubspot-client-bundles/hbase-client-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-client-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-client-bundle/pom.xml b/hubspot-client-bundles/hbase-client-bundle/pom.xml new file mode 100644 index 000000000000..24ce44daf93a --- /dev/null +++ b/hubspot-client-bundles/hbase-client-bundle/pom.xml @@ -0,0 +1,127 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-client-bundle + + + + org.apache.hbase + hbase-openssl + + + + org.apache.hbase + hbase-client + + + + org.apache.hbase + hbase-hadoop-compat + + + org.apache.hbase + hbase-hadoop2-compat + + + + commons-logging + commons-logging + + + org.jruby.joni + joni + + + org.jruby.jcodings + jcodings + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-endpoint + + + * + * + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-logging + org.apache.hbase:hbase-protocol + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-openssl + + org.apache.hbase.thirdparty:* + + org.apache.hbase:hbase-endpoint + + + + com.google.protobuf:protobuf-java + + io.dropwizard.metrics:metrics-core + + commons-io:commons-io + + + + + org.apache.hbase:hbase-endpoint + + org/apache/hadoop/hbase/client/coprocessor/** + org/apache/hadoop/hbase/protobuf/generated/** + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml new file mode 100644 index 000000000000..5c020e374927 --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles + - hbase-client-bundle +provides: + - hbase-mapreduce-bundle diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-mapreduce-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml b/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml new file mode 100644 index 000000000000..233e33750fe1 --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml @@ -0,0 +1,251 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-mapreduce-bundle + + + + + com.hubspot.hbase + hbase-client-bundle + + + + commons-io + commons-io + + + + + + org.apache.hbase + hbase-mapreduce + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-protocol + + + org.apache.hbase + hbase-protocol-shaded + + + org.apache.hbase + hbase-logging + + + com.google.protobuf + protobuf-java + + + org.apache.hbase.thirdparty + hbase-shaded-gson + + + org.apache.hbase.thirdparty + hbase-shaded-protobuf + + + org.apache.hbase.thirdparty + hbase-unsafe + + + org.apache.hbase.thirdparty + hbase-shaded-miscellaneous + + + org.apache.hbase.thirdparty + hbase-shaded-netty + + + + commons-logging + commons-logging + + + + commons-io + commons-io + + + com.sun.jersey + * + + + tomcat + jasper-runtime + + + org.mortbay.jetty + * + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-server + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-protocol + + + org.apache.hbase + hbase-protocol-shaded + + + org.apache.hbase + hbase-logging + + + com.google.protobuf + protobuf-java + + + org.apache.hbase.thirdparty + hbase-shaded-gson + + + org.apache.hbase.thirdparty + hbase-shaded-protobuf + + + org.apache.hbase.thirdparty + hbase-unsafe + + + org.apache.hbase.thirdparty + hbase-shaded-miscellaneous + + + org.apache.hbase.thirdparty + hbase-shaded-netty + + + + + commons-logging + commons-logging + + + + commons-io + commons-io + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + org.glassfish.hk2.external + jakarta.inject + + + jakarta.ws.rs + jakarta.ws.rs-api + + + + + org.apache.hbase + hbase-compression-zstd + + + org.apache.hbase + * + + + + commons-io + commons-io + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + + org.apache.hbase:* + + org.apache.hbase.thirdparty:* + + + + + + + + + diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml new file mode 100644 index 000000000000..26db8c8066b3 --- /dev/null +++ b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + YUM_REPO_UPLOAD_OVERRIDE_CENTOS_8: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + REPO_NAME: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hbase +provides: + - hbase-server-it-bundle + diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-server-it-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-server-it-bundle/pom.xml b/hubspot-client-bundles/hbase-server-it-bundle/pom.xml new file mode 100644 index 000000000000..fa617258f82d --- /dev/null +++ b/hubspot-client-bundles/hbase-server-it-bundle/pom.xml @@ -0,0 +1,168 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-server-it-bundle + + + + org.apache.hbase + hbase-it + test-jar + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-server + test-jar + ${project.version} + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-testing-util + ${project.version} + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + org.apache.hbase:* + + junit:junit + + commons-io:commons-io + + org.apache.hbase.thirdparty:* + com.google.protobuf:protobuf-java + + io.opentelemetry:opentelemetry-api + io.opentelemetry:opentelemetry-context + com.google.protobuf:protobuf-java + io.dropwizard.metrics:metrics-core + + + + + org.apache.kerby:* + + krb5-template.conf + krb5_udp-template.conf + ccache.txt + keytab.txt + + + + + + + + + + diff --git a/hubspot-client-bundles/pom.xml b/hubspot-client-bundles/pom.xml new file mode 100644 index 000000000000..0105ffd28c43 --- /dev/null +++ b/hubspot-client-bundles/pom.xml @@ -0,0 +1,458 @@ + + + 4.0.0 + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + pom + Bundled versions of the hbase client + + + hbase-client-bundle + hbase-mapreduce-bundle + hbase-backup-restore-bundle + hbase-server-it-bundle + + + + org.apache.hadoop.hbase.shaded + + 3.6.3-shaded-SNAPSHOT + + 2.6-hubspot-SNAPSHOT + + + + + + + com.hubspot.hbase + hbase-client-bundle + ${project.version} + + + org.apache.hbase + hbase-client + + + + + com.hubspot.hbase + hbase-mapreduce-bundle + ${project.version} + + + com.hubspot.hbase + hbase-backup-restore-bundle + ${project.version} + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + org.apache.hbase + hbase-openssl + ${project.version} + + + org.apache.hbase + hbase-compression-zstd + ${project.version} + + + org.apache.hbase + hbase-client + ${project.version} + + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-server + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-mapreduce + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-endpoint + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-backup + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-hadoop2-compat + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-it + test-jar + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + create-bundle-with-relocations + + shade + + package + + + true + true + true + true + true + + + com.google.protobuf + ${shade.prefix}.com.google.protobuf + + + com.codahale.metrics + ${shade.prefix}.com.codahale.metrics + + + org.apache.commons.io + ${shade.prefix}.org.apache.commons.io + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + From e5916dc018dc7e62df3ea422c362f6f19d567a15 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Mon, 12 Feb 2024 12:01:54 -0500 Subject: [PATCH 02/37] HubSpot Edit: HBASE-28365: ChaosMonkey batch suspend/resume action assume shell implementation (not yet written upstream) --- .../chaos/actions/RollingBatchSuspendResumeRsAction.java | 4 ++++ .../hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java index 559dec829ee3..78c78c531060 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java @@ -97,6 +97,8 @@ public void perform() throws Exception { suspendRs(server); } catch (Shell.ExitCodeException e) { LOG.warn("Problem suspending but presume successful; code={}", e.getExitCode(), e); + } catch (Exception e) { + LOG.warn("Problem suspending but presume successful", e); } suspendedServers.add(server); break; @@ -106,6 +108,8 @@ public void perform() throws Exception { resumeRs(server); } catch (Shell.ExitCodeException e) { LOG.info("Problem resuming, will retry; code={}", e.getExitCode(), e); + } catch (Exception e) { + LOG.warn("Problem resulting, will retry", e); } break; } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index fb8ab209c3a1..756f0d3846a6 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -86,7 +86,6 @@ private static ExecutorService buildMonkeyThreadPool(final int size) { return Executors.newFixedThreadPool(size, new ThreadFactoryBuilder().setDaemon(false) .setNameFormat("ChaosMonkey-%d").setUncaughtExceptionHandler((t, e) -> { LOG.error("Uncaught exception in thread {}", t.getName(), e); - throw new RuntimeException(e); }).build()); } From 991513e5d0071d7bccbb7cdcb160c4f4ab281c6a Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Sat, 17 Feb 2024 11:40:11 -0500 Subject: [PATCH 03/37] HubSpot Edit: Add retries to verify step of ITBLL --- .../test/IntegrationTestBigLinkedList.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index c1854d87c199..2c4dd96eedab 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -1532,9 +1532,20 @@ protected void runVerify(String outputDir, int numReducers, long expectedNumNode Verify verify = new Verify(); verify.setConf(getConf()); - int retCode = verify.run(iterationOutput, numReducers); - if (retCode > 0) { - throw new RuntimeException("Verify.run failed with return code: " + retCode); + + int retries = getConf().getInt("hbase.itbll.verify.retries", 1); + + while (true) { + int retCode = verify.run(iterationOutput, numReducers); + if (retCode > 0) { + if (retries-- > 0) { + LOG.warn("Verify.run failed with return code: {}. Will retry", retries); + } else { + throw new RuntimeException("Verify.run failed with return code: " + retCode); + } + } else { + break; + } } if (!verify.verify(expectedNumNodes)) { From 0e45bd542ec59ec79014674a37e5f5a5df486c13 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 2 Feb 2024 09:17:58 -0500 Subject: [PATCH 04/37] HubSpot Edit: Add an hbase-site.xml to our bundles that configures ZStdCodec Co-authored-by: Charles Connell --- .../src/main/resources/hbase-site.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml b/hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml new file mode 100644 index 000000000000..629c6f84f30e --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml @@ -0,0 +1,12 @@ + + + + + + hbase.io.compress.zstd.codec + org.apache.hadoop.hbase.io.compress.zstd.ZstdCodec + + From 4af1ad32d3a176be8e13d5ef1dca5b8d4da54c69 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Fri, 19 Apr 2024 10:28:56 -0400 Subject: [PATCH 05/37] HubSpot Edit: Add hdfs stats for local and remote rack bytes read --- .../MetricsRegionServerSource.java | 8 ++++++ .../MetricsRegionServerWrapper.java | 4 +++ .../MetricsRegionServerSourceImpl.java | 4 +++ .../MetricsRegionServerWrapperImpl.java | 25 +++++++++++++++++++ .../MetricsRegionServerWrapperStub.java | 10 ++++++++ 5 files changed, 51 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index c68809a1fddb..c23c222edc54 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -533,6 +533,14 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String ZEROCOPY_BYTES_READ = "zeroCopyBytesRead"; String ZEROCOPY_BYTES_READ_DESC = "The number of bytes read through HDFS zero copy"; + String LOCAL_RACK_BYTES_READ = "localRackBytesRead"; + String LOCAL_RACK_BYTES_READ_DESC = + "The number of bytes read from the same rack of the RegionServer, but not the local HDFS DataNode"; + + String REMOTE_RACK_BYTES_READ = "remoteRackBytesRead"; + String REMOTE_RACK_BYTES_READ_DESC = + "The number of bytes read from a different rack from that of the RegionServer"; + String BLOCKED_REQUESTS_COUNT = "blockedRequestCount"; String BLOCKED_REQUESTS_COUNT_DESC = "The number of blocked requests because of memstore size is " + "larger than blockingMemStoreSize"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 10e71d091f59..67d31ffe64c4 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -544,6 +544,10 @@ public interface MetricsRegionServerWrapper { /** Returns Number of bytes read from the local HDFS DataNode. */ long getLocalBytesRead(); + long getLocalRackBytesRead(); + + long getRemoteRackBytesRead(); + /** Returns Number of bytes read locally through HDFS short circuit. */ long getShortCircuitBytesRead(); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index e0429cfb55d1..b42a02d0e659 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -560,6 +560,10 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()) .addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead()) .addGauge(Interns.info(LOCAL_BYTES_READ, LOCAL_BYTES_READ_DESC), rsWrap.getLocalBytesRead()) + .addGauge(Interns.info(LOCAL_RACK_BYTES_READ, LOCAL_RACK_BYTES_READ_DESC), + rsWrap.getLocalRackBytesRead()) + .addGauge(Interns.info(REMOTE_RACK_BYTES_READ, REMOTE_RACK_BYTES_READ_DESC), + rsWrap.getRemoteRackBytesRead()) .addGauge(Interns.info(SHORTCIRCUIT_BYTES_READ, SHORTCIRCUIT_BYTES_READ_DESC), rsWrap.getShortCircuitBytesRead()) .addGauge(Interns.info(ZEROCOPY_BYTES_READ, ZEROCOPY_BYTES_READ_DESC), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 2bd396242a17..a256e8827a39 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -29,6 +29,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.GlobalStorageStatistics; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; @@ -1052,6 +1054,29 @@ public long getLocalBytesRead() { return FSDataInputStreamWrapper.getLocalBytesRead(); } + @Override + public long getLocalRackBytesRead() { + return getGlobalStorageStatistic("bytesReadDistanceOfOneOrTwo"); + } + + @Override + public long getRemoteRackBytesRead() { + return getGlobalStorageStatistic("bytesReadDistanceOfThreeOrFour") + + getGlobalStorageStatistic("bytesReadDistanceOfFiveOrLarger"); + } + + private static long getGlobalStorageStatistic(String name) { + StorageStatistics stats = GlobalStorageStatistics.INSTANCE.get("hdfs"); + if (stats == null) { + return 0; + } + Long val = stats.getLong(name); + if (val == null) { + return 0; + } + return val; + } + @Override public long getShortCircuitBytesRead() { return FSDataInputStreamWrapper.getShortCircuitBytesRead(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 0e77ae89fef2..84654784c58d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -537,6 +537,16 @@ public long getLocalBytesRead() { return 0; } + @Override + public long getLocalRackBytesRead() { + return 0; + } + + @Override + public long getRemoteRackBytesRead() { + return 0; + } + @Override public long getShortCircuitBytesRead() { return 0; From 2709ece46ea7383359ddc0f9407a5ab1c413c625 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Thu, 18 Apr 2024 08:54:07 -0400 Subject: [PATCH 06/37] HubSpot Edit: Basic healthcheck servlets --- .../apache/hadoop/hbase/master/HMaster.java | 6 + .../master/http/MasterHealthServlet.java | 48 ++++++++ .../hbase/monitoring/HealthCheckServlet.java | 103 ++++++++++++++++++ .../hbase/regionserver/HRegionServer.java | 12 +- .../regionserver/http/RSHealthServlet.java | 95 ++++++++++++++++ 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 9cafbb7cbf9e..21da55d7757b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -137,6 +137,7 @@ import org.apache.hadoop.hbase.master.cleaner.SnapshotCleanerChore; import org.apache.hadoop.hbase.master.hbck.HbckChore; import org.apache.hadoop.hbase.master.http.MasterDumpServlet; +import org.apache.hadoop.hbase.master.http.MasterHealthServlet; import org.apache.hadoop.hbase.master.http.MasterRedirectServlet; import org.apache.hadoop.hbase.master.http.MasterStatusServlet; import org.apache.hadoop.hbase.master.http.api_v1.ResourceConfigFactory; @@ -775,6 +776,11 @@ protected Class getDumpServlet() { return MasterDumpServlet.class; } + @Override + protected Class getHealthServlet() { + return MasterHealthServlet.class; + } + @Override public MetricsMaster getMasterMetrics() { return metricsMaster; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java new file mode 100644 index 000000000000..99f2f08ac8bd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.http; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Optional; +import javax.servlet.http.HttpServletRequest; +import org.apache.hadoop.hbase.ClusterMetrics; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.monitoring.HealthCheckServlet; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class MasterHealthServlet extends HealthCheckServlet { + + public MasterHealthServlet() { + super(HMaster.MASTER); + } + + @Override + protected Optional check(HMaster master, HttpServletRequest req, Connection conn) + throws IOException { + + if (master.isActiveMaster() && master.isOnline()) { + // this will fail if there is a problem with the active master + conn.getAdmin().getClusterMetrics(EnumSet.of(ClusterMetrics.Option.CLUSTER_ID)); + } + + return Optional.empty(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java new file mode 100644 index 000000000000..8d09089b0c64 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import java.io.IOException; +import java.util.Optional; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RpcConnectionRegistry; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public abstract class HealthCheckServlet extends HttpServlet { + + private static final String CLIENT_RPC_TIMEOUT = "healthcheck.hbase.client.rpc.timeout"; + private static final int CLIENT_RPC_TIMEOUT_DEFAULT = 5000; + private static final String CLIENT_RETRIES = "healthcheck.hbase.client.retries"; + private static final int CLIENT_RETRIES_DEFAULT = 2; + private static final String CLIENT_OPERATION_TIMEOUT = + "healthcheck.hbase.client.operation.timeout"; + private static final int CLIENT_OPERATION_TIMEOUT_DEFAULT = 15000; + + private final String serverLookupKey; + + public HealthCheckServlet(String serverLookupKey) { + this.serverLookupKey = serverLookupKey; + } + + @SuppressWarnings("unchecked") + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + T server = (T) getServletContext().getAttribute(serverLookupKey); + try { + check(server, req); + Optional message = check(server, req); + resp.setStatus(200); + resp.getWriter().write(message.orElse("ok")); + } catch (Exception e) { + resp.setStatus(500); + resp.getWriter().write(e.toString()); + } finally { + resp.getWriter().close(); + } + } + + private Optional check(T server, HttpServletRequest req) throws IOException { + if (server == null) { + throw new IOException("Unable to get access to " + serverLookupKey); + } + if (server.isAborted() || server.isStopped() || server.isStopping() || server.isKilled()) { + throw new IOException("The " + serverLookupKey + " is stopping!"); + } + if (!server.getRpcServer().isStarted()) { + throw new IOException("The " + serverLookupKey + "'s RpcServer is not started"); + } + + Configuration conf = new Configuration(server.getConfiguration()); + conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, + RpcConnectionRegistry.class.getName()); + conf.set(RpcConnectionRegistry.BOOTSTRAP_NODES, server.getServerName().getAddress().toString()); + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, + conf.getInt(CLIENT_RPC_TIMEOUT, CLIENT_RPC_TIMEOUT_DEFAULT)); + conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + conf.getInt(CLIENT_RETRIES, CLIENT_RETRIES_DEFAULT)); + conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, + conf.getInt(CLIENT_OPERATION_TIMEOUT, CLIENT_OPERATION_TIMEOUT_DEFAULT)); + + try (Connection conn = ConnectionFactory.createConnection(conf)) { + // this will fail if the server is not accepting requests + if (conn.getClusterId() == null) { + throw new IOException("Could not retrieve clusterId from self via rpc"); + } + + return check(server, req, conn); + } + } + + protected abstract Optional check(T server, HttpServletRequest req, Connection conn) + throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 351b4fef191e..89f528af7573 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -169,6 +169,7 @@ import org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler; import org.apache.hadoop.hbase.regionserver.handler.RegionReplicaFlushHandler; import org.apache.hadoop.hbase.regionserver.http.RSDumpServlet; +import org.apache.hadoop.hbase.regionserver.http.RSHealthServlet; import org.apache.hadoop.hbase.regionserver.http.RSStatusServlet; import org.apache.hadoop.hbase.regionserver.throttle.FlushThroughputControllerFactory; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; @@ -383,7 +384,7 @@ public class HRegionServer extends Thread // A state before we go into stopped state. At this stage we're closing user // space regions. - private boolean stopping = false; + private volatile boolean stopping = false; private volatile boolean killed = false; private volatile boolean shutDown = false; @@ -864,6 +865,10 @@ protected Class getDumpServlet() { return RSDumpServlet.class; } + protected Class getHealthServlet() { + return RSHealthServlet.class; + } + /** * Used by {@link RSDumpServlet} to generate debugging information. */ @@ -2466,6 +2471,7 @@ private void putUpWebUI() throws IOException { try { this.infoServer = new InfoServer(getProcessName(), addr, port, false, this.conf); infoServer.addPrivilegedServlet("dump", "/dump", getDumpServlet()); + infoServer.addPrivilegedServlet("health", "/health", getHealthServlet()); configureInfoServer(); this.infoServer.start(); break; @@ -3193,6 +3199,10 @@ public boolean isStopping() { return this.stopping; } + public boolean isKilled() { + return this.killed; + } + @Override public Configuration getConfiguration() { return conf; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java new file mode 100644 index 000000000000..bc0f35193389 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.http; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import javax.servlet.http.HttpServletRequest; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.monitoring.HealthCheckServlet; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class RSHealthServlet extends HealthCheckServlet { + + private final Map regionUnavailableSince = new ConcurrentHashMap<>(); + + public RSHealthServlet() { + super(HRegionServer.REGIONSERVER); + } + + @Override + protected Optional check(HRegionServer regionServer, HttpServletRequest req, + Connection conn) throws IOException { + long maxUnavailableMillis = Optional.ofNullable(req.getParameter("maxUnavailableMillis")) + .filter(StringUtils::isNumeric).map(Long::parseLong).orElse(Long.MAX_VALUE); + + Instant oldestUnavailableSince = Instant.MAX; + String longestUnavailableRegion = null; + int unavailableCount = 0; + + synchronized (regionUnavailableSince) { + Set regionsPreviouslyUnavailable = new HashSet<>(regionUnavailableSince.keySet()); + + for (HRegion region : regionServer.getOnlineRegionsLocalContext()) { + regionsPreviouslyUnavailable.remove(region.getRegionInfo().getEncodedName()); + if (!region.isAvailable()) { + unavailableCount++; + Instant unavailableSince = regionUnavailableSince + .computeIfAbsent(region.getRegionInfo().getEncodedName(), k -> Instant.now()); + + if (unavailableSince.isBefore(oldestUnavailableSince)) { + oldestUnavailableSince = unavailableSince; + longestUnavailableRegion = region.getRegionInfo().getEncodedName(); + } + + } else { + regionUnavailableSince.remove(region.getRegionInfo().getEncodedName()); + } + } + + regionUnavailableSince.keySet().removeAll(regionsPreviouslyUnavailable); + } + + String message = "ok"; + + if (unavailableCount > 0) { + Duration longestUnavailableRegionTime = + Duration.between(oldestUnavailableSince, Instant.now()); + if (longestUnavailableRegionTime.toMillis() > maxUnavailableMillis) { + throw new IOException("Region " + longestUnavailableRegion + + " has been unavailable too long, since " + oldestUnavailableSince); + } + + message += " - unavailableRegions: " + unavailableCount + ", longestUnavailableDuration: " + + longestUnavailableRegionTime + ", longestUnavailableRegion: " + longestUnavailableRegion; + } + + return Optional.of(message); + + } +} From 2fe5815d344b31262a9f9c7df2466d41fee7c25d Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Thu, 7 Mar 2024 16:57:11 -0500 Subject: [PATCH 07/37] HubSpot Edit: More info when interrupted while waiting on actions --- .../hbase/client/AsyncRequestFutureImpl.java | 63 +++++++++++++++---- .../hbase/client/MultiServerCallable.java | 9 ++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index b34ef863d565..f3e9c0ed0178 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -34,6 +35,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.HConstants; @@ -52,6 +54,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Strings; + /** * The context, and return value, for a single submit/submitAll call. Note on how this class (one AP * submit) works. Initially, all requests are split into groups by server; request is sent to each @@ -195,7 +199,7 @@ public void run() { try { // setup the callable based on the actions, if we don't have one already from the request if (callable == null) { - callable = createCallable(server, tableName, multiAction); + callable = createCallable(server, tableName, multiAction, numAttempt); } RpcRetryingCaller caller = asyncProcess.createCaller(callable, rpcTimeout); @@ -387,10 +391,8 @@ public AsyncRequestFutureImpl(AsyncProcessTask task, List actions, long } else { this.replicaGetIndices = null; } - this.callsInProgress = !hasAnyReplicaGets - ? null - : Collections - .newSetFromMap(new ConcurrentHashMap()); + this.callsInProgress = + Collections.newSetFromMap(new ConcurrentHashMap()); this.asyncProcess = asyncProcess; this.errorsByServer = createServerErrorTracker(); this.errors = new BatchErrors(); @@ -540,7 +542,12 @@ private HRegionLocation getReplicaLocationOrFail(Action action) { private void manageLocationError(Action action, Exception ex) { String msg = - "Cannot get replica " + action.getReplicaId() + " location for " + action.getAction(); + "Cannot get replica " + action.getReplicaId() + " location for " + action.getAction() + ": "; + if (ex instanceof OperationTimeoutExceededException) { + msg += "Operation timeout exceeded."; + } else { + msg += ex == null ? "null cause" : ex.toString(); + } LOG.error(msg); if (ex == null) { ex = new IOException(msg); @@ -1247,20 +1254,31 @@ private String buildDetailedErrorMsg(String string, int index) { @Override public void waitUntilDone() throws InterruptedIOException { + long startTime = EnvironmentEdgeManager.currentTime(); try { if (this.operationTimeout > 0) { // the worker thread maybe over by some exception without decrement the actionsInProgress, // then the guarantee of operationTimeout will be broken, so we should set cutoff to avoid // stuck here forever - long cutoff = (EnvironmentEdgeManager.currentTime() + this.operationTimeout) * 1000L; + long cutoff = (startTime + this.operationTimeout) * 1000L; if (!waitUntilDone(cutoff)) { - throw new SocketTimeoutException("time out before the actionsInProgress changed to zero"); + String msg = "time out before the actionsInProgress changed to zero, with " + + actionsInProgress.get() + " remaining" + getServersInProgress(); + + throw new SocketTimeoutException(msg); } } else { waitUntilDone(Long.MAX_VALUE); } } catch (InterruptedException iex) { - throw new InterruptedIOException(iex.getMessage()); + long duration = EnvironmentEdgeManager.currentTime() - startTime; + String message = "Interrupted after waiting " + duration + "ms of " + operationTimeout + + "ms operation timeout, with " + actionsInProgress.get() + " remaining" + + getServersInProgress(); + if (!Strings.isNullOrEmpty(iex.getMessage())) { + message += ": " + iex.getMessage(); + } + throw new InterruptedIOException(message); } finally { if (callsInProgress != null) { for (CancellableRegionServerCallable clb : callsInProgress) { @@ -1270,6 +1288,29 @@ public void waitUntilDone() throws InterruptedIOException { } } + private String getServersInProgress() { + if (callsInProgress != null) { + Map serversInProgress = new HashMap<>(callsInProgress.size()); + for (CancellableRegionServerCallable callable : callsInProgress) { + if (callable instanceof MultiServerCallable) { + MultiServerCallable multiServerCallable = (MultiServerCallable) callable; + int numAttempt = multiServerCallable.getNumAttempt(); + serversInProgress.compute(multiServerCallable.getServerName(), + (k, v) -> v == null ? numAttempt : Math.max(v, numAttempt)); + } + } + + if (serversInProgress.size() > 0) { + return " on servers: " + serversInProgress.entrySet().stream() + .sorted(Comparator.> comparingInt(Map.Entry::getValue) + .reversed()) + .map(entry -> entry.getKey() + "(" + entry.getValue() + " attempts)") + .collect(Collectors.joining(", ")); + } + } + return ""; + } + private boolean waitUntilDone(long cutoff) throws InterruptedException { boolean hasWait = cutoff != Long.MAX_VALUE; long lastLog = EnvironmentEdgeManager.currentTime(); @@ -1336,10 +1377,10 @@ private ConnectionImplementation.ServerErrorTracker createServerErrorTracker() { * Create a callable. Isolated to be easily overridden in the tests. */ private MultiServerCallable createCallable(final ServerName server, TableName tableName, - final MultiAction multi) { + final MultiAction multi, int numAttempt) { return new MultiServerCallable(asyncProcess.connection, tableName, server, multi, asyncProcess.rpcFactory.newController(), rpcTimeout, tracker, multi.getPriority(), - requestAttributes); + requestAttributes, numAttempt); } private void updateResult(int index, Object result) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java index 6ba0832b26e5..33933dd5684f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java @@ -48,14 +48,17 @@ @InterfaceAudience.Private class MultiServerCallable extends CancellableRegionServerCallable { private MultiAction multiAction; + private final int numAttempt; private boolean cellBlock; MultiServerCallable(final ClusterConnection connection, final TableName tableName, final ServerName location, final MultiAction multi, RpcController rpcController, int rpcTimeout, - RetryingTimeTracker tracker, int priority, Map requestAttributes) { + RetryingTimeTracker tracker, int priority, Map requestAttributes, + int numAttempt) { super(connection, tableName, null, rpcController, rpcTimeout, tracker, priority, requestAttributes); this.multiAction = multi; + this.numAttempt = numAttempt; // RegionServerCallable has HRegionLocation field, but this is a multi-region request. // Using region info from parent HRegionLocation would be a mistake for this class; so // we will store the server here, and throw if someone tries to obtain location/regioninfo. @@ -63,6 +66,10 @@ class MultiServerCallable extends CancellableRegionServerCallable this.cellBlock = isCellBlock(); } + public int getNumAttempt() { + return numAttempt; + } + public void reset(ServerName location, MultiAction multiAction) { this.location = new HRegionLocation(null, location); this.multiAction = multiAction; From cc0df81869bff02004efef80e7edbbc1f2140f5d Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 19 Jun 2024 14:38:18 +0100 Subject: [PATCH 08/37] HubSpot Backport: HBASE-28596: Optimise BucketCache usage upon regions splits/merges. (will be in 2.7.0) --- .../hadoop/hbase/io/HalfStoreFileReader.java | 42 ++++++ .../hadoop/hbase/io/hfile/BlockCache.java | 11 ++ .../hadoop/hbase/io/hfile/BlockCacheUtil.java | 42 ++++++ .../hadoop/hbase/io/hfile/CacheConfig.java | 3 + .../hbase/io/hfile/CombinedBlockCache.java | 5 + .../hadoop/hbase/io/hfile/HFileBlock.java | 13 +- .../hbase/io/hfile/HFilePreadReader.java | 2 +- .../hbase/io/hfile/HFileReaderImpl.java | 42 +++--- .../hbase/io/hfile/bucket/BucketCache.java | 116 +++++++++++----- .../TransitRegionStateProcedure.java | 6 +- .../hbase/regionserver/StoreFileReader.java | 2 +- .../handler/UnassignRegionHandler.java | 8 +- .../hadoop/hbase/TestSplitWithCache.java | 130 ++++++++++++++++++ .../hbase/io/TestHalfStoreFileReader.java | 37 +++-- .../hadoop/hbase/io/hfile/TestPrefetch.java | 8 -- .../io/hfile/TestPrefetchWithBucketCache.java | 70 +++++++++- .../bucket/TestBucketCachePersister.java | 6 + 17 files changed, 447 insertions(+), 96 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java index 2119a3e7cbef..3a4b0437bfca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.IntConsumer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; @@ -29,6 +30,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.hadoop.hbase.io.hfile.HFileReaderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -64,6 +66,8 @@ public class HalfStoreFileReader extends StoreFileReader { private boolean firstKeySeeked = false; + private AtomicBoolean closed = new AtomicBoolean(false); + /** * Creates a half file reader for a hfile referred to by an hfilelink. * @param context Reader context info @@ -349,4 +353,42 @@ public long getFilterEntries() { // Estimate the number of entries as half the original file; this may be wildly inaccurate. return super.getFilterEntries() / 2; } + + /** + * Overrides close method to handle cache evictions for the referred file. If evictionOnClose is + * true, we will seek to the block containing the splitCell and evict all blocks from offset 0 up + * to that block offset if this is a bottom half reader, or the from the split block offset up to + * the end of the file if this is a top half reader. + * @param evictOnClose true if it should evict the file blocks from the cache. + */ + @Override + public void close(boolean evictOnClose) throws IOException { + if (closed.compareAndSet(false, true)) { + if (evictOnClose) { + final HFileReaderImpl.HFileScannerImpl s = + (HFileReaderImpl.HFileScannerImpl) super.getScanner(false, true, false); + final String reference = this.reader.getHFileInfo().getHFileContext().getHFileName(); + final String referred = StoreFileInfo.getReferredToRegionAndFile(reference).getSecond(); + s.seekTo(splitCell); + if (s.getCurBlock() != null) { + long offset = s.getCurBlock().getOffset(); + LOG.trace("Seeking to split cell in reader: {} for file: {} top: {}, split offset: {}", + this, reference, top, offset); + ((HFileReaderImpl) reader).getCacheConf().getBlockCache().ifPresent(cache -> { + int numEvictedReferred = top + ? cache.evictBlocksRangeByHfileName(referred, offset, Long.MAX_VALUE) + : cache.evictBlocksRangeByHfileName(referred, 0, offset); + int numEvictedReference = cache.evictBlocksByHfileName(reference); + LOG.trace( + "Closing reference: {}; referred file: {}; was top? {}; evicted for referred: {};" + + "evicted for reference: {}", + reference, referred, top, numEvictedReferred, numEvictedReference); + }); + } + reader.close(false); + } else { + reader.close(evictOnClose); + } + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 5b11035ebe73..a468752de5cb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -235,4 +235,15 @@ default Optional>> getFullyCachedFiles() { default Optional> getRegionCachedInfo() { return Optional.empty(); } + + /** + * Evict all blocks for the given file name between the passed offset values. + * @param hfileName The file for which blocks should be evicted. + * @param initOffset the initial offset for the range of blocks to be evicted. + * @param endOffset the end offset for the range of blocks to be evicted. + * @return number of blocks evicted. + */ + default int evictBlocksRangeByHfileName(String hfileName, long initOffset, long endOffset) { + return 0; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index 65b886f80ed5..7324701efe58 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.hfile; +import static org.apache.hadoop.hbase.io.hfile.HFileBlock.FILL_HEADER; + import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashSet; @@ -28,8 +30,10 @@ import java.util.concurrent.ConcurrentSkipListSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; +import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.GsonUtil; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -258,6 +262,44 @@ public static int getMaxCachedBlocksByFile(Configuration conf) { return conf == null ? DEFAULT_MAX : conf.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX); } + /** + * Similarly to HFileBlock.Writer.getBlockForCaching(), creates a HFileBlock instance without + * checksum for caching. This is needed for when we cache blocks via readers (either prefetch or + * client read), otherwise we may fail equality comparison when checking against same block that + * may already have been cached at write time. + * @param cacheConf the related CacheConfig object. + * @param block the HFileBlock instance to be converted. + * @return the resulting HFileBlock instance without checksum. + */ + public static HFileBlock getBlockForCaching(CacheConfig cacheConf, HFileBlock block) { + // Calculate how many bytes we need for checksum on the tail of the block. + int numBytes = cacheConf.shouldCacheCompressed(block.getBlockType().getCategory()) + ? 0 + : (int) ChecksumUtil.numBytes(block.getOnDiskDataSizeWithHeader(), + block.getHFileContext().getBytesPerChecksum()); + ByteBuff buff = block.getBufferReadOnly(); + HFileBlockBuilder builder = new HFileBlockBuilder(); + return builder.withBlockType(block.getBlockType()) + .withOnDiskSizeWithoutHeader(block.getOnDiskSizeWithoutHeader()) + .withUncompressedSizeWithoutHeader(block.getUncompressedSizeWithoutHeader()) + .withPrevBlockOffset(block.getPrevBlockOffset()).withByteBuff(buff) + .withFillHeader(FILL_HEADER).withOffset(block.getOffset()).withNextBlockOnDiskSize(-1) + .withOnDiskDataSizeWithHeader(block.getOnDiskDataSizeWithHeader() + numBytes) + .withHFileContext(cloneContext(block.getHFileContext())) + .withByteBuffAllocator(cacheConf.getByteBuffAllocator()).withShared(!buff.hasArray()).build(); + } + + public static HFileContext cloneContext(HFileContext context) { + HFileContext newContext = new HFileContextBuilder().withBlockSize(context.getBlocksize()) + .withBytesPerCheckSum(0).withChecksumType(ChecksumType.NULL) // no checksums in cached data + .withCompression(context.getCompression()) + .withDataBlockEncoding(context.getDataBlockEncoding()) + .withHBaseCheckSum(context.isUseHBaseChecksum()).withCompressTags(context.isCompressTags()) + .withIncludesMvcc(context.isIncludesMvcc()).withIncludesTags(context.isIncludesTags()) + .withColumnFamily(context.getColumnFamily()).withTableName(context.getTableName()).build(); + return newContext; + } + /** * Use one of these to keep a running account of cached blocks by file. Throw it away when done. * This is different than metrics in that it is stats on current state of a cache. See diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 34c97ee64daa..92d7f4eb8903 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -72,6 +72,8 @@ public class CacheConfig implements ConfigurationObserver { */ public static final String EVICT_BLOCKS_ON_CLOSE_KEY = "hbase.rs.evictblocksonclose"; + public static final String EVICT_BLOCKS_ON_SPLIT_KEY = "hbase.rs.evictblocksonsplit"; + /** * Configuration key to prefetch all blocks of a given file into the block cache when the file is * opened. @@ -107,6 +109,7 @@ public class CacheConfig implements ConfigurationObserver { public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false; public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; public static final boolean DEFAULT_EVICT_ON_CLOSE = false; + public static final boolean DEFAULT_EVICT_ON_SPLIT = true; public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false; public static final boolean DEFAULT_PREFETCH_ON_OPEN = false; public static final boolean DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE = false; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 00dc8e4a5551..ef536f9e0be3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -492,4 +492,9 @@ public Optional getBlockSize(BlockCacheKey key) { return l1Result.isPresent() ? l1Result : l2Cache.getBlockSize(key); } + @Override + public int evictBlocksRangeByHfileName(String hfileName, long initOffset, long endOffset) { + return l1Cache.evictBlocksRangeByHfileName(hfileName, initOffset, endOffset) + + l2Cache.evictBlocksRangeByHfileName(hfileName, initOffset, endOffset); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index 16bec1e95888..4c73fc2bcdc7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -697,7 +697,7 @@ public boolean isUnpacked() { * when block is returned to the cache. * @return the offset of this block in the file it was read from */ - long getOffset() { + public long getOffset() { if (offset < 0) { throw new IllegalStateException("HFile block offset not initialized properly"); } @@ -1205,16 +1205,7 @@ void writeBlock(BlockWritable bw, FSDataOutputStream out) throws IOException { * being wholesome (ECC memory or if file-backed, it does checksumming). */ HFileBlock getBlockForCaching(CacheConfig cacheConf) { - HFileContext newContext = new HFileContextBuilder().withBlockSize(fileContext.getBlocksize()) - .withBytesPerCheckSum(0).withChecksumType(ChecksumType.NULL) // no checksums in cached data - .withCompression(fileContext.getCompression()) - .withDataBlockEncoding(fileContext.getDataBlockEncoding()) - .withHBaseCheckSum(fileContext.isUseHBaseChecksum()) - .withCompressTags(fileContext.isCompressTags()) - .withIncludesMvcc(fileContext.isIncludesMvcc()) - .withIncludesTags(fileContext.isIncludesTags()) - .withColumnFamily(fileContext.getColumnFamily()).withTableName(fileContext.getTableName()) - .build(); + HFileContext newContext = BlockCacheUtil.cloneContext(fileContext); // Build the HFileBlock. HFileBlockBuilder builder = new HFileBlockBuilder(); ByteBuff buff; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java index 926237314828..b95ce4bde556 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java @@ -46,7 +46,7 @@ public HFilePreadReader(ReaderContext context, HFileInfo fileInfo, CacheConfig c }); // Prefetch file blocks upon open if requested - if (cacheConf.shouldPrefetchOnOpen() && cacheIfCompactionsOff() && shouldCache.booleanValue()) { + if (cacheConf.shouldPrefetchOnOpen() && shouldCache.booleanValue()) { PrefetchExecutor.request(path, new Runnable() { @Override public void run() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index b6a061043070..db2383db399d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.io.hfile; -import static org.apache.hadoop.hbase.regionserver.CompactSplit.HBASE_REGION_SERVER_ENABLE_COMPACTION; import static org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.BLOCK_CACHE_KEY_KEY; import io.opentelemetry.api.common.Attributes; @@ -42,14 +41,12 @@ import org.apache.hadoop.hbase.SizeCachedKeyValue; import org.apache.hadoop.hbase.SizeCachedNoTagsByteBufferKeyValue; import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue; -import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.IdLock; @@ -159,6 +156,10 @@ public BlockIndexNotLoadedException(Path path) { } } + public CacheConfig getCacheConf() { + return cacheConf; + } + private Optional toStringFirstKey() { return getFirstKey().map(CellUtil::getCellKeyAsString); } @@ -307,7 +308,7 @@ public NotSeekedException(Path path) { } } - protected static class HFileScannerImpl implements HFileScanner { + public static class HFileScannerImpl implements HFileScanner { private ByteBuff blockBuffer; protected final boolean cacheBlocks; protected final boolean pread; @@ -340,6 +341,11 @@ protected static class HFileScannerImpl implements HFileScanner { // Whether we returned a result for curBlock's size in recordBlockSize(). // gets reset whenever curBlock is changed. private boolean providedCurrentBlockSize = false; + + public HFileBlock getCurBlock() { + return curBlock; + } + // Previous blocks that were used in the course of the read protected final ArrayList prevBlocks = new ArrayList<>(); @@ -1292,8 +1298,6 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo BlockCacheKey cacheKey = new BlockCacheKey(path, dataBlockOffset, this.isPrimaryReplicaReader(), expectedBlockType); - boolean cacheable = cacheBlock && cacheIfCompactionsOff(); - boolean useLock = false; IdLock.Entry lockEntry = null; final Span span = Span.current(); @@ -1340,7 +1344,7 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo return cachedBlock; } - if (!useLock && cacheable && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) { + if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) { // check cache again with lock useLock = true; continue; @@ -1351,7 +1355,7 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo span.addEvent("block cache miss", attributes); // Load block from filesystem. HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, pread, - !isCompaction, shouldUseHeap(expectedBlockType, cacheable)); + !isCompaction, shouldUseHeap(expectedBlockType, cacheBlock)); try { validateBlockType(hfileBlock, expectedBlockType); } catch (IOException e) { @@ -1364,25 +1368,30 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo // Don't need the unpacked block back and we're storing the block in the cache compressed if (cacheOnly && cacheCompressed && cacheOnRead) { + HFileBlock blockNoChecksum = BlockCacheUtil.getBlockForCaching(cacheConf, hfileBlock); cacheConf.getBlockCache().ifPresent(cache -> { LOG.debug("Skipping decompression of block {} in prefetch", cacheKey); // Cache the block if necessary - if (cacheable && cacheConf.shouldCacheBlockOnRead(category)) { - cache.cacheBlock(cacheKey, hfileBlock, cacheConf.isInMemory(), cacheOnly); + if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { + cache.cacheBlock(cacheKey, blockNoChecksum, cacheConf.isInMemory(), cacheOnly); } }); if (updateCacheMetrics && hfileBlock.getBlockType().isData()) { HFile.DATABLOCK_READ_COUNT.increment(); } - return hfileBlock; + return blockNoChecksum; } HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader); + HFileBlock unpackedNoChecksum = BlockCacheUtil.getBlockForCaching(cacheConf, unpacked); // Cache the block if necessary cacheConf.getBlockCache().ifPresent(cache -> { - if (cacheable && cacheConf.shouldCacheBlockOnRead(category)) { + if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { // Using the wait on cache during compaction and prefetching. - cache.cacheBlock(cacheKey, cacheCompressed ? hfileBlock : unpacked, + cache.cacheBlock(cacheKey, + cacheCompressed + ? BlockCacheUtil.getBlockForCaching(cacheConf, hfileBlock) + : unpackedNoChecksum, cacheConf.isInMemory(), cacheOnly); } }); @@ -1394,7 +1403,7 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo HFile.DATABLOCK_READ_COUNT.increment(); } - return unpacked; + return unpackedNoChecksum; } } finally { if (lockEntry != null) { @@ -1716,9 +1725,4 @@ public int getMajorVersion() { public void unbufferStream() { fsBlockReader.unbufferStream(); } - - protected boolean cacheIfCompactionsOff() { - return (!StoreFileInfo.isReference(name) && !HFileLink.isHFileLink(name)) - || !conf.getBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION, true); - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 3b08655bcfb3..cd82af74108a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -79,6 +79,7 @@ import org.apache.hadoop.hbase.nio.RefCnt; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.IdReadWriteLock; @@ -222,6 +223,8 @@ public class BucketCache implements BlockCache, HeapSize { // reset after a successful read/write. private volatile long ioErrorStartTime = -1; + private Configuration conf; + /** * A ReentrantReadWriteLock to lock on a particular block identified by offset. The purpose of * this is to avoid freeing the block which is being read. @@ -582,6 +585,30 @@ protected void cacheBlockWithWaitInternal(BlockCacheKey cacheKey, Cacheable cach } } + /** + * If the passed cache key relates to a reference (.), this method looks + * for the block from the referred file, in the cache. If present in the cache, the block for the + * referred file is returned, otherwise, this method returns null. It will also return null if the + * passed cache key doesn't relate to a reference. + * @param key the BlockCacheKey instance to look for in the cache. + * @return the cached block from the referred file, null if there's no such block in the cache or + * the passed key doesn't relate to a reference. + */ + public BucketEntry getBlockForReference(BlockCacheKey key) { + BucketEntry foundEntry = null; + String referredFileName = null; + if (StoreFileInfo.isReference(key.getHfileName())) { + referredFileName = StoreFileInfo.getReferredToRegionAndFile(key.getHfileName()).getSecond(); + } + if (referredFileName != null) { + BlockCacheKey convertedCacheKey = new BlockCacheKey(referredFileName, key.getOffset()); + foundEntry = backingMap.get(convertedCacheKey); + LOG.debug("Got a link/ref: {}. Related cacheKey: {}. Found entry: {}", key.getHfileName(), + convertedCacheKey, foundEntry); + } + return foundEntry; + } + /** * Get the buffer of the block with the specified key. * @param key block's cache key @@ -605,6 +632,9 @@ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat, return re.getData(); } BucketEntry bucketEntry = backingMap.get(key); + if (bucketEntry == null) { + bucketEntry = getBlockForReference(key); + } if (bucketEntry != null) { long start = System.nanoTime(); ReentrantReadWriteLock lock = offsetLock.getLock(bucketEntry.offset()); @@ -613,7 +643,9 @@ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat, // We can not read here even if backingMap does contain the given key because its offset // maybe changed. If we lock BlockCacheKey instead of offset, then we can only check // existence here. - if (bucketEntry.equals(backingMap.get(key))) { + if ( + bucketEntry.equals(backingMap.get(key)) || bucketEntry.equals(getBlockForReference(key)) + ) { // Read the block from IOEngine based on the bucketEntry's offset and length, NOTICE: the // block will use the refCnt of bucketEntry, which means if two HFileBlock mapping to // the same BucketEntry, then all of the three will share the same refCnt. @@ -1750,8 +1782,15 @@ protected String getAlgorithm() { */ @Override public int evictBlocksByHfileName(String hfileName) { + return evictBlocksRangeByHfileName(hfileName, 0, Long.MAX_VALUE); + } + + @Override + public int evictBlocksRangeByHfileName(String hfileName, long initOffset, long endOffset) { fileNotFullyCached(hfileName); - Set keySet = getAllCacheKeysForFile(hfileName); + Set keySet = getAllCacheKeysForFile(hfileName, initOffset, endOffset); + LOG.debug("found {} blocks for file {}, starting offset: {}, end offset: {}", keySet.size(), + hfileName, initOffset, endOffset); int numEvicted = 0; for (BlockCacheKey key : keySet) { if (evictBlock(key)) { @@ -1761,9 +1800,9 @@ public int evictBlocksByHfileName(String hfileName) { return numEvicted; } - private Set getAllCacheKeysForFile(String hfileName) { - return blocksByHFile.subSet(new BlockCacheKey(hfileName, Long.MIN_VALUE), true, - new BlockCacheKey(hfileName, Long.MAX_VALUE), true); + private Set getAllCacheKeysForFile(String hfileName, long init, long end) { + return blocksByHFile.subSet(new BlockCacheKey(hfileName, init), true, + new BlockCacheKey(hfileName, end), true); } /** @@ -2173,25 +2212,20 @@ public void notifyFileCachingCompleted(Path fileName, int totalBlockCount, int d try { final MutableInt count = new MutableInt(); LOG.debug("iterating over {} entries in the backing map", backingMap.size()); - backingMap.entrySet().stream().forEach(entry -> { - if ( - entry.getKey().getHfileName().equals(fileName.getName()) - && entry.getKey().getBlockType().equals(BlockType.DATA) - ) { - long offsetToLock = entry.getValue().offset(); - LOG.debug("found block {} in the backing map. Acquiring read lock for offset {}", - entry.getKey(), offsetToLock); - ReentrantReadWriteLock lock = offsetLock.getLock(offsetToLock); - lock.readLock().lock(); - locks.add(lock); - // rechecks the given key is still there (no eviction happened before the lock acquired) - if (backingMap.containsKey(entry.getKey())) { - count.increment(); - } else { - lock.readLock().unlock(); - locks.remove(lock); - LOG.debug("found block {}, but when locked and tried to count, it was gone."); - } + Set result = getAllCacheKeysForFile(fileName.getName(), 0, Long.MAX_VALUE); + if (result.isEmpty() && StoreFileInfo.isReference(fileName)) { + result = getAllCacheKeysForFile( + StoreFileInfo.getReferredToRegionAndFile(fileName.getName()).getSecond(), 0, + Long.MAX_VALUE); + } + result.stream().forEach(entry -> { + LOG.debug("found block for file {} in the backing map. Acquiring read lock for offset {}", + fileName.getName(), entry.getOffset()); + ReentrantReadWriteLock lock = offsetLock.getLock(entry.getOffset()); + lock.readLock().lock(); + locks.add(lock); + if (backingMap.containsKey(entry) && entry.getBlockType() == BlockType.DATA) { + count.increment(); } }); int metaCount = totalBlockCount - dataBlockCount; @@ -2214,17 +2248,19 @@ public void notifyFileCachingCompleted(Path fileName, int totalBlockCount, int d + "and try the verification again.", fileName.getName()); Thread.sleep(100); notifyFileCachingCompleted(fileName, totalBlockCount, dataBlockCount, size); - } else - if ((getAllCacheKeysForFile(fileName.getName()).size() - metaCount) == dataBlockCount) { - LOG.debug("We counted {} data blocks, expected was {}, there was no more pending in " - + "the cache write queue but we now found that total cached blocks for file {} " - + "is equal to data block count.", count, dataBlockCount, fileName.getName()); - fileCacheCompleted(fileName, size); - } else { - LOG.info("We found only {} data blocks cached from a total of {} for file {}, " - + "but no blocks pending caching. Maybe cache is full or evictions " - + "happened concurrently to cache prefetch.", count, dataBlockCount, fileName); - } + } else if ( + (getAllCacheKeysForFile(fileName.getName(), 0, Long.MAX_VALUE).size() - metaCount) + == dataBlockCount + ) { + LOG.debug("We counted {} data blocks, expected was {}, there was no more pending in " + + "the cache write queue but we now found that total cached blocks for file {} " + + "is equal to data block count.", count, dataBlockCount, fileName.getName()); + fileCacheCompleted(fileName, size); + } else { + LOG.info("We found only {} data blocks cached from a total of {} for file {}, " + + "but no blocks pending caching. Maybe cache is full or evictions " + + "happened concurrently to cache prefetch.", count, dataBlockCount, fileName); + } } } catch (InterruptedException e) { throw new RuntimeException(e); @@ -2250,14 +2286,20 @@ public Optional shouldCacheFile(String fileName) { @Override public Optional isAlreadyCached(BlockCacheKey key) { - return Optional.of(getBackingMap().containsKey(key)); + boolean foundKey = backingMap.containsKey(key); + // if there's no entry for the key itself, we need to check if this key is for a reference, + // and if so, look for a block from the referenced file using this getBlockForReference method. + return Optional.of(foundKey ? true : getBlockForReference(key) != null); } @Override public Optional getBlockSize(BlockCacheKey key) { BucketEntry entry = backingMap.get(key); if (entry == null) { - return Optional.empty(); + // the key might be for a reference tha we had found the block from the referenced file in + // the cache when we first tried to cache it. + entry = getBlockForReference(key); + return entry == null ? Optional.empty() : Optional.of(entry.getOnDiskSizeWithHeader()); } else { return Optional.of(entry.getOnDiskSizeWithHeader()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java index 81397915647d..b1d4483bd9e5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hbase.master.assignment; import static org.apache.hadoop.hbase.io.hfile.CacheConfig.DEFAULT_EVICT_ON_CLOSE; +import static org.apache.hadoop.hbase.io.hfile.CacheConfig.DEFAULT_EVICT_ON_SPLIT; import static org.apache.hadoop.hbase.io.hfile.CacheConfig.EVICT_BLOCKS_ON_CLOSE_KEY; +import static org.apache.hadoop.hbase.io.hfile.CacheConfig.EVICT_BLOCKS_ON_SPLIT_KEY; import static org.apache.hadoop.hbase.master.LoadBalancer.BOGUS_SERVER_NAME; import static org.apache.hadoop.hbase.master.assignment.AssignmentManager.FORCE_REGION_RETAINMENT; @@ -335,7 +337,9 @@ private void closeRegion(MasterProcedureEnv env, RegionStateNode regionNode) thr env.getAssignmentManager().regionClosing(regionNode); CloseRegionProcedure closeProc = isSplit ? new CloseRegionProcedure(this, getRegion(), regionNode.getRegionLocation(), - assignCandidate, true) + assignCandidate, + env.getMasterConfiguration().getBoolean(EVICT_BLOCKS_ON_SPLIT_KEY, + DEFAULT_EVICT_ON_SPLIT)) : new CloseRegionProcedure(this, getRegion(), regionNode.getRegionLocation(), assignCandidate, evictCache); addChildProcedure(closeProc); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java index 09c379227bda..e241bf0a5d34 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java @@ -68,7 +68,7 @@ public class StoreFileReader { protected BloomFilter deleteFamilyBloomFilter = null; private BloomFilterMetrics bloomFilterMetrics = null; protected BloomType bloomFilterType; - private final HFile.Reader reader; + protected final HFile.Reader reader; protected long sequenceID = -1; protected TimeRange timeRange = null; private byte[] lastBloomKey; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java index 2419e709686a..8f8668aa87a8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/UnassignRegionHandler.java @@ -126,11 +126,9 @@ public void process() throws IOException { region.getCoprocessorHost().preClose(abort); } // This should be true only in the case of splits/merges closing the parent regions, as - // there's no point on keep blocks for those region files. As hbase.rs.evictblocksonclose is - // false by default we don't bother overriding it if evictCache is false. - if (evictCache) { - region.getStores().forEach(s -> s.getCacheConfig().setEvictOnClose(true)); - } + // there's no point on keep blocks for those region files. + region.getStores().forEach(s -> s.getCacheConfig().setEvictOnClose(evictCache)); + if (region.close(abort) == null) { // XXX: Is this still possible? The old comment says about split, but now split is done at // master side, so... diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java new file mode 100644 index 000000000000..91e65610f81c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY; +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; +import static org.apache.hadoop.hbase.io.hfile.CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY; +import static org.apache.hadoop.hbase.io.hfile.CacheConfig.EVICT_BLOCKS_ON_SPLIT_KEY; +import static org.apache.hadoop.hbase.io.hfile.CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ MiscTests.class, MediumTests.class }) +public class TestSplitWithCache { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSplitWithCache.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestSplitWithCache.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 1000); + UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); + UTIL.getConfiguration().setBoolean(CACHE_BLOCKS_ON_WRITE_KEY, true); + UTIL.getConfiguration().setBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, true); + UTIL.getConfiguration().set(BUCKET_CACHE_IOENGINE_KEY, "offheap"); + UTIL.getConfiguration().setInt(BUCKET_CACHE_SIZE_KEY, 200); + } + + @Test + public void testEvictOnSplit() throws Exception { + doTest("testEvictOnSplit", true, + (f, m) -> Waiter.waitFor(UTIL.getConfiguration(), 1000, () -> m.get(f) != null), + (f, m) -> Waiter.waitFor(UTIL.getConfiguration(), 1000, () -> m.get(f) == null)); + } + + @Test + public void testDoesntEvictOnSplit() throws Exception { + doTest("testDoesntEvictOnSplit", false, + (f, m) -> Waiter.waitFor(UTIL.getConfiguration(), 1000, () -> m.get(f) != null), + (f, m) -> Waiter.waitFor(UTIL.getConfiguration(), 1000, () -> m.get(f) != null)); + } + + private void doTest(String table, boolean evictOnSplit, + BiConsumer>> predicateBeforeSplit, + BiConsumer>> predicateAfterSplit) throws Exception { + UTIL.getConfiguration().setBoolean(EVICT_BLOCKS_ON_SPLIT_KEY, evictOnSplit); + UTIL.startMiniCluster(1); + try { + TableName tableName = TableName.valueOf(table); + byte[] family = Bytes.toBytes("CF"); + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); + UTIL.getAdmin().createTable(td); + UTIL.waitTableAvailable(tableName); + Table tbl = UTIL.getConnection().getTable(tableName); + List puts = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + Put p = new Put(Bytes.toBytes("row-" + i)); + p.addColumn(family, Bytes.toBytes(1), Bytes.toBytes("val-" + i)); + puts.add(p); + } + tbl.put(puts); + UTIL.getAdmin().flush(tableName); + Collection files = + UTIL.getMiniHBaseCluster().getRegions(tableName).get(0).getStores().get(0).getStorefiles(); + checkCacheForBlocks(tableName, files, predicateBeforeSplit); + UTIL.getAdmin().split(tableName, Bytes.toBytes("row-500")); + Waiter.waitFor(UTIL.getConfiguration(), 30000, + () -> UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2); + UTIL.waitUntilNoRegionsInTransition(); + checkCacheForBlocks(tableName, files, predicateAfterSplit); + } finally { + UTIL.shutdownMiniCluster(); + } + + } + + private void checkCacheForBlocks(TableName tableName, Collection files, + BiConsumer>> checker) { + files.forEach(f -> { + UTIL.getMiniHBaseCluster().getRegionServer(0).getBlockCache().ifPresent(cache -> { + cache.getFullyCachedFiles().ifPresent(m -> { + checker.accept(f.getPath().getName(), m); + }); + assertTrue(cache.getFullyCachedFiles().isPresent()); + }); + }); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java index 13955ccebfec..0ac03b8d4136 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -82,15 +84,19 @@ public static void tearDownAfterClass() throws Exception { */ @Test public void testHalfScanAndReseek() throws IOException { - String root_dir = TEST_UTIL.getDataTestDir().toString(); - Path p = new Path(root_dir, "test"); - Configuration conf = TEST_UTIL.getConfiguration(); FileSystem fs = FileSystem.get(conf); + String root_dir = TEST_UTIL.getDataTestDir().toString(); + Path parentPath = new Path(new Path(root_dir, "parent"), "CF"); + fs.mkdirs(parentPath); + Path splitAPath = new Path(new Path(root_dir, "splita"), "CF"); + Path splitBPath = new Path(new Path(root_dir, "splitb"), "CF"); + Path filePath = StoreFileWriter.getUniqueFile(fs, parentPath); + CacheConfig cacheConf = new CacheConfig(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build(); HFile.Writer w = - HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create(); + HFile.getWriterFactory(conf, cacheConf).withPath(fs, filePath).withFileContext(meta).create(); // write some things. List items = genSomeKeys(); @@ -99,26 +105,35 @@ public void testHalfScanAndReseek() throws IOException { } w.close(); - HFile.Reader r = HFile.createReader(fs, p, cacheConf, true, conf); + HFile.Reader r = HFile.createReader(fs, filePath, cacheConf, true, conf); Cell midKV = r.midKey().get(); byte[] midkey = CellUtil.cloneRow(midKV); - // System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey)); + Path splitFileA = new Path(splitAPath, filePath.getName() + ".parent"); + Path splitFileB = new Path(splitBPath, filePath.getName() + ".parent"); Reference bottom = new Reference(midkey, Reference.Range.bottom); - doTestOfScanAndReseek(p, fs, bottom, cacheConf); + bottom.write(fs, splitFileA); + doTestOfScanAndReseek(splitFileA, fs, bottom, cacheConf); Reference top = new Reference(midkey, Reference.Range.top); - doTestOfScanAndReseek(p, fs, top, cacheConf); + top.write(fs, splitFileB); + doTestOfScanAndReseek(splitFileB, fs, top, cacheConf); r.close(); } private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf) throws IOException { - ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, p).build(); - StoreFileInfo storeFileInfo = - new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, fs.getFileStatus(p), bottom); + Path referencePath = StoreFileInfo.getReferredToFile(p); + FSDataInputStreamWrapper in = new FSDataInputStreamWrapper(fs, referencePath, false, 0); + FileStatus status = fs.getFileStatus(referencePath); + long length = status.getLen(); + ReaderContextBuilder contextBuilder = + new ReaderContextBuilder().withInputStreamWrapper(in).withFileSize(length) + .withReaderType(ReaderContext.ReaderType.PREAD).withFileSystem(fs).withFilePath(p); + ReaderContext context = contextBuilder.build(); + StoreFileInfo storeFileInfo = new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, p, true); storeFileInfo.initHFileInfo(context); final HalfStoreFileReader halfreader = (HalfStoreFileReader) storeFileInfo.createReader(context, cacheConf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index cd2793b8cea0..8e278e40336e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -285,14 +285,6 @@ public void testPrefetchCompressed() throws Exception { conf.setBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, false); } - @Test - public void testPrefetchSkipsRefs() throws Exception { - testPrefetchWhenRefs(true, c -> { - boolean isCached = c != null; - assertFalse(isCached); - }); - } - @Test public void testPrefetchDoesntSkipRefs() throws Exception { testPrefetchWhenRefs(false, c -> { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index db8f2213d0c0..c3954d3cf901 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.io.hfile.BlockCacheFactory.BUCKET_CACHE_BUCKETS_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -39,13 +40,20 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.io.hfile.bucket.BucketEntry; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -135,6 +143,55 @@ public void testPrefetchDoesntOverwork() throws Exception { assertTrue(snapshot.get(key).getCachedTime() < bc.getBackingMap().get(key).getCachedTime()); } + @Test + public void testPrefetchRefsAfterSplit() throws Exception { + conf.setLong(BUCKET_CACHE_SIZE_KEY, 200); + blockCache = BlockCacheFactory.createBlockCache(conf); + cacheConf = new CacheConfig(conf, blockCache); + + Path tableDir = new Path(TEST_UTIL.getDataTestDir(), "testPrefetchRefsAfterSplit"); + RegionInfo region = RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build(); + Path regionDir = new Path(tableDir, region.getEncodedName()); + Path cfDir = new Path(regionDir, "cf"); + HRegionFileSystem regionFS = + HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, region); + Path storeFile = writeStoreFile(100, cfDir); + + // Prefetches the file blocks + LOG.debug("First read should prefetch the blocks."); + readStoreFile(storeFile); + BucketCache bc = BucketCache.getBucketCacheFromCacheConfig(cacheConf).get(); + // Our file should have 6 DATA blocks. We should wait for all of them to be cached + Waiter.waitFor(conf, 300, () -> bc.getBackingMap().size() == 6); + + // split the file and return references to the original file + Random rand = ThreadLocalRandom.current(); + byte[] splitPoint = RandomKeyValueUtil.randomOrderedKey(rand, 50); + HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true); + Path ref = regionFS.splitStoreFile(region, "cf", file, splitPoint, false, + new ConstantSizeRegionSplitPolicy()); + HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true); + // starts reader for the ref. The ref should resolve to the original file blocks + // and not duplicate blocks in the cache. + refHsf.initReader(); + HFile.Reader reader = refHsf.getReader().getHFileReader(); + while (!reader.prefetchComplete()) { + // Sleep for a bit + Thread.sleep(1000); + } + // the ref file blocks keys should actually resolve to the referred file blocks, + // so we should not see additional blocks in the cache. + Waiter.waitFor(conf, 300, () -> bc.getBackingMap().size() == 6); + + BlockCacheKey refCacheKey = new BlockCacheKey(ref.getName(), 0); + Cacheable result = bc.getBlock(refCacheKey, true, false, true); + assertNotNull(result); + BlockCacheKey fileCacheKey = new BlockCacheKey(file.getPath().getName(), 0); + assertEquals(result, bc.getBlock(fileCacheKey, true, false, true)); + assertNull(bc.getBackingMap().get(refCacheKey)); + assertNotNull(bc.getBlockForReference(refCacheKey)); + } + @Test public void testPrefetchInterruptOnCapacity() throws Exception { conf.setLong(BUCKET_CACHE_SIZE_KEY, 1); @@ -270,10 +327,19 @@ private Path writeStoreFile(String fname, int numKVs) throws IOException { return writeStoreFile(fname, meta, numKVs); } + private Path writeStoreFile(int numKVs, Path regionCFDir) throws IOException { + HFileContext meta = new HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build(); + return writeStoreFile(meta, numKVs, regionCFDir); + } + private Path writeStoreFile(String fname, HFileContext context, int numKVs) throws IOException { - Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), fname); + return writeStoreFile(context, numKVs, new Path(TEST_UTIL.getDataTestDir(), fname)); + } + + private Path writeStoreFile(HFileContext context, int numKVs, Path regionCFDir) + throws IOException { StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs) - .withOutputDir(storeFileParentDir).withFileContext(context).build(); + .withOutputDir(regionCFDir).withFileContext(context).build(); Random rand = ThreadLocalRandom.current(); final int rowLen = 32; for (int i = 0; i < numKVs; ++i) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java index 7be959dfad4b..35a60ec93125 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java @@ -49,6 +49,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category({ IOTests.class, MediumTests.class }) public class TestBucketCachePersister { @@ -61,6 +63,8 @@ public class TestBucketCachePersister { public int constructedBlockSize = 16 * 1024; + private static final Logger LOG = LoggerFactory.getLogger(TestBucketCachePersister.class); + public int[] constructedBlockSizes = new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024, 8 * 1024 + 1024, 16 * 1024 + 1024, 28 * 1024 + 1024, 32 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024, 128 * 1024 + 1024 }; @@ -164,6 +168,7 @@ public void testPrefetchBlockEvictionWhilePrefetchRunning() throws Exception { HFile.createReader(fs, storeFile, cacheConf, true, conf); boolean evicted = false; while (!PrefetchExecutor.isCompleted(storeFile)) { + LOG.debug("Entered loop as prefetch for {} is still running.", storeFile); if (bucketCache.backingMap.size() > 0 && !evicted) { Iterator> it = bucketCache.backingMap.entrySet().iterator(); @@ -172,6 +177,7 @@ public void testPrefetchBlockEvictionWhilePrefetchRunning() throws Exception { while (it.hasNext() && !evicted) { if (entry.getKey().getBlockType().equals(BlockType.DATA)) { evicted = bucketCache.evictBlock(it.next().getKey()); + LOG.debug("Attempted eviction for {}. Succeeded? {}", storeFile, evicted); } } } From 9d04d575502fa587950d9a97c3f2beb1c182d54f Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 27 Dec 2024 13:35:13 -0500 Subject: [PATCH 09/37] HubSpot Edit: Upgrade zstd-jni to latest version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4be54a0adf9..ce0a86eb8ec7 100644 --- a/pom.xml +++ b/pom.xml @@ -662,7 +662,7 @@ 1.11.0 1.8.0 1.1.10.4 - 1.5.5-2 + 1.5.7-1 region read lock + * region split --> region close first --> region write lock + * region close --> region write lock + * region bulk load --> region write lock + * + *

+ * read lock is compatible with read lock. ---> no problem with user scan/read region bulk load + * does not cause problem for compaction (no consistency problem, store lock will help the store + * file accounting). They can run almost concurrently at the region level. + *

+ *

+ * The only remaining race condition is between the region close and compaction. So we will + * evaluate, below, how region close intervenes with compaction if compaction does not acquire + * region read lock. + *

+ *

+ * Here are the steps for compaction: + *

    + *
  1. obtain list of StoreFile's
    2. + *
  2. create StoreFileScanner's based on list from #1
    3. + *
  3. perform compaction and save resulting files under tmp dir
    4. + *
  4. swap in compacted files
    5. + *
+ *

+ *

+ * #1 is guarded by store lock. This patch does not change this --> no worse or better For #2, we + * obtain smallest read point (for region) across all the Scanners (for both default compactor and + * stripe compactor). The read points are for user scans. Region keeps the read points for all + * currently open user scanners. Compaction needs to know the smallest read point so that during + * re-write of the hfiles, it can remove the mvcc points for the cells if their mvccs are older + * than the smallest since they are not needed anymore. This will not conflict with compaction. + *

+ *

+ * For #3, it can be performed in parallel to other operations. + *

+ *

+ * For #4 bulk load and compaction don't conflict with each other on the region level (for + * multi-family atomicy). + *

+ *

+ * Region close and compaction are guarded pretty well by the 'writestate'. In HRegion#doClose(), + * we have : + * + * 

+   * synchronized (writestate) {
+   *   // Disable compacting and flushing by background threads for this
+   *   // region.
+   *   canFlush = !writestate.readOnly;
+   *   writestate.writesEnabled = false;
+   *   LOG.debug("Closing " + this + ": disabling compactions & flushes");
+   *   waitForFlushesAndCompactions();
+   * }
+   *
+ * + * {@code waitForFlushesAndCompactions()} would wait for {@code writestate.compacting} to come + * down to 0. and in {@code HRegion.compact()} + * + * 
+   *   try {
+   *     synchronized (writestate) {
+   *       if (writestate.writesEnabled) {
+   *         wasStateSet = true;
+   *         ++writestate.compacting;
+   *       } else {
+   *         String msg = "NOT compacting region " + this + ". Writes disabled.";
+   *         LOG.info(msg);
+   *         status.abort(msg);
+   *         return false;
+   *       }
+   *     }
+   *   }
+   *
+ * + * Also in {@code compactor.performCompaction()}: check periodically to see if a system stop is + * requested + * + * 
+   * if (closeChecker != null && closeChecker.isTimeLimit(store, now)) {
+   *   progress.cancel();
+   *   return false;
+   * }
+   * if (closeChecker != null && closeChecker.isSizeLimit(store, len)) {
+   *   progress.cancel();
+   *   return false;
+   * }
+   *
+ *

+ */ public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController, User user) throws IOException { assert compaction != null && compaction.hasSelection(); @@ -2280,40 +2376,6 @@ public boolean compact(CompactionContext compaction, HStore store, } MonitoredTask status = null; boolean requestNeedsCancellation = true; - /* - * We are trying to remove / relax the region read lock for compaction. Let's see what are the - * potential race conditions among the operations (user scan, region split, region close and - * region bulk load). user scan ---> region read lock region split --> region close first --> - * region write lock region close --> region write lock region bulk load --> region write lock - * read lock is compatible with read lock. ---> no problem with user scan/read region bulk load - * does not cause problem for compaction (no consistency problem, store lock will help the store - * file accounting). They can run almost concurrently at the region level. The only remaining - * race condition is between the region close and compaction. So we will evaluate, below, how - * region close intervenes with compaction if compaction does not acquire region read lock. Here - * are the steps for compaction: 1. obtain list of StoreFile's 2. create StoreFileScanner's - * based on list from #1 3. perform compaction and save resulting files under tmp dir 4. swap in - * compacted files #1 is guarded by store lock. This patch does not change this --> no worse or - * better For #2, we obtain smallest read point (for region) across all the Scanners (for both - * default compactor and stripe compactor). The read points are for user scans. Region keeps the - * read points for all currently open user scanners. Compaction needs to know the smallest read - * point so that during re-write of the hfiles, it can remove the mvcc points for the cells if - * their mvccs are older than the smallest since they are not needed anymore. This will not - * conflict with compaction. For #3, it can be performed in parallel to other operations. For #4 - * bulk load and compaction don't conflict with each other on the region level (for multi-family - * atomicy). Region close and compaction are guarded pretty well by the 'writestate'. In - * HRegion#doClose(), we have : synchronized (writestate) { // Disable compacting and flushing - * by background threads for this // region. canFlush = !writestate.readOnly; - * writestate.writesEnabled = false; LOG.debug("Closing " + this + - * ": disabling compactions & flushes"); waitForFlushesAndCompactions(); } - * waitForFlushesAndCompactions() would wait for writestate.compacting to come down to 0. and in - * HRegion.compact() try { synchronized (writestate) { if (writestate.writesEnabled) { - * wasStateSet = true; ++writestate.compacting; } else { String msg = "NOT compacting region " + - * this + ". Writes disabled."; LOG.info(msg); status.abort(msg); return false; } } Also in - * compactor.performCompaction(): check periodically to see if a system stop is requested if - * (closeChecker != null && closeChecker.isTimeLimit(store, now)) { progress.cancel(); return - * false; } if (closeChecker != null && closeChecker.isSizeLimit(store, len)) { - * progress.cancel(); return false; } - */ try { byte[] cf = Bytes.toBytes(store.getColumnFamilyName()); if (stores.get(cf) != store) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 1df8d0b95807..710c94753093 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -832,7 +832,7 @@ protected List flushCache(final long logCacheFlushId, MemStoreSnapshot sna try { for (Path pathName : pathNames) { lastPathName = pathName; - storeEngine.validateStoreFile(pathName); + storeEngine.validateStoreFile(pathName, false); } return pathNames; } catch (Exception e) { @@ -1118,7 +1118,7 @@ public void deleteChangedReaderObserver(ChangedReadersObserver o) { * block for long periods. *

* During this time, the Store can work as usual, getting values from StoreFiles and writing new - * StoreFiles from the memstore. Existing StoreFiles are not destroyed until the new compacted + * StoreFiles from the MemStore. Existing StoreFiles are not destroyed until the new compacted * StoreFile is completely written-out to disk. *

* The compactLock prevents multiple simultaneous compactions. The structureLock prevents us from @@ -1129,21 +1129,29 @@ public void deleteChangedReaderObserver(ChangedReadersObserver o) { *

* Compaction event should be idempotent, since there is no IO Fencing for the region directory in * hdfs. A region server might still try to complete the compaction after it lost the region. That - * is why the following events are carefully ordered for a compaction: 1. Compaction writes new - * files under region/.tmp directory (compaction output) 2. Compaction atomically moves the - * temporary file under region directory 3. Compaction appends a WAL edit containing the - * compaction input and output files. Forces sync on WAL. 4. Compaction deletes the input files - * from the region directory. Failure conditions are handled like this: - If RS fails before 2, - * compaction wont complete. Even if RS lives on and finishes the compaction later, it will only - * write the new data file to the region directory. Since we already have this data, this will be - * idempotent but we will have a redundant copy of the data. - If RS fails between 2 and 3, the - * region will have a redundant copy of the data. The RS that failed won't be able to finish - * sync() for WAL because of lease recovery in WAL. - If RS fails after 3, the region region - * server who opens the region will pick up the the compaction marker from the WAL and replay it - * by removing the compaction input files. Failed RS can also attempt to delete those files, but - * the operation will be idempotent See HBASE-2231 for details. + * is why the following events are carefully ordered for a compaction: + *

    + *
  1. Compaction writes new files under region/.tmp directory (compaction output)
    2. + *
  2. Compaction atomically moves the temporary file under region directory
    3. + *
  3. Compaction appends a WAL edit containing the compaction input and output files. Forces sync + * on WAL.
    4. + *
  4. Compaction deletes the input files from the region directory.
    5. + *
+ * Failure conditions are handled like this: + *
    + *
  • If RS fails before 2, compaction won't complete. Even if RS lives on and finishes the + * compaction later, it will only write the new data file to the region directory. Since we + * already have this data, this will be idempotent, but we will have a redundant copy of the + * data.
    • + *
  • If RS fails between 2 and 3, the region will have a redundant copy of the data. The RS that + * failed won't be able to finish sync() for WAL because of lease recovery in WAL.
    • + *
  • If RS fails after 3, the region server who opens the region will pick up the compaction + * marker from the WAL and replay it by removing the compaction input files. Failed RS can also + * attempt to delete those files, but the operation will be idempotent
    • + *
+ * See HBASE-2231 for details. * @param compaction compaction details obtained from requestCompaction() - * @return Storefile we compacted into or null if we failed or opted out early. + * @return The storefiles that we compacted into or null if we failed or opted out early. */ public List compact(CompactionContext compaction, ThroughputController throughputController, User user) throws IOException { @@ -1186,7 +1194,7 @@ protected List doCompaction(CompactionRequestImpl cr, throws IOException { // Do the steps necessary to complete the compaction. setStoragePolicyFromFileName(newFiles); - List sfs = storeEngine.commitStoreFiles(newFiles, true); + List sfs = storeEngine.commitStoreFiles(newFiles, true, true); if (this.getCoprocessorHost() != null) { for (HStoreFile sf : sfs) { getCoprocessorHost().postCompact(this, sf, cr.getTracker(), cr, user); @@ -1978,7 +1986,7 @@ public boolean commit(MonitoredTask status) throws IOException { return false; } status.setStatus("Flushing " + this + ": reopening flushed file"); - List storeFiles = storeEngine.commitStoreFiles(tempFiles, false); + List storeFiles = storeEngine.commitStoreFiles(tempFiles, false, false); for (HStoreFile sf : storeFiles) { StoreFileReader r = sf.getReader(); if (LOG.isInfoEnabled()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index 5923befbc9de..8d81c90144ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -36,7 +36,9 @@ import java.util.function.Function; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.hfile.BloomFilterMetrics; import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; @@ -95,6 +97,9 @@ public abstract class StoreEngine openStoreFiles(Collection files, boolean } if (ioe != null) { // close StoreFile readers - boolean evictOnClose = - ctx.getCacheConf() != null ? ctx.getCacheConf().shouldEvictOnClose() : true; + boolean evictOnClose = ctx.getCacheConf() == null || ctx.getCacheConf().shouldEvictOnClose(); for (HStoreFile file : results) { try { if (file != null) { @@ -315,10 +341,8 @@ private List openStoreFiles(Collection files, boolean for (HStoreFile storeFile : results) { if (compactedStoreFiles.contains(storeFile.getPath().getName())) { LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); - storeFile.getReader() - .close(storeFile.getCacheConf() != null - ? storeFile.getCacheConf().shouldEvictOnClose() - : true); + storeFile.getReader().close( + storeFile.getCacheConf() == null || storeFile.getCacheConf().shouldEvictOnClose()); filesToRemove.add(storeFile); } } @@ -380,7 +404,7 @@ private void refreshStoreFilesInternal(Collection newFiles) throw compactedFilesSet.put(sf.getFileInfo(), sf); } - Set newFilesSet = new HashSet(newFiles); + Set newFilesSet = new HashSet<>(newFiles); // Exclude the files that have already been compacted newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); @@ -390,8 +414,8 @@ private void refreshStoreFilesInternal(Collection newFiles) throw return; } - LOG.info("Refreshing store files for " + this + " files to add: " + toBeAddedFiles - + " files to remove: " + toBeRemovedFiles); + LOG.info("Refreshing store files for {} files to add: {} files to remove: {}", this, + toBeAddedFiles, toBeRemovedFiles); Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); for (StoreFileInfo sfi : toBeRemovedFiles) { @@ -401,7 +425,7 @@ private void refreshStoreFilesInternal(Collection newFiles) throw // try to open the files List openedFiles = openStoreFiles(toBeAddedFiles, false); - // propogate the file changes to the underlying store file manager + // propagate the file changes to the underlying store file manager replaceStoreFiles(toBeRemovedStoreFiles, openedFiles, () -> { }, () -> { }); // won't throw an exception @@ -411,11 +435,13 @@ private void refreshStoreFilesInternal(Collection newFiles) throw * Commit the given {@code files}. *

* We will move the file into data directory, and open it. - * @param files the files want to commit - * @param validate whether to validate the store files + * @param files the files want to commit + * @param isCompaction whether this is called from the context of a compaction + * @param validate whether to validate the store files * @return the committed store files */ - public List commitStoreFiles(List files, boolean validate) throws IOException { + public List commitStoreFiles(List files, boolean isCompaction, boolean validate) + throws IOException { List committedFiles = new ArrayList<>(files.size()); HRegionFileSystem hfs = ctx.getRegionFileSystem(); String familyName = ctx.getFamily().getNameAsString(); @@ -423,13 +449,13 @@ public List commitStoreFiles(List files, boolean validate) thr for (Path file : files) { try { if (validate) { - validateStoreFile(file); + validateStoreFile(file, isCompaction); } Path committedPath; // As we want to support writing to data directory directly, here we need to check whether // the store file is already in the right place if (file.getParent() != null && file.getParent().equals(storeDir)) { - // already in the right place, skip renmaing + // already in the right place, skip renaming committedPath = file; } else { // Write-out finished successfully, move into the right spot diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index 332ecd8a95a0..86fe54c98151 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -23,6 +23,12 @@ import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER; import static org.apache.hadoop.hbase.regionserver.compactions.CloseChecker.SIZE_LIMIT_KEY; import static org.apache.hadoop.hbase.regionserver.compactions.CloseChecker.TIME_LIMIT_KEY; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -35,13 +41,17 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -60,6 +70,7 @@ import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; @@ -75,6 +86,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.io.IOUtils; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -86,6 +98,7 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; /** * Test compaction framework and common functions @@ -114,8 +127,6 @@ public class TestCompaction { /** constructor */ public TestCompaction() { - super(); - // Set cache flush size to 1MB conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100); @@ -143,6 +154,12 @@ public void setUp() throws Exception { hcd.setMaxVersions(65536); this.htd.addFamily(hcd); } + if (name.getMethodName().equals("testCompactionWithCorruptBlock")) { + UTIL.getConfiguration().setBoolean("hbase.hstore.validate.read_fully", true); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + hcd.setCompressionType(Compression.Algorithm.GZ); + this.htd.addFamily(hcd); + } this.r = UTIL.createLocalHRegion(htd, null, null); } @@ -354,6 +371,7 @@ public void testCompactionWithCorruptResult() throws Exception { try (FSDataOutputStream stream = fs.create(tmpPath, null, true, 512, (short) 3, 1024L, null)) { stream.writeChars("CORRUPT FILE!!!!"); } + // The complete compaction should fail and the corrupt file should remain // in the 'tmp' directory; assertThrows(IOException.class, () -> store.doCompaction(null, null, null, @@ -361,6 +379,59 @@ public void testCompactionWithCorruptResult() throws Exception { assertTrue(fs.exists(tmpPath)); } + /** + * This test uses a hand-modified HFile, which is loaded in from the resources' path. That file + * was generated from the test support code in this class and then edited to corrupt the + * GZ-encoded block by zeroing-out the first two bytes of the GZip header, the "standard + * declaration" of {@code 1f 8b}, found at offset 33 in the file. I'm not sure why, but it seems + * that in this test context we do not enforce CRC checksums. Thus, this corruption manifests in + * the Decompressor rather than in the reader when it loads the block bytes and compares vs. the + * header. + */ + @Test + public void testCompactionWithCorruptBlock() throws Exception { + createStoreFile(r, Bytes.toString(FAMILY)); + createStoreFile(r, Bytes.toString(FAMILY)); + HStore store = r.getStore(FAMILY); + + Collection storeFiles = store.getStorefiles(); + DefaultCompactor tool = (DefaultCompactor) store.storeEngine.getCompactor(); + CompactionRequestImpl request = new CompactionRequestImpl(storeFiles); + tool.compact(request, NoLimitThroughputController.INSTANCE, null); + + // insert the hfile with a corrupted data block into the region's tmp directory, where + // compaction output is collected. + FileSystem fs = store.getFileSystem(); + Path tmpPath = store.getRegionFileSystem().createTempName(); + try ( + InputStream inputStream = + getClass().getResourceAsStream("TestCompaction_HFileWithCorruptBlock.gz"); + GZIPInputStream gzipInputStream = new GZIPInputStream(Objects.requireNonNull(inputStream)); + OutputStream outputStream = fs.create(tmpPath, null, true, 512, (short) 3, 1024L, null)) { + assertThat(gzipInputStream, notNullValue()); + assertThat(outputStream, notNullValue()); + IOUtils.copyBytes(gzipInputStream, outputStream, 512); + } + LoggerFactory.getLogger(TestCompaction.class).info("Wrote corrupted HFile to {}", tmpPath); + + // The complete compaction should fail and the corrupt file should remain + // in the 'tmp' directory; + try { + store.doCompaction(request, storeFiles, null, EnvironmentEdgeManager.currentTime(), + Collections.singletonList(tmpPath)); + } catch (IOException e) { + Throwable rootCause = e; + while (rootCause.getCause() != null) { + rootCause = rootCause.getCause(); + } + assertThat(rootCause, allOf(instanceOf(IOException.class), + hasProperty("message", containsString("not a gzip file")))); + assertTrue(fs.exists(tmpPath)); + return; + } + fail("Compaction should have failed due to corrupt block"); + } + /** * Create a custom compaction request and be sure that we can track it through the queue, knowing * when the compaction is completed. diff --git a/hbase-server/src/test/resources/org/apache/hadoop/hbase/regionserver/TestCompaction_HFileWithCorruptBlock.gz b/hbase-server/src/test/resources/org/apache/hadoop/hbase/regionserver/TestCompaction_HFileWithCorruptBlock.gz new file mode 100644 index 0000000000000000000000000000000000000000..c93407b455c8c00c2d7745ae4c0faeaa1834a017 GIT binary patch literal 952 zcmV;p14sNHiwFo{hP-D215{;mbVF}#aA9L~X>V>{NJeRFWmjo*XhUyua&>TYLTqni zYXEa`3~_Yw@z!EsVA#sQz;yLL6fiL`H~_^ME-^50FhD@)$>XAoj3TTT&iBZ;FB4jN z!|&Y1BTj2fT-_ro^DEDb3i`T;1eV0UpWWy2zHqN?{HuPZ%mOK{WhXON%@A8|dSHv? zTg7FEU-lTp`jz**EUXgtnSXhH65E`EJjPFWp3e|W-*TYG^08{#p_yw8g3p!o&MZum zN;@|*-+=pEQg`L^J?{;?&lUAY7KX8(`8EH2{x$bAZJR9Fx|S!m2O57AkN-ljC3&OB^cqFSzKx=j6r&h%_^ zLCt>)cx4@`F1A{l9eute;HT(|u>7fQxg5?fs+=4BRnqFW2*w=j6y@SQAC!{L`|;iV ziVNGX{yzS?C4A+4l}^))7gFE92i;vFXl>B8LM*xYPVCwD7JE*zsTr)0wF^}@5zTzn zFTQw+PR`Ctrx#BTWfnQWJLg(mz3lzO*OBjSXEWr@_F?~|zxk!jKURBQ#>?EmfL&dG za0)1K`MQE5%mj!n(Gx%*aso(5NNZqVRY+q`a?^+x^72;fG>l<>B=PLX5f_HG14R** zKvj~fAN>po0%b!b6>pB+%mSO3B;*wCX=^YVFjz=BFnTZqFhm*T7p3bZ79=KTr0QiP zrsU@r=w&1&7N_cY=H;ap1()O(r8=kPRB!U=)mkQ7{Td a!6+C7qhJ(_f Date: Fri, 21 Mar 2025 08:05:03 -0400 Subject: [PATCH 24/37] HubSpot Backport: HBASE-29202 Balancer conditionals make balancer actions more likely to be approved (#6821) (will be in 2.7) Co-authored-by: Ray Mattingly Signed-off-by: Nick Dimiduk --- .../master/balancer/BalancerConditionals.java | 2 +- .../balancer/CandidateGeneratorTestUtil.java | 32 ++++-- ...gTableIsolationAndReplicaDistribution.java | 8 +- .../TestUnattainableBalancerCostGoal.java | 108 ++++++++++++++++++ 4 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java index 021a34bce6b9..b82c68b37da3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -146,7 +146,7 @@ int getViolationCountChange(BalancerClusterState cluster, BalanceAction action) // Reset cluster cluster.doAction(undoAction); - if (isViolatingPre && isViolatingPost) { + if (isViolatingPre == isViolatingPost) { return 0; } else if (!isViolatingPre && isViolatingPost) { return 1; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java index 03bfcce8e150..d2a9d17cdba0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java @@ -51,16 +51,22 @@ public final class CandidateGeneratorTestUtil { private CandidateGeneratorTestUtil() { } + enum ExhaustionType { + COST_GOAL_ACHIEVED, + NO_MORE_MOVES; + } + static void runBalancerToExhaustion(Configuration conf, Map> serverToRegions, Set> expectations, float targetMaxBalancerCost) { - runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000); + runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000, + ExhaustionType.COST_GOAL_ACHIEVED); } static void runBalancerToExhaustion(Configuration conf, Map> serverToRegions, Set> expectations, float targetMaxBalancerCost, - long maxRunningTime) { + long maxRunningTime, ExhaustionType exhaustionType) { // Do the full plan. We're testing with a lot of regions conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); conf.setLong(MAX_RUNNING_TIME_KEY, maxRunningTime); @@ -76,7 +82,7 @@ static void runBalancerToExhaustion(Configuration conf, boolean isBalanced = false; while (!isBalanced) { balancerRuns++; - if (balancerRuns > 1000) { + if (balancerRuns > 10) { throw new RuntimeException("Balancer failed to find balance & meet expectations"); } long start = System.currentTimeMillis(); @@ -111,16 +117,24 @@ static void runBalancerToExhaustion(Configuration conf, } } if (isBalanced) { // Check if the balancer thinks we're done too - LOG.info("All balancer conditions passed. Checking if balancer thinks it's done."); - if (stochasticLoadBalancer.needsBalance(HConstants.ENSEMBLE_TABLE_NAME, cluster)) { - LOG.info("Balancer would still like to run"); - isBalanced = false; + if (exhaustionType == ExhaustionType.COST_GOAL_ACHIEVED) { + // If we expect to achieve the cost goal, then needsBalance should be false + if (stochasticLoadBalancer.needsBalance(HConstants.ENSEMBLE_TABLE_NAME, cluster)) { + LOG.info("Balancer cost goal is not achieved. needsBalance=true"); + isBalanced = false; + } } else { - LOG.info("Balancer is done"); + // If we anticipate running out of moves, then our last balance run should have produced + // nothing + if (regionPlans != null && !regionPlans.isEmpty()) { + LOG.info("Balancer is not out of moves. regionPlans.size()={}", regionPlans.size()); + isBalanced = false; + } } } } - LOG.info("Balancing took {}sec", Duration.ofMillis(balancingMillis).toMinutes()); + LOG.info("Balancer is done. Balancing took {}sec", + Duration.ofMillis(balancingMillis).toMinutes()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java index 3a28ae801e4e..bc31530f4921 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java @@ -104,10 +104,10 @@ public void testTableIsolationAndReplicaDistribution() { conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf); - - runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isMetaTableIsolated, - this::isSystemTableIsolated, CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f, - 60_000); + runBalancerToExhaustion(conf, serverToRegions, + ImmutableSet.of(this::isMetaTableIsolated, this::isSystemTableIsolated, + CandidateGeneratorTestUtil::areAllReplicasDistributed), + 10.0f, 60_000, CandidateGeneratorTestUtil.ExhaustionType.COST_GOAL_ACHIEVED); LOG.info("Meta table regions are successfully isolated, " + "and region replicas are appropriately distributed."); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java new file mode 100644 index 000000000000..ffa2b4a78212 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated; +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * If your minCostNeedsBalance is set too low, then the balancer should still eventually stop making + * moves as further cost improvements become impossible, and balancer plan calculation becomes + * wasteful. This test ensures that the balancer will not get stuck in a loop of continuously moving + * regions. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestUnattainableBalancerCostGoal { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestUnattainableBalancerCostGoal.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestUnattainableBalancerCostGoal.class); + + private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); + private static final TableName NON_SYSTEM_TABLE_NAME = TableName.valueOf("userTable"); + + private static final int NUM_SERVERS = 10; + private static final int NUM_REGIONS = 1000; + private static final float UNACHIEVABLE_COST_GOAL = 0.01f; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + } + + // Create regions + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName = i < 3 ? SYSTEM_TABLE_NAME : NON_SYSTEM_TABLE_NAME; + byte[] startKey = new byte[1]; + startKey[0] = (byte) i; + byte[] endKey = new byte[1]; + endKey[0] = (byte) (i + 1); + + RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey).build(); + allRegions.add(regionInfo); + } + + // Assign all regions to the first server + serverToRegions.put(servers[0], new ArrayList<>(allRegions)); + for (int i = 1; i < NUM_SERVERS; i++) { + serverToRegions.put(servers[i], new ArrayList<>()); + } + } + + @Test + public void testSystemTableIsolation() { + Configuration conf = new Configuration(false); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + runBalancerToExhaustion(conf, serverToRegions, Set.of(this::isSystemTableIsolated), + UNACHIEVABLE_COST_GOAL, 10_000, CandidateGeneratorTestUtil.ExhaustionType.NO_MORE_MOVES); + LOG.info("Meta table regions are successfully isolated."); + } + + private boolean isSystemTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System"); + } +} From 3a29041b186fd3b8974cfcb6b72497c6e2574eab Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 21 Mar 2025 08:10:01 -0400 Subject: [PATCH 25/37] HubSpot Backport: HBASE-29203 There should be a StorefileSize equivalent to the TableSkewCost (#6825) (will be in 2.7) Co-authored-by: Ray Mattingly Signed-off-by: Nick Dimiduk --- .../master/balancer/BalancerClusterState.java | 4 + .../balancer/CostFromRegionLoadFunction.java | 2 +- .../balancer/StochasticLoadBalancer.java | 1 + .../StoreFileTableSkewCostFunction.java | 127 ++++++++++ .../balancer/TestStochasticLoadBalancer.java | 1 + .../TestStoreFileTableSkewCostFunction.java | 239 ++++++++++++++++++ 6 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java index b07287c1ed19..efba0aee733b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java @@ -1085,6 +1085,10 @@ boolean isStopRequested() { return EnvironmentEdgeManager.currentTime() > stopRequestedAt; } + Deque[] getRegionLoads() { + return regionLoads; + } + @Override public String toString() { StringBuilder desc = new StringBuilder("Cluster={servers=["); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java index 199aa10a75fa..bc61ead8da86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java @@ -66,7 +66,7 @@ protected void regionMoved(int region, int oldServer, int newServer) { } @Override - protected final double cost() { + protected double cost() { return cost.cost(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index d184cf52e80f..44e5aad3a6b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -274,6 +274,7 @@ protected List createCostFunctions(Configuration conf) { addCostFunction(costFunctions, localityCost); addCostFunction(costFunctions, rackLocalityCost); addCostFunction(costFunctions, new TableSkewCostFunction(conf)); + addCostFunction(costFunctions, new StoreFileTableSkewCostFunction(conf)); addCostFunction(costFunctions, regionReplicaHostCostFunction); addCostFunction(costFunctions, regionReplicaRackCostFunction); addCostFunction(costFunctions, new ReadRequestCostFunction(conf)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java new file mode 100644 index 000000000000..d37f8caa72e1 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.Collection; +import org.apache.hadoop.conf.Configuration; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Lightweight cost function that mirrors TableSkewCostFunction but aggregates storefile sizes (in + * MB) per table using the CostFromRegionLoadFunction framework. For each table, it computes a + * per-server aggregated storefile size by summing the average storefile size for each region (if + * there are multiple load metrics, it averages them). The imbalance cost (as computed by + * DoubleArrayCost) is then used to drive the balancer to reduce differences between servers. + */ +@InterfaceAudience.Private +public class StoreFileTableSkewCostFunction extends CostFromRegionLoadFunction { + + private static final String STOREFILE_TABLE_SKEW_COST_KEY = + "hbase.master.balancer.stochastic.storefileTableSkewCost"; + private static final float DEFAULT_STOREFILE_TABLE_SKEW_COST = 35; + + // One DoubleArrayCost instance per table. + private DoubleArrayCost[] costsPerTable; + + public StoreFileTableSkewCostFunction(Configuration conf) { + this.setMultiplier( + conf.getFloat(STOREFILE_TABLE_SKEW_COST_KEY, DEFAULT_STOREFILE_TABLE_SKEW_COST)); + } + + @Override + public void prepare(BalancerClusterState cluster) { + // First, set the cluster state and allocate one DoubleArrayCost per table. + this.cluster = cluster; + costsPerTable = new DoubleArrayCost[cluster.numTables]; + for (int tableIdx = 0; tableIdx < cluster.numTables; tableIdx++) { + costsPerTable[tableIdx] = new DoubleArrayCost(); + costsPerTable[tableIdx].prepare(cluster.numServers); + final int tableIndex = tableIdx; + costsPerTable[tableIdx].applyCostsChange(costs -> { + // For each server, compute the aggregated storefile size for this table. + for (int server = 0; server < cluster.numServers; server++) { + double totalStorefileMB = 0; + // Sum over all regions on this server that belong to the given table. + for (int region : cluster.regionsPerServer[server]) { + if (cluster.regionIndexToTableIndex[region] == tableIndex) { + Collection loads = cluster.getRegionLoads()[region]; + double regionCost = 0; + if (loads != null && !loads.isEmpty()) { + // Average the storefile sizes if there are multiple measurements. + for (BalancerRegionLoad rl : loads) { + regionCost += getCostFromRl(rl); + } + regionCost /= loads.size(); + } + totalStorefileMB += regionCost; + } + } + costs[server] = totalStorefileMB; + } + }); + } + } + + @Override + protected void regionMoved(int region, int oldServer, int newServer) { + // Determine the affected table. + int tableIdx = cluster.regionIndexToTableIndex[region]; + costsPerTable[tableIdx].applyCostsChange(costs -> { + // Recompute for the old server if applicable. + updateStoreFilePerServerPerTableCosts(oldServer, tableIdx, costs); + // Recompute for the new server. + updateStoreFilePerServerPerTableCosts(newServer, tableIdx, costs); + }); + } + + private void updateStoreFilePerServerPerTableCosts(int newServer, int tableIdx, double[] costs) { + if (newServer >= 0) { + double totalStorefileMB = 0; + for (int r : cluster.regionsPerServer[newServer]) { + if (cluster.regionIndexToTableIndex[r] == tableIdx) { + Collection loads = cluster.getRegionLoads()[r]; + double regionCost = 0; + if (loads != null && !loads.isEmpty()) { + for (BalancerRegionLoad rl : loads) { + regionCost += getCostFromRl(rl); + } + regionCost /= loads.size(); + } + totalStorefileMB += regionCost; + } + } + costs[newServer] = totalStorefileMB; + } + } + + @Override + protected double cost() { + double totalCost = 0; + // Sum the imbalance cost over all tables. + for (DoubleArrayCost dac : costsPerTable) { + totalCost += dac.cost(); + } + return totalCost; + } + + @Override + protected double getCostFromRl(BalancerRegionLoad rl) { + // Use storefile size in MB as the metric. + return rl.getStorefileSizeMB(); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index 9dc7dab65621..661380814ad9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -531,6 +531,7 @@ public void testDefaultCostFunctionList() { PrimaryRegionCountSkewCostFunction.class.getSimpleName(), MoveCostFunction.class.getSimpleName(), RackLocalityCostFunction.class.getSimpleName(), TableSkewCostFunction.class.getSimpleName(), + StoreFileTableSkewCostFunction.class.getSimpleName(), RegionReplicaHostCostFunction.class.getSimpleName(), RegionReplicaRackCostFunction.class.getSimpleName(), ReadRequestCostFunction.class.getSimpleName(), WriteRequestCostFunction.class.getSimpleName(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java new file mode 100644 index 000000000000..619a055c6502 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.createMockBalancerClusterState; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Size; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category({ MasterTests.class, SmallTests.class }) +public class TestStoreFileTableSkewCostFunction { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStoreFileTableSkewCostFunction.class); + + private static final TableName DEFAULT_TABLE = TableName.valueOf("testTable"); + private static final Map REGION_TO_STORE_FILE_SIZE_MB = new HashMap<>(); + + /** + * Tests that a uniform store file distribution (single table) across servers results in zero + * cost. + */ + @Test + public void testUniformDistribution() { + ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L); + ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L); + ServerName server3 = ServerName.valueOf("server3.example.org", 1234, 1L); + ServerName server4 = ServerName.valueOf("server4.example.org", 1234, 1L); + + Map> serverToRegions = new HashMap<>(); + serverToRegions.put(server1, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server2, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server3, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server4, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect zero cost since all regions (from the same table) are balanced. + assertEquals("Uniform distribution should yield zero cost", 0.0, cost, 1e-6); + } + + /** + * Tests that a skewed store file distribution (single table) results in a positive cost. + */ + @Test + public void testSkewedDistribution() { + ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L); + ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L); + ServerName server3 = ServerName.valueOf("server3.example.org", 1234, 1L); + ServerName server4 = ServerName.valueOf("server4.example.org", 1234, 1L); + + Map> serverToRegions = new HashMap<>(); + // Three servers get regions with 10 store files each, + // while one server gets regions with 30 store files each. + serverToRegions.put(server1, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server2, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server3, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server4, Arrays.asList(createMockRegionInfo(30), createMockRegionInfo(30))); + + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect a positive cost because the distribution is skewed. + assertTrue("Skewed distribution should yield a positive cost", cost > 0.0); + } + + /** + * Tests that an empty cluster (no servers/regions) is handled gracefully. + */ + @Test + public void testEmptyDistribution() { + Map> serverToRegions = new HashMap<>(); + + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect zero cost when there is no load. + assertEquals("Empty distribution should yield zero cost", 0.0, cost, 1e-6); + } + + /** + * Tests that having multiple tables results in a positive cost when each table's regions are not + * balanced across servers – even if the overall load per server is balanced. + */ + @Test + public void testMultipleTablesDistribution() { + // Two servers. + ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L); + ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L); + + // Define two tables. + TableName table1 = TableName.valueOf("testTable1"); + TableName table2 = TableName.valueOf("testTable2"); + + // For table1, all regions are on server1. + // For table2, all regions are on server2. + Map> serverToRegions = new HashMap<>(); + serverToRegions.put(server1, + Arrays.asList(createMockRegionInfo(table1, 10), createMockRegionInfo(table1, 10))); + serverToRegions.put(server2, + Arrays.asList(createMockRegionInfo(table2, 10), createMockRegionInfo(table2, 10))); + + // Although each server gets 20 MB overall, table1 and table2 are not balanced across servers. + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect a positive cost because the skew is computed per table. + assertTrue("Multiple table distribution should yield a positive cost", cost > 0.0); + } + + /** + * Helper method to create a RegionInfo for the default table with the given store file size. + */ + private static RegionInfo createMockRegionInfo(int storeFileSizeMb) { + return createMockRegionInfo(DEFAULT_TABLE, storeFileSizeMb); + } + + /** + * Helper method to create a RegionInfo for a specified table with the given store file size. + */ + private static RegionInfo createMockRegionInfo(TableName table, int storeFileSizeMb) { + long regionId = new Random().nextLong(); + REGION_TO_STORE_FILE_SIZE_MB.put(regionId, storeFileSizeMb); + return RegionInfoBuilder.newBuilder(table).setStartKey(generateRandomByteArray(4)) + .setEndKey(generateRandomByteArray(4)).setReplicaId(0).setRegionId(regionId).build(); + } + + private static byte[] generateRandomByteArray(int n) { + byte[] byteArray = new byte[n]; + new Random().nextBytes(byteArray); + return byteArray; + } + + /** + * A simplified BalancerClusterState which ensures we provide the intended test RegionMetrics data + * when balancing this cluster + */ + private static class DummyBalancerClusterState extends BalancerClusterState { + private final RegionInfo[] testRegions; + + DummyBalancerClusterState(BalancerClusterState bcs) { + super(bcs.clusterState, null, null, null, null); + this.testRegions = bcs.regions; + } + + @Override + Deque[] getRegionLoads() { + @SuppressWarnings("unchecked") + Deque[] loads = new Deque[testRegions.length]; + for (int i = 0; i < testRegions.length; i++) { + Deque dq = new ArrayDeque<>(); + dq.add(new BalancerRegionLoad(createMockRegionMetrics(testRegions[i])) { + }); + loads[i] = dq; + } + return loads; + } + } + + /** + * Creates a mocked RegionMetrics for the given region. + */ + private static RegionMetrics createMockRegionMetrics(RegionInfo regionInfo) { + RegionMetrics regionMetrics = Mockito.mock(RegionMetrics.class); + + // Important + int storeFileSizeMb = REGION_TO_STORE_FILE_SIZE_MB.get(regionInfo.getRegionId()); + when(regionMetrics.getRegionSizeMB()).thenReturn(new Size(storeFileSizeMb, Size.Unit.MEGABYTE)); + when(regionMetrics.getStoreFileSize()) + .thenReturn(new Size(storeFileSizeMb, Size.Unit.MEGABYTE)); + + // Not important + when(regionMetrics.getReadRequestCount()).thenReturn(0L); + when(regionMetrics.getCpRequestCount()).thenReturn(0L); + when(regionMetrics.getWriteRequestCount()).thenReturn(0L); + when(regionMetrics.getMemStoreSize()).thenReturn(new Size(0, Size.Unit.MEGABYTE)); + when(regionMetrics.getCurrentRegionCachedRatio()).thenReturn(0.0f); + return regionMetrics; + } +} From 8e5892c99029217d7a566d24de394c989a58695d Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 21 Mar 2025 08:54:04 -0400 Subject: [PATCH 26/37] HubSpot Edit: I messed up 29202, 29203 backports with incompatibilities. Can squash this, or delete in 2.7 (#167) Co-authored-by: Ray Mattingly --- .../master/balancer/TestStoreFileTableSkewCostFunction.java | 1 - .../master/balancer/TestUnattainableBalancerCostGoal.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java index 619a055c6502..3977ad96dd9a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java @@ -230,7 +230,6 @@ private static RegionMetrics createMockRegionMetrics(RegionInfo regionInfo) { // Not important when(regionMetrics.getReadRequestCount()).thenReturn(0L); - when(regionMetrics.getCpRequestCount()).thenReturn(0L); when(regionMetrics.getWriteRequestCount()).thenReturn(0L); when(regionMetrics.getMemStoreSize()).thenReturn(new Size(0, Size.Unit.MEGABYTE)); when(regionMetrics.getCurrentRegionCachedRatio()).thenReturn(0.0f); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java index ffa2b4a78212..5e95564b6fee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -97,7 +98,7 @@ public static void setup() { public void testSystemTableIsolation() { Configuration conf = new Configuration(false); conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); - runBalancerToExhaustion(conf, serverToRegions, Set.of(this::isSystemTableIsolated), + runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isSystemTableIsolated), UNACHIEVABLE_COST_GOAL, 10_000, CandidateGeneratorTestUtil.ExhaustionType.NO_MORE_MOVES); LOG.info("Meta table regions are successfully isolated."); } From 4996ad7ada9b5ae7ef0c00e7cae55fb07340c3e3 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Mon, 17 Mar 2025 08:34:08 -0400 Subject: [PATCH 27/37] HubSpot Backport: HBASE-29193: Allow ZstdByteBuffDecompressor to take direct ByteBuffer as input and heap ByteBuffer as output, or vice versa (not yet merged upstream) --- .../zstd/ZstdByteBuffDecompressor.java | 67 +++++++------------ .../zstd/TestZstdByteBuffDecompressor.java | 32 +++++++-- pom.xml | 2 +- 3 files changed, 55 insertions(+), 46 deletions(-) diff --git a/hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdByteBuffDecompressor.java b/hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdByteBuffDecompressor.java index ec5315aa4c02..d71d46e2946e 100644 --- a/hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdByteBuffDecompressor.java +++ b/hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdByteBuffDecompressor.java @@ -55,20 +55,8 @@ public class ZstdByteBuffDecompressor implements ByteBuffDecompressor, CanReinit @Override public boolean canDecompress(ByteBuff output, ByteBuff input) { - if (!allowByteBuffDecompression) { - return false; - } - if (output instanceof SingleByteBuff && input instanceof SingleByteBuff) { - ByteBuffer nioOutput = output.nioByteBuffers()[0]; - ByteBuffer nioInput = input.nioByteBuffers()[0]; - if (nioOutput.isDirect() && nioInput.isDirect()) { - return true; - } else if (!nioOutput.isDirect() && !nioInput.isDirect()) { - return true; - } - } - - return false; + return allowByteBuffDecompression && output instanceof SingleByteBuff + && input instanceof SingleByteBuff; } @Override @@ -80,38 +68,35 @@ private int decompressRaw(ByteBuff output, ByteBuff input, int inputLen) throws if (output instanceof SingleByteBuff && input instanceof SingleByteBuff) { ByteBuffer nioOutput = output.nioByteBuffers()[0]; ByteBuffer nioInput = input.nioByteBuffers()[0]; + int origOutputPos = nioOutput.position(); + int n; if (nioOutput.isDirect() && nioInput.isDirect()) { - return decompressDirectByteBuffers(nioOutput, nioInput, inputLen); + n = ctx.decompressDirectByteBuffer(nioOutput, nioOutput.position(), + nioOutput.limit() - nioOutput.position(), nioInput, nioInput.position(), inputLen); } else if (!nioOutput.isDirect() && !nioInput.isDirect()) { - return decompressHeapByteBuffers(nioOutput, nioInput, inputLen); + n = ctx.decompressByteArray(nioOutput.array(), + nioOutput.arrayOffset() + nioOutput.position(), nioOutput.limit() - nioOutput.position(), + nioInput.array(), nioInput.arrayOffset() + nioInput.position(), inputLen); + } else if (nioOutput.isDirect() && !nioInput.isDirect()) { + n = ctx.decompressByteArrayToDirectByteBuffer(nioOutput, nioOutput.position(), + nioOutput.limit() - nioOutput.position(), nioInput.array(), + nioInput.arrayOffset() + nioInput.position(), inputLen); + } else if (!nioOutput.isDirect() && nioInput.isDirect()) { + n = ctx.decompressDirectByteBufferToByteArray(nioOutput.array(), + nioOutput.arrayOffset() + nioOutput.position(), nioOutput.limit() - nioOutput.position(), + nioInput, nioInput.position(), inputLen); + } else { + throw new IllegalStateException("Unreachable line"); } - } - - throw new IllegalStateException("One buffer is direct and the other is not, " - + "or one or more not SingleByteBuffs. This is not supported"); - } - private int decompressDirectByteBuffers(ByteBuffer output, ByteBuffer input, int inputLen) { - int origOutputPos = output.position(); + nioOutput.position(origOutputPos + n); + nioInput.position(input.position() + inputLen); - int n = ctx.decompressDirectByteBuffer(output, output.position(), - output.limit() - output.position(), input, input.position(), inputLen); - - output.position(origOutputPos + n); - input.position(input.position() + inputLen); - return n; - } - - private int decompressHeapByteBuffers(ByteBuffer output, ByteBuffer input, int inputLen) { - int origOutputPos = output.position(); - - int n = ctx.decompressByteArray(output.array(), output.arrayOffset() + output.position(), - output.limit() - output.position(), input.array(), input.arrayOffset() + input.position(), - inputLen); - - output.position(origOutputPos + n); - input.position(input.position() + inputLen); - return n; + return n; + } else { + throw new IllegalStateException( + "At least one buffer is not a SingleByteBuff, this is not supported"); + } } @Override diff --git a/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestZstdByteBuffDecompressor.java b/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestZstdByteBuffDecompressor.java index 86ba921afdbb..94e95e1ae02b 100644 --- a/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestZstdByteBuffDecompressor.java +++ b/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestZstdByteBuffDecompressor.java @@ -60,8 +60,8 @@ public void testCapabilities() { try (ZstdByteBuffDecompressor decompressor = new ZstdByteBuffDecompressor(null)) { assertTrue(decompressor.canDecompress(emptySingleHeapBuff, emptySingleHeapBuff)); assertTrue(decompressor.canDecompress(emptySingleDirectBuff, emptySingleDirectBuff)); - assertFalse(decompressor.canDecompress(emptySingleHeapBuff, emptySingleDirectBuff)); - assertFalse(decompressor.canDecompress(emptySingleDirectBuff, emptySingleHeapBuff)); + assertTrue(decompressor.canDecompress(emptySingleHeapBuff, emptySingleDirectBuff)); + assertTrue(decompressor.canDecompress(emptySingleDirectBuff, emptySingleHeapBuff)); assertFalse(decompressor.canDecompress(emptyMultiHeapBuff, emptyMultiHeapBuff)); assertFalse(decompressor.canDecompress(emptyMultiDirectBuff, emptyMultiDirectBuff)); assertFalse(decompressor.canDecompress(emptySingleHeapBuff, emptyMultiHeapBuff)); @@ -70,7 +70,7 @@ public void testCapabilities() { } @Test - public void testDecompressHeap() throws IOException { + public void testDecompressHeapToHeap() throws IOException { try (ZstdByteBuffDecompressor decompressor = new ZstdByteBuffDecompressor(null)) { ByteBuff output = new SingleByteBuff(ByteBuffer.allocate(64)); ByteBuff input = new SingleByteBuff(ByteBuffer.wrap(COMPRESSED_PAYLOAD)); @@ -81,7 +81,7 @@ public void testDecompressHeap() throws IOException { } @Test - public void testDecompressDirect() throws IOException { + public void testDecompressDirectToDirect() throws IOException { try (ZstdByteBuffDecompressor decompressor = new ZstdByteBuffDecompressor(null)) { ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); ByteBuff input = new SingleByteBuff(ByteBuffer.allocateDirect(COMPRESSED_PAYLOAD.length)); @@ -93,4 +93,28 @@ public void testDecompressDirect() throws IOException { } } + @Test + public void testDecompressDirectToHeap() throws IOException { + try (ZstdByteBuffDecompressor decompressor = new ZstdByteBuffDecompressor(null)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocate(64)); + ByteBuff input = new SingleByteBuff(ByteBuffer.allocateDirect(COMPRESSED_PAYLOAD.length)); + input.put(COMPRESSED_PAYLOAD); + input.rewind(); + int decompressedSize = decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + assertEquals("HBase is fun to use and very fast", + Bytes.toString(output.toBytes(0, decompressedSize))); + } + } + + @Test + public void testDecompressHeapToDirect() throws IOException { + try (ZstdByteBuffDecompressor decompressor = new ZstdByteBuffDecompressor(null)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = new SingleByteBuff(ByteBuffer.wrap(COMPRESSED_PAYLOAD)); + int decompressedSize = decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + assertEquals("HBase is fun to use and very fast", + Bytes.toString(output.toBytes(0, decompressedSize))); + } + } + } diff --git a/pom.xml b/pom.xml index ce0a86eb8ec7..0cff942e8dad 100644 --- a/pom.xml +++ b/pom.xml @@ -662,7 +662,7 @@ 1.11.0 1.8.0 1.1.10.4 - 1.5.7-1 + 1.5.7-2