diff --git a/Dockerfile b/Dockerfile index 91cd6c8..7182042 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,8 @@ -FROM python:3.8.2-alpine3.11 +FROM python:alpine3.16 +ENTRYPOINT [ "python3", "/radosgw_exporter.py" ] +EXPOSE 9242 COPY requirements.txt / +RUN pip install -U pip setuptools RUN pip install --no-cache-dir -r /requirements.txt - COPY radosgw_exporter.py / - -EXPOSE 9242 -ENTRYPOINT [ "python3", "/radosgw_exporter.py" ] \ No newline at end of file diff --git a/LICENCE b/LICENCE new file mode 100644 index 0000000..fc6c82b --- /dev/null +++ b/LICENCE @@ -0,0 +1,13 @@ + Copyright [2022] [sinamoghaddas, MiladYarmohammadi] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index fee7d76..a37f4b0 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,48 @@ ## Introduction This is Rados Gateway Exporter written in python3 for prometheus. You can run it in simple linux or container. -I've check some repos in github and gitlab, but they have some problems with ceph and radosgw. +I've check some repos in GitHub and Gitlab, but they have some problems with ceph and radosgw. ## Requirements -This script needs prometheus-client, radosgw-admin (1.7.1) and python-daemon as modules, and I wrote it with Python 3.8.2. +This script needs prometheus-client, radosgw-admin, python-daemon, and some other libraries as modules, so I wrote it with Python 3.8.2. This is `requirements.txt` : ```bash -prometheus-client -radosgw-admin==1.7.1 -python-daemon +boto==2.49.0 +docutils==0.19 +lockfile==0.12.2 +prometheus-client==0.15.0 +python-daemon==2.3.1 +radosgw-admin==1.7.2 +``` +## Preparing environment +Also you need a S3 user to get data from rados gateway. + +For creating a suitable rados user, use the following instructions: + +In case of not having radosgw-admin command, install the following package: +```bash +pip install radosgw-admin +``` +Then create rados user with these caps: +```json +"caps": [ + { "type": "buckets", + "perm": "read" }, + { "type": "usage", + "perm": "read" }, + { "type": "metadata", + "perm": "read" }, + { "type": "users", + "perm": "read" } +] +``` +By using these commands: +```commandline +radosgw-admin caps add --uid --caps "usage=read" +radosgw-admin caps add --uid --caps "metadata=read" +radosgw-admin caps add --uid --caps "buckets=read" +radosgw-admin caps add --uid --caps "users=read" ``` -Also you need a S3 user to get data from rados gateway. ## Usage ### Docker @@ -20,23 +51,33 @@ Also you need a S3 user to get data from rados gateway. docker run \ --name radosgw_exporter \ -it \ - -p 127.0.0.1:9242:9242 \ - -e ACCESS_KEY='put-your-access-key' \ - -e SECRET_KEY='put-your-secret-key' \ - -e HOST='put-your-radosgw-address' \ - -e PORT='put-your-radosgw-port' \ + -p 9242:9242 \ + -e ACCESS_KEY='PUT_YOUR_ACCESS_KEY' \ + -e SECRET_KEY='PUT_YOUR_SECRET_KEY' \ + -e HOST='PUT_YOUR_RADOSGW_ADDRESS' \ + -e PORT='PUT_YOUR_RADOSGW_PORT' \ moghaddas/radosgw_exporter ``` -### Simple +### Command Line Interface +First, clone the repository: ```bash git clone https://github.com/arvancloud/radosgw_exporter +``` +Then go to the directory +```bash cd radosgw_exporter - +``` +Then install the required packages +```bash pip install -r requirements.txt - - +``` +For printing usage info, use the following command: +```bash ./radosgw_exporter.py -h +``` +The output should be like this: +```bash usage: radosgw_exporter.py [-h] [-H HOST] [-p PORT] [-a ACCESS_KEY] [-s SECRET_KEY] [--insecure] [--debug] [-d] [--signature SIGNATURE] [-t SCRAP_TIMEOUT] [--expose-port EXPOSE_PORT] [--expose-address EXPOSE_ADDRESS] optional arguments: @@ -58,6 +99,14 @@ optional arguments: Exporter port (default 9242) --expose-address EXPOSE_ADDRESS Exporter address (default 0.0.0.0) - +``` +And here is an example for using this app: +```bash ./radosgw_exporter.py -H RADOSGW-HOST -p RADOSGW-PORT -a ACCESS_KEY -s SECRET_KEY --insecure --scrap-timeout 5 --expose-address 127.0.0.1 -``` \ No newline at end of file +``` + + +# links +https://github.com/valerytschopp/python-radosgw-admin + +https://github.com/ceph/ceph/blob/master/doc/radosgw/compression.rst diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cc6f254 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: !!str 3.9 + +services: + node-exporter: + image: moghaddas/radosgw_exporter + ports: + - "9242:9242" + restart: always + environment: + - ACCESS_KEY=PUT_YOUR_ACCESS_KEY + - SECRET_KEY=PUT_YOUR_SECRET_KEY + - HOST=PUT_YOUR_RADOSGW_ADDRESS + - PORT=PUT_YOUR_RADOSGW_PORT \ No newline at end of file diff --git a/radosgw_exporter.py b/radosgw_exporter.py index 92a9246..7ed7271 100755 --- a/radosgw_exporter.py +++ b/radosgw_exporter.py @@ -12,41 +12,15 @@ from prometheus_client import Gauge from prometheus_client import Summary - -""" -# base -pip install radosgw-admin - -# create rados user with this caps -"caps": [ - { "type": "buckets", - "perm": "*" }, - { "type": "usage", - "perm": "read" }, - { "type": "metadata", - "perm": "read" }, - { "type": "users", - "perm": "*" } -] - -radosgw-admin caps add --uid --caps "buckets=read,write" -radosgw-admin caps add --uid --caps "users=read,write" - - -# exporter -pip install prometheus_client - -# links -https://github.com/valerytschopp/python-radosgw-admin -https://github.com/ceph/ceph/blob/master/doc/radosgw/compression.rst -""" - # call argparse parser = argparse.ArgumentParser() -parser.add_argument('-H', '--host', help='Host ip or url without http/https and port (required, env HOST)',default=os.environ.get('HOST', None), required=False) -parser.add_argument('-p', '--port', help='Port address (default 443, env PORT)',default=os.environ.get('PORT', 443)) -parser.add_argument('-a', '--access-key', help='Access Key of S3 (required, env ACCESS_KEY)', default=os.environ.get('ACCESS_KEY', None), required=False) -parser.add_argument('-s', '--secret-key', help='Secret Key of S3 (required, env SECRET_KEY)', default=os.environ.get('SECRET_KEY', None), required=False) +parser.add_argument('-H', '--host', help='Host ip or url without http/https and port (required, env HOST)', + default=os.environ.get('HOST', None), required=False) +parser.add_argument('-p', '--port', help='Port address (default 443, env PORT)', default=os.environ.get('PORT', 443)) +parser.add_argument('-a', '--access-key', help='Access Key of S3 (required, env ACCESS_KEY)', + default=os.environ.get('ACCESS_KEY', None), required=False) +parser.add_argument('-s', '--secret-key', help='Secret Key of S3 (required, env SECRET_KEY)', + default=os.environ.get('SECRET_KEY', None), required=False) parser.add_argument('--insecure', help='Disable ssl/tls verification (default true)', default=True, action='store_true') parser.add_argument('--debug', help='Enable debug mode', action='store_true') parser.add_argument('-d', '--daemon', help='Enable daemon mode (default false)', action='store_true', default=False) @@ -72,6 +46,7 @@ # create a metric to track time spent and requests made. REQUEST_TIME = Summary('request_processing_seconds', 'Time spent processing request') + # check args def radosgw_args(): # check host @@ -79,7 +54,7 @@ def radosgw_args(): print("host variable doesn't exist.\n") parser.print_help(sys.stderr) sys.exit(1) - + # check access key if not access_key: print("access-key doesn't exist.\n") @@ -94,9 +69,9 @@ def radosgw_args(): # check expose address and port sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((expose_address,expose_port)) + result = sock.connect_ex((expose_address, expose_port)) if result == 0: - print(f"expose address or port aren't usefull: {expose_address}:{expose_port}") + print(f"expose address or port aren't useful: {expose_address}:{expose_port}") print(f"use other address port\n") parser.print_help(sys.stderr) sys.exit(1) @@ -120,13 +95,21 @@ def radosgw_metrics(): # set help and type date for metrics radosgw_bucket_total_metric = Gauge('radosgw_bucket_total', 'Total number of buckets') radosgw_bucket_owner_metric = Gauge('radosgw_bucket_owner', 'Name of Owner(UID) in string', ['owner']) - radosgw_bucket_num_object_metric = Gauge('radosgw_bucket_num_object_total', 'Number of Object in bucket', ['owner', 'name']) + radosgw_bucket_num_object_metric = Gauge('radosgw_bucket_num_object_total', 'Number of Object in bucket', + ['owner', 'name']) radosgw_bucket_size_metric = Gauge('radosgw_bucket_size', 'Size of bucket in bytes', ['owner', 'name']) radosgw_bucket_size_kb_metric = Gauge('radosgw_bucket_size_kb', 'Size of bucket in kilobytes', ['owner', 'name']) - radosgw_bucket_size_actual_metric = Gauge('radosgw_bucket_size_actual', 'Actual size of bucket in bytes', ['owner', 'name']) - radosgw_bucket_size_kb_actual_metric = Gauge('radosgw_bucket_size_kb_actual', 'Actual size of bucket in kilobytes', ['owner', 'name']) - radosgw_bucket_size_utilized_metric = Gauge('radosgw_bucket_size_utilized', 'Utilized size (total size of compressed data) of bucket in bytes', ['owner', 'name']) - radosgw_bucket_size_kb_utilized_metric = Gauge('radosgw_bucket_size_kb_utilized', 'Utilized size (total size of compressed data) of bucket in kilobytes', ['owner', 'name']) + radosgw_bucket_size_actual_metric = Gauge('radosgw_bucket_size_actual', 'Actual size of bucket in bytes', + ['owner', 'name']) + radosgw_bucket_size_kb_actual_metric = Gauge('radosgw_bucket_size_kb_actual', 'Actual size of bucket in kilobytes', + ['owner', 'name']) + radosgw_bucket_size_utilized_metric = Gauge('radosgw_bucket_size_utilized', + 'Utilized size (total size of compressed data) of bucket in bytes', + ['owner', 'name']) + radosgw_bucket_size_kb_utilized_metric = Gauge('radosgw_bucket_size_kb_utilized', + 'Utilized size (total size of compressed data) of bucket in ' + 'kilobytes', + ['owner', 'name']) radosgw_scrap_timeout_metric = Gauge('radosgw_scrap_timeout_seconds', 'Scrap interval in seconds') @@ -135,16 +118,24 @@ def radosgw_metrics(): def radosgw_collector(): try: # create connection to radosgw - rgwadmin = radosgw.connection.RadosGWAdminConnection(host = host, - port = port, - access_key = access_key, - secret_key = secret_key, - is_secure = not is_secure, - debug = debug, - aws_signature = signature) - + try: + rgwadmin = radosgw.connection.RadosGWAdminConnection(host=host, + port=port, + access_key=access_key, + secret_key=secret_key, + is_secure=not is_secure, + debug=debug, + aws_signature=signature) + except Exception as e: + print(e) + return 1 + # get buckets and details of them - buckets = rgwadmin.get_buckets() + try: + buckets = rgwadmin.get_buckets() + except Exception as e: + print(e) + return 1 # total bucket counter bucket_total = 0 @@ -192,15 +183,18 @@ def radosgw_collector(): radosgw_bucket_size_utilized_metric.labels(bucket_owner, bucket_name).set(bucket_size_utilized) radosgw_bucket_size_kb_utilized_metric.labels(bucket_owner, bucket_name).set(bucket_size_kb_utilized) - - except radosgw.exception.NoSuchBucket as err: - print(f"problem occured: {err}") + except radosgw.exception.NoSuchBucket as e: + print(f"problem occurred: {e}") -# main fucntion of script +# main function of script def radosgw_runner(): try: - start_http_server(expose_port, expose_address) + try: + start_http_server(expose_port, expose_address) + except Exception as e: + print(e) + return 1 print(f"exporter start listening on {expose_address}:{expose_port}") print(f"scrap every {scrap_timeout} seconds") print(f"debug mode is {debug}\n") @@ -209,10 +203,10 @@ def radosgw_runner(): print(f"exporter connect to {host}:{port}") print(f"insecure option is {is_secure}") - # call fuction of metric creation + # call function of metric creation radosgw_metrics() radosgw_scrap_timeout_metric.set(scrap_timeout) - + while True: if debug: now = datetime.datetime.now() @@ -227,20 +221,21 @@ def radosgw_runner(): print(f"start timeout for {scrap_timeout}") time.sleep(scrap_timeout) - except : - print(f"problem occured") + except Exception as e: + print(f"problem occurred") + print(e) # main if __name__ == "__main__": - # check radosgs args + # check radosgw args radosgw_args() if is_daemon: with daemon.DaemonContext( - working_directory='/tmp', - umask=0o002 - ) as context: - radosgw_runner() + working_directory='/tmp', + umask=0o002 + ) as context: + radosgw_runner() else: - radosgw_runner() \ No newline at end of file + radosgw_runner() diff --git a/requirements.txt b/requirements.txt index 64a6254..7bfb0cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -prometheus-client -radosgw-admin==1.7.1 -python-daemon \ No newline at end of file +boto==2.49.0 +docutils==0.19 +lockfile==0.12.2 +prometheus-client==0.15.0 +python-daemon==2.3.1 +radosgw-admin==1.7.2