app changed to use a file with url, not single url ; also some unit-tests added but not completely

Author: bissquit

README.md

@@ -2,17 +2,23 @@
 
 ## Usage
 
-Multiple installation scenarios are provided.
+Multiple installation scenarios are provided. To test exporter locally without real GE servers try to run a fake GE server instead:
+
+```shell script
+python3 fake_ge_server.py && echo 'http://localhost:8081' > $(pwd)/urls.txt
+```
+Don't forget to clone repo first!
 
 ### Docker
 
-Don't forget to pass valid url link to your Gradle Enterprise server:
+
 
 ```shell script
-docker run -it --rm \
-  -p 8080:8080 \
-  -e APP_URL_LINK=http://localhost \
-  -e APP_PORT=8080 \
+PORT=8080 ; docker run -it --rm \
+  -p ${PORT}:${PORT} \
+  -v "$(pwd)/":"/app/" \
+  -e APP_FILE_PATH=/app/urls.txt \
+  -e APP_PORT=${PORT} \
   -e APP_CHECK_INTERVAL=60 \
   -e LOG_LEVEL=DEBUG \
   bissquit/gradle-server-exporter:latest
@@ -28,22 +34,50 @@ docker-compose up -d --build
 
 ### k8s
 
-Use [k8s-handle](https://github.com/2gis/k8s-handle) to deploy exporter to k8s environment.
+Use [k8s-handle](https://github.com/2gis/k8s-handle) to deploy exporter to k8s environment:
+
+```shell script
+k8s-handle apply -s env-name
+```
 
 Render templates without deployment:
 
 ```shell script
 k8s-handle render -s env-name
 ```
 
-Deploy:
-
-```shell script
-k8s-handle apply -s env-name
+# Help
+
+Exporter looks for the file contains url(s) to your Gradle Enterprise Servers. Each line is [well-formatted](https://validators.readthedocs.io/en/latest/#module-validators.url) url.
+You may pass options both via command line arguments or environment variables:
+
+|Command line argument|Environment variable|Description|
+| ----------- | ----------- | ----------- |
+|-h, --help|-|show help message|
+|-f, --file|`APP_FILE_PATH`|Absolute path to file. Each line is url link (default: Empty string)|
+|-p, --port|`APP_PORT`|Port to be listened (default: 8080)|
+|-t, --time|`APP_CHECK_INTERVAL`|Default time range in seconds to check metrics (default: 60)|
+|-|`LOG_LEVEL`|Log level based on Python [logging](https://docs.python.org/3/library/logging.html) module. expected values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)|
+
+Metrics example:
+
+```text
+gradle_ingest_queue_pending{url="http://localhost:8081"} 0
+gradle_ingest_queue_requested{url="http://localhost:8081"} 0
+gradle_ingest_queue_ageMins{url="http://localhost:8081"} 0
+gradle_ingest_queue_requestWaitTimeSecs{url="http://localhost:8081"} 0
+gradle_ingest_queue_incomingRate1m{url="http://localhost:8081"} 0.03221981766544038
+gradle_ingest_queue_incomingRate5m{url="http://localhost:8081"} 0.02219163413405735
+gradle_ingest_queue_incomingRate15m{url="http://localhost:8081"} 0.021373141599789678
+gradle_ingest_queue_processingRate1m{url="http://localhost:8081"} 0.03399783025186821
+gradle_ingest_queue_processingRate5m{url="http://localhost:8081"} 0.022374841163558885
+gradle_ingest_queue_processingRate15m{url="http://localhost:8081"} 0.021459615070953553
 ```
 
-## Setup environment
-Setup is quite simple
+
+## Dev environment
+
+Setup environment is quite simple:
 ```shell script
 make env
 make test

deploy/config.yaml

@@ -28,4 +28,4 @@ common:
 
 env-name:
   app_name: gradle-server-exporter
-  url_link: http://localhost
+  file_path: /app/urls.txt

deploy/templates/deployment.yaml.j2

@@ -26,8 +26,8 @@ spec:
       - name: {{ app_name }}
         image: {{ image_path }}:{{ image_version }}
         env:
-        - name: APP_URL_LINK
-          value: "{{ url_link }}"
+        - name: APP_FILE_PATH
+          value: "{{ file_path }}"
         - name: APP_PORT
           value: "{{ app_port }}"
         - name: APP_LOOP_TIME

fake_ge_server.py

@@ -0,0 +1,24 @@
+from aiohttp import web
+
+
+async def handle(request):
+    name = request.match_info.get('name', "Anonymous")
+    text = '''{
+    "pending" : 0,
+    "requested" : 0,
+    "ageMins" : 0,
+    "requestWaitTimeSecs" : 0,
+    "incomingRate1m" : 0.03221981766544038,
+    "incomingRate5m" : 0.02219163413405735,
+    "incomingRate15m" : 0.021373141599789678,
+    "processingRate1m" : 0.03399783025186821,
+    "processingRate5m" : 0.022374841163558885,
+    "processingRate15m" : 0.021459615070953553
+}'''
+    return web.Response(text=text, headers={'Content-Type': 'application/json'})
+
+app = web.Application()
+app.add_routes([web.get('/', handle)])
+
+if __name__ == '__main__':
+    web.run_app(app, port=8081)

gradle_server_exporter.py

@@ -1,10 +1,12 @@
 import os
+import sys
 import json
 import aiohttp
 from aiohttp import web
 import asyncio
 import logging
 import argparse
+import validators
 
 logging.basicConfig(level=os.getenv("LOG_LEVEL", logging.INFO), format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -14,6 +16,10 @@ def parse_args():
     # You may either use command line argument or env variables
     parser = argparse.ArgumentParser(prog='gradle_server_exporter',
                                      description='Prometheus exporter for Gradle Enterprise Server ')
+    parser.add_argument('-f', '--file',
+                        default=os.getenv("APP_FILE_PATH", ''),
+                        type=str,
+                        help='Absolute path to file. Each line is url link (default: Empty string)')
     parser.add_argument('-p', '--port',
                         default=os.getenv("APP_PORT", 8080),
                         type=int,
@@ -22,10 +28,6 @@ def parse_args():
                         default=os.getenv("APP_CHECK_INTERVAL", 60),
                         type=int,
                         help='Default time range in seconds to check metrics (default: 60)')
-    parser.add_argument('-l', '--link',
-                        default=os.getenv("APP_URL_LINK", False),
-                        type=str,
-                        help='Put source ip address into labels set (default: False)')
     return parser.parse_args()
 
 
@@ -75,10 +77,10 @@ def validate_json(json_data):
     return None
 
 
-def generate_metrics(json_data):
+def generate_metrics(json_data, url):
     metrics_str = ''
     for k, v in json_data.items():
-        metrics_str += f'gradle_ingest_queue_{k} {v}\n'
+        metrics_str += f'gradle_ingest_queue_{k}{{url="{url}"}} {v}\n'
     return metrics_str
 
 
@@ -92,9 +94,16 @@ async def cleanup_background_tasks(app):
 
 
 async def metrics_checker(app):
+    if not app['urls_list']:
+        logging.critical('Empty urls list is accepted. Nothing to check. Exiting...')
+        sys.exit(1)
+
     while True:
-        json_data = await get_data(app['args'].link)
-        app['metrics_str'] = generate_metrics(json_data=json_data)
+        app['metrics_str'] = ''
+        for url in app['urls_list']:
+            json_data = await get_data(url=url)
+            app['metrics_str'] += generate_metrics(json_data=json_data,
+                                                   url=url)
         await asyncio.sleep(app['args'].time)
 
 
@@ -106,12 +115,63 @@ async def return_metrics(request):
     return web.Response(text=request.app['metrics_str'])
 
 
+class HandleFileData:
+    def __init__(self, path):
+        self.path = path
+
+    def return_urls_list(self):
+        strs_list = self.read_file(path=self.path)
+        urls_list = self.normalize_urls_list(strs_list=strs_list)
+        return urls_list
+
+    def normalize_urls_list(self, strs_list):
+        invalid_urls_counter = 0
+        urls_list = []
+
+        for line in strs_list:
+            line_striped = line.strip()
+            if line_striped == '':
+                pass
+            # ignore line in the list if it doesn't look like a valid url
+            elif self.normalize_url(url=line_striped) is False:
+                invalid_urls_counter += 1
+            else:
+                urls_list.append(line_striped)
+
+        if invalid_urls_counter > 0:
+            logger.warning(f'{invalid_urls_counter} url(s) has been removed due to invalid format. Check file {self.path}')
+            logger.info('Read more about url format at https://validators.readthedocs.io/en/latest/#module-validators.url')
+
+        return urls_list
+
+    @staticmethod
+    def read_file(path):
+        try:
+            logger.info(f'Reading file {path}')
+            with open(path) as file:
+                # Return all lines in the file, as a list where each line is an item
+                strs_list = file.readlines()
+        except OSError as error:
+            logger.critical(f'Could not read file {path}: {error}')
+            strs_list = []
+        return strs_list
+
+    @staticmethod
+    def normalize_url(url):
+        result = True
+        if not validators.url(url):
+            logger.warning(f'String {url} is not a valid url. Skipping...')
+            result = False
+        return result
+
+
 def main():
     args = parse_args()
     app = web.Application()
     # For storing global-like variables, feel free to save them in an Application instance
     app['metrics_str'] = 'Initialization'
     app['args'] = args
+    app['urls_list'] = HandleFileData(args.file).return_urls_list()
     app.add_routes([web.get('/metrics', return_metrics)])
     # https://docs.aiohttp.org/en/stable/web_advanced.html#background-tasks
     app.on_startup.append(start_background_tasks)

requirements.txt

@@ -1 +1,2 @@
 aiohttp==3.7.4.post0
+validators==0.18.2

tests/test_gradle_server_exporter.py

@@ -2,12 +2,14 @@
 import aiohttp
 import json
 import yarl
+import io
 from multidict import CIMultiDictProxy, CIMultiDict
 from gradle_server_exporter import \
     get_data, \
     validate_json, \
     parse_args, \
-    generate_metrics
+    generate_metrics, \
+    HandleFileData
 
 
 class MockResponse:
@@ -55,6 +57,18 @@ async def json(self):
         return json.loads(self._text)
 
 
+# emulates incorrect directory path
+class MockOSReadFile:
+    def __init__(self, path):
+        self.path = path
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        raise OSError
+
+
 @pytest.mark.asyncio
 async def test_get_data(mocker):
     json_data = {
@@ -105,13 +119,94 @@ def test_parse_args():
 
 
 def test_generate_metrics():
+    fake_url_str = 'http://fake.url'
     json_data = {
         "pending": 0,
         "requested": 0
     }
-    result = generate_metrics(json_data=json_data)
-    assert result == 'gradle_ingest_queue_pending 0\ngradle_ingest_queue_requested 0\n'
+    result = generate_metrics(json_data=json_data,
+                              url=fake_url_str)
+    assert result == f'gradle_ingest_queue_pending{{url="{fake_url_str}"}} 0\ngradle_ingest_queue_requested{{url="{fake_url_str}"}} 0\n'
 
     json_data = json.loads('{}')
-    result = generate_metrics(json_data=json_data)
+    result = generate_metrics(json_data=json_data,
+                              url=fake_url_str)
     assert result == ''
+
+
+def test_normalize_url():
+    invalid_urls_list = [
+        'http:/google.com',
+        'httpa://google.com',
+        'htt:/google.com',
+        'http//google.com',
+        'ttp://google.com',
+        'http://google',
+        'http://google.com:44a',
+        'google.com'
+    ]
+
+    for url in invalid_urls_list:
+        result = HandleFileData('').normalize_url(url=url)
+        assert result is False
+
+    valid_urls_list = [
+        'http://google.com',
+        'https://google.com',
+        'http://google.com:443',
+        'http://google.com/path',
+        'http://google.com:443/path?'
+    ]
+
+    for url in valid_urls_list:
+        result = HandleFileData('').normalize_url(url=url)
+        assert result is True
+
+
+@pytest.mark.asyncio
+def test_read_file(mocker):
+    fake_path_str = '/fake/path'
+
+    file_data_str = 'http://google.ru \nhttps://ya.ru\n'
+    # creates file-like obj in memory with appropriate methods like read() and write()
+    file = io.StringIO(file_data_str)
+    mocker.patch("builtins.open", return_value=file)
+    client = HandleFileData('').read_file(fake_path_str)
+    assert client == ['http://google.ru \n', 'https://ya.ru\n']
+
+    file_data_str = '\n\n'
+    file = io.StringIO(file_data_str)
+    mocker.patch("builtins.open", return_value=file)
+    client = HandleFileData('').read_file(fake_path_str)
+    assert client == ['\n', '\n']
+
+    file_data_str = ''
+    file = io.StringIO(file_data_str)
+    mocker.patch("builtins.open", return_value=file)
+    client = HandleFileData('').read_file(fake_path_str)
+    assert client == []
+
+    resp = MockOSReadFile(fake_path_str)
+    mocker.patch("builtins.open", return_value=resp)
+    client = HandleFileData('').read_file(fake_path_str)
+    assert client == []
+
+
+def test_normalize_urls_list():
+    file_data_str = [
+        'htt://google.ru \n',
+        'https://ya.ru\n',
+        '\n',
+        ' \n',
+        '  \n',
+        '\n ',
+        '\n  ',
+        '\n\n',
+        '\n\n ',
+        '\n\n  ',
+        '   ',
+        'google.ru',
+        '  http://site.tld '
+    ]
+    result = HandleFileData('').normalize_urls_list(strs_list=file_data_str)
+    assert result == ['https://ya.ru', 'http://site.tld']