diff --git a/server/tests-py/remote_relationship_tests/.gitignore b/server/tests-py/remote_relationship_tests/.gitignore new file mode 100644 index 0000000000000..195d5c734a596 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/.gitignore @@ -0,0 +1,3 @@ +__pycache__ +test_output +.previous_work_dir diff --git a/server/tests-py/remote_relationship_tests/Readme.md b/server/tests-py/remote_relationship_tests/Readme.md new file mode 100644 index 0000000000000..1ad329ee627e8 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/Readme.md @@ -0,0 +1,50 @@ + + +### Setup ### + +The test setup includes two Postgres databases with [sportsdb](https://www.thesportsdb.com/) schema and data, and two GraphQL engines running on the Postgres databases. Then one of the GraphQL engines is added as a remote schema to another GraphQL engine. + +The data will be same in both the databases. But the tables will reside in different database schema in-order to avoid GraphQL schema conflicts. + +The Python script `test_with_sportsdb.py` will help in setting up the databases, starting the Hasura GraphQL engines, and setting up relationships (both local and remote). This script will run databases on docker, and the GraphQL engines are run with `stack exec`. + +#### Setup GraphQL Engines #### +- Install dependencies for the Python script in a virtual environment +```sh +$ python3 -m venv venv +$ source venv/bin/activate +$ pip3 install -r requirements.txt +``` +- Inorder to start GraphQL engines with sportsdb, run +```sh +python3 test_with_sportsdb.py +``` + +This will setup Postgres databases and runs the main and remote GraphQL servers +Pressing enter will teardown both Postgres database + +The initial setup will take some time. The subsequent ones will be faster. The Postgres data is bind mounted from a volume in the host, which will be reused. + +### Benchmarking ### + +We may employ https://github.com/hasura/graphql-bench to do the benchmarks. + +Create the `bench.yaml` file +``` +- name: events_remote_affilications + warmup_duration: 60 + duration: 300 + candidates: + - name: hge-with-remote + url: http://127.0.0.1:8081/v1/graphql + query: events_remote_affiliations + queries_file: queries.graphql + rps: + - 20 + - 40 +``` + +To run the benchmark, do +```sh +cat bench.yaml | docker run -i --rm --net=host -v $(pwd)/queries.graphql:/graphql-bench/ws/queries.graphql hasura/graphql-bench:v0.3 +``` diff --git a/server/tests-py/remote_relationship_tests/example-bench.yaml b/server/tests-py/remote_relationship_tests/example-bench.yaml new file mode 100644 index 0000000000000..9e5843b0c1335 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/example-bench.yaml @@ -0,0 +1,11 @@ +- name: events_affilications + warmup_duration: 60 + duration: 300 + candidates: + - name: hge-with-remote + url: http://127.0.0.1:8081/v1/graphql + query: events_remote_affiliations + queries_file: queries.graphql + rps: + - 20 + - 40 diff --git a/server/tests-py/remote_relationship_tests/port_allocator.py b/server/tests-py/remote_relationship_tests/port_allocator.py new file mode 100644 index 0000000000000..a788743140278 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/port_allocator.py @@ -0,0 +1,27 @@ +import threading +import socket + + +class PortAllocator: + + def __init__(self): + self.allocated_ports = set() + self.lock = threading.Lock() + + def get_unused_port(self, start): + port = start + if self.is_port_open(port) or port in self.allocated_ports: + return self.get_unused_port(port + 1) + else: + return port + + def is_port_open(self, port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + res = sock.connect_ex(('127.0.0.1', port)) + return res == 0 + + def allocate_port(self, start): + with self.lock: + port = self.get_unused_port(start) + self.allocated_ports.add(port) + return port diff --git a/server/tests-py/remote_relationship_tests/queries.graphql b/server/tests-py/remote_relationship_tests/queries.graphql new file mode 100644 index 0000000000000..17e6f4de40ef1 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/queries.graphql @@ -0,0 +1,27 @@ +query events_affiliations { + hge_events(limit: 50){ + event_status + last_update + affiliations_events_by_event_id(limit: 2) { + affiliation{ + publisher{ + publisher_name + } + } + } + } +} + +query events_remote_affiliations { + hge_events(limit: 50){ + event_status + last_update + remote_affiliations_events_by_event_id(limit: 2) { + affiliation{ + publisher{ + publisher_name + } + } + } + } +} diff --git a/server/tests-py/remote_relationship_tests/requirements.txt b/server/tests-py/remote_relationship_tests/requirements.txt new file mode 100644 index 0000000000000..1425cd5c9778f --- /dev/null +++ b/server/tests-py/remote_relationship_tests/requirements.txt @@ -0,0 +1,5 @@ +requests-cache +docker +psycopg2 +colorama +inflection diff --git a/server/tests-py/remote_relationship_tests/run_hge.py b/server/tests-py/remote_relationship_tests/run_hge.py new file mode 100644 index 0000000000000..9014a6be51e3f --- /dev/null +++ b/server/tests-py/remote_relationship_tests/run_hge.py @@ -0,0 +1,397 @@ +import os +import subprocess +import requests +import inflection +import json +import time +import signal +import docker +from requests.exceptions import ConnectionError +from colorama import Fore, Style + +class HGEError(Exception): + pass + + +class HGE: + + default_graphql_env = { + 'HASURA_GRAPHQL_ENABLE_TELEMETRY': 'false', + 'EVENT_WEBHOOK_HEADER': "MyEnvValue", + 'HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES': 'true', + 'HASURA_GRAPHQL_CONSOLE_ASSETS_DIR' : '../../../../console/static/dist/', + 'HASURA_GRAPHQL_ENABLE_CONSOLE' : 'true' + } + + def __init__(self, pg, port_allocator, admin_secret=None, docker_image=None, log_file='hge.log'): + self.pg = pg + self.log_file = log_file + self.tix_file = self.log_file[:-4] + '.tix' + self.admin_secret = admin_secret + self.docker_image = docker_image + self.introspection = None + self.obj_fk_rels = set() + self.arr_fk_rels = set() + self.port_allocator = port_allocator + self.url = None + self.proc = None + self.container = None + + @classmethod + def do_stack_build(cls): + print(Fore.YELLOW + "Performing Stack build first" + Style.RESET_ALL) + subprocess.check_call(['stack','build']) + + def get_hge_env(self): + hge_env = { + **os.environ, + **self.default_graphql_env.copy(), + 'HASURA_GRAPHQL_DATABASE_URL': self.pg.url, + 'HASURA_GRAPHQL_SERVER_PORT': str(self.port), + 'HPCTIXFILE' : self.tix_file + } + if self.admin_secret: + hge_env['HASURA_GRAPHQL_ADMIN_SECRET'] = self.admin_secret + return hge_env + + def run(self): + if self.docker_image: + self.run_with_docker() + else: + self.run_with_stack() + + def run_with_docker(self): + self.port = self.port_allocator.allocate_port(8080) + hge_env = self.get_hge_env() + process_args = ['graphql-engine', 'serve'] + docker_ports = {str(self.port) + '/tcp': ('127.0.0.1', self.port)} + self.docker_client = docker.from_env() + print("Running GraphQL Engine docker with image:", + self.docker_image, '(port:{})'.format(self.port)) + self.container = self.docker_client.containers.run( + self.docker_image, + command=process_args, + detach=True, + ports=docker_ports, + environment=hge_env, + network_mode='host', + volumes={} + ) + self.url = 'http://localhost:' + str(self.port) + print("Waiting for GraphQL Engine to be running.", end='') + self.wait_for_start() + + def run_with_stack(self): + self.port = self.port_allocator.allocate_port(8080) + hge_env = self.get_hge_env() + process_args = ['stack', 'exec', 'graphql-engine', '--', 'serve'] + self.log_fp = open(self.log_file, 'w') + self.proc = subprocess.Popen( + process_args, + env=hge_env, + shell=False, + bufsize=-1, + stdout=self.log_fp, + stderr=subprocess.STDOUT + ) + self.url = 'http://localhost:' + str(self.port) + print("Waiting for GraphQL Engine to be running.", end='') + self.wait_for_start() + + def check_if_process_is_running(self): + if self.proc.poll() is not None: + with open(self.log_file) as fr: + raise HGEError( + "GraphQL engine failed with error: " + fr.read()) + + def check_if_container_is_running(self): + self.container.reload() + if self.container.status == 'exited': + raise HGEError( + "GraphQL engine failed with error: \n" + + self.container.logs(stdout=True, stderr=True).decode('ascii') + ) + + def wait_for_start(self, timeout=60): + if timeout <= 0: + raise HGEError("Timeout waiting for graphql process to start") + if self.proc: + self.check_if_process_is_running() + elif self.container: + self.check_if_container_is_running() + try: + q = { 'query': 'query { __typename }' } + r = requests.post(self.url + '/v1/graphql',json.dumps(q),headers=self.admin_auth_headers()) + if r.status_code == 200: + print() + return + except ConnectionError: + pass + print(".", end="", flush=True), + sleep_time = 0.5 + time.sleep(sleep_time) + self.wait_for_start(timeout - sleep_time) + + def teardown(self): + if getattr(self, 'log_fp', None): + self.log_fp.close() + self.log_fp = None + if self.proc: + self.cleanup_process() + elif self.container: + self.cleanup_docker() + + def cleanup_process(self): + print(Fore.YELLOW + "Stopping graphql engine at port:", self.port, Style.RESET_ALL) + self.proc.send_signal(signal.SIGINT) + self.proc.wait() + self.proc = None + + def cleanup_docker(self): + cntnr_info = "HGE docker container " + self.container.name + " " + repr(self.container.image) + print(Fore.YELLOW + "Stopping " + cntnr_info + Style.RESET_ALL) + self.container.stop() + print(Fore.YELLOW + "Removing " + cntnr_info + Style.RESET_ALL) + self.container.remove() + self.container = None + + def admin_auth_headers(self): + headers = {} + if self.admin_secret: + headers['X-Hasura-Admin-Secret'] = self.admin_secret + return headers + + def v1q(self, q, exp_status=200): + resp = requests.post(self.url + '/v1/query', json.dumps(q), headers=self.admin_auth_headers()) + assert resp.status_code == exp_status, (resp.status_code, resp.json()) + return resp.json() + + def graphql_admin_q(self, q, exp_status = 200): + resp = requests.post(self.url + '/v1/graphql', q, headers=self.admin_auth_headers()) + assert resp.status_code == exp_status, (resp.status_code, resp.json()) + return resp.json() + + def track_all_tables_in_schema(self, schema='public'): + print("Track all tables in schema ", schema) + all_tables = self.pg.get_all_tables_in_a_schema(schema) + all_tables = [ {'schema': schema, 'name': t} + for t in all_tables ] + return self.track_tables(all_tables) + + def run_bulk(self, queries, exp_status = 200): + bulk_q = { + 'type': 'bulk', + 'args': queries + } + return self.v1q(bulk_q, exp_status) + + def track_tables(self, tables, exp_status=200): + queries = [] + for table in tables: + q = { + 'type' : 'track_table', + 'args' : table + } + queries.append(q) + return self.run_bulk(queries, exp_status) + + def track_table(self, table, exp_status=200): + q = self.mk_track_table_q(table) + return self.v1q(q, exp_status) + + def mk_track_table_q(self, table): + return { + 'type' : 'track_table', + 'args' : table + } + + def add_remote_schema(self, name, remote_url, headers={}, client_hdrs=False): + def hdr_name_val_pair(headers): + nvp = [] + for (k,v) in headers.items(): + nvp.append({'name': k, 'value': v}) + return nvp + if len(headers) > 0: + client_hdrs = True + q = { + 'type' : 'add_remote_schema', + 'args': { + 'name': name, + 'comment': name, + 'definition': { + 'url': remote_url, + 'headers': hdr_name_val_pair(headers), + 'forward_client_headers': client_hdrs + } + } + } + return self.v1q(q) + + + def create_remote_obj_rel_to_itself(self, tables_schema, remote, remote_tables_schema): + print("Creating remote relationship to the tables in schema {} to itself using remote {}".format(tables_schema, remote)) + fk_constrnts = self.pg.get_all_fk_constraints(tables_schema) + for (s,_,t,c,fs,ft,fc) in fk_constrnts: + table_cols = self.pg.get_all_columns_of_a_table(t, s) + if not 'id' in table_cols: + continue + rel_name = 'remote_' + inflection.singularize(t) + '_via_' + c + query ={ + 'type': 'create_remote_relationship', + 'args' : { + 'name' : rel_name, + 'table' : { + 'schema': s, + 'name': t + }, + 'remote_schema': remote, + 'hasura_fields': ['id', c], + 'remote_field': { + remote_tables_schema + '_' + ft + '_by_pk' : { + 'arguments': { + 'id': '$' + c + }, + 'field': { + inflection.pluralize(t) + '_by_' + c: { + 'arguments' : { + 'where': { + c : { + '_eq': '$id' + } + } + } + } + } + } + } + } + } + print(query) + self.v1q(query) + + def create_remote_obj_fk_ish_relationships(self, tables_schema, remote, remote_tables_schema): + print("Creating object foreign key ish relationships for tables in schema {} using remote {}".format(tables_schema, remote)) + fk_constrnts = self.pg.get_all_fk_constraints(tables_schema) + for (s,_,t,c,fs,ft,fc) in fk_constrnts: + rel_name = inflection.singularize(ft) + if c.endswith('_id'): + rel_name = c[:-3] + rel_name = 'remote_' + rel_name + query ={ + 'type': 'create_remote_relationship', + 'args' : { + 'name' : rel_name, + 'table' : { + 'schema': s, + 'name': t + }, + 'remote_schema': remote, + 'hasura_fields': [c], + 'remote_field': { + remote_tables_schema + '_' + ft + '_by_pk' : { + 'arguments' : { + 'id': '$' + c + } + } + } + + } + } + print(query) + self.v1q(query) + + def create_obj_fk_relationships(self, schema='public'): + print("Creating object foreign key relationships for tables in schema ", schema) + fk_constrnts = self.pg.get_all_fk_constraints(schema) + queries = [] + for (s,_,t,c,fs,ft,fc) in fk_constrnts: + rel_name = inflection.singularize(ft) + if c.endswith('_id'): + rel_name = c[:-3] + table_cols = self.pg.get_all_columns_of_a_table(t, s) + if rel_name in table_cols: + rel_name += '_' + inflection.singularize(ft) + queries.append({ + 'type' : 'create_object_relationship', + 'args': { + 'table': { + 'schema': s, + 'name': t + }, + 'name': rel_name, + 'using': { + 'foreign_key_constraint_on': c + } + } + }) + self.obj_fk_rels.add(((s,t),rel_name)) + return self.run_bulk(queries) + + def create_remote_arr_fk_ish_relationships(self, tables_schema, remote, remote_tables_schema): + fk_constrnts = self.pg.get_all_fk_constraints(tables_schema) + for (s,_,t,c,fs,ft,fc) in fk_constrnts: + rel_name = 'remote_' + inflection.pluralize(t) + '_by_' + c + query ={ + 'type': 'create_remote_relationship', + 'args' : { + 'name' : rel_name, + 'table' : { + 'schema': fs, + 'name': ft + }, + 'remote_schema': remote, + 'hasura_fields': ['id'], + 'remote_field': { + remote_tables_schema + '_' + t : { + 'arguments' : { + 'where': { + c : { + '_eq': '$id' + } + } + } + } + } + } + } + print(query) + self.v1q(query) + + def create_arr_fk_relationships(self, schema='public'): + print("Creating array foreign key relationships for tables in schema ", schema) + fk_constrnts = self.pg.get_all_fk_constraints(schema) + queries = [] + for (s,cn,t,c,fs,ft,fc) in fk_constrnts: + rel_name = inflection.pluralize(t) + '_by_' + c + queries.append({ + 'type' : 'create_array_relationship', + 'args': { + 'table': { + 'schema': fs, + 'name': ft + }, + 'name': rel_name, + 'using': { + 'foreign_key_constraint_on': { + 'table': { + 'schema': s, + 'name': t + }, + 'column': c + } + } + } + }) + self.arr_fk_rels.add(((fs,ft),rel_name)) + return self.run_bulk(queries) + + def run_sql(self, sql, exp_status=200): + return self.v1q(self.mk_run_sql_q(sql)) + + def mk_run_sql_q(self, sql): + return { + 'type' : 'run_sql', + 'args': { + 'sql' : sql + } + } diff --git a/server/tests-py/remote_relationship_tests/run_postgres.py b/server/tests-py/remote_relationship_tests/run_postgres.py new file mode 100644 index 0000000000000..1188e96884df4 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/run_postgres.py @@ -0,0 +1,190 @@ +import docker +import time +from contextlib import contextmanager +import psycopg2 +from psycopg2.sql import SQL, Identifier +from colorama import Fore, Style +import os + + +class PostgresError(Exception): + pass + + +class Postgres: + + def __init__(self, docker_image, db_data_dir, port_allocator, url): + self.port_allocator = port_allocator + self.docker_image = docker_image + self.db_data_dir = os.path.abspath(db_data_dir) + self.url = url + + def setup(self): + if self.docker_image and not self.url: + self.start_postgres_docker() + + def start_postgres_docker(self): + if self.url: + print("Postgres is already running") + return + self.port = self.port_allocator.allocate_port(5433) + self.user = 'hge_test' + self.password = 'hge_pass' + self.database = 'hge_test' + env = { + 'POSTGRES_USER' : self.user, + 'POSTGRES_PASSWORD' : self.password, + 'POSTGRES_DB' : self.database + } + docker_ports = {'5432/tcp': ('127.0.0.1', self.port)} + docker_vols = { + self.db_data_dir: { + 'bind': '/var/lib/postgresql/data', + 'mode': 'rw' + } + } + + self.docker_client = docker.from_env() + print("Running postgres docker with image:", + self.docker_image, '(port:{})'.format(self.port)) + cntnr = self.docker_client.containers.run( + self.docker_image, + detach=True, + ports=docker_ports, + environment=env, + volumes = docker_vols + ) + self.pg_container = cntnr + self.url = 'postgresql://' + self.user + ':' + self.password + '@localhost:' + str(self.port) + '/' + self.database + print("Waiting for database to be up and running.", end="", flush=True) + self.wait_for_db_start() + print("") + + def check_if_container_is_running(self): + self.pg_container.reload() + if self.pg_container.status == 'exited': + raise PostgresError( + "Postgres docker failed with error: \n" + + self.pg_container.logs(stdout=True, stderr=True).decode('ascii') + ) + + def wait_for_db_start(self, timeout=60): + if self.pg_container: + self.check_if_container_is_running() + if timeout > 0: + try: + self.run_sql('select 1') + return + except Exception as e: + if timeout < 5: + print("\nWaiting for database to be up and running:" + repr(e), end=""), + else: + print(".", end="", flush=True), + sleep_time = 0.5 + time.sleep(sleep_time) + self.wait_for_db_start(timeout-sleep_time) + else: + raise PostgresError("Timeout waiting for database to start") + + @contextmanager + def cursor(self): + with psycopg2.connect(self.url) as conn: + with conn.cursor() as cursor: + yield cursor + + def get_all_fk_constraints(self, schema='public'): + with self.cursor() as cursor: + cursor.execute(''' + SELECT + tc.table_schema, + tc.constraint_name, + tc.table_name, + kcu.column_name, + ccu.table_schema AS foreign_table_schema, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name + FROM + information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' and tc.table_schema=%s + ''',(schema,)) + return cursor.fetchall() + + def move_tables_to_schema(self, cur_schema, target_schema): + print("Moving tables in schema {} to schema {}".format(cur_schema, target_schema)) + table_names = self.get_all_tables_in_a_schema(cur_schema) + with self.cursor() as cursor: + cursor.execute(SQL('CREATE SCHEMA IF NOT EXISTS {} ').format(Identifier(target_schema))) + for table in table_names: + cursor.execute(SQL('''ALTER TABLE {}.{} SET SCHEMA {};''').format( Identifier(cur_schema), Identifier(table), Identifier(target_schema) )) + + def get_all_columns_of_a_table(self, table_name, table_schema='public'): + columns = [] + with self.cursor() as cursor: + cursor.execute(''' + SELECT column_name + FROM information_schema.columns + WHERE table_name= %s and table_schema=%s; + ''', (table_name, table_schema)) + for row in cursor.fetchall(): + columns.append(row[0]) + return columns + + def get_all_tables_with_column(self, column, schema='public'): + tables = [] + with self.cursor() as cursor: + cursor.execute(''' + SELECT table_name + FROM information_schema.columns + WHERE column_name= %s and table_schema=%s; + ''', (column,schema)) + for row in cursor.fetchall(): + tables.append(row[0]) + return tables + + def set_id_as_primary_key_for_tables(self, schema='public'): + print("Setting id as primary key for all tables in schema ", schema) + tables_with_id_col = self.get_all_tables_with_column('id', schema) + with self.cursor() as cursor: + for table in tables_with_id_col: + self.set_id_as_primary_key_for_table(cursor, table, schema) + + def set_id_as_primary_key_for_table(self, cursor, table, schema='public'): + cursor.execute(SQL('''ALTER TABLE {}.{} ADD PRIMARY KEY (id);''').format( Identifier(schema), Identifier(table) )) + + def get_all_tables_in_a_schema(self, schema='public'): + tables = [] + with self.cursor() as cursor: + cursor.execute(''' + SELECT table_name + FROM information_schema.tables + WHERE table_schema=%s; + ''', (schema,)) + for row in cursor.fetchall(): + tables.append(row[0]) + return tables + + def run_sql(self, sql): + with self.cursor() as cursor: + cursor.execute(sql) + + def run_sql_from_file(self, sql_file): + print("Running sql from file:", sql_file) + self.run_sql(open(sql_file, 'r').read()) + + def teardown(self): + self.cleanup_docker() + + def cleanup_docker(self): + if getattr(self, 'pg_container', None): + cntnr_info = "Postgres docker container " + self.pg_container.name + " " + repr(self.pg_container.image) + print(Fore.YELLOW + "Stopping " + cntnr_info + Style.RESET_ALL) + self.pg_container.stop() + print(Fore.YELLOW + "Removing " + cntnr_info + Style.RESET_ALL) + self.pg_container.remove() + self.pg_container = None diff --git a/server/tests-py/remote_relationship_tests/test_with_sportsdb.py b/server/tests-py/remote_relationship_tests/test_with_sportsdb.py new file mode 100644 index 0000000000000..44963f21427a0 --- /dev/null +++ b/server/tests-py/remote_relationship_tests/test_with_sportsdb.py @@ -0,0 +1,264 @@ +import requests +from zipfile import ZipFile +import requests_cache +import os +import threading +from port_allocator import PortAllocator +from run_postgres import Postgres +from run_hge import HGE +from colorama import Fore, Style +import argparse +import sys + + +def _first_true(iterable, default=False, pred=None): + return next(filter(pred, iterable), default) + + +class HGETestSetup: + + sportsdb_url='http://www.sportsdb.org/modules/sd/assets/downloads/sportsdb_sample_postgresql.zip' + + default_work_dir = 'test_output' + + previous_work_dir_file = '.previous_work_dir' + + def __init__(self, pg_urls, pg_docker_image, hge_docker_image=None, hge_admin_secret=None, skip_stack_build=False): + self.pg_url, self.remote_pg_url = pg_urls or (None, None) + self.pg_docker_image = pg_docker_image + self.hge_docker_image = hge_docker_image + self.hge_admin_secret = hge_admin_secret + self.skip_stack_build = skip_stack_build + self.graphql_queries_file = os.path.abspath('queries.graphql') + self.port_allocator = PortAllocator() + self.set_work_dir() + self.init_pgs() + self.init_hges() + self.set_previous_work_dir() + + def get_previous_work_dir(self): + try: + with open(self.previous_work_dir_file) as f: + return f.read() + except FileNotFoundError: + return None + + def set_previous_work_dir(self): + with open(self.previous_work_dir_file, 'w') as f: + return f.write(self.work_dir) + + def get_work_dir(self): + default_work_dir = self.get_previous_work_dir() or self.default_work_dir + return os.environ.get('WORK_DIR') \ + or input(Fore.YELLOW + '(Set WORK_DIR environmental variable to avoid this)\n' + + 'Please specify the work directory. (default:{}):'.format(default_work_dir) + + Style.RESET_ALL).strip() \ + or default_work_dir + + + def set_work_dir(self): + self.work_dir = self.get_work_dir() + print ("WORK_DIR: ", self.work_dir) + os.makedirs(self.work_dir, exist_ok=True) + requests_cache.install_cache(self.work_dir + '/sportsdb_cache') + + def init_pgs(self): + pg_confs = [ + ('sportsdb_data', self.pg_url), + ('remote_sportsdb_data', self.remote_pg_url) + ] + self.pg, self.remote_pg = [ + Postgres( + port_allocator=self.port_allocator, docker_image=self.pg_docker_image, + db_data_dir= self.work_dir + '/' + data_dir, url=url + ) + for (data_dir, url) in pg_confs + ] + + def init_hges(self): + hge_confs = [ + (self.pg, 'hge.log'), + (self.remote_pg, 'remote_hge.log') + ] + self.hge, self.remote_hge = [ + HGE( + pg=pg, port_allocator=self.port_allocator, admin_secret=self.hge_admin_secret, + log_file= self.work_dir + '/' + log_file, docker_image=self.hge_docker_image) + for (pg, log_file) in hge_confs + ] + + def setup_graphql_engines(self): + + if not self.hge_docker_image and not self.skip_stack_build: + HGE.do_stack_build() + + def run_concurrently(threads): + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + def run_concurrently_fns(*fns): + threads = [threading.Thread(target=fn) for fn in fns] + return run_concurrently(threads) + + def set_hge(hge, schema, hge_type): + pg = hge.pg + # Schema and data + pg.run_sql_from_file(sql_file) + pg.set_id_as_primary_key_for_tables(schema='public') + pg.move_tables_to_schema('public', schema) + + # Metadata stuff + hge.track_all_tables_in_schema(schema) + hge.create_obj_fk_relationships(schema) + hge.create_arr_fk_relationships(schema) + + run_concurrently_fns( + self.pg.start_postgres_docker, + self.remote_pg.start_postgres_docker) + print("Postgres url:", self.pg.url) + print("Remote Postgres url:", self.remote_pg.url) + + self.remote_hge.run() + self.hge.run() + + # Skip if the tables are already present + tables = self.pg.get_all_tables_in_a_schema('hge') + if len(tables) > 0: + return + + # Download sportsdb + zip_file = self.download_sportsdb_zip(self.work_dir+ '/sportsdb.zip') + sql_file = self.unzip_sql_file(zip_file) + + # Create the required tables and move them to required schemas + hge_thread = threading.Thread( + target=set_hge, args=(self.hge, 'hge', 'Main')) + remote_hge_thread = threading.Thread( + target=set_hge, args=(self.remote_hge, 'remote_hge', 'Remote')) + run_concurrently([hge_thread, remote_hge_thread]) + + # Add remote_hge as remote schema + self.hge.add_remote_schema( + 'remote_hge', self.remote_hge.url + '/v1/graphql', + self.remote_hge.admin_auth_headers()) + + tables = self.pg.get_all_tables_in_a_schema('hdb_catalog') + if 'hdb_remote_relationship' not in tables: + return + + # Create remote relationships + self.hge.create_remote_obj_rel_to_itself('hge', 'remote_hge', 'remote_hge') + self.hge.create_remote_arr_fk_ish_relationships('hge', 'remote_hge', 'remote_hge') + self.hge.create_remote_obj_fk_ish_relationships('hge', 'remote_hge', 'remote_hge') + + + def teardown(self): + for res in [self.hge, self.remote_hge, self.pg, self.remote_pg]: + res.teardown() + + def download_sportsdb_zip(self, filename, url=sportsdb_url): + with requests.get(url, stream=True) as r: + r.raise_for_status() + total=0 + print() + with open(filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + total += len(chunk) + print("\rDownloaded: ", int(total/1024) , 'KB', end='') + f.write(chunk) + print('\nDB Zip File:', filename) + return filename + + def unzip_sql_file(self, zip_file): + with ZipFile(zip_file, 'r') as zip: + sql_file = zip.infolist()[0] + print('DB SQL file:', sql_file.filename) + zip.extract(sql_file, self.work_dir) + return self.work_dir + '/' + sql_file.filename + + +class HGETestSetupWithArgs(HGETestSetup): + + default_pg_docker_image = 'circleci/postgres:11.5-alpine-postgis' + + def __init__(self): + self.set_arg_parse_options() + self.parse_args() + super().__init__( + pg_urls = self.pg_urls, + pg_docker_image = self.pg_docker_image, + hge_docker_image = self.hge_docker_image, + hge_admin_secret = self.hge_admin_secret, + skip_stack_build = self.skip_stack_build + ) + + def set_arg_parse_options(self): + self.arg_parser = argparse.ArgumentParser() + self.set_pg_options() + self.set_hge_options() + + def parse_args(self): + self.parsed_args = self.arg_parser.parse_args() + self.set_pg_confs() + self.set_hge_confs() + + def set_pg_confs(self): + self.pg_urls, self.pg_docker_image = self.get_exclusive_params([ + ('pg_urls', 'HASURA_BENCH_PG_URLS'), + ('pg_docker_image', 'HASURA_BENCH_PG_DOCKER_IMAGE') + ]) + if self.pg_urls: + self.pg_urls = self.pg_urls.split(',') + else: + self.pg_docker_image = self.pg_docker_image or self.default_pg_docker_image + + def set_hge_confs(self): + self.hge_docker_image = self.get_param('hge_docker_image', 'HASURA_BENCH_DOCKER_IMAGE') + self.hge_admin_secret = self.get_param('hge_admin_secret', 'HASURA_BENCH_HGE_ADMIN_SECRET') + self.skip_stack_build = self.parsed_args.skip_stack_build + + def set_pg_options(self): + self.arg_parser.add_argument('--pg-urls', metavar='HASURA_BENCH_PG_URLS', help='Postgres database urls to be used for tests, given as comma separated values', required=False) + self.arg_parser.add_argument('--pg-docker-image', metavar='HASURA_BENCH_PG_DOCKER_IMAGE', help='Postgres docker image to be used for tests', required=False) + + def set_hge_options(self): + self.arg_parser.add_argument('--hge-docker-image', metavar='HASURA_BENCH_HGE_DOCKER_IMAGE', help='GraphQl engine docker image to be used for tests', required=False) + self.arg_parser.add_argument('--hge-admin-secret', metavar='HASURA_BENCH_HGE_ADMIN_SECRET', help='Admin secret set for GraphQL engines. By default, no admin secret is set', required=False) + self.arg_parser.add_argument('--skip-stack-build', help='Skip stack build if this option is set', action='store_true', required=False) + + + def get_param(self, attr, env): + return _first_true([getattr(self.parsed_args, attr), os.getenv(env)]) + + def get_exclusive_params(self, params_loc): + excl_param = None + params_out = [] + for (attr, env) in params_loc: + param = self.get_param(attr, env) + params_out.append(param) + if param: + if not excl_param: + excl_param = (param, attr, env) + else: + (param1, attr1, env1) = excl_param + def loc(a, e): + arg = '--' + a.replace('_','-') + return arg + '(env: ' + e + ')' + print(loc(attr, env), 'and', loc(attr1, env1), 'should not be defined together') + sys.exit(1) + return params_out + + +if __name__ == "__main__": + test_setup = HGETestSetupWithArgs() + try: + test_setup.setup_graphql_engines() + print("Hasura GraphQL engine is running on URL:",test_setup.hge.url+ '/v1/graphql') + input(Fore.BLUE+'Press Enter to stop GraphQL engine' + Style.RESET_ALL) + finally: + test_setup.teardown() +