Skip to content

Commit 974e6ed

Browse files
odesenfansodesenfans
committed
Feature: separate API process (#393)
Problem: Users experience latency issues with the CCN API. This is caused by some parts of the code being synchronous: inherently for filesystem accesses, by design for DB accesses. Solution: run multiple API processes. We now use gunicorn to spawn several API worker processes. The API is now spawned as an independent container to make it easier to configure. The API port and number of workers can be configured by setting the following environment variables: - `CCN_CONFIG_API_PORT` - `CCN_CONFIG_API_NB_WORKERS`.
1 parent 9b0e5e5 commit 974e6ed

File tree

10 files changed

+203
-173
lines changed

10 files changed

+203
-173
lines changed

‎deployment/samples/docker-compose/docker-compose.yml‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,12 @@ volumes:
88
services:
99
pyaleph:
1010
restart: always
11-
image: alephim/pyaleph-node:v0.5.0-rc1
11+
image: alephim/pyaleph-node:latest
1212
command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
13-
ports:
14-
- "127.0.0.1:8000:8000/tcp"
15-
- "4024:4024/tcp"
1613
volumes:
1714
- ./config.yml:/opt/pyaleph/config.yml
1815
- ./keys:/opt/pyaleph/keys
1916
- pyaleph-local-storage:/var/lib/pyaleph
20-
2117
depends_on:
2218
- postgres
2319
- ipfs
@@ -29,6 +25,27 @@ services:
2925
options:
3026
max-size: 50m
3127

28+
pyaleph-api:
29+
restart: always
30+
image: alephim/pyaleph-node:latest
31+
command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
32+
entrypoint: ["bash", "deployment/scripts/run_aleph_ccn_api.sh"]
33+
ports:
34+
- "4024:4024/tcp"
35+
volumes:
36+
- ./config.yml:/opt/pyaleph/config.yml
37+
- pyaleph-local-storage:/var/lib/pyaleph
38+
environment:
39+
CCN_CONFIG_API_PORT: 4024
40+
CCN_CONFIG_API_NB_WORKERS: 8
41+
depends_on:
42+
- pyaleph
43+
networks:
44+
- pyaleph
45+
logging:
46+
options:
47+
max-size: 50m
48+
3249
p2p-service:
3350
restart: always
3451
image: alephim/p2p-service:0.1.2

‎deployment/samples/docker-monitoring/docker-compose.yml‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,8 @@ volumes:
1010
services:
1111
pyaleph:
1212
restart: always
13-
image: alephim/pyaleph-node:v0.5.0-rc1
13+
image: alephim/pyaleph-node:latest
1414
command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
15-
ports:
16-
- "127.0.0.1:8000:8000/tcp"
17-
- "4024:4024/tcp"
1815
volumes:
1916
- ./config.yml:/opt/pyaleph/config.yml
2017
- ./keys:/opt/pyaleph/keys
@@ -30,6 +27,27 @@ services:
3027
options:
3128
max-size: 50m
3229

30+
pyaleph-api:
31+
restart: always
32+
image: alephim/pyaleph-node:latest
33+
command: --config /opt/pyaleph/config.yml --key-dir /opt/pyaleph/keys -v
34+
entrypoint: ["bash", "deployment/scripts/run_aleph_ccn_api.sh"]
35+
ports:
36+
- "4024:4024/tcp"
37+
volumes:
38+
- ./config.yml:/opt/pyaleph/config.yml
39+
- pyaleph-local-storage:/var/lib/pyaleph
40+
environment:
41+
CCN_CONFIG_API_PORT: 4024
42+
CCN_CONFIG_API_NB_WORKERS: 8
43+
depends_on:
44+
- pyaleph
45+
networks:
46+
- pyaleph
47+
logging:
48+
options:
49+
max-size: 50m
50+
3351
p2p-service:
3452
restart: always
3553
image: alephim/p2p-service:0.1.2
@@ -106,7 +124,7 @@ services:
106124
volumes:
107125
- ./prometheus.yml:/etc/prometheus/prometheus.yml
108126
networks:
109-
- pyaleph
127+
- pyaleph-api
110128
- grafana
111129

112130
grafana:

‎deployment/scripts/run_aleph_ccn.sh‎

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,36 +26,7 @@ while test $# -gt 0; do
2626
shift
2727
done
2828

29-
function get_config()
30-
{
31-
config_key="$1"
32-
config_value=$(python3 "${SCRIPT_DIR}/get_config_value.py" --config-file "${CONFIG_FILE}" "${config_key}")
33-
echo "${config_value}"
34-
}
35-
36-
function wait_for_it()
37-
{
38-
"${SCRIPT_DIR}"/wait-for-it.sh "$@"
39-
}
40-
41-
POSTGRES_HOST=$(get_config postgres.host)
42-
POSTGRES_PORT=$(get_config postgres.port)
43-
IPFS_HOST=$(get_config ipfs.host)
44-
IPFS_PORT=$(get_config ipfs.port)
45-
RABBITMQ_HOST=$(get_config rabbitmq.host)
46-
RABBITMQ_PORT=$(get_config rabbitmq.port)
47-
REDIS_HOST=$(get_config redis.host)
48-
REDIS_PORT=$(get_config redis.port)
49-
P2P_SERVICE_HOST=$(get_config p2p.daemon_host)
50-
P2P_SERVICE_CONTROL_PORT=$(get_config p2p.control_port)
51-
52-
if [ "$(get_config ipfs.enabled)" = "True" ]; then
53-
wait_for_it -h "${IPFS_HOST}" -p "${IPFS_PORT}"
54-
fi
55-
56-
wait_for_it -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}"
57-
wait_for_it -h "${RABBITMQ_HOST}" -p "${RABBITMQ_PORT}"
58-
wait_for_it -h "${REDIS_HOST}" -p "${REDIS_PORT}"
59-
wait_for_it -h "${P2P_SERVICE_HOST}" -p "${P2P_SERVICE_CONTROL_PORT}"
29+
source ${SCRIPT_DIR}/wait_for_services.sh
30+
wait_for_services "${CONFIG_FILE}"
6031

6132
exec pyaleph "${PYALEPH_ARGS[@]}"

‎deployment/scripts/run_aleph_ccn_api.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
# Starts an Aleph Core Channel Node API server.
3+
4+
set -euo pipefail
5+
6+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
7+
CONFIG_FILE="/var/pyaleph/config.yml"
8+
9+
while test $# -gt 0; do
10+
case "$1" in
11+
--help)
12+
help
13+
;;
14+
--config)
15+
CONFIG_FILE="$2"
16+
shift
17+
;;
18+
esac
19+
shift
20+
done
21+
22+
source ${SCRIPT_DIR}/wait_for_services.sh
23+
wait_for_services "${CONFIG_FILE}"
24+
25+
NB_WORKERS="${CCN_CONFIG_API_NB_WORKERS:-4}"
26+
PORT=${CCN_CONFIG_API_PORT:-4024}
27+
28+
echo "Starting aleph.im CCN API server on port ${PORT} (${NB_WORKERS} workers)"
29+
30+
exec gunicorn \
31+
"aleph.api_entrypoint:create_app" \
32+
--bind 0.0.0.0:${PORT} \
33+
--worker-class aiohttp.worker.GunicornUVLoopWebWorker \
34+
--workers ${NB_WORKERS} \
35+
--access-logfile "-"

‎deployment/scripts/wait_for_services.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
2+
3+
function get_config() {
4+
config_file="$1"
5+
config_key="$2"
6+
config_value=$(python3 "${SCRIPT_DIR}/get_config_value.py" --config-file "${CONFIG_FILE}" "${config_key}")
7+
echo "${config_value}"
8+
}
9+
10+
function wait_for_it() {
11+
"${SCRIPT_DIR}"/wait-for-it.sh "$@"
12+
}
13+
14+
function wait_for_services() {
15+
config_file="$1"
16+
17+
POSTGRES_HOST=$(get_config "${config_file}" postgres.host)
18+
POSTGRES_PORT=$(get_config "${config_file}" postgres.port)
19+
IPFS_HOST=$(get_config "${config_file}" ipfs.host)
20+
IPFS_PORT=$(get_config "${config_file}" ipfs.port)
21+
RABBITMQ_HOST=$(get_config "${config_file}" rabbitmq.host)
22+
RABBITMQ_PORT=$(get_config "${config_file}" rabbitmq.port)
23+
REDIS_HOST=$(get_config "${config_file}" redis.host)
24+
REDIS_PORT=$(get_config "${config_file}" redis.port)
25+
P2P_SERVICE_HOST=$(get_config "${config_file}" p2p.daemon_host)
26+
P2P_SERVICE_CONTROL_PORT=$(get_config "${config_file}" p2p.control_port)
27+
28+
if [ "$(get_config "${config_file}" ipfs.enabled)" = "True" ]; then
29+
wait_for_it -h "${IPFS_HOST}" -p "${IPFS_PORT}"
30+
fi
31+
32+
wait_for_it -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}"
33+
wait_for_it -h "${RABBITMQ_HOST}" -p "${RABBITMQ_PORT}"
34+
wait_for_it -h "${REDIS_HOST}" -p "${REDIS_PORT}"
35+
wait_for_it -h "${P2P_SERVICE_HOST}" -p "${P2P_SERVICE_CONTROL_PORT}"
36+
}

‎setup.cfg‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ install_requires =
5050
cosmospy==6.0.0
5151
dataclasses_json==0.5.6
5252
eth_account==0.8.0
53+
gunicorn==20.1.0
5354
hexbytes==0.2.2
5455
msgpack==1.0.3 # required by aiocache
5556
multiaddr==0.0.9 # for libp2p-stubs
@@ -72,6 +73,7 @@ install_requires =
7273
substrate-interface==1.3.4
7374
ujson==5.1.0 # required by aiocache
7475
urllib3==1.26.11
76+
uvloop==0.17.0
7577
web3==6.0.0b9
7678

7779
dependency_links =

‎src/aleph/api_entrypoint.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import logging
2+
from pathlib import Path
3+
4+
import sentry_sdk
5+
from aiohttp import web
6+
from configmanager import Config
7+
8+
import aleph.config
9+
from aleph.db.connection import make_engine, make_session_factory
10+
from aleph.services.cache.node_cache import NodeCache
11+
from aleph.services.ipfs import IpfsService
12+
from aleph.services.ipfs.common import make_ipfs_client
13+
from aleph.services.p2p import init_p2p_client
14+
from aleph.services.storage.fileystem_engine import FileSystemStorageEngine
15+
from aleph.storage import StorageService
16+
from aleph.toolkit.monitoring import setup_sentry
17+
from aleph.web import create_aiohttp_app
18+
from aleph.web.controllers.app_state_getters import (
19+
APP_STATE_CONFIG,
20+
APP_STATE_MQ_CONN,
21+
APP_STATE_NODE_CACHE,
22+
APP_STATE_P2P_CLIENT,
23+
APP_STATE_SESSION_FACTORY,
24+
APP_STATE_STORAGE_SERVICE,
25+
)
26+
27+
28+
async def configure_aiohttp_app(
29+
config: Config,
30+
) -> web.Application:
31+
with sentry_sdk.start_transaction(name=f"init-api-server"):
32+
p2p_client = await init_p2p_client(config, service_name=f"api-server-aiohttp")
33+
34+
engine = make_engine(
35+
config,
36+
echo=config.logging.level.value == logging.DEBUG,
37+
application_name=f"aleph-api",
38+
)
39+
session_factory = make_session_factory(engine)
40+
41+
node_cache = NodeCache(
42+
redis_host=config.redis.host.value, redis_port=config.redis.port.value
43+
)
44+
45+
ipfs_client = make_ipfs_client(config)
46+
ipfs_service = IpfsService(ipfs_client=ipfs_client)
47+
storage_service = StorageService(
48+
storage_engine=FileSystemStorageEngine(folder=config.storage.folder.value),
49+
ipfs_service=ipfs_service,
50+
node_cache=node_cache,
51+
)
52+
53+
app = create_aiohttp_app()
54+
55+
app[APP_STATE_CONFIG] = config
56+
app[APP_STATE_P2P_CLIENT] = p2p_client
57+
# Reuse the connection of the P2P client to avoid opening two connections
58+
app[APP_STATE_MQ_CONN] = p2p_client.mq_client.connection
59+
app[APP_STATE_NODE_CACHE] = node_cache
60+
app[APP_STATE_STORAGE_SERVICE] = storage_service
61+
app[APP_STATE_SESSION_FACTORY] = session_factory
62+
63+
return app
64+
65+
66+
async def create_app() -> web.Application:
67+
config = aleph.config.app_config
68+
69+
# TODO: make the config file path configurable
70+
config_file = Path.cwd() / "config.yml"
71+
config.yaml.load(str(config_file))
72+
73+
logging.basicConfig(level=config.logging.level.value)
74+
75+
if config.sentry.dsn.value:
76+
setup_sentry(config)
77+
78+
return await configure_aiohttp_app(config=config)

0 commit comments

Comments
 (0)