Skip to content

Commit 4417844

Browse files
authored
Merge pull request #55 from osint-dev-team/develop
Add workers. RabbitMQ queue.
2 parents a4b42ed + d0f17dd commit 4417844

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1220
-48
lines changed

.dockerignore

+2
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,5 @@
44
!server.py
55
!requirements.txt
66
!docker/server/wait-for-it.sh
7+
!docker/consumer/wait-for-it.sh
8+
!docker/consumer/consumer.py

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
.idea/
33
.DS_Store
44
results/*.json
5+
src/drivers/chromedriver_*
56

67
# Byte-compiled / optimized / DLL files
78
__pycache__/

Makefile

+10-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,20 @@
11
.PHONY: up up_build test clean
22

33
up:
4-
docker-compose up -d
4+
docker-compose up -d --scale consumer=5
55

6-
up_build:
7-
docker-compose up -d --build --force-recreate
6+
up_log:
7+
docker-compose up --scale consumer=5
8+
9+
build:
10+
docker-compose up -d --build --force-recreate --scale consumer=5
11+
12+
build_log:
13+
docker-compose up --build --force-recreate --scale consumer=5
814

915
tests:
1016
python3 -W ignore:ResourceWarning -m unittest discover -v -b
1117

1218
clean:
1319
docker-compose down
20+
docker volume rm osint-framework_postgres -f

README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ python3 cli.py -h
4848

4949
To run the framework as a web service via docker and docker-compose:
5050
```bash
51-
make up
51+
make up_log
5252
```
5353
or
5454
```bash
55-
docker-compose up
55+
docker-compose up --scale consumer=5
5656
```
5757
## As a separated module
5858
Basic:

assets/screenshots/docker.png

416 KB
Loading

docker-compose.yml

+51
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ services:
44
db:
55
container_name: osint-framework-db
66
image: postgres:alpine
7+
restart: unless-stopped
78
environment:
89
POSTGRES_HOST_AUTH_METHOD: trust
910
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
@@ -14,21 +15,68 @@ services:
1415
interval: 30s
1516
timeout: 5s
1617
retries: 5
18+
ports:
19+
- "127.0.0.1:5432:5432"
1720
volumes:
1821
- postgres:/data/postgres
1922
networks:
2023
- postgres
24+
rabbitmq:
25+
container_name: osint-framework-rabbitmq
26+
image: rabbitmq:alpine
27+
restart: unless-stopped
28+
healthcheck:
29+
test: nc -z localhost 5672
30+
interval: 30s
31+
timeout: 5s
32+
retries: 5
33+
ports:
34+
- "127.0.0.1:5672:5672"
35+
networks:
36+
- rabbitmq
37+
consumer:
38+
image: osint-framework-consumer:1.0
39+
restart: on-failure
40+
depends_on:
41+
- db
42+
- rabbitmq
43+
healthcheck:
44+
test: nc -z osint-framework-rabbitmq 5672
45+
interval: 30s
46+
timeout: 5s
47+
retries: 5
48+
environment:
49+
POSTGRES_DATABASE: ${POSTGRES_DATABASE:-osint}
50+
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
51+
POSTGRES_USER: ${POSTGRES_USER:-osint_framework}
52+
POSTGRES_HOST: ${POSTGRES_HOST:-osint-framework-db}
53+
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
54+
RABBITMQ_HOST: ${RABBITMQ_HOST:-osint-framework-rabbitmq}
55+
RABBITMQ_PORT: ${RABBITMQ_PORT:-5672}
56+
LOG_HANDLER: ${LOG_HANDLER:-stream}
57+
build:
58+
context: .
59+
target: osint-framework-consumer
60+
dockerfile: docker/consumer/Dockerfile
61+
networks:
62+
- postgres
63+
- rabbitmq
2164
server:
2265
container_name: osint-framework-server
2366
image: osint-framework-server:1.0
67+
restart: on-failure
2468
depends_on:
2569
- db
70+
- rabbitmq
2671
environment:
2772
POSTGRES_DATABASE: ${POSTGRES_DATABASE:-osint}
2873
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
2974
POSTGRES_USER: ${POSTGRES_USER:-osint_framework}
3075
POSTGRES_HOST: ${POSTGRES_HOST:-osint-framework-db}
3176
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
77+
RABBITMQ_HOST: ${RABBITMQ_HOST:-osint-framework-rabbitmq}
78+
RABBITMQ_PORT: ${RABBITMQ_PORT:-5672}
79+
LOG_HANDLER: ${LOG_HANDLER:-stream}
3280
build:
3381
context: .
3482
target: osint-framework-server
@@ -42,8 +90,11 @@ services:
4290
- "8888:8888"
4391
networks:
4492
- postgres
93+
- rabbitmq
4594
networks:
4695
postgres:
4796
driver: bridge
97+
rabbitmq:
98+
driver: bridge
4899
volumes:
49100
postgres:

docker/consumer/Dockerfile

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
FROM python:3.8-alpine as osint-framework-consumer
2+
3+
LABEL org.label-schema.name="OSINT Framework Consumer" \
4+
org.label-schema.description="OSINT Framework Consumer" \
5+
org.label-schema.license="GPL-2.0"
6+
7+
COPY . /app/
8+
COPY docker/server/wait-for-it.sh /app/wait-for-it.sh
9+
COPY docker/consumer/consumer.py /app/consumer.py
10+
WORKDIR /app
11+
RUN apk add --no-cache --virtual .build_deps build-base libffi-dev gcc musl-dev && \
12+
apk add --no-cache postgresql-dev bash curl && \
13+
# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
14+
apk add --no-cache unzip libexif udev chromium chromium-chromedriver xvfb && \
15+
pip install --no-cache-dir -r requirements.txt && \
16+
# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
17+
pip install --no-cache-dir pyvirtualdisplay && \
18+
apk del .build_deps && \
19+
chmod +x wait-for-it.sh
20+
21+
# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
22+
ENV DISPLAY=:99
23+
24+
ENTRYPOINT ["./wait-for-it.sh", "-t", "5", "osint-framework-rabbitmq:5672", "--", "python3", "consumer.py"]

docker/consumer/consumer.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env python3
2+
3+
4+
from logging import basicConfig, INFO
5+
6+
from src.queue.consumer import Consumer
7+
8+
basicConfig(level=INFO)
9+
10+
if __name__ == "__main__":
11+
consumer = Consumer()
12+
consumer.start_consuming()

docker/consumer/wait-for-it.sh

+182
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#!/usr/bin/env bash
2+
# Use this script to test if a given TCP host/port are available
3+
4+
WAITFORIT_cmdname=${0##*/}
5+
6+
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
7+
8+
usage()
9+
{
10+
cat << USAGE >&2
11+
Usage:
12+
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
13+
-h HOST | --host=HOST Host or IP under test
14+
-p PORT | --port=PORT TCP port under test
15+
Alternatively, you specify the host and port as host:port
16+
-s | --strict Only execute subcommand if the test succeeds
17+
-q | --quiet Don't output any status messages
18+
-t TIMEOUT | --timeout=TIMEOUT
19+
Timeout in seconds, zero for no timeout
20+
-- COMMAND ARGS Execute command with args after the test finishes
21+
USAGE
22+
exit 1
23+
}
24+
25+
wait_for()
26+
{
27+
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
28+
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
29+
else
30+
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
31+
fi
32+
WAITFORIT_start_ts=$(date +%s)
33+
while :
34+
do
35+
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
36+
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
37+
WAITFORIT_result=$?
38+
else
39+
(echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
40+
WAITFORIT_result=$?
41+
fi
42+
if [[ $WAITFORIT_result -eq 0 ]]; then
43+
WAITFORIT_end_ts=$(date +%s)
44+
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
45+
break
46+
fi
47+
sleep 1
48+
done
49+
return $WAITFORIT_result
50+
}
51+
52+
wait_for_wrapper()
53+
{
54+
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
55+
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
56+
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
57+
else
58+
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
59+
fi
60+
WAITFORIT_PID=$!
61+
trap "kill -INT -$WAITFORIT_PID" INT
62+
wait $WAITFORIT_PID
63+
WAITFORIT_RESULT=$?
64+
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
65+
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
66+
fi
67+
return $WAITFORIT_RESULT
68+
}
69+
70+
# process arguments
71+
while [[ $# -gt 0 ]]
72+
do
73+
case "$1" in
74+
*:* )
75+
WAITFORIT_hostport=(${1//:/ })
76+
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
77+
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
78+
shift 1
79+
;;
80+
--child)
81+
WAITFORIT_CHILD=1
82+
shift 1
83+
;;
84+
-q | --quiet)
85+
WAITFORIT_QUIET=1
86+
shift 1
87+
;;
88+
-s | --strict)
89+
WAITFORIT_STRICT=1
90+
shift 1
91+
;;
92+
-h)
93+
WAITFORIT_HOST="$2"
94+
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
95+
shift 2
96+
;;
97+
--host=*)
98+
WAITFORIT_HOST="${1#*=}"
99+
shift 1
100+
;;
101+
-p)
102+
WAITFORIT_PORT="$2"
103+
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
104+
shift 2
105+
;;
106+
--port=*)
107+
WAITFORIT_PORT="${1#*=}"
108+
shift 1
109+
;;
110+
-t)
111+
WAITFORIT_TIMEOUT="$2"
112+
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
113+
shift 2
114+
;;
115+
--timeout=*)
116+
WAITFORIT_TIMEOUT="${1#*=}"
117+
shift 1
118+
;;
119+
--)
120+
shift
121+
WAITFORIT_CLI=("$@")
122+
break
123+
;;
124+
--help)
125+
usage
126+
;;
127+
*)
128+
echoerr "Unknown argument: $1"
129+
usage
130+
;;
131+
esac
132+
done
133+
134+
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
135+
echoerr "Error: you need to provide a host and port to test."
136+
usage
137+
fi
138+
139+
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
140+
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
141+
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
142+
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
143+
144+
# Check to see if timeout is from busybox?
145+
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
146+
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
147+
148+
WAITFORIT_BUSYTIMEFLAG=""
149+
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
150+
WAITFORIT_ISBUSY=1
151+
# Check if busybox timeout uses -t flag
152+
# (recent Alpine versions don't support -t anymore)
153+
if timeout &>/dev/stdout | grep -q -e '-t '; then
154+
WAITFORIT_BUSYTIMEFLAG="-t"
155+
fi
156+
else
157+
WAITFORIT_ISBUSY=0
158+
fi
159+
160+
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
161+
wait_for
162+
WAITFORIT_RESULT=$?
163+
exit $WAITFORIT_RESULT
164+
else
165+
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
166+
wait_for_wrapper
167+
WAITFORIT_RESULT=$?
168+
else
169+
wait_for
170+
WAITFORIT_RESULT=$?
171+
fi
172+
fi
173+
174+
if [[ $WAITFORIT_CLI != "" ]]; then
175+
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
176+
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
177+
exit $WAITFORIT_RESULT
178+
fi
179+
exec "${WAITFORIT_CLI[@]}"
180+
else
181+
exit $WAITFORIT_RESULT
182+
fi

docker/server/Dockerfile

+7
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,16 @@ COPY docker/server/wait-for-it.sh /app/wait-for-it.sh
99
WORKDIR /app
1010
RUN apk add --no-cache --virtual .build_deps build-base libffi-dev gcc musl-dev && \
1111
apk add --no-cache postgresql-dev bash curl && \
12+
# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
13+
apk add --no-cache unzip libexif udev chromium chromium-chromedriver xvfb && \
1214
pip install --no-cache-dir -r requirements.txt && \
15+
# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
16+
pip install --no-cache-dir pyvirtualdisplay && \
1317
apk del .build_deps && \
1418
chmod +x wait-for-it.sh
1519

20+
# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
21+
ENV DISPLAY=:99
22+
1623
EXPOSE 8888
1724
ENTRYPOINT ["./wait-for-it.sh", "-t", "5", "osint-framework-db:5432", "--", "python3", "server.py"]

0 commit comments

Comments
 (0)