Skip to content

Commit e7e6024

Browse files
regisbregisb
committed
🌅
0 parents  commit e7e6024

19 files changed

+377
-0
lines changed

‎.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.*.swp
2+
!.gitignore
3+
TODO
4+
__pycache__
5+
*.egg-info/
6+
/build/
7+
/dist/

‎MANIFEST.in

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
recursive-include tutorvision/patches *
2+
recursive-include tutorvision/templates *

‎README.rst

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
vision plugin for `Tutor <https://docs.tutor.overhang.io>`__
2+
===================================================================================
3+
4+
TODO:
5+
- Collect data with Vector
6+
- Collect tracking logs
7+
- Collect nginx logs
8+
- Send logs to clickhouse
9+
- Make it optional to mount /var/run/docker.sock
10+
- adjust vector verbosity
11+
- log everything to file instead of console? -> tmp volume
12+
- Provision clickhouse
13+
- make database name a tutor config
14+
- make clickhouse host a tutor config
15+
- specify TTL for tables?
16+
- Expose data with redash
17+
- Provision dashboards
18+
- Utility tools for authentication
19+
- Kubernetes compatibility
20+
- Sweet readme
21+
22+
Installation
23+
------------
24+
25+
::
26+
27+
pip install git+https://github.com/overhangio/tutor-vision
28+
29+
Usage
30+
-----
31+
32+
::
33+
34+
tutor plugins enable vision
35+
tutor local quickstart
36+
37+
To access the analytics frontend, open http(s)://vision.<YOUR_LMS_HOST> in your browser. When running locally, this will be http://vision.local.overhang.io. The email address and password required for logging in are::
38+
39+
tutor config printvalue VISION_REDASH_ROOT_EMAIL
40+
tutor config printvalue VISION_REDASH_ROOT_PASSWORD
41+
42+
Development
43+
-----------
44+
45+
To reload Vector configuration after changes to vector.toml, run::
46+
47+
tutor config save && tutor local exec vision-vector sh kill -s HUP
48+
49+
License
50+
-------
51+
52+
This software is licensed under the terms of the AGPLv3.

‎setup.py

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import io
2+
import os
3+
from setuptools import setup, find_packages
4+
5+
here = os.path.abspath(os.path.dirname(__file__))
6+
7+
with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
8+
readme = f.read()
9+
10+
about = {}
11+
with io.open(
12+
os.path.join(here, "tutorvision", "__about__.py"),
13+
"rt",
14+
encoding="utf-8",
15+
) as f:
16+
exec(f.read(), about)
17+
18+
setup(
19+
name="tutor-vision",
20+
version=about["__version__"],
21+
url="https://github.com/overhangio/tutor-vision",
22+
project_urls={
23+
"Code": "https://github.com/overhangio/tutor-vision",
24+
"Issue tracker": "https://github.com/overhangio/tutor-vision/issues",
25+
},
26+
license="AGPLv3",
27+
author="Overhang.IO",
28+
description="vision plugin for Tutor",
29+
long_description=readme,
30+
packages=find_packages(exclude=["tests*"]),
31+
include_package_data=True,
32+
python_requires=">=3.5",
33+
install_requires=["tutor-openedx"],
34+
entry_points={
35+
"tutor.plugin.v0": [
36+
"vision = tutorvision.plugin"
37+
]
38+
},
39+
classifiers=[
40+
"Development Status :: 3 - Alpha",
41+
"Intended Audience :: Developers",
42+
"License :: OSI Approved :: GNU Affero General Public License v3",
43+
"Operating System :: OS Independent",
44+
"Programming Language :: Python",
45+
"Programming Language :: Python :: 3.5",
46+
"Programming Language :: Python :: 3.6",
47+
"Programming Language :: Python :: 3.7",
48+
"Programming Language :: Python :: 3.8",
49+
"Programming Language :: Python :: 3.9",
50+
"Programming Language :: Python :: 3.10",
51+
],
52+
)

‎tutorvision/__about__.py

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__version__ = "0.1.0"

‎tutorvision/__init__.py

Whitespace-only changes.

‎tutorvision/patches/.gitignore

Whitespace-only changes.

‎tutorvision/patches/caddyfile

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Vision
2+
{{ VISION_REDASH_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} {
3+
reverse_proxy nginx:80
4+
}

‎tutorvision/patches/local-docker-compose-jobs-services

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
vision-clickhouse-job:
2+
image: {{ VISION_CLICKHOUSE_DOCKER_IMAGE }}
3+
depends_on: {{ [("vision-clickhouse", VISION_RUN_CLICKHOUSE)]|list_if }}
4+
vision-redash-job:
5+
image: {{ VISION_REDASH_DOCKER_IMAGE }}
6+
command: create_db
7+
env_file: ../plugins/vision/apps/redash/env
8+
depends_on:
9+
- vision-postgres
10+
- vision-redis

‎tutorvision/patches/local-docker-compose-services

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
####### vision plugin
2+
3+
# log collection
4+
vision-vector:
5+
image: docker.io/timberio/vector:0.11.X-alpine
6+
volumes:
7+
- ../plugins/vision/apps/vector/vector.toml:/etc/vector/vector.toml:ro
8+
- /var/run/docker.sock:/var/run/docker.sock:ro
9+
environment:
10+
- DOCKER_HOST=/var/run/docker.sock
11+
restart: unless-stopped
12+
13+
{% if VISION_RUN_CLICKHOUSE %}
14+
# log storage
15+
vision-clickhouse:
16+
image: {{ VISION_CLICKHOUSE_DOCKER_IMAGE }}
17+
volumes:
18+
- ../../data/vision/clickhouse:/var/lib/clickhouse
19+
ulimits:
20+
nofile:
21+
soft: 262144
22+
hard: 262144
23+
restart: unless-stopped
24+
{% endif %}
25+
26+
# frontend
27+
vision-redash-server:
28+
image: {{ VISION_REDASH_DOCKER_IMAGE }}
29+
command: server
30+
env_file: ../plugins/vision/apps/redash/env
31+
environment:
32+
REDASH_WEB_WORKERS: 4
33+
restart: unless-stopped
34+
depends_on:
35+
- vision-postgres
36+
- vision-redis
37+
vision-redash-scheduler:
38+
image: {{ VISION_REDASH_DOCKER_IMAGE }}
39+
command: scheduler
40+
env_file: ../plugins/vision/apps/redash/env
41+
environment:
42+
QUEUES: "celery"
43+
WORKERS_COUNT: 1
44+
restart: unless-stopped
45+
depends_on:
46+
- vision-postgres
47+
- vision-redis
48+
vision-redash-scheduled-worker:
49+
image: {{ VISION_REDASH_DOCKER_IMAGE }}
50+
command: worker
51+
env_file: ../plugins/vision/apps/redash/env
52+
environment:
53+
QUEUES: "scheduled_queries,schemas"
54+
WORKERS_COUNT: 1
55+
restart: unless-stopped
56+
depends_on:
57+
- vision-postgres
58+
- vision-redis
59+
vision-redash-adhoc-worker:
60+
image: {{ VISION_REDASH_DOCKER_IMAGE }}
61+
command: worker
62+
env_file: ../plugins/vision/apps/redash/env
63+
environment:
64+
QUEUES: "queries"
65+
WORKERS_COUNT: 2
66+
restart: unless-stopped
67+
depends_on:
68+
- vision-postgres
69+
- vision-redis
70+
vision-redis:
71+
image: docker.io/redis:5.0-alpine
72+
restart: unless-stopped
73+
vision-postgres:
74+
image: docker.io/postgres:9.6-alpine
75+
environment:
76+
POSTGRES_USER: "{{ VISION_REDASH_POSTGRESQL_USER }}"
77+
POSTGRES_PASSWORD: "{{ VISION_REDASH_POSTGRESQL_PASSWORD }}"
78+
POSTGRES_DB: "{{ VISION_REDASH_POSTGRESQL_DB }}"
79+
volumes:
80+
- ../../data/vision/redash/postgres:/var/lib/postgresql/data
81+
restart: unless-stopped

‎tutorvision/patches/nginx-extra

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Vision
2+
upstream vision-backend {
3+
server vision-redash-server:5000 fail_timeout=0;
4+
}
5+
server {
6+
listen 80;
7+
server_name {{ VISION_REDASH_HOST }};
8+
9+
# Disables server version feedback on pages and in headers
10+
server_tokens off;
11+
12+
client_max_body_size 10m;
13+
14+
location / {
15+
proxy_set_header Host $http_host;
16+
proxy_redirect off;
17+
proxy_pass http://vision-backend;
18+
}
19+
}

‎tutorvision/plugin.py

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
from glob import glob
2+
import os
3+
4+
from .__about__ import __version__
5+
6+
HERE = os.path.abspath(os.path.dirname(__file__))
7+
8+
templates = os.path.join(HERE, "templates")
9+
10+
config = {
11+
"add": {
12+
"REDASH_POSTGRESQL_PASSWORD": "{{ 20|random_string }}",
13+
"REDASH_COOKIE_SECRET": "{{ 20|random_string }}",
14+
"REDASH_ROOT_PASSWORD": "{{ 20|random_string }}",
15+
"REDASH_SECRET_KEY": "{{ 20|random_string }}",
16+
},
17+
"defaults": {
18+
"CLICKHOUSE_DOCKER_IMAGE": "docker.io/yandex/clickhouse-server:20.8.14.4",
19+
"RUN_CLICKHOUSE": True,
20+
"CLICKHOUSE_HOST": "vision-clickhouse",
21+
"CLICKHOUSE_HTTP_PORT": 8123,
22+
"CLICKHOUSE_PORT": 9000,
23+
"CLICKHOUSE_DATABASE": "openedx_{{ ID }}",
24+
"REDASH_DOCKER_IMAGE": "docker.io/redash/redash:8.0.0.b32245",
25+
"REDASH_POSTGRESQL_USER": "redash",
26+
"REDASH_POSTGRESQL_DB": "redash",
27+
"REDASH_HOST": "vision.{{ LMS_HOST }}",
28+
"REDASH_ROOT_USERNAME": "admin",
29+
"REDASH_ROOT_EMAIL": "{{ CONTACT_EMAIL }}",
30+
},
31+
}
32+
33+
hooks = {"init": ["vision-clickhouse", "vision-redash"]}
34+
35+
36+
def patches():
37+
all_patches = {}
38+
for path in glob(os.path.join(HERE, "patches", "*")):
39+
with open(path) as patch_file:
40+
name = os.path.basename(path)
41+
content = patch_file.read()
42+
all_patches[name] = content
43+
return all_patches

‎tutorvision/templates/vision/apps/.gitignore

Whitespace-only changes.

‎tutorvision/templates/vision/apps/redash/env

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
PYTHONUNBUFFERED=0
2+
REDASH_LOG_LEVEL=INFO
3+
REDASH_REDIS_URL=redis://vision-redis:6379/0
4+
REDASH_COOKIE_SECRET="{{ VISION_REDASH_COOKIE_SECRET }}"
5+
REDASH_SECRET_KEY="{{ VISION_REDASH_SECRET_KEY }}"
6+
REDASH_DATABASE_URL="postgresql://{{ VISION_REDASH_POSTGRESQL_USER }}:{{ VISION_REDASH_POSTGRESQL_PASSWORD }}@vision-postgres/{{ VISION_REDASH_POSTGRESQL_DB }}"
7+
REDASH_MAIL_SERVER="{{ SMTP_HOST }}"
8+
REDASH_MAIL_PORT="{{ SMTP_PORT }}"
9+
REDASH_MAIL_USE_TLS="{{ SMTP_USE_TLS }}"
10+
REDASH_MAIL_USE_SSL="{{ SMTP_USE_SSL }}"
11+
REDASH_MAIL_USERNAME="{{ SMTP_USERNAME }}"
12+
REDASH_MAIL_PASSWORD="{{ SMTP_PASSWORD }}"
13+
REDASH_MAIL_DEFAULT_SENDER="{{ CONTACT_EMAIL }}"
14+
REDASH_HOST="{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ VISION_REDASH_HOST }}"

‎tutorvision/templates/vision/apps/vector/vector.toml

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Vector's API for introspection
2+
[api]
3+
enabled = true
4+
address = "127.0.0.1:8686"
5+
6+
### Sources
7+
8+
# Capture logs from all containers
9+
[sources.containers]
10+
type = "docker_logs"
11+
12+
### Transforms
13+
[transforms.openedx_containers]
14+
type = "filter"
15+
inputs = ["containers"]
16+
condition.type = "check_fields"
17+
condition."label.com.docker.compose.regex" = "(lms|cms)"
18+
19+
[transforms.nginx_containers]
20+
type = "filter"
21+
inputs = ["containers"]
22+
condition.type = "check_fields"
23+
condition."label.com.docker.compose.eq" = "nginx"
24+
25+
[transforms.tracking_raw]
26+
type = "regex_parser"
27+
inputs = ["openedx_containers"]
28+
drop_failed = true
29+
drop_field = false
30+
field = "message"
31+
# 2021-03-09 13:08:41,292 INFO 21 [tracking] [user 3] [ip 172.18.0.1] logger.py:42 - {...}
32+
patterns = ["^.* \\[tracking\\] [^{}]* (?P<tracking_message>\\{.*\\})$"]
33+
34+
[transforms.tracking]
35+
type = "remap"
36+
inputs = ["tracking_raw"]
37+
# Time formats: https://docs.rs/chrono/0.4.19/chrono/format/strftime/index.html#specifiers
38+
source = '''
39+
.message = .tracking_message
40+
.tracking_message = parse_json(.tracking_message)
41+
.time = parse_timestamp(.tracking_message.time, format="%+")
42+
del(.tracking_message)
43+
'''
44+
45+
### Sinks
46+
47+
# Log all events to stdout, for debugging
48+
[sinks.out]
49+
type = "console"
50+
inputs = ["tracking"]
51+
encoding.codec = "json"
52+
53+
# Send logs to clickhouse
54+
[sinks.clickhouse]
55+
type = "clickhouse"
56+
encoding.only_fields = ["time", "message"]
57+
# Required: https://github.com/timberio/vector/issues/5797
58+
encoding.timestamp_format = "unix"
59+
inputs = ["tracking"]
60+
endpoint = "http://{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_HTTP_PORT }}"
61+
database = "{{ VISION_CLICKHOUSE_DATABASE }}"
62+
table = "tracking"
63+
healthcheck = true
64+
65+
{{ patch("vision-vector-toml") }}

‎tutorvision/templates/vision/build/.gitignore

Whitespace-only changes.

‎tutorvision/templates/vision/hooks/.gitignore

Whitespace-only changes.

‎tutorvision/templates/vision/hooks/vision-clickhouse/init

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
clickhouse-client --host {{ VISION_CLICKHOUSE_HOST }} --port {{ VISION_CLICKHOUSE_PORT }} \
2+
--query "CREATE DATABASE IF NOT EXISTS {{ VISION_CLICKHOUSE_DATABASE }}"
3+
4+
# TODO add PARTITION BY?
5+
clickhouse-client --host {{ VISION_CLICKHOUSE_HOST }} --port {{ VISION_CLICKHOUSE_PORT}} --database {{ VISION_CLICKHOUSE_DATABASE }} \
6+
--query 'CREATE TABLE IF NOT EXISTS tracking (
7+
`time` DateTime,
8+
`message` String
9+
) ENGINE MergeTree ORDER BY time'
10+
11+
# TODO add materialized view https://youtu.be/pZkKsfr8n3M?t=1731
12+
clickhouse-client --host {{ VISION_CLICKHOUSE_HOST }} --port {{ VISION_CLICKHOUSE_PORT}} --database {{ VISION_CLICKHOUSE_DATABASE }} \
13+
--query 'CREATE VIEW IF NOT EXISTS events AS
14+
SELECT
15+
time,
16+
JSONExtractString(message, 'name') as name,
17+
JSONExtract(message, 'context', 'course_id', 'String') as course_id,
18+
JSONExtract(message, 'context', 'user_id', 'Int64') as user_id
19+
FROM tracking
20+
WHERE JSONExtractString(message, 'event_source')='browser'
21+
ORDER BY time'

‎tutorvision/templates/vision/hooks/vision-redash/init

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
./manage.py database create_tables
2+
./manage.py users create_root --password={{ VISION_REDASH_ROOT_PASSWORD }} {{ VISION_REDASH_ROOT_EMAIL }} {{ VISION_REDASH_ROOT_USERNAME }} || echo "Skipping admin user creation"
3+
4+
(./manage.py ds list | grep datalake && echo "datalake data source already exists") || \
5+
(echo "creating datalake data source..." && \
6+
./manage.py ds new --type=clickhouse --options='{"url":"http://{{ VISION_CLICKHOUSE_HOST }}:{{ VISION_CLICKHOUSE_HTTP_PORT }}", "dbname": "{{ VISION_CLICKHOUSE_DATABASE }}"}' datalake)

0 commit comments

Comments
 (0)