Skip to content

Commit 09de692

Browse files
author
matthias_schaub
committed
Merge branch 'dev'
2 parents 4fc3aaf + 6b8c527 commit 09de692

39 files changed

+1458
-1292
lines changed

docker-compose.yaml

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,13 @@ services:
5656
command: mapswipe_workers --verbose run --schedule m --only_new_results
5757
volumes:
5858
- ./api-data:/var/lib/mapswipe_workers/api-data/:rw
59-
- ./api-data/agg_res_by_task_id:/var/lib/mapswipe_workers/api-data/agg_res_by_task_id:rw
60-
- ./api-data/agg_res_by_project_id_and_date:/var/lib/mapswipe_workers/api-data/agg_res_by_project_id_and_date:rw
61-
- ./api-data/agg_progress_by_project_id_and_date:/var/lib/mapswipe_workers/api-data/agg_progress_by_project_id_and_date:rw
62-
- ./api-data/agg_res_by_user_id_and_date:/var/lib/mapswipe_workers/api-data/agg_res_by_user_id_and_date:rw
63-
restart: unless-stopped
59+
- ./api-data/agg_results:/var/lib/mapswipe_workers/api-data/agg_results:rw
60+
- ./api-data/groups:/var/lib/mapswipe_workers/api-data/groups:rw
61+
- ./api-data/history:/var/lib/mapswipe_workers/api-data/history:rw
62+
- ./api-data/projects:/var/lib/mapswipe_workers/api-data/projects:rw
63+
- ./api-data/results:/var/lib/mapswipe_workers/api-data/results:rw
64+
- ./api-data/tasks:/var/lib/mapswipe_workers/api-data/tasks:rw
65+
restart: "no"
6466
networks:
6567
- mapswipe_workers
6668
- postgres
@@ -73,35 +75,34 @@ services:
7375
POSTGERS_PASSWORD: '${POSTGRES_PASSWORD}'
7476
POSTGRES_USER: 'mapswipe_workers'
7577
POSTGRES_DB: 'mapswipe'
78+
PGDATA: '/var/lib/postgresql/mapswipe'
7679
WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
7780
GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
7881
# Set options for WAL-G (backup tool)
7982
command: postgres -c archive_mode=on -c archive_timeout=60 -c archive_command="/archive_command.sh %p"
8083
volumes:
81-
- ./postgres-data:/var/lib/postgresql/data
84+
- ./postgres-data:/var/lib/postgresql/mapswipe
8285
restart: unless-stopped
8386
ports:
8487
- "5432:5432"
8588
networks:
8689
- postgres
8790

88-
postgres_restore_backup:
89-
container_name: postgres_restore_backup
91+
postgres_recovery:
92+
container_name: postgres_recovery
9093
build:
91-
context: postgres/
92-
dockerfile: Dockerfile-restore_backup
93-
args:
94-
WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
95-
GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
94+
context: postgres/recovery/
9695
environment:
9796
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
9897
POSTGRES_USER: 'mapswipe_workers'
9998
POSTGRES_DB: 'mapswipe'
100-
command: postgres
99+
PGDATA: '/var/lib/postgresql/mapswipe'
100+
WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
101+
GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
101102
volumes:
102-
- ./postgres-data:/var/lib/postgresql/restored_cluster
103+
- ./pgdata_restored:/var/lib/postgresql/mapswipe
103104
ports:
104-
- "5432:5432"
105+
- "5445:5432"
105106

106107
firebase_deploy:
107108
container_name: firebase_deploy

docs/source/cli.md

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ This document describes how to use the command line interface of MapSwipe Worker
44

55
In our current deployment setup the commands of the MapSwipe Workers CLI are hard-coded in the Docker-Compose File.
66

7+
You can run these commands also using docker-compose:
8+
```
9+
docker-compose run mapswipe_workers mapswipe_workers --help
10+
```
11+
712

813
```
914
Usage: mapswipe_workers [OPTIONS] COMMAND [ARGS]...
@@ -56,8 +61,26 @@ Usage: mapswipe_workers generate-stats [OPTIONS]
5661
Options:
5762
-s, --schedule [m|h|d] Generate stats every 10 minutes (m), every hour (h)
5863
or every day (d).
59-
--only_new_results Generate stats for all projects or only for those
60-
updated or with new results.
64+
--project_id_list TEXT provide project id strings as a list stats will be
65+
generated only for these projects.
66+
Use it like '["project_a", "project_b"]'
67+
--help Show this message and exit.
68+
```
69+
70+
## Generate Statistics for all projects
71+
72+
Ideally you run this using a separate docker container. e.g. like this:
73+
74+
```
75+
docker-compose run mapswipe_workers mapswipe_workers generate-stats-all-projects
76+
```
77+
78+
```
79+
Usage: mapswipe_workers generate-stats-all-projects [OPTIONS]
80+
81+
Options:
82+
-s, --schedule [m|h|d] Generate stats every 10 minutes (m), every hour (h)
83+
or every day (d).
6184
--help Show this message and exit.
6285
```
6386

firebase/functions/index.js

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,27 @@ exports.counter = functions.database.ref('/v2/results/{projectId}/{groupId}/{use
1919
const promises = []
2020
const result = snapshot.after.val()
2121

22+
// if result ref does not contain all required attributes we don't updated counters
23+
// e.g. due to some error when uploading from client
24+
if (!result.hasOwnProperty('results')) {
25+
console.log('no results attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
26+
console.log('will not update counters')
27+
return null
28+
} else if (!result.hasOwnProperty('timestamp')) {
29+
console.log('no timestamp attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
30+
console.log('will not update counters')
31+
return null
32+
} else if (!result.hasOwnProperty('endTime')) {
33+
console.log('no endTime attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
34+
console.log('will not update counters')
35+
return null
36+
} else if (!result.hasOwnProperty('startTime')) {
37+
console.log('no startTime attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
38+
console.log('will not update counters')
39+
return null
40+
}
41+
42+
2243
// Firebase Realtime Database references
2344
const groupFinishedCountRef = admin.database().ref('/v2/groups/'+context.params.projectId+'/'+context.params.groupId+'/finishedCount')
2445
const groupRequiredCountRef = admin.database().ref('/v2/groups/'+context.params.projectId+'/'+context.params.groupId+'/requiredCount')

manager_dashboard/manager_dashboard/create.html

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@
3030
integrity="sha512-GffPMF3RvMeYyc1LWMHtK8EbPv0iNZ8/oTtHPx9/cc2ILxQ+u905qIwdpULaqDkyBKgOaB57QTMg7ztg8Jm2Og=="
3131
crossorigin=""></script>
3232
<script src='https://npmcdn.com/@turf/turf/turf.min.js'></script>
33+
34+
<script
35+
src="https://browser.sentry-cdn.com/5.7.1/bundle.min.js"
36+
integrity="sha384-KMv6bBTABABhv0NI+rVWly6PIRvdippFEgjpKyxUcpEmDWZTkDOiueL5xW+cztZZ"
37+
crossorigin="anonymous"></script>
38+
<script>
39+
Sentry.init({ dsn: 'https://[email protected]/1788403' });
40+
</script>
3341
</head>
3442
<body onload="initMap()">
3543

@@ -167,7 +175,7 @@ <h3>Project Type Specific Information</h3>
167175
<div id="geometryMap"></div>
168176
<p id="geometryInfo"></p>
169177
<p id="geometryContent"></p>
170-
<span>Upload your project area as GeoJSON File. Make sure that you provide a single polygon geometry.</span>
178+
<span>Upload your project area as GeoJSON File (max. 2.5MB). Make sure that you provide a single polygon geometry.</span>
171179
</li>
172180
<li>
173181
<label for="tileServerBuildArea">Tile Server Name</label>
@@ -262,7 +270,7 @@ <h3>Project Type Specific Information</h3>
262270
<div id="geometryChangeDetectionMap"></div>
263271
<p id="geometryChangeDetectionInfo"></p>
264272
<p id="geometryChangeDetectionContent"></p>
265-
<span>Upload your project area as GeoJSON File. Make sure that you provide a single polygon geometry.</span>
273+
<span>Upload your project area as GeoJSON File (max. 2.5MB). Make sure that you provide a single polygon geometry.</span>
266274
</li>
267275
<h4>Tile Server A</h4>
268276
<li>

manager_dashboard/manager_dashboard/index.html

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,15 @@
2121
<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
2222
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
2323
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
24+
25+
<script
26+
src="https://browser.sentry-cdn.com/5.7.1/bundle.min.js"
27+
integrity="sha384-KMv6bBTABABhv0NI+rVWly6PIRvdippFEgjpKyxUcpEmDWZTkDOiueL5xW+cztZZ"
28+
crossorigin="anonymous"></script>
29+
<script>
30+
Sentry.init({ dsn: 'https://[email protected]/1788403' });
31+
</script>
32+
2433
</head>
2534
<body>
2635

manager_dashboard/manager_dashboard/js/forms.js

Lines changed: 71 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ function displayImportForm() {
7171

7272
function openFile(event) {
7373
var input = event.target;
74+
7475
var info_element_id = event.target.id + 'Info'
7576
var content_element_id = event.target.id + 'Content'
7677
var map_element_id = event.target.id + 'Map'
@@ -91,67 +92,77 @@ function openFile(event) {
9192
var zoomLevel = parseInt(document.getElementById('zoomLevelChangeDetection').value)
9293
}
9394

94-
var reader = new FileReader();
95-
reader.onload = function(){
96-
97-
try {
98-
var text = reader.result;
99-
var geojsonData = JSON.parse(text)
100-
101-
// check number of features
102-
numberOfFeatures = geojsonData['features'].length
103-
console.log('number of features: ' + numberOfFeatures)
104-
if (numberOfFeatures > 1) {
105-
throw 'too many features: ' + numberOfFeatures
106-
}
107-
info_output.innerHTML += 'Number of Features: ' + numberOfFeatures + '<br>';
108-
info_output.style.display = 'block'
109-
110-
// check input geometry type
111-
feature = geojsonData['features'][0]
112-
type = turf.getType(feature)
113-
console.log('geometry type: ' + type)
114-
if (type !== 'Polygon' & type !== 'MultiPolygon') {
115-
throw 'wrong geometry type: ' + type
116-
}
117-
info_output.innerHTML += 'Feature Type: ' + type + '<br>';
118-
info_output.style.display = 'block'
119-
120-
// check project size
121-
122-
area = turf.area(feature)/1000000 // area in square kilometers
123-
maxArea = (20 - zoomLevel) * (20 - zoomLevel) * 1250
124-
console.log('project size: ' + area + ' sqkm')
125-
if (area > 5000) {
126-
throw 'project is to large: ' + area + ' sqkm; ' + 'max allowed size for this zoom level: ' + maxArea + ' sqkm'
127-
}
128-
info_output.innerHTML += 'Project Size: ' + area + ' sqkm<br>';
129-
info_output.style.display = 'block'
130-
131-
// add feature to map
132-
layer.clearLayers()
133-
layer.addData(geojsonData);
134-
map.fitBounds(layer.getBounds());
135-
console.log('added input geojson feature')
136-
137-
// add text to html object
138-
info_output.innerHTML += 'Project seems to be valid :)';
139-
info_output.style.display = 'block'
140-
141-
if (event.target.id === 'geometry') {
142-
BuildAreaGeometry = text
143-
} else {
144-
ChangeDetectionGeometry = text
145-
}
146-
147-
148-
}
149-
catch(err) {
150-
info_output.innerHTML = '<b>Error reading GeoJSON file</b><br>' + err;
151-
info_output.style.display = 'block'
152-
}
153-
};
95+
// Check file size before loading
96+
var filesize = input.files[0].size;
97+
if (filesize > 2.5 * 1024 * 1024) {
98+
var err='filesize is too big (max 2.5MB): ' + filesize/(1000*1000)
99+
info_output.innerHTML = '<b>Error reading GeoJSON file</b><br>' + err;
100+
info_output.style.display = 'block'
101+
} else {
102+
info_output.innerHTML += 'File Size is valid <br>';
103+
info_output.style.display = 'block'
104+
105+
var reader = new FileReader();
106+
reader.onload = function(){
107+
108+
try {
109+
var text = reader.result;
110+
var geojsonData = JSON.parse(text)
111+
112+
// check number of features
113+
numberOfFeatures = geojsonData['features'].length
114+
console.log('number of features: ' + numberOfFeatures)
115+
if (numberOfFeatures > 1) {
116+
throw 'too many features: ' + numberOfFeatures
117+
}
118+
info_output.innerHTML += 'Number of Features: ' + numberOfFeatures + '<br>';
119+
info_output.style.display = 'block'
120+
121+
// check input geometry type
122+
feature = geojsonData['features'][0]
123+
type = turf.getType(feature)
124+
console.log('geometry type: ' + type)
125+
if (type !== 'Polygon' & type !== 'MultiPolygon') {
126+
throw 'wrong geometry type: ' + type
127+
}
128+
info_output.innerHTML += 'Feature Type: ' + type + '<br>';
129+
info_output.style.display = 'block'
130+
131+
// check project size
132+
133+
area = turf.area(feature)/1000000 // area in square kilometers
134+
maxArea = (20 - zoomLevel) * (20 - zoomLevel) * 1250
135+
console.log('project size: ' + area + ' sqkm')
136+
if (area > 5000) {
137+
throw 'project is to large: ' + area + ' sqkm; ' + 'max allowed size for this zoom level: ' + maxArea + ' sqkm'
138+
}
139+
info_output.innerHTML += 'Project Size: ' + area + ' sqkm<br>';
140+
info_output.style.display = 'block'
141+
142+
// add feature to map
143+
layer.clearLayers()
144+
layer.addData(geojsonData);
145+
map.fitBounds(layer.getBounds());
146+
console.log('added input geojson feature')
147+
148+
// add text to html object
149+
info_output.innerHTML += 'Project seems to be valid :)';
150+
info_output.style.display = 'block'
151+
152+
if (event.target.id === 'geometry') {
153+
BuildAreaGeometry = text
154+
} else {
155+
ChangeDetectionGeometry = text
156+
}
157+
158+
}
159+
catch(err) {
160+
info_output.innerHTML = '<b>Error reading GeoJSON file</b><br>' + err;
161+
info_output.style.display = 'block'
162+
}
163+
};
154164
reader.readAsText(input.files[0]);
165+
}
155166
};
156167

157168
function openImageFile(event) {

manager_dashboard/manager_dashboard/manage.html

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,15 @@
2121
<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
2222
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
2323
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
24+
25+
<script
26+
src="https://browser.sentry-cdn.com/5.7.1/bundle.min.js"
27+
integrity="sha384-KMv6bBTABABhv0NI+rVWly6PIRvdippFEgjpKyxUcpEmDWZTkDOiueL5xW+cztZZ"
28+
crossorigin="anonymous"></script>
29+
<script>
30+
Sentry.init({ dsn: 'https://[email protected]/1788403' });
31+
</script>
32+
2433
</head>
2534
<body>
2635

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
repos:
2+
- repo: https://github.com/psf/black
3+
rev: stable
4+
hooks:
5+
- id: black
6+
- repo: https://gitlab.com/pycqa/flake8
7+
rev: 3.7.9
8+
hooks:
9+
- id: flake8
10+
args: ["--ignore=E501"]

mapswipe_workers/Dockerfile

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,31 @@ RUN apt-get update
77
RUN apt-get install gdal-bin
88

99
# create directories for config, logs and data
10-
RUN mkdir -p /usr/share/config/mapswipe_workers/
11-
RUN mkdir -p /usr/local/mapswipe_workers/
12-
RUN mkdir -p /var/log/mapswipe_workers/
13-
RUN mkdir -p /var/lib/mapswipe_workers/
14-
RUN mkdir -p /var/lib/mapswipe_workers/api-data
15-
RUN mkdir -p /var/lib/mapswipe_workers/api-data/agg_res_by_task_id/
16-
RUN mkdir -p /var/lib/mapswipe_workers/api-data/agg_res_by_project_id_and_date/
17-
RUN mkdir -p /var/lib/mapswipe_workers/api-data/agg_progress_by_project_id_and_date/
18-
RUN mkdir -p /var/lib/mapswipe_workers/api-data/agg_res_by_user_id_and_date/
10+
ARG config_dir=/usr/share/config/mapswipe_workers/
11+
ARG repo_dir=/usr/local/mapswipe_workers/
12+
ARG data_dir=/var/lib/mapswipe_workers/
13+
ARG logs_dir=/var/log/mapswipe_workers/
14+
15+
RUN mkdir -p $config_dir
16+
RUN mkdir -p $repo_dir
17+
RUN mkdir -p $logs_dir
18+
RUN mkdir -p $data_dir
19+
RUN mkdir -p $data_dir"api-data"
20+
RUN mkdir -p $data_dir"api-data/agg_results/"
21+
RUN mkdir -p $data_dir"api-data/groups/"
22+
RUN mkdir -p $data_dir"api-data/history/"
23+
RUN mkdir -p $data_dir"api-data/projects/"
24+
RUN mkdir -p $data_dir"api-data/results/"
25+
RUN mkdir -p $data_dir"api-data/tasks/"
1926

2027
# copy mapswipe workers repo from local repo
21-
WORKDIR /usr/local/mapswipe_workers/
28+
WORKDIR $repo_dir
2229
COPY mapswipe_workers/ mapswipe_workers/
2330
COPY sample_data/ sample_data/
2431
COPY tests/ tests/
2532
COPY requirements.txt .
2633
COPY setup.py .
27-
COPY config /usr/share/config/mapswipe_workers
34+
COPY config $config_dir
2835

2936
# Install dependencies and mapswipe-workers
3037
# RUN python setup.py install

0 commit comments

Comments
 (0)