Skip to content

Commit 168b062

Browse files
author
Marcelo Rodrigues Da Silva Soares
committed
Merge branch 'dev' into 'master'
Dev > Master See merge request dbdev/dbaas!2
2 parents c6bd17a + 94e2962 commit 168b062

File tree

31 files changed

+1932
-233
lines changed

31 files changed

+1932
-233
lines changed

‎Makefile

+9-10
Original file line numberDiff line numberDiff line change
@@ -228,27 +228,27 @@ docker_stop:
228228
# a gcp possui mais de um env
229229
# para acessar os secrets corretos, garantimos que estamos no projeto correto
230230
set_env:
231-
@echo "Project env:${PROJECT_ENV}"; \
232-
if [ "${PROJECT_ENV}" = "DEV" ]; then \
231+
@echo "Project env:${ENV}"; \
232+
if [ "${ENV}" = "DEV" ]; then \
233233
echo 'changing project to DEV'; \
234234
LOWERCASE_ENV="dev"; \
235235
gcloud config set project gglobo-dbaaslab-dev-qa; \
236236
exit 0; \
237-
elif [ "${PROJECT_ENV}" = "PROD" ]; then \
237+
elif [ "${ENV}" = "PROD" ]; then \
238238
echo 'changing project to PROD'; \
239239
LOWERCASE_ENV="prod"; \
240240
gcloud config set project gglobo-dbaas-hub; \
241241
exit 0; \
242242
else\
243-
echo "PROJECT_ENV not found. Exiting";\
244-
echo "please call like this: make set_env PROJECT_ENV=PROD or DEV";\
243+
echo "ENV not found. Exiting";\
244+
echo "please call like this: make set_env ENV=PROD or DEV";\
245245
exit 1;\
246246
fi\
247247

248248
get_last_tag:
249249
@echo "exemplo de uso make get_last_tag ENV=DEV"; \
250250
echo "exemplo de uso make get_last_tag ENV=PROD"; \
251-
make set_env PROJECT_ENV=${ENV}; \
251+
make set_env ENV=${ENV}; \
252252
SECRET_NAME="DBDEV_${ENV}_DBAAS_IMAGE_VERSION"; \
253253
echo "Getting secret $${SECRET_NAME}"; \
254254
gcloud secrets versions access "latest" --secret "$${SECRET_NAME}"; \
@@ -258,7 +258,7 @@ set_new_tag:
258258
@echo "tag usada:${TAG}"
259259
@echo "exemplo de uso make set_tag TAG=v1.02 ENV=DEV"
260260
@echo "exemplo de uso make set_tag TAG=v1.034 ENV=PROD"
261-
@make set_env PROJECT_ENV=${ENV}; \
261+
@make set_env ENV=${ENV}; \
262262
if [ $$? -eq 0 ]; then \
263263
echo "env set"; \
264264
else \
@@ -282,7 +282,6 @@ docker_deploy_gcp:
282282
@echo "make docker_deploy_gcp TAG=v1.02 ENV=PROD"
283283
make docker_deploy_build TAG=${TAG}
284284
make docker_deploy_push TAG=${TAG}
285-
make set_new_tag ENV=${ENV} TAG=${TAG}
286285

287286
docker_deploy_build:
288287
@echo "tag usada:${TAG}"
@@ -344,15 +343,15 @@ gcp_deploy_dev_script:
344343

345344
# Mysql utilities
346345
gcp_mysql_dev_cli:
347-
make set_env PROJECT_ENV=DEV; \
346+
make set_env ENV=DEV; \
348347
HOST=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_HOST"); \
349348
USER=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_USER"); \
350349
DB=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_NAME"); \
351350
PASS=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_PASSWORD"); \
352351
mysql -u$$USER -p$$PASS -h$$HOST
353352

354353
gcp_mysql_dev_dump:
355-
make set_env PROJECT_ENV=DEV; \
354+
make set_env ENV=DEV; \
356355
HOST=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_HOST"); \
357356
USER=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_USER"); \
358357
DB=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_NAME"); \

‎dbaas/admin/templates/admin/base.html

-3
Original file line numberDiff line numberDiff line change
@@ -176,9 +176,6 @@
176176
{% for page in flatpages %}
177177
<a href="{{ page.url }}">{{ page.title }}</a>&nbsp;&nbsp;&nbsp;
178178
{% endfor %}
179-
<a href="{{ iaas_status }}" target="_blank">IaaS status</a>&nbsp;&nbsp;&nbsp;
180-
<a href="{{ iaas_quota }}" target="_blank">IaaS quota</a>&nbsp;&nbsp;&nbsp;
181-
<a href="{{ sofia_main_dashboard }}" target="_blank">Sofia Dashboard</a>&nbsp;&nbsp;&nbsp;
182179
</p>
183180
</div>
184181
{% endblock %}

‎dbaas/admin/templatetags/config_tags.py

+9
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,12 @@ def get_config(conf_name=None):
1919
"description": c.description,
2020
"hash": c.hash
2121
}
22+
23+
24+
@register.filter
25+
def is_dba(user, name):
26+
list_validation = Configuration.get_by_name_as_list('list_validation_custom_views')
27+
is_dba = user.team_set.filter(role__name="role_dba").exists()
28+
if is_dba or name in list_validation:
29+
return True
30+
return False

‎dbaas/backup/tasks.py

+57-49
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def mysql_binlog_save(client, instance):
6767
def lock_instance(driver, instance, client):
6868
try:
6969
LOG.debug('Locking instance {}'.format(instance))
70-
driver.lock_database(client)
70+
driver.lock_database_ssh(instance, client)
7171
LOG.debug('Instance {} is locked'.format(instance))
7272
return True
7373
except Exception as e:
@@ -78,7 +78,7 @@ def lock_instance(driver, instance, client):
7878
def unlock_instance(driver, instance, client):
7979
try:
8080
LOG.debug('Unlocking instance {}'.format(instance))
81-
driver.unlock_database(client)
81+
driver.unlock_database_ssh(instance, client)
8282
LOG.debug('Instance {} is unlocked'.format(instance))
8383
return True
8484
except Exception as e:
@@ -230,70 +230,78 @@ def make_instance_gcp_snapshot_backup(
230230

231231
snapshot_final_status = Snapshot.SUCCESS
232232

233-
locked = None
234-
client = None
235233
driver = infra.get_driver()
234+
client = driver.get_client(instance)
236235
try:
237-
client = driver.get_client(instance)
238236
locked = lock_instance(driver, instance, client)
239237
if not locked:
240238
snapshot_final_status = Snapshot.WARNING
241239

242240
if 'MySQL' in type(driver).__name__:
243241
mysql_binlog_save(client, instance)
244242

245-
has_snapshot = Snapshot.objects.filter(
243+
has_snapshot_with_status_warning = Snapshot.objects.filter(
246244
status=Snapshot.WARNING, instance=instance, end_at__year=datetime.now().year,
247245
end_at__month=datetime.now().month, end_at__day=datetime.now().day
248246
)
249-
backup_hour_list = Configuration.get_by_name_as_list('make_database_backup_hour')
250-
if not snapshot_final_status == Snapshot.WARNING and not has_snapshot:
251-
cont = 0
252-
for _ in range(backup_retry_attempts):
253-
cont += 1
254-
try:
255-
code = 201
256-
response, data = provider.new_take_snapshot(persist=persist)
257247

258-
if response.status_code < 400:
259-
break
248+
if has_snapshot_with_status_warning:
249+
raise Exception("Backup with WARNING already created today.")
260250

261-
if cont >= 3:
262-
raise IndexError
251+
if snapshot_final_status == Snapshot.WARNING:
252+
raise Exception("Snapshot has status WARNING, check the logs.")
263253

264-
except IndexError as e:
265-
response, content = e
266-
if response.status_code == 503:
267-
errormsg = "{} - 503 error creating snapshot for instance: {}. It will try again in 30 seconds. ".format(
268-
strftime("%d/%m/%Y %H:%M:%S"), instance
269-
)
270-
LOG.error(errormsg)
271-
if task:
272-
task.add_detail(errormsg)
273-
sleep(30)
274-
else:
275-
raise e
254+
cont = 0
255+
for _ in range(backup_retry_attempts):
256+
cont += 1
257+
try:
258+
code = 201
259+
response, data = provider.new_take_snapshot(persist=persist)
276260

277-
if response.status_code < 400:
278-
while code != 200:
279-
sleep(20)
280-
snap_response, snap_status = provider.take_snapshot_status(data['identifier'])
281-
if snap_response.status_code in [200, 202]:
282-
unlock_instance(driver, instance, client)
283-
if snap_response.status_code == 200:
284-
break
285-
if snap_response.status_code >= 400:
286-
raise error
287-
code = snap_response.status_code
288-
289-
snapshot.done(snap_status)
290-
snapshot.save()
291-
else:
292-
errormsg = response['message']
293-
set_backup_error(infra, snapshot, errormsg)
261+
if response.status_code < 400:
262+
break
263+
264+
if cont >= 3:
265+
raise IndexError
266+
267+
except IndexError as e:
268+
response, content = e
269+
if response.status_code == 503:
270+
errormsg = "{} - 503 error creating snapshot for instance: {}. It will try again in 30 seconds. ".format(
271+
strftime("%d/%m/%Y %H:%M:%S"), instance
272+
)
273+
LOG.error(errormsg)
274+
if task:
275+
task.add_detail(errormsg)
276+
sleep(30)
277+
else:
278+
raise e
279+
280+
count = 0
281+
if response.status_code < 400:
282+
while code != 200:
283+
sleep(20)
284+
snap_response, snap_status = provider.take_snapshot_status(data['identifier'])
285+
if snap_response.status_code in [200, 202]:
286+
unlock_instance(driver, instance, client)
287+
if snap_response.status_code == 200:
288+
break
289+
if snap_response.status_code >= 400:
290+
raise error
291+
if count >= 90:
292+
errormsg = "Timeout"
293+
set_backup_error(infra, snapshot, errormsg)
294+
raise Exception(errormsg)
295+
else:
296+
count += 1
297+
code = snap_response.status_code
298+
299+
snapshot.done(snap_status)
300+
snapshot.save()
294301
else:
295-
if str(current_hour) in backup_hour_list:
296-
raise Exception("Backup with WARNING already created today.")
302+
errormsg = response['message']
303+
set_backup_error(infra, snapshot, errormsg)
304+
raise Exception(errormsg)
297305

298306
except Exception as e:
299307
errormsg = "Error creating snapshot: {}".format(e)

‎dbaas/dashboard/admin/__init__.py

-8
This file was deleted.

‎dbaas/dashboard/admin/dashboard.py

Whitespace-only changes.

‎dbaas/dashboard/admin/dashboard_sofia.py

Whitespace-only changes.

‎dbaas/dashboard/templates/dashboard/sofia_dashboard.html

-31
This file was deleted.

‎dbaas/dashboard/urls.py

-2
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55

66
url(r"^$", "dashboard", name="dashboard.index"),
77

8-
url(r'^sofia_dashboard/$', "sofia_dashboard"),
9-
108
url(r'^databaseinfra/(?P<infra_id>\d+)/?$',
119
'databaseinfra', name='databaseinfra.index'),
1210

‎dbaas/dbaas/settings.py

-1
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,6 @@ def LOCAL_FILES(path):
252252
"django.core.context_processors.tz",
253253
"django.contrib.messages.context_processors.messages",
254254
'django.core.context_processors.request',
255-
'dbaas.views.external_links',
256255
)
257256

258257
# backend for haystack

‎dbaas/dbaas/templates/adminplus/index.html

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{% extends "admin/index.html" %}
2+
{% load config_tags %}
23

34
{% block sidebar %}
45
{{ block.super }}
@@ -20,7 +21,9 @@
2021
</thead>
2122
<tbody>
2223
{% for path, name in custom_list %}
23-
<tr><td><a href="{{ path }}">{{ name }}</a></td></tr>
24+
{% if user|is_dba:name %}
25+
<tr><td><a href="{{ path }}">{{ name }}</a></td></tr>
26+
{% endif %}
2427
{% endfor %}
2528
</tbody>
2629
</table>

‎dbaas/dbaas/views.py

-13
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,5 @@
1-
from system.models import Configuration
21
from django.views.generic import TemplateView
32

43

5-
def external_links(request):
6-
iaas_status = Configuration.get_by_name('iaas_status')
7-
iaas_quota = Configuration.get_by_name('iaas_quota')
8-
sofia_grafana_url = Configuration.get_by_name('sofia_grafana_url')
9-
sofia_grafana_datasource = Configuration.get_by_name('sofia_grafana_datasource')
10-
sofia_dashboard = "{}?var-datasource={}".format(sofia_grafana_url, sofia_grafana_datasource)
11-
12-
return {'iaas_status': iaas_status,
13-
'iaas_quota': iaas_quota,
14-
'sofia_main_dashboard': sofia_dashboard}
15-
16-
174
class DeployView(TemplateView):
185
template_name = 'deploy/deploy.html'

‎dbaas/dbaas_services/analyzing/views.py

+14-12
Original file line numberDiff line numberDiff line change
@@ -42,28 +42,30 @@ def post(self, request, *args, **kwargs):
4242

4343
class DatabaseReport(ListView):
4444

45-
def has_perm(self, request):
45+
def has_permission(self, request):
4646
UserMiddleware.set_current_user(request.user)
4747
from_teams = [x.role for x in Team.objects.filter(users=request.user)]
4848
role_dba = Role.objects.get(name='role_dba')
4949

50-
if role_dba not in from_teams:
51-
return render(request, 'databases/index.html', {"has_perm": False})
50+
if role_dba in from_teams:
51+
return render(request, 'databases/index.html', {"has_perm": True})
52+
5253

5354
def get(self, request, *args, **kwargs):
54-
self.has_perm(request)
55+
if self.has_permission(request):
56+
return render(request, 'databases/index.html', {"has_perm": True})
57+
else:
58+
return HttpResponseRedirect(reverse('admin:index'))
5559

56-
return render(request, 'databases/index.html', {"has_perm": True})
5760

5861
def post(self, request, *args, **kwargs):
59-
self.has_perm(request)
62+
if self.has_permission(request):
63+
database_report = request.POST.get("database_report", "")
64+
if database_report == 'database_report':
65+
return self.default_database_report()
66+
else:
67+
return self.vm_by_line_database_report()
6068

61-
database_report = request.POST.get("database_report", "")
62-
63-
if database_report == 'database_report':
64-
return self.default_database_report()
65-
else:
66-
return self.vm_by_line_database_report()
6769

6870
def vm_by_line_database_report(self):
6971

‎dbaas/drivers/base.py

+6
Original file line numberDiff line numberDiff line change
@@ -214,9 +214,15 @@ def get_client(self, instance):
214214
def lock_database(self, client):
215215
raise NotImplementedError()
216216

217+
def lock_database_ssh(self, instance, client):
218+
return self.lock_database(client)
219+
217220
def unlock_database(self, client):
218221
raise NotImplementedError()
219222

223+
def unlock_database_ssh(self, instance, client):
224+
return self.unlock_database(client)
225+
220226
def check_instance_is_eligible_for_backup(self, instance):
221227
raise NotImplementedError()
222228

0 commit comments

Comments
 (0)