From ad3b1828b4b52af1aff9a9e7fa29518327197384 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 17 Feb 2023 11:42:44 +0100 Subject: [PATCH 01/18] Created the view for submitting mirrors --- .gitignore | 1 + mirrors/urls_mirrorlist.py | 1 + mirrors/views/mirrorlist.py | 120 ++++++++++++++++++++++++++- requirements.txt | 2 +- templates/mirrors/mirror_submit.html | 34 ++++++++ 5 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 templates/mirrors/mirror_submit.html diff --git a/.gitignore b/.gitignore index df22a1ed0..8e538fe1b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ local_settings.py archweb.db archweb.db-* database.db +/*.tar.gz tags collected_static/ testing/ diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index c4df468a5..43999e7e3 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -3,6 +3,7 @@ urlpatterns = [ path('', views.generate_mirrorlist, name='mirrorlist'), + path('submit/', views.submit_mirror, name='mirrorsubmit'), re_path(r'^all/$', views.find_mirrors, {'countries': ['all']}), re_path(r'^all/(?P[A-z]+)/$', views.find_mirrors_simple, name='mirrorlist_simple') ] diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index dadb64ca9..a6a178563 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -2,17 +2,104 @@ from django import forms from django.db.models import Q -from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.forms.widgets import ( + Select, + SelectMultiple, + CheckboxSelectMultiple, + TextInput, + EmailInput +) from django.shortcuts import get_object_or_404, redirect, render from django.views.decorators.csrf import csrf_exempt from django_countries import countries -from ..models import MirrorUrl, MirrorProtocol +from ..models import Mirror, MirrorUrl, MirrorProtocol from ..utils import get_mirror_statuses import random +# This is populated later, and re-populated every refresh +# This was the only way to get 3 different examples without +# changing the models.py +url_examples = [] + + +class MirrorRequestForm(forms.ModelForm): + upstream = forms.ModelChoiceField( + queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), + required=False) + + class Meta: + model = Mirror + fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', + 'isos', 'rsync_user', 'rsync_password', 'notes') + + def __init__(self, *args, **kwargs): + super(MirrorRequestForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['name'].widget.attrs.update({'placeholder': 'Ex: mirror.argentina.co'}) + fields['rsync_user'].widget.attrs.update({'placeholder': 'Optional'}) + fields['rsync_password'].widget.attrs.update({'placeholder': 'Optional'}) + fields['notes'].widget.attrs.update({'placeholder': 'Ex: Hosted by ISP GreatISO.bg'}) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row=u'%(label)s %(field)s%(help_text)s', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + + +class MirrorUrlForm(forms.ModelForm): + class Meta: + model = MirrorUrl + fields = ('url', 'country', 'bandwidth', 'active') + + def __init__(self, *args, **kwargs): + global url_examples + + super(MirrorUrlForm, self).__init__(*args, **kwargs) + fields = self.fields + + if len(url_examples) == 0: + url_examples = [ + 'Ex: http://mirror.argentina.co/archlinux', + 'Ex: https://mirror.argentina.co/archlinux', + 'Ex: rsync://mirror.argentina.co/archlinux' + ] + + fields['url'].widget.attrs.update({'placeholder': url_examples.pop()}) + + def clean_url(self): + # is this a valid-looking URL? + url_parts = urlparse(self.cleaned_data["url"]) + if not url_parts.scheme: + raise forms.ValidationError("No URL scheme (protocol) provided.") + if not url_parts.netloc: + raise forms.ValidationError("No URL host provided.") + if url_parts.params or url_parts.query or url_parts.fragment: + raise forms.ValidationError( + "URL parameters, query, and fragment elements are not supported.") + # ensure we always save the URL with a trailing slash + path = url_parts.path + if not path.endswith('/'): + path += '/' + url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) + return url + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row=u'%(label)s %(field)s%(help_text)s', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + + class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False, widget=SelectMultiple(attrs={'size': '12'})) protocol = forms.MultipleChoiceField(required=False, widget=CheckboxSelectMultiple) @@ -127,4 +214,33 @@ def find_mirrors_simple(request, protocol): proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) +def submit_mirror(request): + # if request.method == 'POST' or len(request.GET) > 0: + # data = request.POST if request.method == 'POST' else request.GET + # form1 = MirrorUrlForm(data=data) + # if form.is_valid(): + # countries = form.cleaned_data['country'] + # protocols = form.cleaned_data['protocol'] + # use_status = form.cleaned_data['use_mirror_status'] + # ipv4 = '4' in form.cleaned_data['ip_version'] + # ipv6 = '6' in form.cleaned_data['ip_version'] + # return find_mirrors(request, countries, protocols, + # use_status, ipv4, ipv6) + # else: + form1 = MirrorRequestForm() + url1 = MirrorUrlForm() + url2 = MirrorUrlForm() + url3 = MirrorUrlForm() + + return render( + request, + 'mirrors/mirror_submit.html', + { + 'submission_form1': form1, + 'url1': url1, + 'url2': url2, + 'url3': url3 + } + ) + # vim: set ts=4 sw=4 et: diff --git a/requirements.txt b/requirements.txt index dc1937968..61a02a8c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ --e git+git://github.com/fredj/cssmin.git@master#egg=cssmin +cssmin==0.2.0 Django==4.0.1 IPy==1.1 Markdown==3.3.4 diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html new file mode 100644 index 000000000..614eeca1d --- /dev/null +++ b/templates/mirrors/mirror_submit.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} +{% load package_extras %} +{% block title %}Arch Linux - Pacman Mirrorlist Generator{% endblock %} + +{% block content %} +
+ +

Mirror Request

+ +

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

+ +

Available mirrors

+ +

Below are direct links to the two different tiers you will need to be acquainted with.

+ +

+ +

Mirror information

+ +

Before you can submit a Tier 1 request the mirror in question must first be a registered Tier 2 for a certain amount of time with proven reliablity. Once the submitted information is verified the mirror will be visible under the appropriate tier list above. This process usually takes 5 minutes.

+ +
+ {{ submission_form1.as_div }} + {{ url1.as_div }} + {{ url2.as_div }} + {{ url3.as_div }} +

+
+
+{% endblock %} + From 7dc2168f7d12ab600d84eb3f4fda43287af90bd3 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 17 Feb 2023 20:59:08 +0100 Subject: [PATCH 02/18] Added POST --- mirrors/views/mirrorlist.py | 69 ++++++++++++++++++++-------- templates/mirrors/mirror_submit.html | 7 +++ 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index a6a178563..87e828930 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -1,4 +1,5 @@ from operator import attrgetter, itemgetter +from urllib.parse import urlparse, urlunsplit from django import forms from django.db.models import Q @@ -13,7 +14,7 @@ from django.views.decorators.csrf import csrf_exempt from django_countries import countries -from ..models import Mirror, MirrorUrl, MirrorProtocol +from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync from ..utils import get_mirror_statuses import random @@ -27,8 +28,12 @@ class MirrorRequestForm(forms.ModelForm): upstream = forms.ModelChoiceField( - queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), - required=False) + queryset=Mirror.objects.filter( + tier__gte=0, + tier__lte=1 + ), + required=False + ) class Meta: model = Mirror @@ -100,6 +105,21 @@ def as_div(self): errors_on_separate_row=True) +class MirrorRsyncForm(forms.ModelForm): + class Meta: + model = MirrorRsync + fields = ('ip',) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row=u'%(label)s %(field)s%(help_text)s', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + + class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False, widget=SelectMultiple(attrs={'size': '12'})) protocol = forms.MultipleChoiceField(required=False, widget=CheckboxSelectMultiple) @@ -215,22 +235,30 @@ def find_mirrors_simple(request, protocol): return find_mirrors(request, protocols=[proto]) def submit_mirror(request): - # if request.method == 'POST' or len(request.GET) > 0: - # data = request.POST if request.method == 'POST' else request.GET - # form1 = MirrorUrlForm(data=data) - # if form.is_valid(): - # countries = form.cleaned_data['country'] - # protocols = form.cleaned_data['protocol'] - # use_status = form.cleaned_data['use_mirror_status'] - # ipv4 = '4' in form.cleaned_data['ip_version'] - # ipv6 = '6' in form.cleaned_data['ip_version'] - # return find_mirrors(request, countries, protocols, - # use_status, ipv4, ipv6) - # else: - form1 = MirrorRequestForm() - url1 = MirrorUrlForm() - url2 = MirrorUrlForm() - url3 = MirrorUrlForm() + if request.method == 'POST' or len(request.GET) > 0: + data = request.POST if request.method == 'POST' else request.GET + + form1 = MirrorRequestForm(data=data) + url1 = MirrorUrlForm(data=data) + url2 = MirrorUrlForm(data=data) + url3 = MirrorUrlForm(data=data) + rsync = MirrorRsyncForm(data=data) + + if form1.is_valid() and url1.is_valid() and url2.is_valid() and url3.is_valid() and rsync.is_valid(): + print("Successful") + # countries = form.cleaned_data['country'] + # protocols = form.cleaned_data['protocol'] + # use_status = form.cleaned_data['use_mirror_status'] + # ipv4 = '4' in form.cleaned_data['ip_version'] + # ipv6 = '6' in form.cleaned_data['ip_version'] + # return find_mirrors(request, countries, protocols, + # use_status, ipv4, ipv6) + else: + form1 = MirrorRequestForm() + url1 = MirrorUrlForm() + url2 = MirrorUrlForm() + url3 = MirrorUrlForm() + rsync = MirrorRsyncForm() return render( request, @@ -239,7 +267,8 @@ def submit_mirror(request): 'submission_form1': form1, 'url1': url1, 'url2': url2, - 'url3': url3 + 'url3': url3, + 'rsync' : rsync } ) diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html index 614eeca1d..4e537e6f9 100644 --- a/templates/mirrors/mirror_submit.html +++ b/templates/mirrors/mirror_submit.html @@ -20,6 +20,7 @@

Available mirrors

Mirror information

+ {% if not submission_form1.is_valid %}

Before you can submit a Tier 1 request the mirror in question must first be a registered Tier 2 for a certain amount of time with proven reliablity. Once the submitted information is verified the mirror will be visible under the appropriate tier list above. This process usually takes 5 minutes.

@@ -27,8 +28,14 @@

Mirror information

{{ url1.as_div }} {{ url2.as_div }} {{ url3.as_div }} + {{ rsync.as_div }}

+ {% else %} + + Your request have successfully been submitted and should be visible within 5 minutes in the mirrorlist. + + {% endif %} {% endblock %} From 3904bac6e14b184fc438c9fe91c92a4c53dcc7d4 Mon Sep 17 00:00:00 2001 From: Nina Nick <5041877+ninchester@users.noreply.github.com> Date: Fri, 17 Feb 2023 22:27:09 +0100 Subject: [PATCH 03/18] Added save functionality for mirrors --- mirrors/views/mirrorlist.py | 76 ++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index 87e828930..3054dc5ed 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -19,7 +19,6 @@ import random - # This is populated later, and re-populated every refresh # This was the only way to get 3 different examples without # changing the models.py @@ -234,41 +233,64 @@ def find_mirrors_simple(request, protocol): proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) + def submit_mirror(request): if request.method == 'POST' or len(request.GET) > 0: data = request.POST if request.method == 'POST' else request.GET - - form1 = MirrorRequestForm(data=data) - url1 = MirrorUrlForm(data=data) - url2 = MirrorUrlForm(data=data) - url3 = MirrorUrlForm(data=data) - rsync = MirrorRsyncForm(data=data) - - if form1.is_valid() and url1.is_valid() and url2.is_valid() and url3.is_valid() and rsync.is_valid(): - print("Successful") - # countries = form.cleaned_data['country'] - # protocols = form.cleaned_data['protocol'] - # use_status = form.cleaned_data['use_mirror_status'] - # ipv4 = '4' in form.cleaned_data['ip_version'] - # ipv6 = '6' in form.cleaned_data['ip_version'] - # return find_mirrors(request, countries, protocols, - # use_status, ipv4, ipv6) + + mirror_form = MirrorRequestForm(data=data) + mirror_url1_form = MirrorUrlForm(data={ + "url": data.get("url1-url", None), + "country": data.get("url1-country", None), + "bandwidth": data.get("url1-bandwidth", None), + "active": data.get("url1-active", None)}) + mirror_url2_form = MirrorUrlForm(data={ + "url": data.get("url2-url", None), + "country": data.get("url2-country", None), + "bandwidth": data.get("url2-bandwidth", None), + "active": data.get("url2-active", None)}) + mirror_url3_form = MirrorUrlForm(data={ + "url": data.get("url3-url", None), + "country": data.get("url3-country", None), + "bandwidth": data.get("url3-bandwidth", None), + "active": data.get("url3-active", None)}) + rsync_form = MirrorRsyncForm(data=data) + + if mirror_form.is_valid() \ + and mirror_url1_form.is_valid() \ + and mirror_url2_form.is_valid() \ + and mirror_url3_form.is_valid() \ + and rsync_form.is_valid(): + # TODO make this in a transaction + mirror = mirror_form.save() + mirror_url1 = mirror_url1_form.save(commit=False) + mirror_url2 = mirror_url2_form.save(commit=False) + mirror_url3 = mirror_url3_form.save(commit=False) + rsync = rsync_form.save(commit=False) + mirror_url1.mirror = mirror + mirror_url1.save() + mirror_url2.mirror = mirror + mirror_url2.save() + mirror_url3.mirror = mirror + mirror_url3.save() + rsync.mirror = mirror + rsync.save() else: - form1 = MirrorRequestForm() - url1 = MirrorUrlForm() - url2 = MirrorUrlForm() - url3 = MirrorUrlForm() - rsync = MirrorRsyncForm() + mirror_form = MirrorRequestForm() + mirror_url1_form = MirrorUrlForm(prefix="url1") + mirror_url2_form = MirrorUrlForm(prefix="url2") + mirror_url3_form = MirrorUrlForm(prefix="url3") + rsync_form = MirrorRsyncForm() return render( request, 'mirrors/mirror_submit.html', { - 'submission_form1': form1, - 'url1': url1, - 'url2': url2, - 'url3': url3, - 'rsync' : rsync + 'submission_form1': mirror_form, + 'url1': mirror_url1_form, + 'url2': mirror_url2_form, + 'url3': mirror_url3_form, + 'rsync': rsync_form } ) From b02b3124c901b96da9e28d2e08791a9641e23a43 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 11:27:34 +0100 Subject: [PATCH 04/18] Added sending of email on new mirror requests, improved validation checks --- mirrors/views/mirrorlist.py | 116 ++++++++++++------ templates/mirrors/mirror_submit.html | 21 ++-- .../mirrors/new_mirror_mail_template.txt | 9 ++ 3 files changed, 96 insertions(+), 50 deletions(-) create mode 100644 templates/mirrors/new_mirror_mail_template.txt diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index 3054dc5ed..0bfbf9f59 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -1,8 +1,12 @@ from operator import attrgetter, itemgetter from urllib.parse import urlparse, urlunsplit +from functools import partial from django import forms +from django.db import DatabaseError, transaction from django.db.models import Q +from django.core.mail import send_mail +from django.template import loader from django.forms.widgets import ( Select, SelectMultiple, @@ -37,7 +41,7 @@ class MirrorRequestForm(forms.ModelForm): class Meta: model = Mirror fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', - 'isos', 'rsync_user', 'rsync_password', 'notes') + 'isos', 'active', 'public', 'rsync_user', 'rsync_password', 'notes') def __init__(self, *args, **kwargs): super(MirrorRequestForm, self).__init__(*args, **kwargs) @@ -109,6 +113,11 @@ class Meta: model = MirrorRsync fields = ('ip',) + def __init__(self, *args, **kwargs): + super(MirrorRsyncForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['ip'].widget.attrs.update({'placeholder': 'Ex: 1.2.4.5'}) + def as_div(self): "Returns this form rendered as HTML s." return self._html_output( @@ -234,47 +243,70 @@ def find_mirrors_simple(request, protocol): return find_mirrors(request, protocols=[proto]) +def mail_mirror_admins(data): + template = loader.get_template('mirrors/new_mirror_mail_template.txt') + for mirror_maintainer in ['anton@hvornum.se', 'pitastrudl@gmail.com']: + send_mail('A mirror entry was submitted: \'%s\'' % data.get('name'), + template.render(data), + 'Arch Mirror Notification ', + [mirror_maintainer], + fail_silently=True) + + def submit_mirror(request): if request.method == 'POST' or len(request.GET) > 0: data = request.POST if request.method == 'POST' else request.GET + # data is immutable, need to be copied and modified to enforce + # active and public is False. + tmp = data.copy() + tmp['active'] = False + tmp['public'] = False + data = tmp + mirror_form = MirrorRequestForm(data=data) - mirror_url1_form = MirrorUrlForm(data={ - "url": data.get("url1-url", None), - "country": data.get("url1-country", None), - "bandwidth": data.get("url1-bandwidth", None), - "active": data.get("url1-active", None)}) - mirror_url2_form = MirrorUrlForm(data={ - "url": data.get("url2-url", None), - "country": data.get("url2-country", None), - "bandwidth": data.get("url2-bandwidth", None), - "active": data.get("url2-active", None)}) - mirror_url3_form = MirrorUrlForm(data={ - "url": data.get("url3-url", None), - "country": data.get("url3-country", None), - "bandwidth": data.get("url3-bandwidth", None), - "active": data.get("url3-active", None)}) + mirror_url1_form = MirrorUrlForm(data=data, prefix="url1") + if data.get('url2-url') != '': + mirror_url2_form = MirrorUrlForm(data=data, prefix="url2") + else: + mirror_url2_form = MirrorUrlForm(prefix="url2") + if data.get('url3-url') != '': + mirror_url3_form = MirrorUrlForm(data=data, prefix="url3") + else: + mirror_url3_form = MirrorUrlForm(prefix="url3") rsync_form = MirrorRsyncForm(data=data) - if mirror_form.is_valid() \ - and mirror_url1_form.is_valid() \ - and mirror_url2_form.is_valid() \ - and mirror_url3_form.is_valid() \ - and rsync_form.is_valid(): - # TODO make this in a transaction - mirror = mirror_form.save() - mirror_url1 = mirror_url1_form.save(commit=False) - mirror_url2 = mirror_url2_form.save(commit=False) - mirror_url3 = mirror_url3_form.save(commit=False) - rsync = rsync_form.save(commit=False) - mirror_url1.mirror = mirror - mirror_url1.save() - mirror_url2.mirror = mirror - mirror_url2.save() - mirror_url3.mirror = mirror - mirror_url3.save() - rsync.mirror = mirror - rsync.save() + mirror_url2_form.fields['url'].required = False + mirror_url3_form.fields['url'].required = False + rsync_form.fields['ip'].required = False + + if mirror_form.is_valid() and mirror_url1_form.is_valid(): + try: + with transaction.atomic(): + transaction.on_commit(partial(mail_mirror_admins, data)) + + mirror = mirror_form.save() + mirror_url1 = mirror_url1_form.save(commit=False) + mirror_url1.mirror = mirror + mirror_url1.save() + + if data.get('url2-url') != '' and mirror_url2_form.is_valid(): + mirror_url2 = mirror_url2_form.save(commit=False) + mirror_url2.mirror = mirror + mirror_url2.save() + if data.get('url3-url') != '' and mirror_url3_form.is_valid(): + mirror_url3 = mirror_url3_form.save(commit=False) + mirror_url3.mirror = mirror + mirror_url3.save() + + if data.get('ip') != '' and rsync.is_valid(): + rsync = rsync_form.save(commit=False) + rsync.mirror = mirror + rsync.save() + + except DatabaseError as error: + print(error) + else: mirror_form = MirrorRequestForm() mirror_url1_form = MirrorUrlForm(prefix="url1") @@ -282,15 +314,19 @@ def submit_mirror(request): mirror_url3_form = MirrorUrlForm(prefix="url3") rsync_form = MirrorRsyncForm() + mirror_url2_form.fields['url'].required = False + mirror_url3_form.fields['url'].required = False + rsync_form.fields['ip'].required = False + return render( request, 'mirrors/mirror_submit.html', { - 'submission_form1': mirror_form, - 'url1': mirror_url1_form, - 'url2': mirror_url2_form, - 'url3': mirror_url3_form, - 'rsync': rsync_form + 'mirror_form': mirror_form, + 'mirror_url1_form': mirror_url1_form, + 'mirror_url2_form': mirror_url2_form, + 'mirror_url3_form': mirror_url3_form, + 'rsync_form': rsync_form } ) diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html index 4e537e6f9..80d26a20c 100644 --- a/templates/mirrors/mirror_submit.html +++ b/templates/mirrors/mirror_submit.html @@ -20,21 +20,22 @@

Available mirrors

Mirror information

- {% if not submission_form1.is_valid %} + {% if mirror_form.is_valid or mirror_url1_form.is_valid %} + + Your request have successfully been submitted and should be visible within 5 minutes in the mirrorlist. + + {% else %}

Before you can submit a Tier 1 request the mirror in question must first be a registered Tier 2 for a certain amount of time with proven reliablity. Once the submitted information is verified the mirror will be visible under the appropriate tier list above. This process usually takes 5 minutes.

- {{ submission_form1.as_div }} - {{ url1.as_div }} - {{ url2.as_div }} - {{ url3.as_div }} - {{ rsync.as_div }} + {{ mirror_form.as_div }} + {{ mirror_url1_form.as_div }} + {{ mirror_url2_form.as_div }} + {{ mirror_url3_form.as_div }} +

If you are registring a Tier 1 mirror, you need to supply the static IP to the machine that will be using rsync towards the Tier 0 mirror. This is so we can allow it to sync. + {{ rsync_form.as_div }}

- {% else %} - - Your request have successfully been submitted and should be visible within 5 minutes in the mirrorlist. - {% endif %} {% endblock %} diff --git a/templates/mirrors/new_mirror_mail_template.txt b/templates/mirrors/new_mirror_mail_template.txt new file mode 100644 index 000000000..9b474ca42 --- /dev/null +++ b/templates/mirrors/new_mirror_mail_template.txt @@ -0,0 +1,9 @@ +{% autoescape off %}A new mirror has been requested called "{{ name }}". As the mirror administrator, check in on https://archlinux.org/mirrors/{{ name }}/ after a few minutes and check: + + * Is the Completion % more than 98% for Tier 2 and 100% for Tier 1? + * Does it have HTTP or HTTPS protocols? + +If so, go to https://archlinux.org/admin/mirrors/, find the new mirror and mark it as Active and Public! +If the mirror hasn't synced in a couple of days, email the mirror admin asking if we can assist in any way. + +{% endautoescape %} From 0937f1358e048b5a7b11c7b6983c95927439a5bb Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 17:49:11 +0100 Subject: [PATCH 05/18] Added automatic deny if ERRORS > LIMIT during TIME_FRAME. --- mirrors/views/mirrorlist.py | 121 +++++++++++++----- sitestatic/archweb.css | 9 ++ .../mirror_submit_error_upgrading.html | 32 +++++ 3 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 templates/mirrors/mirror_submit_error_upgrading.html diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index 0bfbf9f59..c428580f0 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -1,6 +1,7 @@ from operator import attrgetter, itemgetter from urllib.parse import urlparse, urlunsplit from functools import partial +from datetime import timedelta from django import forms from django.db import DatabaseError, transaction @@ -14,12 +15,13 @@ TextInput, EmailInput ) +from django.utils.timezone import now from django.shortcuts import get_object_or_404, redirect, render from django.views.decorators.csrf import csrf_exempt from django_countries import countries -from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync -from ..utils import get_mirror_statuses +from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync, MirrorLog +from ..utils import get_mirror_statuses, get_mirror_errors import random @@ -27,7 +29,8 @@ # This was the only way to get 3 different examples without # changing the models.py url_examples = [] - +TIER_1_MAX_ERROR_RATE = 2 +TIER_1_ERROR_TIME_RANGE = 30 class MirrorRequestForm(forms.ModelForm): upstream = forms.ModelChoiceField( @@ -47,9 +50,10 @@ def __init__(self, *args, **kwargs): super(MirrorRequestForm, self).__init__(*args, **kwargs) fields = self.fields fields['name'].widget.attrs.update({'placeholder': 'Ex: mirror.argentina.co'}) + fields['alternate_email'].widget.attrs.update({'placeholder': 'Optional'}) fields['rsync_user'].widget.attrs.update({'placeholder': 'Optional'}) fields['rsync_password'].widget.attrs.update({'placeholder': 'Optional'}) - fields['notes'].widget.attrs.update({'placeholder': 'Ex: Hosted by ISP GreatISO.bg'}) + fields['notes'].widget.attrs.update({'placeholder': 'Optional (Ex: Hosted by ISP GreatISO.bg)'}) def as_div(self): "Returns this form rendered as HTML s." @@ -252,6 +256,46 @@ def mail_mirror_admins(data): [mirror_maintainer], fail_silently=True) +def validate_tier_1(data): + if data.get('tier') != '1': + return None + + # If there is no Tier 2 with the same name, + # We invalidate this Tier 1. + if not len((found_mirror := Mirror.objects.filter(name=data.get('name'), tier=2, active=True, public=True))): + return False + + main_url = MirrorUrl.objects.filter( + url__startswith=data.get('url1-url'), + mirror=found_mirror[0] + ) + + # If the Tier 2 and Tier 1 does not have matching URL, + # it requires manual intervention as it's not a direct upgrade. + if len(main_url) <= 0: + return False + + # DEBUG entry: + MirrorLog.objects.create( + url=main_url[0], + location=None, + check_time=now(), + last_sync=None, + duration=0.2, is_success=False, error="Test - 404" + ) + + error_logs = get_mirror_errors(mirror_id=found_mirror[0].id, cutoff=timedelta(days=TIER_1_ERROR_TIME_RANGE), + show_all=True) + + if error_logs: + num_o_errors = 0 + for error in error_logs: + num_o_errors += error['error_count'] + + if num_o_errors >= TIER_1_MAX_ERROR_RATE: + return False + + return found_mirror def submit_mirror(request): if request.method == 'POST' or len(request.GET) > 0: @@ -265,6 +309,7 @@ def submit_mirror(request): data = tmp mirror_form = MirrorRequestForm(data=data) + mirror_url1_form = MirrorUrlForm(data=data, prefix="url1") if data.get('url2-url') != '': mirror_url2_form = MirrorUrlForm(data=data, prefix="url2") @@ -274,38 +319,52 @@ def submit_mirror(request): mirror_url3_form = MirrorUrlForm(data=data, prefix="url3") else: mirror_url3_form = MirrorUrlForm(prefix="url3") + rsync_form = MirrorRsyncForm(data=data) mirror_url2_form.fields['url'].required = False mirror_url3_form.fields['url'].required = False rsync_form.fields['ip'].required = False - if mirror_form.is_valid() and mirror_url1_form.is_valid(): - try: - with transaction.atomic(): - transaction.on_commit(partial(mail_mirror_admins, data)) - - mirror = mirror_form.save() - mirror_url1 = mirror_url1_form.save(commit=False) - mirror_url1.mirror = mirror - mirror_url1.save() - - if data.get('url2-url') != '' and mirror_url2_form.is_valid(): - mirror_url2 = mirror_url2_form.save(commit=False) - mirror_url2.mirror = mirror - mirror_url2.save() - if data.get('url3-url') != '' and mirror_url3_form.is_valid(): - mirror_url3 = mirror_url3_form.save(commit=False) - mirror_url3.mirror = mirror - mirror_url3.save() - - if data.get('ip') != '' and rsync.is_valid(): - rsync = rsync_form.save(commit=False) - rsync.mirror = mirror - rsync.save() - - except DatabaseError as error: - print(error) + if data.get('tier') == '1': + if existing_mirror := validate_tier_1(data): + existing_mirror.update(tier=1) + else: + return render( + request, + 'mirrors/mirror_submit_error_upgrading.html', + { + 'TIER_1_ERROR_TIME_RANGE': TIER_1_ERROR_TIME_RANGE, + 'TIER_1_MAX_ERROR_RATE': TIER_1_MAX_ERROR_RATE, + } + ) + else: + if mirror_form.is_valid() and mirror_url1_form.is_valid(): + try: + with transaction.atomic(): + transaction.on_commit(partial(mail_mirror_admins, data)) + + mirror = mirror_form.save() + mirror_url1 = mirror_url1_form.save(commit=False) + mirror_url1.mirror = mirror + mirror_url1.save() + + if data.get('url2-url') != '' and mirror_url2_form.is_valid(): + mirror_url2 = mirror_url2_form.save(commit=False) + mirror_url2.mirror = mirror + mirror_url2.save() + if data.get('url3-url') != '' and mirror_url3_form.is_valid(): + mirror_url3 = mirror_url3_form.save(commit=False) + mirror_url3.mirror = mirror + mirror_url3.save() + + if data.get('ip') != '' and rsync_form.is_valid(): + rsync = rsync_form.save(commit=False) + rsync.mirror = mirror + rsync.save() + + except DatabaseError as error: + print(error) else: mirror_form = MirrorRequestForm() @@ -314,6 +373,8 @@ def submit_mirror(request): mirror_url3_form = MirrorUrlForm(prefix="url3") rsync_form = MirrorRsyncForm() + mirror_form.fields['active'].widget = forms.HiddenInput() + mirror_form.fields['public'].widget = forms.HiddenInput() mirror_url2_form.fields['url'].required = False mirror_url3_form.fields['url'].required = False rsync_form.fields['ip'].required = False diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index aa2c62089..49aebadf3 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -72,6 +72,15 @@ code { padding: 0.15em 0.25em; } +.error { + font-family: monospace, monospace; + padding: 0.15em 0.25em; + overflow: auto; + color: #a94442; + background: #f2dede; + border: 1px solid #ebccd1; +} + pre { font-family: monospace, monospace; border: 1px solid #bdb; diff --git a/templates/mirrors/mirror_submit_error_upgrading.html b/templates/mirrors/mirror_submit_error_upgrading.html new file mode 100644 index 000000000..caaeb7fcd --- /dev/null +++ b/templates/mirrors/mirror_submit_error_upgrading.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% load package_extras %} +{% block title %}Arch Linux - Pacman Mirrorlist Generator{% endblock %} + +{% block content %} +
+ +

Mirror Request

+ +

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

+ +

Available mirrors

+ +

Below are direct links to the two different tiers you will need to be acquainted with.

+ +

+ +

Mirror information

+ + 
+    
+    The mirror does not meet the minimum requirements to be upgraded to a Tier 1 mirror.
+    Please make sure it has been a reliable Tier 2 mirror with at most {{ TIER_1_MAX_ERROR_RATE }} errors over the last {{ TIER_1_ERROR_TIME_RANGE }} days.
+
+ + Please read the requirements on https://wiki.archlinux.org/title/DeveloperWiki:NewMirrors#Tier_1_requirements +
+{% endblock %} + From ea4d9bce0aa30dcc496746d99bb9324772f0e551 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 18:11:58 +0100 Subject: [PATCH 06/18] Shortened URL's according to comments --- templates/mirrors/mirror_submit.html | 2 +- templates/mirrors/mirror_submit_error_upgrading.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html index 80d26a20c..413770a1a 100644 --- a/templates/mirrors/mirror_submit.html +++ b/templates/mirrors/mirror_submit.html @@ -7,7 +7,7 @@

Mirror Request

-

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

+

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

Available mirrors

diff --git a/templates/mirrors/mirror_submit_error_upgrading.html b/templates/mirrors/mirror_submit_error_upgrading.html index caaeb7fcd..8d09a9187 100644 --- a/templates/mirrors/mirror_submit_error_upgrading.html +++ b/templates/mirrors/mirror_submit_error_upgrading.html @@ -7,7 +7,7 @@

Mirror Request

-

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

+

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

Available mirrors

From 46cf8e06f25539ec208f74c4d38b1f43755a4d73 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 20:15:04 +0100 Subject: [PATCH 07/18] Removed hardcoded emails --- mirrors/views/mirrorlist.py | 9 +++++++-- templates/mirrors/mirror_submit.html | 2 +- templates/mirrors/mirror_submit_error_upgrading.html | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index c428580f0..cd7a26f1f 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -19,6 +19,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.views.decorators.csrf import csrf_exempt from django_countries import countries +from django.contrib.auth.models import Group, User from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync, MirrorLog from ..utils import get_mirror_statuses, get_mirror_errors @@ -249,11 +250,15 @@ def find_mirrors_simple(request, protocol): def mail_mirror_admins(data): template = loader.get_template('mirrors/new_mirror_mail_template.txt') - for mirror_maintainer in ['anton@hvornum.se', 'pitastrudl@gmail.com']: + + mirror_maintainer_group = Group.objects.filter(name='Mirror Maintainers') + mirror_maintainers = User.objects.filter(is_active=True).filter(groups__in=mirror_maintainer_group) + + for maintainer in mirror_maintainers: send_mail('A mirror entry was submitted: \'%s\'' % data.get('name'), template.render(data), 'Arch Mirror Notification ', - [mirror_maintainer], + [maintainer.email], fail_silently=True) def validate_tier_1(data): diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html index 413770a1a..d4db37da6 100644 --- a/templates/mirrors/mirror_submit.html +++ b/templates/mirrors/mirror_submit.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% load package_extras %} -{% block title %}Arch Linux - Pacman Mirrorlist Generator{% endblock %} +{% block title %}Arch Linux - Submit New Mirror{% endblock %} {% block content %}
diff --git a/templates/mirrors/mirror_submit_error_upgrading.html b/templates/mirrors/mirror_submit_error_upgrading.html index 8d09a9187..0876969aa 100644 --- a/templates/mirrors/mirror_submit_error_upgrading.html +++ b/templates/mirrors/mirror_submit_error_upgrading.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% load package_extras %} -{% block title %}Arch Linux - Pacman Mirrorlist Generator{% endblock %} +{% block title %}Arch Linux - Submit New Mirror{% endblock %} {% block content %}
From 32689a947d2c57cfca05af4fd3e7402ef913da45 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 20:21:16 +0100 Subject: [PATCH 08/18] Added MIN_DAYS as Tier 2 before Tier 1 requests being accepted. --- mirrors/views/mirrorlist.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index cd7a26f1f..3e4303632 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -32,6 +32,7 @@ url_examples = [] TIER_1_MAX_ERROR_RATE = 2 TIER_1_ERROR_TIME_RANGE = 30 +TIER_1_MIN_DAYS_AS_TIER_2 = 60 class MirrorRequestForm(forms.ModelForm): upstream = forms.ModelChoiceField( @@ -261,18 +262,21 @@ def mail_mirror_admins(data): [maintainer.email], fail_silently=True) -def validate_tier_1(data): +def validate_tier_1_request(data): if data.get('tier') != '1': return None # If there is no Tier 2 with the same name, # We invalidate this Tier 1. - if not len((found_mirror := Mirror.objects.filter(name=data.get('name'), tier=2, active=True, public=True))): + if not len((tier_2_mirror := Mirror.objects.filter(name=data.get('name'), tier=2, active=True, public=True))): + return False + + if tier_2_mirror[0].created - now() < timedelta(days=TIER_1_MIN_DAYS_AS_TIER_2): return False main_url = MirrorUrl.objects.filter( url__startswith=data.get('url1-url'), - mirror=found_mirror[0] + mirror=tier_2_mirror[0] ) # If the Tier 2 and Tier 1 does not have matching URL, @@ -288,8 +292,9 @@ def validate_tier_1(data): last_sync=None, duration=0.2, is_success=False, error="Test - 404" ) + # /DEBUG enry - error_logs = get_mirror_errors(mirror_id=found_mirror[0].id, cutoff=timedelta(days=TIER_1_ERROR_TIME_RANGE), + error_logs = get_mirror_errors(mirror_id=tier_2_mirror[0].id, cutoff=timedelta(days=TIER_1_ERROR_TIME_RANGE), show_all=True) if error_logs: @@ -300,6 +305,9 @@ def validate_tier_1(data): if num_o_errors >= TIER_1_MAX_ERROR_RATE: return False + # Final check, is the mirror old enough to qualify for Tier 1? + print(tier_2_mirror[0].created) + return found_mirror def submit_mirror(request): @@ -332,7 +340,7 @@ def submit_mirror(request): rsync_form.fields['ip'].required = False if data.get('tier') == '1': - if existing_mirror := validate_tier_1(data): + if existing_mirror := validate_tier_1_request(data): existing_mirror.update(tier=1) else: return render( From 594545ff8b1462b0130071b90cfb2cac3b2d6dd5 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 20:47:48 +0100 Subject: [PATCH 09/18] Added captcha on submission --- mirrors/views/mirrorlist.py | 17 +++++++++++++++++ requirements.txt | 3 ++- settings.py | 1 + sitestatic/archweb.css | 9 +++++++++ templates/mirrors/mirror_submit.html | 1 + urls.py | 5 +++++ 6 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index 3e4303632..eee0146db 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -20,6 +20,7 @@ from django.views.decorators.csrf import csrf_exempt from django_countries import countries from django.contrib.auth.models import Group, User +from captcha.fields import CaptchaField from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync, MirrorLog from ..utils import get_mirror_statuses, get_mirror_errors @@ -34,6 +35,19 @@ TIER_1_ERROR_TIME_RANGE = 30 TIER_1_MIN_DAYS_AS_TIER_2 = 60 +class CaptchaForm(forms.Form): + captcha = CaptchaField() + + def as_div(self): + "Returns this form rendered as HTML s." + + return self._html_output( + normal_row=u'
%(label)s
%(field)s%(help_text)s
', + error_row=u'%s', + row_ender='
', + help_text_html=u' %s', + errors_on_separate_row=True) + class MirrorRequestForm(forms.ModelForm): upstream = forms.ModelChoiceField( queryset=Mirror.objects.filter( @@ -311,6 +325,8 @@ def validate_tier_1_request(data): return found_mirror def submit_mirror(request): + captcha_form = CaptchaForm() + if request.method == 'POST' or len(request.GET) > 0: data = request.POST if request.method == 'POST' else request.GET @@ -396,6 +412,7 @@ def submit_mirror(request): request, 'mirrors/mirror_submit.html', { + 'captcha' : captcha_form, 'mirror_form': mirror_form, 'mirror_url1_form': mirror_url1_form, 'mirror_url2_form': mirror_url2_form, diff --git a/requirements.txt b/requirements.txt index 6576e808c..bfddc7eb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ Markdown==3.3.7 bencode.py==4.0.0 django-countries==7.3.2 django-extensions==3.1.3 +django-simple-captcha==0.5.17 jsmin==3.0.1 pgpdump==1.5 parse==1.19.0 @@ -15,4 +16,4 @@ feedparser==6.0.10 bleach==5.0.0 requests==2.25.1 xtarfile==0.1.0 -zstandard==0.17.0 +zstandard==0.17.0 \ No newline at end of file diff --git a/settings.py b/settings.py index 3488a22e7..16dc01958 100644 --- a/settings.py +++ b/settings.py @@ -135,6 +135,7 @@ 'public', 'releng', 'visualize', + 'captcha' ) # Logging configuration for not getting overspammed diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index 49aebadf3..f07a1cdb8 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -1206,3 +1206,12 @@ ul.signoff-list { .pgp-key-ids { display: inline-block; } + +.captcha { + display: flex; +} + +.captcha-input { + display: flex; + flex-direction: column; +} \ No newline at end of file diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html index d4db37da6..58c3881e6 100644 --- a/templates/mirrors/mirror_submit.html +++ b/templates/mirrors/mirror_submit.html @@ -34,6 +34,7 @@

Mirror information

{{ mirror_url3_form.as_div }}

If you are registring a Tier 1 mirror, you need to supply the static IP to the machine that will be using rsync towards the Tier 0 mirror. This is so we can allow it to sync. {{ rsync_form.as_div }} + {{ captcha.as_div }}

{% endif %} diff --git a/urls.py b/urls.py index f5edec5cb..e8f5d6b4c 100644 --- a/urls.py +++ b/urls.py @@ -110,6 +110,11 @@ path('logout/', auth_views.LogoutView.as_view(template_name='registration/logout.html'), name='logout'), ]) +# Captcha +urlpatterns.extend([ + path('captcha/', include('captcha.urls')), +]) + # django-toolbar if settings.DEBUG_TOOLBAR: # pragma: no cover import debug_toolbar From 001d6e4ab5e602b666006ca60c64fc1696b045a3 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 20:55:10 +0100 Subject: [PATCH 10/18] Added captcha validation (forgot) --- mirrors/views/mirrorlist.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index eee0146db..54f78a888 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -325,11 +325,12 @@ def validate_tier_1_request(data): return found_mirror def submit_mirror(request): - captcha_form = CaptchaForm() if request.method == 'POST' or len(request.GET) > 0: data = request.POST if request.method == 'POST' else request.GET + captcha_form = CaptchaForm(data=data) + # data is immutable, need to be copied and modified to enforce # active and public is False. tmp = data.copy() @@ -368,7 +369,7 @@ def submit_mirror(request): } ) else: - if mirror_form.is_valid() and mirror_url1_form.is_valid(): + if captcha_form.is_valid() and mirror_form.is_valid() and mirror_url1_form.is_valid(): try: with transaction.atomic(): transaction.on_commit(partial(mail_mirror_admins, data)) @@ -396,6 +397,7 @@ def submit_mirror(request): print(error) else: + captcha_form = CaptchaForm() mirror_form = MirrorRequestForm() mirror_url1_form = MirrorUrlForm(prefix="url1") mirror_url2_form = MirrorUrlForm(prefix="url2") From 9ea8db5464ad24ae0e6711da6c35eda82f23c926 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 21:23:23 +0100 Subject: [PATCH 11/18] Added entries to authors --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index cbf41a06d..1c13ff06d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -37,3 +37,5 @@ Thayer Williams Thomas Bächler Tom Willemsen Tyler Dence +Anton Hvornum +Nina Nick <5041877+ninchester@users.noreply.github.com> \ No newline at end of file From 7ef39df891007c2afadf6532b1a3a9ac39d09f63 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 21:25:46 +0100 Subject: [PATCH 12/18] Reverted requirements change for cssmin --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bfddc7eb0..e8614ba9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -cssmin==0.2.0 +-e git+git://github.com/fredj/cssmin.git@master#egg=cssmin Django==4.1.6 IPy==1.1 Markdown==3.3.7 From 0825b1bc4808912661daf567149706bdf663e423 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 21:28:00 +0100 Subject: [PATCH 13/18] Changed the wording on the HTML page, no promised time frame is given. --- templates/mirrors/mirror_submit.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html index 58c3881e6..13a6eb0a8 100644 --- a/templates/mirrors/mirror_submit.html +++ b/templates/mirrors/mirror_submit.html @@ -22,10 +22,10 @@

Mirror information

{% if mirror_form.is_valid or mirror_url1_form.is_valid %} - Your request have successfully been submitted and should be visible within 5 minutes in the mirrorlist. + Your request have successfully been submitted and we will mark the mirror as public and active once we have manually verified the mirror. {% else %} -

Before you can submit a Tier 1 request the mirror in question must first be a registered Tier 2 for a certain amount of time with proven reliablity. Once the submitted information is verified the mirror will be visible under the appropriate tier list above. This process usually takes 5 minutes.

+

Before you can submit a Tier 1 request the mirror in question must first be a registered Tier 2 for a certain amount of time with proven reliablity. Once the submitted information is verified by the Arch Linux mirror maintainers the mirror will be visible under the appropriate tier list above.

{{ mirror_form.as_div }} From 365a14efd114fd84a8f0a9efcb2a71106a158545 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 21:32:16 +0100 Subject: [PATCH 14/18] Removed debug code and fixed linting --- mirrors/views/mirrorlist.py | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index 54f78a888..664ed799c 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -9,11 +9,8 @@ from django.core.mail import send_mail from django.template import loader from django.forms.widgets import ( - Select, SelectMultiple, - CheckboxSelectMultiple, - TextInput, - EmailInput + CheckboxSelectMultiple ) from django.utils.timezone import now from django.shortcuts import get_object_or_404, redirect, render @@ -22,7 +19,7 @@ from django.contrib.auth.models import Group, User from captcha.fields import CaptchaField -from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync, MirrorLog +from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync from ..utils import get_mirror_statuses, get_mirror_errors import random @@ -35,6 +32,7 @@ TIER_1_ERROR_TIME_RANGE = 30 TIER_1_MIN_DAYS_AS_TIER_2 = 60 + class CaptchaForm(forms.Form): captcha = CaptchaField() @@ -48,6 +46,7 @@ def as_div(self): help_text_html=u' %s', errors_on_separate_row=True) + class MirrorRequestForm(forms.ModelForm): upstream = forms.ModelChoiceField( queryset=Mirror.objects.filter( @@ -271,10 +270,11 @@ def mail_mirror_admins(data): for maintainer in mirror_maintainers: send_mail('A mirror entry was submitted: \'%s\'' % data.get('name'), - template.render(data), - 'Arch Mirror Notification ', - [maintainer.email], - fail_silently=True) + template.render(data), + 'Arch Mirror Notification ', + [maintainer.email], + fail_silently=True) + def validate_tier_1_request(data): if data.get('tier') != '1': @@ -298,16 +298,6 @@ def validate_tier_1_request(data): if len(main_url) <= 0: return False - # DEBUG entry: - MirrorLog.objects.create( - url=main_url[0], - location=None, - check_time=now(), - last_sync=None, - duration=0.2, is_success=False, error="Test - 404" - ) - # /DEBUG enry - error_logs = get_mirror_errors(mirror_id=tier_2_mirror[0].id, cutoff=timedelta(days=TIER_1_ERROR_TIME_RANGE), show_all=True) @@ -322,10 +312,10 @@ def validate_tier_1_request(data): # Final check, is the mirror old enough to qualify for Tier 1? print(tier_2_mirror[0].created) - return found_mirror + return tier_2_mirror -def submit_mirror(request): +def submit_mirror(request): if request.method == 'POST' or len(request.GET) > 0: data = request.POST if request.method == 'POST' else request.GET @@ -349,7 +339,7 @@ def submit_mirror(request): mirror_url3_form = MirrorUrlForm(data=data, prefix="url3") else: mirror_url3_form = MirrorUrlForm(prefix="url3") - + rsync_form = MirrorRsyncForm(data=data) mirror_url2_form.fields['url'].required = False @@ -414,7 +404,7 @@ def submit_mirror(request): request, 'mirrors/mirror_submit.html', { - 'captcha' : captcha_form, + 'captcha': captcha_form, 'mirror_form': mirror_form, 'mirror_url1_form': mirror_url1_form, 'mirror_url2_form': mirror_url2_form, From 5846eea267b84ceb9183426a6455053ff36ef31f Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 21:36:01 +0100 Subject: [PATCH 15/18] Changed to a more sane error limit for Tier 1, 10 errors over 30 days should be ok as some errors such as faulty DNS lookups do occur on some hosts momentarily. That we have to account for --- mirrors/views/mirrorlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index 664ed799c..f207431a7 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -28,7 +28,7 @@ # This was the only way to get 3 different examples without # changing the models.py url_examples = [] -TIER_1_MAX_ERROR_RATE = 2 +TIER_1_MAX_ERROR_RATE = 10 TIER_1_ERROR_TIME_RANGE = 30 TIER_1_MIN_DAYS_AS_TIER_2 = 60 From 120f5e4d492008404b3db5cd5753dae81001a824 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 21:38:49 +0100 Subject: [PATCH 16/18] Reverting .gitignore - See #449 instead --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 924777246..7c21fe7e6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ local_settings.py archweb.db archweb.db-* database.db -/*.tar.gz tags collected_static/ testing/ From cd7ccc271fb534c763c0013d5470313793d1159d Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 22:08:37 +0100 Subject: [PATCH 17/18] Added a test using GET values --- mirrors/tests/test_mirror_submission.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 mirrors/tests/test_mirror_submission.py diff --git a/mirrors/tests/test_mirror_submission.py b/mirrors/tests/test_mirror_submission.py new file mode 100644 index 000000000..dba38d1fa --- /dev/null +++ b/mirrors/tests/test_mirror_submission.py @@ -0,0 +1,3 @@ +def test_mirror_registration(client, mirrorurl): + response = client.get('/mirrorlist/submit/?name=test3&tier=2&upstream=1&admin_email=anton%40hvornum.se&alternate_email=&isos=on&rsync_user=&rsync_password=¬es=&active=True&public=True&url1-url=rsync%3A%2F%2Ftest3.com%2Farchlinux&url1-country=SE&url1-bandwidth=1234&url1-active=on&url2-url=&url2-country=&url2-bandwidth=&url2-active=on&url3-url=&url3-country=&url3-bandwidth=&url3-active=on&ip=&captcha_0=d5a017cc3851fb59898167f666759c99b42afd52&captcha_1=tdof') + assert response.status_code == 200 From 0da5082eeb27e65e06ad9f40c1099febd1d12447 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sat, 18 Feb 2023 22:47:10 +0100 Subject: [PATCH 18/18] Fixed requirements? --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e8614ba9b..d1cf1d2b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ --e git+git://github.com/fredj/cssmin.git@master#egg=cssmin +-e git+https://github.com/fredj/cssmin.git@master#egg=cssmin Django==4.1.6 IPy==1.1 Markdown==3.3.7