Client ID registration (#444) (#445)

kaysiz · web-flow · commit 1efb824f020e · 2025-05-15T16:23:49.000+02:00
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -18,6 +18,7 @@ jobs:
       AWS_REGION: ${{ secrets.AWS_REGION }}
       GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       LAUNCH_DARKLY_KEY: ${{ secrets.LAUNCH_DARKLY_KEY_DEV }}
+      DB_HOST: 127.0.0.1 # Will not work with 'localhost', since that will try a Unix socket connection (!)
     services:
       elasticsearch7:
         image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0
@@ -30,6 +31,16 @@ jobs:
           http.cors.allow-origin: "*"
         ports:
           - 9200:9200
+      db:
+        image: mysql:8.0
+        env:
+          MYSQL_DATABASE: "rorapi"
+          MYSQL_USER: "ror_user"
+          MYSQL_PASSWORD: "password"
+          MYSQL_ROOT_PASSWORD: "password"
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
     steps:
       - name: Checkout ror-api code
         uses: actions/checkout@v2
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,6 +9,7 @@ jobs:
       ELASTIC_PASSWORD: "changeme"
       ELASTIC7_HOST: "localhost"
       ELASTIC7_PORT: "9200"
+      DB_HOST: 127.0.0.1
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.AWS_REGION }}
@@ -26,6 +27,16 @@ jobs:
           http.cors.allow-origin: "*"
         ports:
           - 9200:9200
+      db:
+        image: mysql:8.0
+        env:
+          MYSQL_DATABASE: "rorapi"
+          MYSQL_USER: "ror_user"
+          MYSQL_PASSWORD: "password"
+          MYSQL_ROOT_PASSWORD: "password"
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
     steps:
       - name: Checkout ror-api code
         uses: actions/checkout@v2
diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml
@@ -10,6 +10,7 @@ jobs:
       ELASTIC_PASSWORD: "changeme"
       ELASTIC7_HOST: "localhost"
       ELASTIC7_PORT: "9200"
+      DB_HOST: 127.0.0.1
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.AWS_REGION }}
@@ -27,6 +28,16 @@ jobs:
           http.cors.allow-origin: "*"
         ports:
           - 9200:9200
+      db:
+        image: mysql:8.0
+        env:
+          MYSQL_DATABASE: "rorapi"
+          MYSQL_USER: "ror_user"
+          MYSQL_PASSWORD: "password"
+          MYSQL_ROOT_PASSWORD: "password"
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
     steps:
       - name: Checkout ror-api code
         uses: actions/checkout@v2
diff --git a/Dockerfile b/Dockerfile
@@ -16,7 +16,7 @@ RUN mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak && \
     mv /etc/apt/sources.list.d.bak /etc/apt/sources.list.d && \
     apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \
     apt-get clean && \
-    apt-get install ntp wget unzip tzdata python3-pip libmagic1 -y && \
+    apt-get install ntp wget unzip tzdata python3-pip libmagic1 default-libmysqlclient-dev libcairo2-dev pkg-config -y && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Enable Passenger and Nginx and remove the default site
@@ -54,6 +54,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 RUN pip3 install yapf
 
 # collect static files for Django
+ENV DJANGO_SKIP_DB_CHECK=True
 RUN python manage.py collectstatic --noinput
 
 # Expose web
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -19,6 +19,18 @@ services:
       timeout: 1s
     volumes:
       - ./esdata:/usr/share/elasticsearch/data
+  db:
+    image: mysql:8.0
+    volumes:
+      - mysql_data:/var/lib/mysql
+    env_file:
+      - .env
+    ports:
+      - "3306:3306"
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
+      timeout: 20s
+      retries: 10
   web:
     container_name: rorapiweb
     env_file: .env
@@ -31,3 +43,6 @@ services:
       - ./rorapi:/home/app/webapp/rorapi
     depends_on:
       - elasticsearch7
+      - db
+volumes:
+  mysql_data:
diff --git a/requirements.txt b/requirements.txt
@@ -24,4 +24,8 @@ update_address @ git+https://github.com/ror-community/update_address.git
 launchdarkly-server-sdk==7.6.1
 jsonschema==3.2.0
 python-magic
-iso639-lang
+iso639-lang
+mysqlclient==2.2.7
+bleach==6.0.0
+pycountry==22.3.5
+django-ses==3.5.0
diff --git a/rorapi/common/urls.py b/rorapi/common/urls.py
@@ -3,7 +3,7 @@
 from rest_framework.documentation import include_docs_urls
 from  . import views
 from rorapi.common.views import (
-    HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate)
+    HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView,ValidateClientView)
 
 urlpatterns = [
     # Health check
@@ -14,6 +14,8 @@
     path('generateaddress/<str:geonamesid>', GenerateAddress.as_view()),
     url(r"^generateid$", GenerateId.as_view()),
     re_path(r"^(?P<version>(v1|v2))\/bulkupdate$", BulkUpdate.as_view()),
+    re_path(r"^(?P<version>(v1|v2))\/register$", ClientRegistrationView.as_view()),
+    path('validate-client-id/<str:client_id>/', ValidateClientView.as_view()),
     url(r"^(?P<version>(v1|v2))\/indexdata/(?P<branch>.*)", IndexData.as_view()),
     url(r"^(?P<version>(v1|v2))\/indexdatadump\/(?P<filename>v(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P<dataenv>(test|prod))$", IndexDataDump.as_view()),
     url(r"^(?P<version>(v1|v2))\/", include(views.organizations_router.urls)),
diff --git a/rorapi/common/views.py b/rorapi/common/views.py
@@ -37,10 +37,87 @@
 import os
 import update_address as ua
 from rorapi.management.commands.generaterorid import check_ror_id
-from rorapi.management.commands.generaterorid import check_ror_id
 from rorapi.management.commands.indexror import process_files
 from django.core import management
 import rorapi.management.commands.indexrordump
+from django.core.mail import EmailMultiAlternatives
+from django.utils.timezone import now
+from rorapi.v2.models import Client
+from rorapi.v2.serializers import ClientSerializer
+
+class ClientRegistrationView(APIView):
+    def post(self, request, version='v2'):
+        serializer = ClientSerializer(data=request.data)
+        if serializer.is_valid():
+            client = serializer.save()
+
+            subject = 'ROR API client ID'
+            from_email = "ROR API Support <api@ror.org>"
+            recipient_list = [client.email]
+
+            html_content = self._get_html_content(client.client_id)
+            text_content = self._get_text_content(client.client_id)
+
+            msg = EmailMultiAlternatives(subject, text_content, from_email, recipient_list)
+            msg.attach_alternative(html_content, "text/html")
+            msg.send()
+
+            return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED)
+
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+    def _get_text_content(self, client_id):
+        return f"""
+            Thank you for registering for a ROR API client ID!
+
+            Your ROR API client ID is:
+            {client_id}
+
+            This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.
+
+            In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example:
+
+            curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford
+
+            Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.
+
+            We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io/docs/client-id
+
+            If you have questions, please see ROR documentation or contact us at support@ror.org
+
+            Cheers,
+            The ROR Team
+            support@ror.org
+            https://ror.org
+        """
+
+
+    def _get_html_content(self, client_id):
+        return f"""
+            <div style="font-family: Arial, sans-serif; line-height: 1.5;">
+                <p>Thank you for registering for a ROR API client ID!</p>
+                <p><strong>Your ROR API client ID is:</strong></p>
+                <pre style="background:#f4f4f4;padding:10px;">{client_id}</pre>
+                <p>This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.</p>
+                <p>In order to receive a rate limit of <strong>2000 requests per 5 minute period</strong>, please include this client ID with your ROR API requests, in a custom HTTP header named <code>Client-Id</code>, for example:</p>
+                <pre style="background:#f4f4f4;padding:10px;">curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford</pre>
+                <p>Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.</p>
+                <p>We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new one.</p>
+                <p>For more information about ROR API client IDs, see <a href="https://ror.readme.io/docs/client-id/">our documentation</a>.</p>
+                <p>If you have questions, please see the ROR documentation or contact us at <a href="mailto:support@ror.org">support@ror.org</a>.</p>
+                <p>Cheers,<br>
+                The ROR Team<br>
+                <a href="mailto:support@ror.org">support@ror.org</a><br>
+                <a href="https://ror.org">https://ror.org</a></p>
+            </div>
+        """
+
+
+class ValidateClientView(APIView):
+    def get(self, request, client_id):
+        client_exists = Client.objects.filter(client_id=client_id).exists()
+
+        return Response({'valid': client_exists}, status=status.HTTP_200_OK)
 
 class OurTokenPermission(BasePermission):
     """
diff --git a/rorapi/management/commands/generaterorid.py b/rorapi/management/commands/generaterorid.py
@@ -26,3 +26,10 @@ def check_ror_id(version):
         check_ror_id(version)
     return ror_id
 
+
+def generate_ror_client_id():
+    """Generates a random ROR client ID.
+    """
+
+    n = random.randint(0, 2**160 - 1)
+    return base32_crockford.encode(n).lower().zfill(32)
diff --git a/rorapi/migrations/0001_create_client_model.py b/rorapi/migrations/0001_create_client_model.py
@@ -0,0 +1,30 @@
+# Generated by Django 2.2.28 on 2025-03-11 07:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Client',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('email', models.EmailField(max_length=255)),
+                ('name', models.CharField(blank=True, max_length=255)),
+                ('institution_name', models.CharField(blank=True, max_length=255)),
+                ('institution_ror', models.URLField(blank=True, max_length=255)),
+                ('country_code', models.CharField(blank=True, max_length=2)),
+                ('ror_use', models.TextField(blank=True, max_length=500)),
+                ('client_id', models.CharField(editable=False, max_length=32, unique=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('last_request_at', models.DateTimeField(blank=True, null=True)),
+                ('request_count', models.IntegerField(default=0)),
+            ],
+        ),
+    ]
diff --git a/rorapi/migrations/0002_auto_20250326_1054.py b/rorapi/migrations/0002_auto_20250326_1054.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2025-03-26 10:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rorapi', '0001_create_client_model'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='client',
+            name='email',
+            field=models.EmailField(max_length=255, unique=True),
+        ),
+    ]
diff --git a/rorapi/migrations/0003_auto_20250415_1207.py b/rorapi/migrations/0003_auto_20250415_1207.py
@@ -0,0 +1,43 @@
+# Generated by Django 2.2.28 on 2025-04-15 12:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rorapi', '0002_auto_20250326_1054'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='client',
+            name='country_code',
+            field=models.CharField(blank=True, max_length=2, null=True),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='email',
+            field=models.EmailField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='institution_name',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='institution_ror',
+            field=models.URLField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='name',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='ror_use',
+            field=models.TextField(blank=True, max_length=500, null=True),
+        ),
+    ]
diff --git a/rorapi/migrations/__init__.py b/rorapi/migrations/__init__.py
diff --git a/rorapi/settings.py b/rorapi/settings.py
@@ -11,6 +11,7 @@
 """
 
 import os
+import sys
 import json
 import sentry_sdk
 import boto3
@@ -71,7 +72,7 @@
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'django_prometheus.middleware.PrometheusAfterMiddleware',
+    'django_prometheus.middleware.PrometheusAfterMiddleware'
 ]
 
 ROOT_URLCONF = 'rorapi.common.urls'
@@ -105,7 +106,23 @@
 # Database
 # https://docs.djangoproject.com/en/2.2/ref/settings/#databases
 
-DATABASES = {}
+if 'collectstatic' in sys.argv and os.environ.get('DJANGO_SKIP_DB_CHECK') == 'True':
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.dummy'
+        }
+    }
+else:
+    DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': os.environ.get('DB_NAME', 'rorapi'),
+        'USER': os.environ.get('DB_USER', 'root'),
+        'PASSWORD': os.environ.get('DB_PASSWORD', 'password'),
+        'HOST': os.environ.get('DB_HOST', 'db'),
+        'PORT': os.environ.get('DB_PORT', '3306'),
+    }
+}
 
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
@@ -273,3 +290,13 @@
 GRID_REMOVED_IDS = []
 
 LAUNCH_DARKLY_KEY = os.environ.get('LAUNCH_DARKLY_KEY')
+
+# Toggle for behavior-based rate limiting
+import os
+ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True"
+
+# Email settings for Django
+EMAIL_BACKEND = 'django_ses.SESBackend'
+AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
+AWS_SES_REGION_NAME = os.environ.get('AWS_REGION', 'eu-west-1')
diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py
diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py
diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py
diff --git a/rorapi/v2/tests.py b/rorapi/v2/tests.py
