[#38] code migration to use pydantic v2

Makefile

@@ -33,4 +33,4 @@ test:
 
 .PHONY: pre-post
 pre-post:
-	docker-compose run catalog-trigger python /app/triggers/management/change_streams_pre_and_post.py
+	docker-compose run catalog-trigger python /app/api/models/management/change_streams_pre_and_post.py

api/adapters/hydroshare.py

@@ -1,39 +1,43 @@
 import requests
 from datetime import datetime
 from typing import List, Optional, Union
-from pydantic import BaseModel, EmailStr, HttpUrl
+from pydantic import BaseModel, EmailStr
 
-from api.adapters.base import AbstractRepositoryMetadataAdapter, AbstractRepositoryRequestHandler
+from api.adapters.base import (
+    AbstractRepositoryMetadataAdapter,
+    AbstractRepositoryRequestHandler,
+)
 from api.adapters.utils import RepositoryType, register_adapter
 from api.exceptions import RepositoryException
 from api.models import schema
 from api.models.catalog import DatasetMetadataDOC
 from api.models.user import Submission, SubmissionType
+from api.models.schema import HttpUrlStr
 
 
 class Creator(BaseModel):
-    name: Optional[str]
-    email: Optional[EmailStr]
-    organization: Optional[str]
-    homepage: Optional[HttpUrl]
-    address: Optional[str]
+    name: Optional[str] = None
+    email: Optional[EmailStr] = None
+    organization: Optional[str] = None
+    homepage: Optional[HttpUrlStr] = None
+    address: Optional[str] = None
     identifiers: Optional[dict] = {}
 
     def to_dataset_creator(self):
         if self.name:
-            creator = schema.Creator.construct()
+            creator = schema.Creator.model_construct()
             creator.name = self.name
             if self.email:
                 creator.email = self.email
             if self.organization:
-                affiliation = schema.Organization.construct()
+                affiliation = schema.Organization.model_construct()
                 affiliation.name = self.organization
                 creator.affiliation = affiliation
             _ORCID_identifier = self.identifiers.get("ORCID", "")
             if _ORCID_identifier:
                 creator.identifier = _ORCID_identifier
         else:
-            creator = schema.Organization.construct()
+            creator = schema.Organization.model_construct()
             creator.name = self.organization
             if self.homepage:
                 creator.url = self.homepage
@@ -45,20 +49,20 @@ def to_dataset_creator(self):
 
 class Award(BaseModel):
     funding_agency_name: str
-    title: Optional[str]
-    number: Optional[str]
-    funding_agency_url: Optional[HttpUrl]
+    title: Optional[str] = None
+    number: Optional[str] = None
+    funding_agency_url: Optional[HttpUrlStr] = None
 
     def to_dataset_grant(self):
-        grant = schema.Grant.construct()
+        grant = schema.Grant.model_construct()
         if self.title:
             grant.name = self.title
         else:
             grant.name = self.funding_agency_name
         if self.number:
             grant.identifier = self.number
 
-        funder = schema.Organization.construct()
+        funder = schema.Organization.model_construct()
         funder.name = self.funding_agency_name
         if self.funding_agency_url:
             funder.url = self.funding_agency_url
@@ -72,7 +76,7 @@ class TemporalCoverage(BaseModel):
     end: datetime
 
     def to_dataset_temporal_coverage(self):
-        temp_cov = schema.TemporalCoverage.construct()
+        temp_cov = schema.TemporalCoverage.model_construct()
         if self.start:
             temp_cov.startDate = self.start
             if self.end:
@@ -81,48 +85,50 @@ def to_dataset_temporal_coverage(self):
 
 
 class SpatialCoverageBox(BaseModel):
-    name: Optional[str]
+    name: Optional[str] = None
     northlimit: float
     eastlimit: float
     southlimit: float
     westlimit: float
 
     def to_dataset_spatial_coverage(self):
-        place = schema.Place.construct()
+        place = schema.Place.model_construct()
         if self.name:
             place.name = self.name
 
-        place.geo = schema.GeoShape.construct()
-        place.geo.box = f"{self.northlimit} {self.eastlimit} {self.southlimit} {self.westlimit}"
+        place.geo = schema.GeoShape.model_construct()
+        place.geo.box = (
+            f"{self.northlimit} {self.eastlimit} {self.southlimit} {self.westlimit}"
+        )
         return place
 
 
 class SpatialCoveragePoint(BaseModel):
-    name: Optional[str]
+    name: Optional[str] = None
     north: float
     east: float
 
     def to_dataset_spatial_coverage(self):
-        place = schema.Place.construct()
+        place = schema.Place.model_construct()
         if self.name:
             place.name = self.name
-        place.geo = schema.GeoCoordinates.construct()
+        place.geo = schema.GeoCoordinates.model_construct()
         place.geo.latitude = self.north
         place.geo.longitude = self.east
         return place
 
 
 class ContentFile(BaseModel):
     file_name: str
-    url: HttpUrl
+    url: HttpUrlStr
     size: int
     content_type: str
     logical_file_type: str
     modified_time: datetime
     checksum: str
 
     def to_dataset_media_object(self):
-        media_object = schema.MediaObject.construct()
+        media_object = schema.MediaObject.model_construct()
         media_object.contentUrl = self.url
         media_object.encodingFormat = self.content_type
         media_object.contentSize = f"{self.size/1000.00} KB"
@@ -138,13 +144,13 @@ class Relation(BaseModel):
     def to_dataset_part_relation(self, relation_type: str):
         relation = None
         if relation_type == "IsPartOf" and self.type.endswith("is part of"):
-            relation = schema.IsPartOf.construct()
+            relation = schema.IsPartOf.model_construct()
         elif relation_type == "HasPart" and self.type.endswith("resource includes"):
-            relation = schema.HasPart.construct()
+            relation = schema.HasPart.model_construct()
         else:
             return relation
 
-        description, url = self.value.rsplit(',', 1)
+        description, url = self.value.rsplit(",", 1)
         relation.description = description.strip()
         relation.url = url.strip()
         relation.name = self.value
@@ -153,10 +159,10 @@ def to_dataset_part_relation(self, relation_type: str):
 
 class Rights(BaseModel):
     statement: str
-    url: HttpUrl
+    url: HttpUrlStr
 
     def to_dataset_license(self):
-        _license = schema.License.construct()
+        _license = schema.License.model_construct()
         _license.name = self.statement
         _license.url = self.url
         return _license
@@ -170,7 +176,9 @@ def get_metadata(self, record_id: str):
         def make_request(url, file_list=False) -> Union[dict, List[dict]]:
             response = requests.get(url)
             if response.status_code != 200:
-                raise RepositoryException(status_code=response.status_code, detail=response.text)
+                raise RepositoryException(
+                    status_code=response.status_code, detail=response.text
+                )
             if not file_list:
                 return response.json()
 
@@ -180,7 +188,9 @@ def make_request(url, file_list=False) -> Union[dict, List[dict]]:
             while response.json()["next"]:
                 response = requests.get(response.json()["next"])
                 if response.status_code != 200:
-                    raise RepositoryException(status_code=response.status_code, detail=response.text)
+                    raise RepositoryException(
+                        status_code=response.status_code, detail=response.text
+                    )
                 content_files.extend(response.json()["results"])
             return content_files
 
@@ -219,21 +229,21 @@ def update_submission(submission: Submission, repo_record_id: str) -> Submission
 class _HydroshareResourceMetadata(BaseModel):
     title: str
     abstract: str
-    url: HttpUrl
-    identifier: HttpUrl
+    url: HttpUrlStr
+    identifier: HttpUrlStr
     creators: List[Creator]
     created: datetime
     modified: datetime
-    published: Optional[datetime]
-    subjects: Optional[List[str]]
+    published: Optional[datetime] = None
+    subjects: Optional[List[str]] = []
     language: str
     rights: Rights
-    awards: Optional[List[Award]]
-    spatial_coverage: Optional[Union[SpatialCoverageBox, SpatialCoveragePoint]]
-    period_coverage: Optional[TemporalCoverage]
-    relations: Optional[List[Relation]]
+    awards: Optional[List[Award]] = []
+    spatial_coverage: Optional[Union[SpatialCoverageBox, SpatialCoveragePoint]] = None
+    period_coverage: Optional[TemporalCoverage] = None
+    relations: Optional[List[Relation]] = []
     citation: str
-    content_files: Optional[List[ContentFile]]
+    content_files: Optional[List[ContentFile]] = []
 
     def to_dataset_creators(self):
         creators = []
@@ -287,13 +297,13 @@ def to_dataset_license(self):
 
     @staticmethod
     def to_dataset_provider():
-        provider = schema.Organization.construct()
+        provider = schema.Organization.model_construct()
         provider.name = RepositoryType.HYDROSHARE
         provider.url = "https://www.hydroshare.org/"
         return provider
 
     def to_catalog_dataset(self):
-        dataset = DatasetMetadataDOC.construct()
+        dataset = DatasetMetadataDOC.model_construct()
         dataset.provider = self.to_dataset_provider()
         dataset.name = self.title
         dataset.description = self.abstract

api/authentication/fastapi_resource_server.py

@@ -47,24 +47,24 @@ def fetch_jwks(well_known: dict) -> dict:
 
 
 class JwtDecodeOptions(BaseModel):
-    verify_signature: Optional[bool]
-    verify_aud: Optional[bool]
-    verify_iat: Optional[bool]
-    verify_exp: Optional[bool]
-    verify_nbf: Optional[bool]
-    verify_iss: Optional[bool]
-    verify_sub: Optional[bool]
-    verify_jti: Optional[bool]
-    verify_at_hash: Optional[bool]
-    require_aud: Optional[bool]
-    require_iat: Optional[bool]
-    require_exp: Optional[bool]
-    require_nbf: Optional[bool]
-    require_iss: Optional[bool]
-    require_sub: Optional[bool]
-    require_jti: Optional[bool]
-    require_at_hash: Optional[bool]
-    leeway: Optional[int]
+    verify_signature: Optional[bool] = None
+    verify_aud: Optional[bool] = None
+    verify_iat: Optional[bool] = None
+    verify_exp: Optional[bool] = None
+    verify_nbf: Optional[bool] = None
+    verify_iss: Optional[bool] = None
+    verify_sub: Optional[bool] = None
+    verify_jti: Optional[bool] = None
+    verify_at_hash: Optional[bool] = None
+    require_aud: Optional[bool] = None
+    require_iat: Optional[bool] = None
+    require_exp: Optional[bool] = None
+    require_nbf: Optional[bool] = None
+    require_iss: Optional[bool] = None
+    require_sub: Optional[bool] = None
+    require_jti: Optional[bool] = None
+    require_at_hash: Optional[bool] = None
+    leeway: Optional[int] = None
 
 
 class OidcResourceServer(SecurityBase):

api/config/__init__.py

@@ -1,9 +1,11 @@
 from functools import lru_cache
 from typing import Any
 
-from pydantic import BaseSettings, HttpUrl
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from dotenv import load_dotenv
 
+from api.models.schema import HttpUrlStr
+
 # had to use load_dotenv() to get the env variables to work during testing
 load_dotenv()
 
@@ -17,21 +19,22 @@ class Settings(BaseSettings):
     testing: bool = False
 
     oidc_issuer: str
-    hydroshare_meta_read_url: HttpUrl
-    hydroshare_file_read_url: HttpUrl
+    hydroshare_meta_read_url: HttpUrlStr
+    hydroshare_file_read_url: HttpUrlStr
     search_relevance_score_threshold: float = 1.4
 
     def __init__(self, **data: Any) -> None:
         super().__init__(**data)
         if self.testing:
             self.database_name = f"{self.database_name}"
+        self.hydroshare_meta_read_url = self.hydroshare_meta_read_url
+        self.hydroshare_file_read_url = self.hydroshare_file_read_url
 
     @property
     def db_connection_string(self):
         return f"{self.db_protocol}://{self.db_username}:{self.db_password}@{self.db_host}/?retryWrites=true&w=majority"
 
-    class Config:
-        env_file = ".env"
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
 
 
 @lru_cache()

api/main.py

@@ -1,5 +1,6 @@
 import asyncio
 import os
+from contextlib import asynccontextmanager
 
 import uvicorn
 from beanie import init_beanie
@@ -21,7 +22,13 @@
 from api.exceptions import RepositoryException
 
 
-app = FastAPI()
+@asynccontextmanager
+async def lifespan(app_: FastAPI):
+    await startup_db_client()
+    yield
+    await shutdown_db_client()
+
+app = FastAPI(lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,
@@ -43,15 +50,13 @@ async def validation_exception_handler(request, exc: ValidationError):
                              status_code=status.HTTP_400_BAD_REQUEST)
 
 
-@app.on_event("startup")
 async def startup_db_client():
     settings = get_settings()
     app.mongodb_client = AsyncIOMotorClient(settings.db_connection_string)
     app.mongodb = app.mongodb_client[settings.database_name]
     await init_beanie(database=app.mongodb, document_models=[DatasetMetadataDOC, User, Submission])
 
 
-@app.on_event("shutdown")
 async def shutdown_db_client():
     app.mongodb_client.close()
 
@@ -85,7 +90,8 @@ def handle_exit(self, sig: int, frame) -> None:
 async def main():
     """Run FastAPI"""
 
-    server = Server(config=uvicorn.Config(app, workers=1, loop="asyncio", host="0.0.0.0", port=8000, forwarded_allow_ips="*"))
+    server = Server(config=uvicorn.Config(app, workers=1, loop="asyncio", host="0.0.0.0", port=8000,
+                                          forwarded_allow_ips="*"))
     api = asyncio.create_task(server.serve())
 
     await asyncio.wait([api])