diff --git a/README.md b/README.md index 96f97fbb..8e5a393d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ All models are in `nowcasting_datamodel.models.py`. The diagram below shows how the different tables are connected. ![Models](diagram.png) -![Models](diagram_pv.png) ### connection.py diff --git a/diagram_pv.png b/diagram_pv.png deleted file mode 100644 index 56472f6a..00000000 Binary files a/diagram_pv.png and /dev/null differ diff --git a/nowcasting_datamodel/fake.py b/nowcasting_datamodel/fake.py index 6b7d8605..7b393079 100644 --- a/nowcasting_datamodel/fake.py +++ b/nowcasting_datamodel/fake.py @@ -12,7 +12,6 @@ MetricSQL, MetricValueSQL, MLModelSQL, - PVSystemSQL, national_gb_label, ) from nowcasting_datamodel.models.forecast import ForecastSQL, ForecastValueSQL @@ -32,11 +31,6 @@ def make_fake_location(gsp_id: int) -> LocationSQL: return LocationSQL(label=f"GSP_{gsp_id}", gsp_id=gsp_id) -def make_fake_pv_system() -> PVSystemSQL: - """Make fake location with gsp id""" - return PVSystemSQL(pv_system_id=1, provider="pvoutput.org", latitude=55, longitude=0) - - def make_fake_input_data_last_updated() -> InputDataLastUpdatedSQL: """Make fake input data last updated""" now = datetime.now(tz=timezone.utc) diff --git a/nowcasting_datamodel/models/__init__.py b/nowcasting_datamodel/models/__init__.py index c1184b15..af145b4e 100644 --- a/nowcasting_datamodel/models/__init__.py +++ b/nowcasting_datamodel/models/__init__.py @@ -8,8 +8,7 @@ 5. Input data status, shows when the data was collected (models.py) 6. Forecasts, a forecast that is made for one gsp, for several time steps into the future (models.py) -7. PV system for storing PV data (pv.py) -8. PV yield for storing PV data (pv.py) + Current these models have a primary index of 'id'. This keeps things very simple at the start. @@ -25,4 +24,3 @@ from .gsp import * # noqa F403 from .metric import * # noqa F403 from .models import * # noqa F403 -from .pv import * # noqa F403 diff --git a/nowcasting_datamodel/models/base.py b/nowcasting_datamodel/models/base.py index a68bdaf5..3a904120 100644 --- a/nowcasting_datamodel/models/base.py +++ b/nowcasting_datamodel/models/base.py @@ -3,4 +3,3 @@ from sqlalchemy.orm import declarative_base Base_Forecast = declarative_base() # noqa -Base_PV = declarative_base() # noqa diff --git a/nowcasting_datamodel/models/pv.py b/nowcasting_datamodel/models/pv.py deleted file mode 100644 index 76fee6f8..00000000 --- a/nowcasting_datamodel/models/pv.py +++ /dev/null @@ -1,155 +0,0 @@ -""" Pydantic and Sqlalchemy models for the database - -7. PV system for storing PV data (pv.py) -8. PV yield for storing PV data (pv.py) - -""" - -import logging -from datetime import datetime -from typing import Optional - -from pydantic import Field, field_validator -from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String -from sqlalchemy.orm import relationship - -from nowcasting_datamodel.models.base import Base_PV -from nowcasting_datamodel.models.utils import CreatedMixin, EnhancedBaseModel -from nowcasting_datamodel.utils import datetime_with_timezone - -logger = logging.getLogger(__name__) -pv_output = "pvoutput.org" -solar_sheffield_passiv = "solar_sheffield_passiv" -providers = [pv_output, solar_sheffield_passiv] - -######## -# 7. PV Metadata -######## - - -class PVSystemSQL(Base_PV, CreatedMixin): - """Metadata for PV data""" - - __tablename__ = "pv_system" - - id = Column(Integer, primary_key=True) - pv_system_id = Column(Integer, index=True) - provider = Column(String) - latitude = Column(Float) - longitude = Column(Float) - name = Column(String, nullable=True) - orientation = Column(Float, nullable=True) - status_interval_minutes = Column(Integer, nullable=True) - installed_capacity_kw = Column( - Float, - nullable=True, - ) - ml_capacity_kw = Column( - Float, - nullable=True, - ) - ocf_id = Column(Integer, nullable=True) - correct_data = Column(Boolean, default=True) - - pv_yield = relationship("PVYieldSQL", back_populates="pv_system") - - -class PVSystem(EnhancedBaseModel): - """Metadata for PV data""" - - pv_system_id: int = Field(..., description="The PV system id") - provider: str = Field(..., description="The provider of the PV system") - latitude: float = Field(None, description="The latitude of the PV system") - longitude: float = Field(None, description="The longitude of the PV system") - name: Optional[str] = Field(None, description="The PV system name") - orientation: Optional[float] = Field(None, description="The orientation of the PV system") - status_interval_minutes: Optional[float] = Field( - None, description="The number of minutes for the pv data to be refreshed" - ) - installed_capacity_kw: Optional[float] = Field( - None, description="The capacity of the pv system in kw." - ) - ml_capacity_kw: Optional[float] = Field( - None, - description="The capacity of the pv system in kw, user for ML models. " - "This may be different from the installed capacity", - ) - correct_data: Optional[bool] = Field( - True, description="If the data from the pv system is not broken in some way" - ) - ocf_id: Optional[int] = Field(None, description="The PV system id that is unique to OCF") - - @field_validator("provider") - def validate_provider(cls, v): - """Validate the provider""" - if v not in providers: - raise Exception(f"Provider ({v}) must be in {providers}") - return v - - def to_orm(self) -> PVSystemSQL: - """Change model to PVSystemSQL""" - return PVSystemSQL( - pv_system_id=self.pv_system_id, - provider=self.provider, - latitude=self.latitude, - longitude=self.longitude, - name=self.name, - orientation=self.orientation, - status_interval_minutes=self.status_interval_minutes, - correct_data=self.correct_data, - installed_capacity_kw=self.installed_capacity_kw, - ml_capacity_kw=self.ml_capacity_kw, - ocf_id=self.ocf_id, - ) - - -######## -# 8. PV Yield -######## - - -class PVYieldSQL(Base_PV, CreatedMixin): - """PV Yield data""" - - __tablename__ = "pv_yield" - - id = Column(Integer, primary_key=True) - datetime_utc = Column(DateTime, index=True) - solar_generation_kw = Column(Float) - - # many (forecasts) to one (location) - pv_system = relationship("PVSystemSQL", back_populates="pv_yield") - pv_system_id = Column(Integer, ForeignKey("pv_system.id"), index=True) - - Index("ix_datetime_utc", datetime_utc.desc()) - - -class PVYield(EnhancedBaseModel): - """PV Yield data""" - - datetime_utc: datetime = Field(..., description="The timestamp of the pv system") - solar_generation_kw: float = Field(..., description="The provider of the PV system") - pv_system: Optional[PVSystem] = Field( - None, - description="The PV system associated with this model", - ) - - @field_validator("datetime_utc", mode="before") - def normalize_datetime_utc(cls, v): - """Normalize datetime_utc field""" - return datetime_with_timezone(cls, v) - - @field_validator("solar_generation_kw") - def validate_solar_generation_kw(cls, v): - """Validate the solar_generation_kw field""" - if v < 0: - logger.debug(f"Changing solar_generation_kw ({v}) to 0") - v = 0 - return v - - def to_orm(self) -> PVYieldSQL: - """Change model to PVYieldSQL""" - return PVYieldSQL( - datetime_utc=self.datetime_utc, - solar_generation_kw=self.solar_generation_kw, - ) diff --git a/nowcasting_datamodel/read/read.py b/nowcasting_datamodel/read/read.py index 95870cdf..108d6182 100644 --- a/nowcasting_datamodel/read/read.py +++ b/nowcasting_datamodel/read/read.py @@ -18,7 +18,6 @@ InputDataLastUpdatedSQL, LocationSQL, MLModelSQL, - PVSystemSQL, StatusSQL, national_gb_label, ) @@ -757,31 +756,3 @@ def get_all_locations(session: Session, gsp_ids: List[int] = None) -> List[Locat locations = [nation] + locations return locations - - -def get_pv_system( - session: Session, pv_system_id: int, provider: Optional[str] = "pvoutput.org" -) -> PVSystemSQL: - """ - Get model object from name and version - - :param session: database session - :param pv_system_id: pv system id - :param provider: the pv provider, defaulted to pvoutput.org - - return: PVSystem object - """ - - # start main query - query = session.query(PVSystemSQL) - - # filter on pv_system_id and provider - query = query.filter(PVSystemSQL.pv_system_id == pv_system_id) - query = query.filter(PVSystemSQL.provider == provider) - - # get all results - pv_systems = query.all() - - pv_system = pv_systems[0] - - return pv_system diff --git a/nowcasting_datamodel/read/read_pv.py b/nowcasting_datamodel/read/read_pv.py deleted file mode 100644 index 004c0c06..00000000 --- a/nowcasting_datamodel/read/read_pv.py +++ /dev/null @@ -1,212 +0,0 @@ -""" Read pv functions """ - -import logging -from datetime import datetime, timezone -from typing import List, Optional, Union - -from sqlalchemy import desc -from sqlalchemy.orm import Session, joinedload - -from nowcasting_datamodel.models import PVSystemSQL, PVYieldSQL - -logger = logging.getLogger(__name__) - - -def get_pv_systems( - session: Session, - pv_systems_ids: Optional[List[int]] = None, - provider: Optional[str] = "pvoutput.org", - correct_data: Optional[bool] = None, -) -> List[PVSystemSQL]: - """ - Get all pv systems - - :param session: - :param pv_systems_ids: optional list of ids - :param provider: optional provider name - :param correct_data: Filters on incorrect_data in pv_systems - :return: list of pv systems - """ - - # start main query - query = session.query(PVSystemSQL) - query = query.distinct(PVSystemSQL.id) - - # only select correct data pv systems - if correct_data is not None: - query = query.filter(PVSystemSQL.correct_data == correct_data) - - # filter on pv_system_id and provider - if pv_systems_ids is not None: - query = query.filter(PVSystemSQL.pv_system_id.in_(pv_systems_ids)) - - if provider is not None: - query = query.filter(PVSystemSQL.provider == provider) - - # order by 'created_utc' desc, so we get the latest one - query = query.order_by(PVSystemSQL.id, desc(PVSystemSQL.created_utc)) - - # get all results - pv_systems = query.all() - - return pv_systems - - -def get_latest_pv_yield( - session: Session, - pv_systems: List[PVSystemSQL], - append_to_pv_systems: bool = False, - start_datetime_utc: Optional[datetime] = None, - start_created_utc: Optional[datetime] = None, - correct_data: Optional[bool] = None, -) -> Union[List[PVYieldSQL], List[PVSystemSQL]]: - """ - Get the last pv yield data - - :param session: database sessions - :param pv_systems: list of pv systems - :param append_to_pv_systems: append pv yield to pv systems, or return pv systems. - If appended the yield is access by 'pv_system.last_pv_yield' - :param start_created_utc: search filters > on 'created_utc'. Can be None - :param start_datetime_utc: search filters > on 'datetime_utc'. Can be None - :param correct_data: Filters on incorrect_data in pv_systems - :return: either list of pv yields, or pv systems - """ - - logger.info("Getting latest pv yield") - - pv_systems_ids = [pv_system.id for pv_system in pv_systems] - - # start main query - query = session.query(PVYieldSQL) - query = query.join(PVSystemSQL) - query = query.where( - PVSystemSQL.id == PVYieldSQL.pv_system_id, - ) - - # only select on results per pv system - query = query.distinct(PVSystemSQL.id) - - # select only th epv systems we want - query = query.where(PVSystemSQL.id.in_(pv_systems_ids)) - - # only select correct data pv systems - if correct_data is not None: - query = query.filter(PVSystemSQL.correct_data == correct_data) - - # filter on datetime utc - if start_datetime_utc is not None: - query = query.filter(PVYieldSQL.datetime_utc >= start_datetime_utc) - - # filter on created utc - if start_created_utc is not None: - query = query.filter(PVYieldSQL.created_utc >= start_created_utc) - - # order by 'created_utc' desc, so we get the latest one - query = query.order_by( - PVSystemSQL.id, desc(PVYieldSQL.datetime_utc), desc(PVYieldSQL.created_utc) - ) - - # get all results - pv_yields: List[PVYieldSQL] = query.all() - - # add utc timezone - for pv_yield in pv_yields: - pv_yield.datetime_utc = pv_yield.datetime_utc.replace(tzinfo=timezone.utc) - - if not append_to_pv_systems: - return pv_yields - else: - logger.info("Will be returning pv systems") - - # get list of pvsystems with last pv yields - pv_systems_with_pv_yields = [] - for pv_yield in pv_yields: - pv_system = pv_yield.pv_system - pv_system.last_pv_yield = pv_yield - - pv_systems_with_pv_yields.append(pv_system) - - # add pv systems that dont have any pv yields - pv_systems_with_pv_yields_ids = [pv_system.id for pv_system in pv_systems_with_pv_yields] - - logger.debug(f"Found {len(pv_systems_with_pv_yields_ids)} pv systems with pv yields") - - pv_systems_with_no_pv_yields = [] - for pv_system in pv_systems: - if pv_system.id not in pv_systems_with_pv_yields_ids: - pv_system.last_pv_yield = None - - pv_systems_with_no_pv_yields.append(pv_system) - - logger.debug(f"Found {len(pv_systems_with_no_pv_yields)} pv systems with no pv yields") - - all_pv_systems = pv_systems_with_pv_yields + pv_systems_with_no_pv_yields - - return all_pv_systems - - -def get_pv_yield( - session: Session, - pv_systems_ids: Optional[List[int]] = None, - start_utc: Optional[datetime] = None, - end_utc: Optional[datetime] = None, - correct_data: Optional[bool] = None, - providers: Optional[List[str]] = None, - distinct: Optional[bool] = False, -) -> Union[List[PVYieldSQL], List[PVSystemSQL]]: - """ - Get the last pv yield data - - :param session: database sessions - :param pv_systems_ids: list of pv systems ids - :param end_utc: search filters < on 'datetime_utc'. Can be None - :param start_utc: search filters >= on 'datetime_utc'. Can be None - :param correct_data: Filters on incorrect_data in pv_systems - :param providers: optional list of provider names - :param distinct: if True, only return distinct pv yields - :return: either list of pv yields, or pv systems - """ - - # start main query - query = session.query(PVYieldSQL) - - if distinct: - query = query.distinct(PVSystemSQL.id, PVYieldSQL.datetime_utc) - - query = query.join(PVSystemSQL) - query = query.options(joinedload(PVYieldSQL.pv_system)) - - # only select correct data pv systems - if correct_data is not None: - query = query.filter(PVSystemSQL.correct_data == correct_data) - - # select only th pv systems we want - if pv_systems_ids is not None: - query = query.where(PVSystemSQL.pv_system_id.in_(pv_systems_ids)) - - # filter on start time - if start_utc is not None: - query = query.filter(PVYieldSQL.datetime_utc >= start_utc) - - # filter on end time - if end_utc is not None: - query = query.filter(PVYieldSQL.datetime_utc < end_utc) - - if providers is not None: - query = query.filter(PVSystemSQL.provider.in_(providers)) - - # order by 'created_utc' desc, so we get the latest one - query = query.order_by( - PVSystemSQL.id, - PVYieldSQL.datetime_utc, - PVYieldSQL.created_utc.desc(), - ) - - # get all results - pv_yields: List[PVYieldSQL] = query.all() - - for pv_yield in pv_yields: - pv_yield.datetime_utc = pv_yield.datetime_utc.replace(tzinfo=timezone.utc) - - return pv_yields diff --git a/nowcasting_datamodel/save/save.py b/nowcasting_datamodel/save/save.py index 731a4991..ab447129 100644 --- a/nowcasting_datamodel/save/save.py +++ b/nowcasting_datamodel/save/save.py @@ -6,7 +6,6 @@ from sqlalchemy.orm.session import Session -from nowcasting_datamodel.models import PVSystem, PVSystemSQL from nowcasting_datamodel.models.forecast import ForecastSQL from nowcasting_datamodel.save.adjust import add_adjust_to_forecasts from nowcasting_datamodel.save.update import ( @@ -82,46 +81,6 @@ def save( session.commit() -def save_pv_system(session: Session, pv_system: PVSystem) -> PVSystemSQL: - """ - Get model object from name and version - - :param session: database session - :param pv_system: pv system sql object - - return: PVSystem object - - """ - - # start main query - query = session.query(PVSystemSQL) - - # filter on pv_system_id and provider - query = query.filter(PVSystemSQL.pv_system_id == pv_system.pv_system_id) - query = query.filter(PVSystemSQL.provider == pv_system.provider) - - # get all results - pv_systems = query.all() - - if len(pv_systems) == 0: - logger.debug( - f"Model for provider {pv_system.provider} and pv_system_id {pv_system.pv_system_id }" - f"does not exist so going to add it" - ) - - session.add(pv_system.to_orm()) - session.commit() - - else: - logger.debug( - f"Model for provider {pv_system.provider} and pv_system_id {pv_system.pv_system_id} " - f"is already there so not going to add it" - ) - pv_system = pv_systems[0] - - return pv_system - - def save_all_forecast_values_seven_days( session: Session, forecasts: List[ForecastSQL], remove_non_distinct: bool = False ): diff --git a/test-docker-compose.yml b/test-docker-compose.yml index 23f76871..bddd55da 100644 --- a/test-docker-compose.yml +++ b/test-docker-compose.yml @@ -2,7 +2,8 @@ version: "3" services: postgres_forecast: - image: postgres:14.5 + container_name: postgres_forecast + image: postgres:17 restart: always environment: - POSTGRES_USER=postgres @@ -10,27 +11,16 @@ services: ports: - "5432:5432" - postgres_pv: - image: postgres:14.5 - restart: always - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - ports: - - "5433:5432" - datamodel: build: context: . dockerfile: infrastructure/docker/Dockerfile environment: - DB_URL=postgresql://postgres:postgres@postgres_forecast:5432/postgres - - DB_URL_PV=postgresql://postgres:postgres@postgres_pv:5432/postgres - GIT_PYTHON_REFRESH=quiet - LOG_LEVEL=DEBUG depends_on: - postgres_forecast - - postgres_pv volumes: - ./tests/:/app/tests - ./nowcasting_datamodel/:/app/nowcasting_datamodel @@ -43,7 +33,6 @@ services: TESTING: 1 environment: - DB_URL=postgresql://postgres:postgres@postgres_forecast:5432/postgres - - DB_URL_PV=postgresql://postgres:postgres@postgres_pv:5432/postgres - GIT_PYTHON_REFRESH=quiet - LOG_LEVEL=DEBUG command: > @@ -55,7 +44,6 @@ services: && cp coverage.xml ./tests/" depends_on: - postgres_forecast - - postgres_pv - datamodel volumes: - ./tests/:/app/tests diff --git a/tests/conftest.py b/tests/conftest.py index 8ac64929..b44fa423 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,6 @@ from nowcasting_datamodel.fake import make_fake_forecasts, make_fake_me_latest from nowcasting_datamodel.models import MetricValueSQL from nowcasting_datamodel.models.forecast import ForecastSQL -from nowcasting_datamodel.models.pv import Base_PV @pytest.fixture @@ -78,34 +77,6 @@ def db_session(db_connection): connection.close() -@pytest.fixture -def db_connection_pv(): - url = os.getenv("DB_URL_PV", "sqlite:///test_pv.db") - - connection = DatabaseConnection(url=url, base=Base_PV) - Base_PV.metadata.create_all(connection.engine) - - yield connection - - Base_PV.metadata.drop_all(connection.engine) - - -@pytest.fixture(scope="function", autouse=True) -def db_session_pv(db_connection_pv): - """Creates a new database session for a test.""" - - connection = db_connection_pv.engine.connect() - t = connection.begin() - - with db_connection_pv.get_session() as s: - s.begin() - yield s - s.rollback() - - t.rollback() - connection.close() - - @pytest.fixture def latest_me(db_session) -> List[MetricValueSQL]: # create diff --git a/tests/read/conftest.py b/tests/read/conftest.py deleted file mode 100644 index a11d3f0d..00000000 --- a/tests/read/conftest.py +++ /dev/null @@ -1,61 +0,0 @@ -from datetime import datetime - -import pytest - -from nowcasting_datamodel.models import PVSystem, PVSystemSQL, PVYield - - -@pytest.fixture() -def pv_systems(db_session_pv): - pv_system_sql_1: PVSystemSQL = PVSystem( - pv_system_id=1, provider="pvoutput.org", status_interval_minutes=5 - ).to_orm() - pv_system_sql_2: PVSystemSQL = PVSystem( - pv_system_id=2, provider="pvoutput.org", status_interval_minutes=5 - ).to_orm() - - # add to database - db_session_pv.add(pv_system_sql_1) - db_session_pv.add(pv_system_sql_2) - - db_session_pv.commit() - - return [pv_system_sql_1, pv_system_sql_2] - - -@pytest.fixture() -def pv_yields_and_systems(db_session_pv): - pv_yield_1 = PVYield(datetime_utc=datetime(2022, 1, 1), solar_generation_kw=1) - pv_yield_1_sql = pv_yield_1.to_orm() - - pv_yield_2 = PVYield(datetime_utc=datetime(2022, 1, 2), solar_generation_kw=2) - pv_yield_2_sql = pv_yield_2.to_orm() - - pv_yield_3 = PVYield(datetime_utc=datetime(2022, 1, 1), solar_generation_kw=3) - pv_yield_3_sql = pv_yield_3.to_orm() - - pv_system_sql_1: PVSystemSQL = PVSystem( - pv_system_id=1, provider="pvoutput.org", status_interval_minutes=5 - ).to_orm() - pv_system_sql_2: PVSystemSQL = PVSystem( - pv_system_id=2, provider="pvoutput.org", status_interval_minutes=5 - ).to_orm() - - # add pv system to yield object - pv_yield_1_sql.pv_system = pv_system_sql_1 - pv_yield_2_sql.pv_system = pv_system_sql_1 - pv_yield_3_sql.pv_system = pv_system_sql_2 - - # add to database - db_session_pv.add(pv_yield_1_sql) - db_session_pv.add(pv_yield_2_sql) - db_session_pv.add(pv_yield_3_sql) - db_session_pv.add(pv_system_sql_1) - db_session_pv.add(pv_system_sql_2) - - db_session_pv.commit() - - return { - "pv_yields": [pv_yield_1_sql, pv_yield_2_sql, pv_yield_3_sql], - "pv_systems": [pv_system_sql_1, pv_system_sql_2], - } diff --git a/tests/read/test_read.py b/tests/read/test_read.py index 6de19df0..881bed83 100644 --- a/tests/read/test_read.py +++ b/tests/read/test_read.py @@ -9,12 +9,10 @@ make_fake_forecast, make_fake_forecasts, make_fake_national_forecast, - make_fake_pv_system, ) from nowcasting_datamodel.models import ( InputDataLastUpdatedSQL, LocationSQL, - PVSystem, Status, national_gb_label, ) @@ -35,10 +33,8 @@ get_latest_national_forecast, get_latest_status, get_location, - get_pv_system, update_latest_input_data_last_updated, ) -from nowcasting_datamodel.save.save import save_pv_system logger = logging.getLogger(__name__) @@ -403,20 +399,6 @@ def test_get_national_latest_forecast(db_session): assert forecast_values_read == f2 -def test_get_pv_system(db_session_pv): - pv_system = PVSystem.model_validate(make_fake_pv_system(), from_attributes=True) - save_pv_system(session=db_session_pv, pv_system=pv_system) - - pv_system_get = get_pv_system( - session=db_session_pv, provider=pv_system.provider, pv_system_id=pv_system.pv_system_id - ) - # this get defaulted to True when adding to the database - pv_system.correct_data = True - assert PVSystem.model_validate(pv_system, from_attributes=True) == PVSystem.model_validate( - pv_system_get, from_attributes=True - ) - - def test_get_latest_input_data_last_updated_multiple_entries(db_session): now = datetime.now(tz=None) yesterday = now - timedelta(hours=24) diff --git a/tests/read/test_read_pv.py b/tests/read/test_read_pv.py deleted file mode 100644 index 311b3700..00000000 --- a/tests/read/test_read_pv.py +++ /dev/null @@ -1,164 +0,0 @@ -import logging -from datetime import datetime, timezone - -from nowcasting_datamodel.models import PVSystem, PVSystemSQL, pv_output, solar_sheffield_passiv -from nowcasting_datamodel.read.read_pv import get_latest_pv_yield, get_pv_systems, get_pv_yield -from nowcasting_datamodel.save.save import save_pv_system - -logger = logging.getLogger(__name__) - - -def test_save_pv_system(db_session_pv): - pv_system_1 = PVSystem(pv_system_id=1, provider="pvoutput.org", status_interval_minutes=5) - - save_pv_system(session=db_session_pv, pv_system=pv_system_1) - - pv_systems_get = get_pv_systems(session=db_session_pv) - assert len(pv_systems_get) == 1 - - -def test_save_pv_system_repeat(db_session_pv, pv_systems): - # add extra systems that is already there - save_pv_system(session=db_session_pv, pv_system=pv_systems[0]) - - pv_systems_get = get_pv_systems(session=db_session_pv) - assert len(pv_systems_get) == 2 - - -def test_save_pv_system_correct_data(db_session_pv, pv_systems): - pv_systems[0].correct_data = False - pv_systems_get = get_pv_systems(session=db_session_pv, correct_data=True) - assert len(pv_systems_get) == 1 - - pv_systems_get = get_pv_systems( - session=db_session_pv, pv_systems_ids=[pv_systems_get[0].id], correct_data=True - ) - - assert len(pv_systems_get) == 1 - - -def test_get_pv_system(db_session_pv, pv_systems): - pv_systems_get = get_pv_systems(session=db_session_pv) - assert len(pv_systems_get) == 2 - - pv_systems_get = get_pv_systems(session=db_session_pv, pv_systems_ids=[pv_systems_get[0].id]) - - assert len(pv_systems_get) == 1 - - -def test_get_latest_pv_yield(db_session_pv, pv_yields_and_systems): - [pv_system_sql_1, pv_system_sql_2] = pv_yields_and_systems["pv_systems"] - - pv_yields = get_latest_pv_yield( - session=db_session_pv, pv_systems=[pv_system_sql_1, pv_system_sql_2] - ) - - # read database - # this is 3 for when using sqlite as 'distinct' does work - assert len(pv_yields) == 2 - - assert pv_yields[0].datetime_utc == datetime(2022, 1, 2, tzinfo=timezone.utc) - assert pv_yields[1].datetime_utc == datetime(2022, 1, 1, tzinfo=timezone.utc) - - pv_systems = db_session_pv.query(PVSystemSQL).order_by(PVSystemSQL.created_utc).all() - pv_yields[0].pv_system.id = pv_systems[0].id - - -def test_get_latest_pv_yield_correct_data(db_session_pv, pv_yields_and_systems): - [pv_system_sql_1, pv_system_sql_2] = pv_yields_and_systems["pv_systems"] - pv_system_sql_1.correct_data = False - - pv_yields = get_latest_pv_yield( - session=db_session_pv, - pv_systems=[pv_system_sql_1, pv_system_sql_2], - correct_data=True, - ) - - # read database - # this is 2 for when using sqlite as 'distinct' does work - assert len(pv_yields) == 1 - - -def test_get_latest_pv_yield_filter(db_session_pv, pv_yields_and_systems): - [pv_system_sql_1, pv_system_sql_2] = pv_yields_and_systems["pv_systems"] - - pv_yields = get_latest_pv_yield( - session=db_session_pv, - pv_systems=[pv_system_sql_1, pv_system_sql_2], - start_datetime_utc=datetime(2022, 1, 2), - start_created_utc=datetime(2022, 1, 2), - ) - - # read database - assert len(pv_yields) == 1 - - assert pv_yields[0].datetime_utc == datetime(2022, 1, 2, tzinfo=timezone.utc) - - pv_systems = db_session_pv.query(PVSystemSQL).order_by(PVSystemSQL.created_utc).all() - pv_yields[0].pv_system.id = pv_systems[0].id - - -def test_get_latest_pv_yield_append_no_yields(db_session_pv, pv_systems): - [pv_system_sql_1, pv_system_sql_2] = pv_systems - - pv_systems = get_latest_pv_yield( - session=db_session_pv, - pv_systems=[pv_system_sql_1, pv_system_sql_2], - append_to_pv_systems=True, - ) - - assert pv_systems[0].last_pv_yield is None - assert len(pv_systems) == 2 - - -def test_get_latest_pv_yield_append(db_session_pv, pv_yields_and_systems): - [pv_system_sql_1, pv_system_sql_2] = pv_yields_and_systems["pv_systems"] - - assert pv_system_sql_1.pv_system_id == 1 - assert pv_system_sql_2.pv_system_id == 2 - - pv_systems = get_latest_pv_yield( - session=db_session_pv, - pv_systems=[pv_system_sql_1, pv_system_sql_2], - append_to_pv_systems=True, - ) - assert pv_systems[0].last_pv_yield is not None - # this is 3 for when using sqlite as 'distinct' does work - assert len(pv_systems) == 2 - - -def test_read_pv_yield(db_session_pv, pv_yields_and_systems): - assert len(get_pv_yield(session=db_session_pv, pv_systems_ids=[1])) == 2 - assert len(get_pv_yield(session=db_session_pv, pv_systems_ids=[1, 2])) == 3 - - -def test_read_pv_yield_providers(db_session_pv, pv_yields_and_systems): - assert len(get_pv_yield(session=db_session_pv, providers=[pv_output])) == 3 - assert len(get_pv_yield(session=db_session_pv, providers=[solar_sheffield_passiv])) == 0 - - -def test_read_pv_yield_correct_data(db_session_pv, pv_yields_and_systems): - pv_yields_and_systems["pv_systems"][0].correct_data = False - - assert len(get_pv_yield(session=db_session_pv, pv_systems_ids=[1], correct_data=True)) == 0 - assert len(get_pv_yield(session=db_session_pv, pv_systems_ids=[1, 2], correct_data=True)) == 1 - - -def test_read_pv_yield_start_utc(db_session_pv, pv_yields_and_systems): - pv_yields = get_pv_yield( - session=db_session_pv, pv_systems_ids=[1], start_utc=datetime(2022, 1, 2) - ) - assert len(pv_yields) == 1 - assert pv_yields[0].datetime_utc == datetime(2022, 1, 2, tzinfo=timezone.utc) - - -def test_read_pv_yield_end_utc(db_session_pv, pv_yields_and_systems): - pv_yields = get_pv_yield( - session=db_session_pv, pv_systems_ids=[1, 2], end_utc=datetime(2022, 1, 2), distinct=True - ) - assert len(pv_yields) == 2 - - print(pv_yields) - - assert pv_yields[0].datetime_utc == datetime(2022, 1, 1, tzinfo=timezone.utc) - assert pv_yields[1].datetime_utc == datetime(2022, 1, 1, tzinfo=timezone.utc) diff --git a/tests/save/test_save.py b/tests/save/test_save.py index b993e124..920155c5 100644 --- a/tests/save/test_save.py +++ b/tests/save/test_save.py @@ -11,8 +11,8 @@ ForecastValueSevenDaysSQL, ForecastValueSQL, ) -from nowcasting_datamodel.models.pv import PVSystem, PVSystemSQL -from nowcasting_datamodel.save.save import save, save_all_forecast_values_seven_days, save_pv_system + +from nowcasting_datamodel.save.save import save, save_all_forecast_values_seven_days @freeze_time("2024-01-01 00:00:00") @@ -132,23 +132,6 @@ def test_save_no_adjuster(db_session): assert forecast_latest_values[0].gsp_id == forecast_latest_values[1].gsp_id -def test_save_pv_system(db_session_pv): - pv_systems = db_session_pv.query(PVSystemSQL).all() - assert len(pv_systems) == 0 - - pv_system = PVSystem(pv_system_id=2, provider="pvoutput.org", latitude=55, longitude=0) - - save_pv_system(session=db_session_pv, pv_system=pv_system) - - pv_systems = db_session_pv.query(PVSystemSQL).all() - assert len(pv_systems) == 1 - - save_pv_system(session=db_session_pv, pv_system=pv_system) - - pv_systems = db_session_pv.query(PVSystemSQL).all() - assert len(pv_systems) == 1 - - def test_save_all_forecast_values_seven_days(db_session): now = datetime.now(tz=timezone.utc) forecasts = make_fake_forecasts(gsp_ids=range(0, 3), session=db_session, t0_datetime_utc=now) diff --git a/tests/test_fake_pv.py b/tests/test_fake_pv.py deleted file mode 100644 index 9546d4c9..00000000 --- a/tests/test_fake_pv.py +++ /dev/null @@ -1,24 +0,0 @@ -from datetime import datetime - -import pytest - -from nowcasting_datamodel.fake import make_fake_pv_system -from nowcasting_datamodel.models import PVSystem, PVSystemSQL, PVYield - - -def test_make_fake_pv_system(): - pv_system_sql: PVSystemSQL = make_fake_pv_system() - pv_system = PVSystem.model_validate(pv_system_sql) - _ = PVSystem.to_orm(pv_system) - - -def test_pv_system_error(): - with pytest.raises(Exception): - _ = PVSystem(pv_system_id=1, provider="fake.com", latitude=55, longitude=0) - - -def test_make_fake_pv_yield(): - pv_yield = PVYield(datetime_utc=datetime(2022, 1, 1), solar_generation_kw=-1) - assert pv_yield.solar_generation_kw == 0 - - _ = pv_yield.to_orm()