Merge pull request #11 from openclimatefix/add-gsp

add gsp model

Author: peterdudfield

diagram.png

@@ -9,12 +9,12 @@
     ForecastSQL,
     ForecastValueSQL,
     InputDataLastUpdatedSQL,
-    LocationSQL,
     MLModelSQL,
     PVSystemSQL,
     national_gb_label,
 )
-from nowcasting_datamodel.read import get_location, get_model
+from nowcasting_datamodel.models.gsp import LocationSQL
+from nowcasting_datamodel.read.read import get_location, get_model
 
 
 def make_fake_location(gsp_id: int) -> LocationSQL:

nowcasting_datamodel/fake.py

@@ -0,0 +1,48 @@
+"""empty message
+
+Revision ID: d2f031f24593
+Revises: 14e1747b9710
+Create Date: 2022-03-21 15:56:09.757720
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d2f031f24593"
+down_revision = "14e1747b9710"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():  # noqa 103
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "gsp_yield",
+        sa.Column("created_utc", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("datetime_utc", sa.DateTime(), nullable=True),
+        sa.Column("solar_generation_kw", sa.String(), nullable=True),
+        sa.Column("regime", sa.String(), nullable=True),
+        sa.Column("location_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["location_id"],
+            ["location.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_gsp_yield_datetime_utc"), "gsp_yield", ["datetime_utc"], unique=False)
+    op.create_index(
+        "ix_gsp_yield_datetime_utc_desc", "gsp_yield", [sa.text("datetime_utc DESC")], unique=False
+    )
+    op.create_index(op.f("ix_gsp_yield_location_id"), "gsp_yield", ["location_id"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():  # noqa 103
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_gsp_yield_location_id"), table_name="gsp_yield")
+    op.drop_index("ix_gsp_yield_datetime_utc_desc", table_name="gsp_yield")
+    op.drop_index(op.f("ix_gsp_yield_datetime_utc"), table_name="gsp_yield")
+    op.drop_table("gsp_yield")
+    # ### end Alembic commands ###

nowcasting_datamodel/migrations/forecast/versions/d2f031f24593_.py

@@ -20,5 +20,6 @@
 The primary keys could be 'gsp_id' and 'target_datetime_utc'.
 """
 
+from .gsp import *  # noqa F403
 from .models import *  # noqa F403
 from .pv import *  # noqa F403

nowcasting_datamodel/models/__init__.py

@@ -0,0 +1,122 @@
+""" Pydantic and Sqlalchemy models for the database
+
+2. Location objects, where the forecast is for
+8. GSP yield for storing GSP yield data
+
+"""
+import logging
+from datetime import datetime
+from typing import Optional
+
+from pydantic import Field, validator
+from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, String
+from sqlalchemy.orm import relationship
+
+from nowcasting_datamodel.models.base import Base_Forecast
+from nowcasting_datamodel.models.utils import CreatedMixin, EnhancedBaseModel
+from nowcasting_datamodel.utils import datetime_must_have_timezone
+
+logger = logging.getLogger(__name__)
+
+########
+# 2. Location
+########
+
+
+class LocationSQL(Base_Forecast):
+    """Location that the forecast is for"""
+
+    __tablename__ = "location"
+
+    id = Column(Integer, primary_key=True)
+    label = Column(String)
+    gsp_id = Column(Integer)
+    gsp_name = Column(String, nullable=True)
+    gsp_group = Column(String, nullable=True)
+    region_name = Column(String, nullable=True)
+
+    forecast = relationship("ForecastSQL", back_populates="location")
+    gsp_yield = relationship("GSPYieldSQL", back_populates="location")
+
+
+class Location(EnhancedBaseModel):
+    """Location that the forecast is for"""
+
+    label: str = Field(..., description="")
+    gsp_id: Optional[int] = Field(None, description="The Grid Supply Point (GSP) id", index=True)
+    gsp_name: Optional[str] = Field(None, description="The GSP name")
+    gsp_group: Optional[str] = Field(None, description="The GSP group name")
+    region_name: Optional[str] = Field(None, description="The GSP region name")
+
+    rm_mode = True
+
+    def to_orm(self) -> LocationSQL:
+        """Change model to LocationSQL"""
+        return LocationSQL(
+            label=self.label,
+            gsp_id=self.gsp_id,
+            gsp_name=self.gsp_name,
+            gsp_group=self.gsp_group,
+            region_name=self.region_name,
+        )
+
+
+class GSPYieldSQL(Base_Forecast, CreatedMixin):
+    """GSP Yield data"""
+
+    __tablename__ = "gsp_yield"
+
+    id = Column(Integer, primary_key=True)
+    datetime_utc = Column(DateTime, index=True)
+    solar_generation_kw = Column(String)
+    regime = Column(String, nullable=True)
+
+    # many (forecasts) to one (location)
+    location = relationship("LocationSQL", back_populates="gsp_yield")
+    location_id = Column(Integer, ForeignKey("location.id"), index=True)
+
+    Index("ix_gsp_yield_datetime_utc_desc", datetime_utc.desc())
+
+
+class GSPYield(EnhancedBaseModel):
+    """GSP Yield data"""
+
+    datetime_utc: datetime = Field(..., description="The timestamp of the gsp yield")
+    solar_generation_kw: float = Field(..., description="The amount of solar generation")
+    regime: str = Field(
+        "in-day", description="When the GSP data is pulled, can be 'in-day' or 'day-after'"
+    )
+
+    _normalize_target_time = validator("datetime_utc", allow_reuse=True)(
+        datetime_must_have_timezone
+    )
+
+    gsp: Optional[Location] = Field(
+        None,
+        description="The GSP associated with this model",
+    )
+
+    @validator("solar_generation_kw")
+    def validate_solar_generation_kw(cls, v):
+        """Validate the solar_generation_kw field"""
+        if v < 0:
+            logger.debug(f"Changing solar_generation_kw ({v}) to 0")
+            v = 0
+        return v
+
+    @validator("regime")
+    def validate_regime(cls, v):
+        """Validate the solar_generation_kw field"""
+        if v not in ["day-after", "in-da"]:
+            message = f"Regime ({v}) not in 'day-after', 'in-da'"
+            logger.debug(message)
+            raise Exception(message)
+        return v
+
+    def to_orm(self) -> GSPYieldSQL:
+        """Change model to GSPYieldSQL"""
+        return GSPYieldSQL(
+            datetime_utc=self.datetime_utc,
+            solar_generation_kw=self.solar_generation_kw,
+            regime=self.regime,
+        )

nowcasting_datamodel/models/gsp.py

@@ -2,7 +2,6 @@
 
 The following class are made
 1. Reusable classes
-2. Location objects, where the forecast is for
 3. Model object, what forecast model is used
 4. ForecastValue objects, specific values of a forecast and time
 5. Input data status, shows when the data was collected
@@ -18,6 +17,7 @@
 from sqlalchemy.orm import relationship
 
 from nowcasting_datamodel.models.base import Base_Forecast
+from nowcasting_datamodel.models.gsp import Location
 from nowcasting_datamodel.models.utils import CreatedMixin, EnhancedBaseModel
 from nowcasting_datamodel.utils import datetime_must_have_timezone
 
@@ -28,41 +28,6 @@
 ########
 # 2. Location
 ########
-class LocationSQL(Base_Forecast):
-    """Location that the forecast is for"""
-
-    __tablename__ = "location"
-
-    id = Column(Integer, primary_key=True)
-    label = Column(String)
-    gsp_id = Column(Integer)
-    gsp_name = Column(String, nullable=True)
-    gsp_group = Column(String, nullable=True)
-    region_name = Column(String, nullable=True)
-
-    forecast = relationship("ForecastSQL", back_populates="location")
-
-
-class Location(EnhancedBaseModel):
-    """Location that the forecast is for"""
-
-    label: str = Field(..., description="")
-    gsp_id: Optional[int] = Field(None, description="The Grid Supply Point (GSP) id", index=True)
-    gsp_name: Optional[str] = Field(None, description="The GSP name")
-    gsp_group: Optional[str] = Field(None, description="The GSP group name")
-    region_name: Optional[str] = Field(None, description="The GSP region name")
-
-    rm_mode = True
-
-    def to_orm(self) -> LocationSQL:
-        """Change model to LocationSQL"""
-        return LocationSQL(
-            label=self.label,
-            gsp_id=self.gsp_id,
-            gsp_name=self.gsp_name,
-            gsp_group=self.gsp_group,
-            region_name=self.region_name,
-        )
 
 
 ########

nowcasting_datamodel/models/models.py

@@ -0,0 +1 @@
+""" init for read functions """

nowcasting_datamodel/read/__init__.py

@@ -0,0 +1,73 @@
+""" Read pv functions """
+from typing import List, Union
+
+from sqlalchemy import desc
+from sqlalchemy.orm import Session
+
+from nowcasting_datamodel.models import GSPYieldSQL, LocationSQL
+
+
+def get_latest_gsp_yield(
+    session: Session, gsps: List[LocationSQL], append_to_gsps: bool = False, regime: str = "in-day"
+) -> Union[List[GSPYieldSQL], List[LocationSQL]]:
+    """
+    Get the last gsp yield data
+
+    :param session: database sessions
+    :param gsps: list of gsps
+    :param append_to_gsps: append gsp yield to pv systems, or return pv systems.
+        If appended the yield is access by 'pv_system.last_gsp_yield'
+    :param regime: What regime the data is in, either 'in-day' or 'day-after'
+    :return: either list of gsp yields, or pv systems
+    """
+
+    gsp_ids = [gsp.id for gsp in gsps]
+
+    # start main query
+    query = session.query(GSPYieldSQL)
+    query = query.join(LocationSQL)
+    query = query.where(
+        LocationSQL.id == GSPYieldSQL.location_id,
+    )
+
+    # filter on regime
+    query = query.where(GSPYieldSQL.regime == regime)
+
+    # only select on results per pv system
+    query = query.distinct(LocationSQL.id)
+
+    # select only th epv systems we want
+    query = query.where(LocationSQL.id.in_(gsp_ids))
+
+    # order by 'created_utc' desc, so we get the latest one
+    query = query.order_by(
+        LocationSQL.id, desc(GSPYieldSQL.datetime_utc), desc(GSPYieldSQL.created_utc)
+    )
+
+    # get all results
+    gsp_yields: List[GSPYieldSQL] = query.all()
+
+    if not append_to_gsps:
+        return gsp_yields
+    else:
+        # get list of pvsystems with last pv yields
+        gsp_systems_with_gsp_yields = []
+        for gsp_yield in gsp_yields:
+            gsp = gsp_yield.location
+            gsp.last_gsp_yield = gsp_yield
+
+            gsp_systems_with_gsp_yields.append(gsp)
+
+        # add pv systems that dont have any pv yields
+        gsp_systems_with_gsp_yields_ids = [gsp.id for gsp in gsp_systems_with_gsp_yields]
+
+        gsp_systems_with_no_gsp_yields = []
+        for gsp in gsps:
+            if gsp.id not in gsp_systems_with_gsp_yields_ids:
+                gsp.last_gsp_yield = None
+
+                gsp_systems_with_no_gsp_yields.append(gsp)
+
+        all_gsp_systems = gsp_systems_with_gsp_yields_ids + gsp_systems_with_no_gsp_yields
+
+        return all_gsp_systems