Internal: make file size column not nullable

Problem: the size column of the `files` table is nullable for no
reason.

Solution: make it non-nullable.

Author: odesenfans

deployment/migrations/versions/0015_039c56d3b33e_file_size_not_null.py

@@ -0,0 +1,28 @@
+"""file size not null
+
+Revision ID: 039c56d3b33e
+Revises: daa92b500049
+Create Date: 2023-04-13 17:13:01.353182
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "039c56d3b33e"
+down_revision = "daa92b500049"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column("files", "size", existing_type=sa.BIGINT(), nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column("files", "size", existing_type=sa.BIGINT(), nullable=True)
+    # ### end Alembic commands ###

src/aleph/chains/chaindata.py

@@ -1,29 +1,31 @@
 import asyncio
 import json
-from typing import Dict, Optional, List, Any, Mapping, Set, Type, cast
+from typing import Dict, Optional, List, Any, Mapping, Set, cast
 
 from aleph_message.models import StoreContent, ItemType, Chain, MessageType
 from pydantic import ValidationError
 
 from aleph.chains.common import LOGGER
 from aleph.config import get_config
 from aleph.db.accessors.chains import upsert_chain_tx
-from aleph.db.accessors.files import upsert_tx_file_pin, upsert_stored_file
+from aleph.db.accessors.files import upsert_tx_file_pin, upsert_file
 from aleph.db.accessors.pending_txs import upsert_pending_tx
-from aleph.db.models import ChainTxDb, MessageDb, StoredFileDb
+from aleph.db.models import ChainTxDb, MessageDb
 from aleph.exceptions import (
     InvalidContent,
     AlephStorageException,
     ContentCurrentlyUnavailable,
 )
 from aleph.schemas.chains.indexer_response import MessageEvent, GenericMessageEvent
+from aleph.schemas.chains.tezos_indexer_response import (
+    MessageEventPayload as TezosMessageEventPayload,
+)
 from aleph.storage import StorageService
 from aleph.toolkit.timestamp import utc_now
 from aleph.types.chain_sync import ChainSyncProtocol
 from aleph.types.db_session import DbSessionFactory, DbSession
 from aleph.types.files import FileType
 from aleph.utils import get_sha256
-from aleph.schemas.chains.tezos_indexer_response import MessageEventPayload as TezosMessageEventPayload
 
 INCOMING_MESSAGE_AUTHORIZED_FIELDS = [
     "item_hash",
@@ -140,12 +142,12 @@ async def _get_tx_messages_off_chain_protocol(
                 with self.session_factory() as session:
                     # Some chain data files are duplicated, and can be treated in parallel,
                     # hence the upsert.
-                    stored_file = StoredFileDb(
-                        hash=sync_file_content.hash,
-                        type=FileType.FILE,
+                    upsert_file(
+                        session=session,
+                        file_hash=sync_file_content.hash,
+                        file_type=FileType.FILE,
                         size=len(sync_file_content.raw_value),
                     )
-                    upsert_stored_file(session=session, file=stored_file)
                     upsert_tx_file_pin(
                         session=session,
                         file_hash=file_hash,

src/aleph/db/accessors/files.py

@@ -147,9 +147,7 @@ def get_address_files_for_api(
     return session.execute(select_stmt).all()
 
 
-def upsert_file(
-    session: DbSession, file_hash: str, size: Optional[int], file_type: FileType
-):
+def upsert_file(session: DbSession, file_hash: str, size: int, file_type: FileType):
     upsert_file_stmt = (
         insert(StoredFileDb)
         .values(hash=file_hash, size=size, type=file_type)
@@ -158,15 +156,6 @@ def upsert_file(
     session.execute(upsert_file_stmt)
 
 
-def upsert_stored_file(
-    session: DbSession,
-    file: StoredFileDb,
-) -> None:
-    upsert_file(
-        session=session, file_hash=file.hash, size=file.size, file_type=file.type
-    )
-
-
 def get_file(session: DbSession, file_hash: str) -> Optional[StoredFileDb]:
     select_stmt = select(StoredFileDb).where(StoredFileDb.hash == file_hash)
     return session.execute(select_stmt).scalar_one_or_none()

src/aleph/db/models/files.py

@@ -37,7 +37,7 @@ class StoredFileDb(Base):
 
     # size: int = Column(BigInteger, nullable=False)
     # TODO: compute the size from local storage
-    size: Optional[int] = Column(BigInteger, nullable=True)
+    size: int = Column(BigInteger, nullable=False)
     type: FileType = Column(ChoiceType(FileType), nullable=False)
 
     pins: List["FilePinDb"] = relationship("FilePinDb", back_populates="file")

src/aleph/handlers/content/store.py

@@ -23,13 +23,13 @@
     insert_message_file_pin,
     get_file_tag,
     upsert_file_tag,
-    upsert_stored_file,
     delete_file_pin,
     refresh_file_tag,
     is_pinned_file,
     get_message_file_pin,
+    upsert_file,
 )
-from aleph.db.models import MessageDb, StoredFileDb
+from aleph.db.models import MessageDb
 from aleph.exceptions import AlephStorageException, UnknownHashError
 from aleph.handlers.content.content_handler import ContentHandler
 from aleph.storage import StorageService
@@ -121,7 +121,9 @@ async def fetch_related_content(
 
         ipfs_enabled = config.ipfs.enabled.value
         do_standard_lookup = True
-        size = None
+
+        # Sentinel value, the code below always sets a value but mypy does not see it.
+        size: int = -1
 
         if engine == ItemType.ipfs and ipfs_enabled:
             if item_type_from_hash(item_hash) != ItemType.ipfs:
@@ -190,12 +192,12 @@ async def fetch_related_content(
 
             size = len(file_content)
 
-        stored_file = StoredFileDb(
-            hash=item_hash,
-            type=FileType.DIRECTORY if is_folder else FileType.FILE,
+        upsert_file(
+            session=session,
+            file_hash=item_hash,
+            file_type=FileType.DIRECTORY if is_folder else FileType.FILE,
             size=size,
         )
-        upsert_stored_file(session=session, file=stored_file)
 
     async def check_dependencies(self, session: DbSession, message: MessageDb) -> None:
         content = _get_store_content(message)

tests/db/test_files.py

@@ -18,11 +18,12 @@
 @pytest.mark.asyncio
 async def test_is_pinned_file(session_factory: DbSessionFactory):
     def is_pinned(_session_factory, _file_hash) -> bool:
-        with session_factory() as session:
-            return is_pinned_file(session=session, file_hash=_file_hash)
+        with _session_factory() as _session:
+            return is_pinned_file(session=_session, file_hash=_file_hash)
 
     file = StoredFileDb(
         hash="QmTm7g1Mh3BhrQPjnedVQ5g67DR7cwhyMN3MvFt1JPPdWd",
+        size=27,
         type=FileType.FILE,
     )
 
@@ -49,10 +50,12 @@ def is_pinned(_session_factory, _file_hash) -> bool:
 async def test_upsert_file_tag(session_factory: DbSessionFactory):
     original_file = StoredFileDb(
         hash="QmTm7g1Mh3BhrQPjnedVQ5g67DR7cwhyMN3MvFt1JPPdWd",
+        size=32,
         type=FileType.FILE,
     )
     new_version = StoredFileDb(
         hash="QmTm7g1Mh3BhrQPjnedVQ5g67DR7cwhyMN3MvFt1JPPdWe",
+        size=413,
         type=FileType.FILE,
     )
 
@@ -122,10 +125,12 @@ async def test_refresh_file_tag(session_factory: DbSessionFactory):
     files = [
         StoredFileDb(
             hash="QmTm7g1Mh3BhrQPjnedVQ5g67DR7cwhyMN3MvFt1JPPdWd",
+            size=123,
             type=FileType.FILE,
         ),
         StoredFileDb(
             hash="QmTm7g1Mh3BhrQPjnedVQ5g67DR7cwhyMN3MvFt1JPPdWe",
+            size=678,
             type=FileType.FILE,
         ),
     ]