diff --git a/.dockerignore b/.dockerignore index 21e0e7c..aff88c4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ # Add build files !requirements** !README.md +!fix_reward_field.py # Add code !registrydao diff --git a/Dockerfile b/Dockerfile index aad2dbc..9063270 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,21 @@ -FROM dipdup/dipdup:6.1.2 +FROM dipdup/dipdup:7.5.10 + +USER root + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + python3-dev \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/root/.cargo/bin:${PATH}" COPY requirements.txt . RUN pip3 install -r requirements.txt +COPY fix_reward_field.py . +RUN python3 fix_reward_field.py + COPY . . diff --git a/README.md b/README.md index 8ac63af..28f6f21 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Homebase Indexer (v3) -Minimal DipDup (v6.1) + Hasura + Postgres indexer for Homebase/RegistryDAO. It indexes on-chain events into Postgres and exposes them via Hasura GraphQL. +Minimal DipDup (v7.x) + Hasura + Postgres indexer for Homebase/RegistryDAO. It indexes on-chain events into Postgres and exposes them via Hasura GraphQL. ## Quick Start (Docker) - Start services: `docker-compose up -d` -- Hasura Console: open `http://localhost:9088` +- Hasura Console: open `http://localhost:9013` - Track tables: Hasura → Data → public → Track All tables (Optional) - Logs (indexer): `docker-compose logs -f indexer` @@ -17,7 +17,7 @@ Minimal DipDup (v6.1) + Hasura + Postgres indexer for Homebase/RegistryDAO. It i - `registrydao/` Python package (handlers, utils, models, hooks, types, sql) - `hasura/` Hasura metadata (optional) - `dipdup.yml` DipDup configuration (contracts, datasources, indexes) -- `docker-compose.yml` Postgres, Hasura (mapped to `http://localhost:9088`), indexer +- `docker-compose.yml` Postgres, Hasura (mapped to `http://localhost:9013`), indexer ## Development Notes - Format with Black and isort (see `requirements.txt`) @@ -26,14 +26,9 @@ Minimal DipDup (v6.1) + Hasura + Postgres indexer for Homebase/RegistryDAO. It i ## Configuration - Review `dipdup.yml` before enabling new indexes (check `first_level`, network) -- Optional: `tzkt-proxy` and `vector` services are included in compose - -## TzKT Proxy (1.16 Compatibility) -- Why: TzKT 1.16 replaces the SignalR events hub (`/v1/events`) with a native WebSocket endpoint (`/v1/ws`). Some clients (including current DipDup usage) still call `/v1/events`. -- How: The `tzkt-proxy` Nginx service rewrites `/v1/events` and `/v1/events/*` to the new WebSocket path and upgrades the connection, while proxying other `/v1/*` REST calls unchanged. -- Wiring: In `docker-compose.yml`, the proxy is reachable as `tzkt-mainnet` and `tzkt-ghostnet` (network aliases). `dipdup.yml` points datasources to `http://tzkt-mainnet` and `http://tzkt-ghostnet`, enabling streaming against TzKT 1.16. -- Config: See `nginx-tzkt.conf`. Adjust upstreams if using a self-hosted TzKT; keep server names consistent with `dipdup.yml`. +- Optional: `vector` service for log shipping to Axiom is included in compose +- DipDup 7.x has native WebSocket support for TzKT 1.16+, connecting directly to `https://api.tzkt.io` and `https://api.ghostnet.tzkt.io` # Deprecation Notices -https://x.com/dipdup_io/status/1955649362069201026 \ No newline at end of file +https://x.com/dipdup_io/status/1955649362069201026 diff --git a/dipdup.yml b/dipdup.yml index 2aa3a99..5e988e0 100644 --- a/dipdup.yml +++ b/dipdup.yml @@ -1,16 +1,17 @@ -spec_version: 1.2 +spec_version: 2.0 package: registrydao - database: kind: postgres - url: !env DATABASE_URL - schema_name: ${PG_SCHEMA:-changeme} - + host: ${PG_HOST:-db} + port: ${PG_PORT:-5432} + user: ${PG_USER:-indexer} + password: ${PG_PASSWORD:-qwerty} + database: ${PG_DB:-indexer_db} + schema_name: ${PG_SCHEMA:-public} sentry: dsn: ${SENTRY_DSN} environment: prod - debug: True - + debug: true advanced: reindex: manual: wipe @@ -18,165 +19,154 @@ advanced: rollback: wipe config_modified: wipe schema_modified: wipe - contracts: registry_mainnet: + kind: tezos address: KT1MFxwTan4ptw6PSc3KK6e1xfzMrCb382tw typename: registry - registry_ghostnet: - address: KT1QZtF8vVUvZYRxttRwgftc4EaQHZWgzXNp + registry_shadownet: + kind: tezos + address: KT1MgGJX3V6stxaxkynfRR6ShJTEKFy9tLnk typename: registry - datasources: tzkt_mainnet: - kind: tzkt - url: http://tzkt-mainnet - - tzkt_ghostnet: - kind: tzkt - url: http://tzkt-ghostnet - + kind: tezos.tzkt + url: https://api.mainnet.tzkt.io + tzkt_shadownet: + kind: tezos.tzkt + url: https://api.shadownet.tzkt.io templates: registry_dao: - kind: operation + kind: tezos.tzkt.operations datasource: types: - - transaction - - origination + - transaction + - origination contracts: - - + - handlers: - - callback: on_origination - pattern: - - type: origination - originated_contract: - - callback: on_propose - pattern: - - type: transaction - destination: - entrypoint: propose - - callback: on_flush - pattern: - - type: transaction - destination: - entrypoint: flush - - callback: on_vote - pattern: - - type: transaction - destination: - entrypoint: vote - - callback: on_drop_proposal - pattern: - - type: transaction - destination: - entrypoint: drop_proposal - - callback: on_unstake_vote - pattern: - - type: transaction - destination: - entrypoint: unstake_vote - - callback: on_freeze - pattern: - - type: transaction - destination: - entrypoint: freeze - - callback: on_unfreeze - pattern: - - type: transaction - destination: - entrypoint: unfreeze - + - callback: on_origination + pattern: + - type: origination + originated_contract: + - callback: on_propose + pattern: + - destination: + entrypoint: propose + - callback: on_flush + pattern: + - destination: + entrypoint: flush + - callback: on_vote + pattern: + - destination: + entrypoint: vote + - callback: on_drop_proposal + pattern: + - destination: + entrypoint: drop_proposal + - callback: on_unstake_vote + pattern: + - destination: + entrypoint: unstake_vote + - callback: on_freeze + pattern: + - destination: + entrypoint: freeze + - callback: on_unfreeze + pattern: + - destination: + entrypoint: unfreeze indexes: factory_mainnet: - kind: operation + kind: tezos.tzkt.operations datasource: tzkt_mainnet - first_level: 30000 + first_level: 2900000 types: - - origination + - transaction + - origination + contracts: + - registry_mainnet handlers: - - callback: on_factory_origination - pattern: - - type: origination - similar_to: registry_mainnet - - callback: on_propose - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: propose - - callback: on_vote - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: vote - - callback: on_drop_proposal - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: drop_proposal - - callback: on_unstake_vote - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: unstake_vote - - callback: on_flush - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: flush - - callback: on_freeze - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: freeze - - callback: on_unfreeze - pattern: - - type: transaction - destination: registry_mainnet - entrypoint: unfreeze - - factory_ghostnet: - kind: operation - datasource: tzkt_ghostnet - first_level: 8916100 + - callback: on_origination + pattern: + - type: origination + originated_contract: registry_mainnet + - callback: on_factory_origination + pattern: + - type: origination + similar_to: registry_mainnet + - callback: on_propose + pattern: + - destination: registry_mainnet + entrypoint: propose + - callback: on_vote + pattern: + - destination: registry_mainnet + entrypoint: vote + - callback: on_drop_proposal + pattern: + - destination: registry_mainnet + entrypoint: drop_proposal + - callback: on_unstake_vote + pattern: + - destination: registry_mainnet + entrypoint: unstake_vote + - callback: on_flush + pattern: + - destination: registry_mainnet + entrypoint: flush + - callback: on_freeze + pattern: + - destination: registry_mainnet + entrypoint: freeze + - callback: on_unfreeze + pattern: + - destination: registry_mainnet + entrypoint: unfreeze + factory_shadownet: + kind: tezos.tzkt.operations + datasource: tzkt_shadownet + first_level: 0 types: - - origination + - transaction + - origination + contracts: + - registry_shadownet handlers: - - callback: on_factory_origination - pattern: - - type: origination - similar_to: registry_ghostnet - - callback: on_propose - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: propose - - callback: on_vote - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: vote - - callback: on_drop_proposal - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: drop_proposal - - callback: on_unstake_vote - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: unstake_vote - - callback: on_flush - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: flush - - callback: on_freeze - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: freeze - - callback: on_unfreeze - pattern: - - type: transaction - destination: registry_ghostnet - entrypoint: unfreeze + - callback: on_origination + pattern: + - type: origination + originated_contract: registry_shadownet + - callback: on_factory_origination + pattern: + - type: origination + similar_to: registry_shadownet + - callback: on_propose + pattern: + - destination: registry_shadownet + entrypoint: propose + - callback: on_vote + pattern: + - destination: registry_shadownet + entrypoint: vote + - callback: on_drop_proposal + pattern: + - destination: registry_shadownet + entrypoint: drop_proposal + - callback: on_unstake_vote + pattern: + - destination: registry_shadownet + entrypoint: unstake_vote + - callback: on_flush + pattern: + - destination: registry_shadownet + entrypoint: flush + - callback: on_freeze + pattern: + - destination: registry_shadownet + entrypoint: freeze + - callback: on_unfreeze + pattern: + - destination: registry_shadownet + entrypoint: unfreeze diff --git a/docker-compose.yml b/docker-compose.yml index d958af9..a83341e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,28 +12,16 @@ services: - vector-data:/var/lib/vector restart: unless-stopped - - tzkt-proxy: - image: nginx:1.25-alpine - restart: unless-stopped - networks: - v3-homebase-indexer: - aliases: - - tzkt-mainnet - - tzkt-ghostnet - volumes: - - ./nginx-tzkt.conf:/etc/nginx/conf.d/default.conf:ro - hasura: depends_on: - db ports: - - '0.0.0.0:9088:8080' + - '0.0.0.0:9013:8080' environment: - 'HASURA_GRAPHQL_DATABASE_URL=postgres://indexer:qwerty@db:5432/indexer_db' - HASURA_GRAPHQL_ENABLE_CONSOLE=true - HASURA_GRAPHQL_DEV_MODE=true - - HASURA_GRAPHQL_ADMIN_SECRET=qanvse4NVSmv2gFqw3M0vWXaCJyIMllbV75Z5gTWq3neRuYgJLCLmQmw8 + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_GRAPHQL_ADMIN_SECRET} - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=indexer - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true - 'HASURA_GRAPHQL_CORS_DOMAIN=https://tezos-homebase.io, https://*.tezos-homebase.io, https://*.netlify.app, http://localhost:3000, http://localhost:8080, http://localhost:8088, http://localhost:8000, http://localhost:8001, http://localhost:8002, http://localhost:8003, http://localhost:8004, http://localhost:8005, http://localhost:8006, http://localhost:8007, http://localhost:8008, http://localhost:8009, https://3000.code-app-tezosbase.w3d.run/, https://3000.code-app-tezosbase.w3d.run' @@ -41,17 +29,17 @@ services: volumes: - ./hasura/metadata:/hasura-metadata networks: - - v3-homebase-indexer + - v3-indexer db: environment: - POSTGRES_USER=indexer - POSTGRES_PASSWORD=qwerty - POSTGRES_DB=indexer_db - image: postgres + image: postgres:16 networks: - - v3-homebase-indexer + - v3-indexer volumes: - - v3_indexer_db:/var/lib/postgresql/data + - v3-indexer-db:/var/lib/postgresql/data indexer: build: . @@ -70,14 +58,14 @@ services: - SENTRY_DSN=${SENTRY_DSN} - SENTRY_ENVIRONMENT=${SENTRY_ENV:-development} networks: - - v3-homebase-indexer + - v3-indexer volumes: - v3_indexer_db: + v3-indexer-db: driver: local vector-data: driver: local networks: - v3-homebase-indexer: - driver: bridge \ No newline at end of file + v3-indexer: + driver: bridge diff --git a/fix_reward_field.py b/fix_reward_field.py new file mode 100644 index 0000000..f8df760 --- /dev/null +++ b/fix_reward_field.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +""" +Patch for DipDup v7 to handle missing 'reward' field in TzKT API responses +""" + +import os +import sys + +def patch_tzkt_model(): + """Patch the TzktBlockData.from_json method to handle missing 'reward' field""" + + file_path = '/opt/dipdup/src/dipdup/models/tezos_tzkt.py' + + # Read the original file + with open(file_path, 'r') as f: + content = f.read() + + # Replace the problematic line to handle missing 'reward' field + # The original line: reward=block_json['reward'], + # Should become: reward=block_json.get('reward', 0), + + old_pattern = "reward=block_json['reward']," + new_pattern = "reward=block_json.get('reward', 0)," + + if old_pattern in content: + content = content.replace(old_pattern, new_pattern) + + # Write the patched file + with open(file_path, 'w') as f: + f.write(content) + + print(f"Successfully patched {file_path}") + return True + else: + print(f"Pattern not found or already patched in {file_path}") + return False + +if __name__ == '__main__': + try: + if patch_tzkt_model(): + sys.exit(0) + else: + # Not an error if already patched + sys.exit(0) + except Exception as e: + print(f"Error patching file: {e}") + sys.exit(1) + + + + diff --git a/nginx-tzkt.conf b/nginx-tzkt.conf deleted file mode 100644 index ebd0ead..0000000 --- a/nginx-tzkt.conf +++ /dev/null @@ -1,95 +0,0 @@ -# /etc/nginx/conf.d/tzkt-proxy.conf -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -server { - listen 80; - server_name tzkt-mainnet; - - # --- WebSocket hub: handle BOTH `/v1/events` and `/v1/events/...` --- - # Exact match (no trailing slash): /v1/events?id=... - location = /v1/events { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host api.tzkt.io; - proxy_ssl_server_name on; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_buffering off; - - proxy_pass https://api.tzkt.io/v1/ws; - } - - # Prefix match (with trailing slash): /v1/events/negotiate, /v1/events/... - location ^~ /v1/events/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host api.tzkt.io; - proxy_ssl_server_name on; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_buffering off; - - proxy_pass https://api.tzkt.io/v1/ws/; - } - - # --- Pass-through for all other REST endpoints under /v1/ --- - location /v1/ { - proxy_http_version 1.1; - proxy_set_header Host api.tzkt.io; - proxy_ssl_server_name on; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - - proxy_pass https://api.tzkt.io; - } -} - -server { - listen 80; - server_name tzkt-ghostnet; - - # --- WebSocket hub: handle BOTH `/v1/events` and `/v1/events/...` --- - # Exact match (no trailing slash): /v1/events?id=... - location = /v1/events { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host api.tzkt.io; - proxy_ssl_server_name on; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_buffering off; - - proxy_pass https://api.ghostnet.tzkt.io/v1/ws; - } - - # Prefix match (with trailing slash): /v1/events/negotiate, /v1/events/... - location ^~ /v1/events/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host api.tzkt.io; - proxy_ssl_server_name on; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_buffering off; - - proxy_pass https://api.ghostnet.tzkt.io/v1/ws/; - } - - # --- Pass-through for all other REST endpoints under /v1/ --- - location /v1/ { - proxy_http_version 1.1; - proxy_set_header Host api.tzkt.io; - proxy_ssl_server_name on; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - - proxy_pass https://api.ghostnet.tzkt.io; - } -} \ No newline at end of file diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index c71ff07..0000000 --- a/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -# Optional: Can be removed -server { - listen 80; - - location / { - proxy_pass http://hasura:8080; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - - # === CORS magic ===================================================== - # If the request comes with an Origin header, echo it back and - # allow credentials. Browsers accept this; wildcard "*" would fail. - if ($http_origin ~* "^https?://") { - add_header Access-Control-Allow-Origin "$http_origin" always; - add_header Access-Control-Allow-Credentials "true" always; - add_header Access-Control-Allow-Headers "Content-Type, Authorization" always; - add_header Access-Control-Allow-Methods "GET,POST,PUT,PATCH,DELETE,OPTIONS" always; - } - - # Short-circuit pre-flight so Hasura never sees it. - if ($request_method = OPTIONS) { - add_header Content-Length 0; - add_header Content-Type "text/plain; charset=utf-8"; - return 204; - } - } -} diff --git a/registrydao/handlers/on_drop_proposal.py b/registrydao/handlers/on_drop_proposal.py index d712698..3fbced2 100644 --- a/registrydao/handlers/on_drop_proposal.py +++ b/registrydao/handlers/on_drop_proposal.py @@ -1,7 +1,7 @@ from registrydao.utils.ledger import update_ledger from typing import Optional from datetime import datetime -from dipdup.models import OperationData, Origination, Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models @@ -21,9 +21,9 @@ async def on_drop_proposal( dao = await models.DAO.get(address=dao_address) status = await models.ProposalStatus.get(description='dropped') - proposal = await models.Proposal.get(key=drop_proposal.data.parameter_json, dao=dao) + proposal = await models.Proposal.get(key=drop_proposal.parameter.__root__, dao=dao) await models.ProposalStatusUpdates.get_or_create(status=status, proposal=proposal, timestamp=drop_proposal.data.timestamp, level=drop_proposal.data.level) except Exception as e: print("Error in on_drop_proposal: " + str(drop_proposal.data.target_address)) - print(e) \ No newline at end of file + print(e) diff --git a/registrydao/handlers/on_factory_origination.py b/registrydao/handlers/on_factory_origination.py index ea4fb3f..c3d474a 100644 --- a/registrydao/handlers/on_factory_origination.py +++ b/registrydao/handlers/on_factory_origination.py @@ -1,33 +1,96 @@ from registrydao.utils.ctx import extract_network_from_ctx -from dipdup.models import OperationData, Transaction, Origination, BigMapDiff, BigMapData, BigMapAction +from registrydao.utils.http import fetch +from registrydao.constants import NETWORK_MAP +from dipdup.models.tezos_tzkt import TzktOrigination as Origination, TzktOperationData from dipdup.context import HandlerContext -from typing import cast - - -from registrydao.types.registry.storage import RegistryStorage +from typing import cast, Union async def on_factory_origination( ctx: HandlerContext, - registry_origination: Origination[RegistryStorage], + registry_origination: Union[Origination, TzktOperationData], ) -> None: try: - originated_contract = cast(str, registry_origination.data.originated_contract_address) + # Handle both TzktOrigination (with .data) and TzktOperationData (direct) + # Check if it's a TzktOrigination wrapper by checking if it has 'data' attribute + if hasattr(registry_origination, 'data') and registry_origination.data is not None: + # TzktOrigination wrapper + origination_data = registry_origination.data + originated_contract = cast(str, origination_data.originated_contract_address) + origination_level = origination_data.level + else: + # TzktOperationData directly (for similar_to pattern) + # TzktOperationData doesn't have .data, it IS the data + originated_contract = cast(str, registry_origination.originated_contract_address) + origination_level = registry_origination.level + index_name = f'registry_dao_{originated_contract}' + ctx.logger.info(f"on_factory_origination called for {originated_contract} at level {origination_level}") + network = extract_network_from_ctx(ctx) - if index_name not in ctx.config.indexes: - await ctx.add_contract( - name=originated_contract, - address=originated_contract, - typename='registry', + try: + eps = await fetch( + f"https://api.{NETWORK_MAP[network]}.tzkt.io/v1/contracts/{originated_contract}/entrypoints" ) - await ctx.add_index( - name=index_name, - template='registry_dao', - values=dict(contract=originated_contract, datasource=f'tzkt_{network}'), + if isinstance(eps, list): + ep_set = set(ep.get('name', ep) if isinstance(ep, dict) else ep for ep in eps) + else: + ep_set = set() + required = {"propose", "vote", "flush", "freeze", "unfreeze", "drop_proposal"} + if not required.issubset(ep_set): + ctx.logger.info( + "Skipping index for %s: missing required entrypoints; got: %s", + originated_contract, + sorted(list(ep_set))[:10], + ) + return + except Exception as e: + ctx.logger.info( + "Skipping index for %s due to entrypoint verification error: %s", + originated_contract, + e, ) + return + + try: + exists = index_name in ctx.config.indexes + except Exception: + exists = False + + if not exists: + try: + from dipdup.database import get_connection + async with get_connection() as conn: + result = await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM dipdup_index WHERE name = $1)", + index_name + ) + exists = result if result else False + except Exception: + exists = False + + if not exists: + try: + await ctx.add_contract( + name=originated_contract, + address=originated_contract, + typename='registry', + kind='tezos', + ) + await ctx.add_index( + name=index_name, + template='registry_dao', + values=dict(contract=originated_contract, datasource=f'tzkt_{network}'), + first_level=origination_level, + ) + except Exception as e: + ctx.logger.info('Skipping index creation for %s: %s', originated_contract, e) except Exception as e: - print("Error in on_factory_origination: " + cast(str, registry_origination.data.originated_contract_address)) - print(e) \ No newline at end of file + ctx.logger.error(f"Error in on_factory_origination: {e}") + ctx.logger.error(f"Handler received: {type(registry_origination)}") + if hasattr(registry_origination, 'data'): + ctx.logger.error(f"Data attributes: {dir(registry_origination.data)}") + else: + ctx.logger.error(f"Direct attributes: {dir(registry_origination)}") diff --git a/registrydao/handlers/on_flush.py b/registrydao/handlers/on_flush.py index 7c1a055..0015afd 100644 --- a/registrydao/handlers/on_flush.py +++ b/registrydao/handlers/on_flush.py @@ -1,7 +1,7 @@ from registrydao.utils.extra import update_extra from registrydao.utils.ledger import update_ledger -from dipdup.models import Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models @@ -9,22 +9,19 @@ from registrydao.types.registry.parameter.flush import FlushParameter from registrydao.types.registry.storage import RegistryStorage -def extract_key(proposal_key_list_item) -> str: - return proposal_key_list_item['bytes'] - async def on_flush( ctx: HandlerContext, flush: Transaction[FlushParameter, RegistryStorage], ) -> None: try: - non_flushed_or_executed_keys = list(map(extract_key, flush.data.storage['proposals'])) + non_flushed_or_executed_keys = list(flush.storage.proposals.keys()) dao_address = flush.data.target_address dao = await models.DAO.get(address=dao_address) await update_ledger(dao_address, flush.data.diffs) - await update_extra(dao_address, flush.data.storage['extra']['handler_storage']) + await update_extra(dao_address, flush.storage.extra.handler_storage) - dao.guardian = flush.data.storage["guardian"] + dao.guardian = flush.storage.guardian await dao.save() executed_status = await models.ProposalStatus.get(description='executed') @@ -47,4 +44,4 @@ async def on_flush( await models.ProposalStatusUpdates.get_or_create(status=dropped_status, proposal=all_proposals[i], timestamp=flush.data.timestamp, level=flush.data.level) except Exception as e: print("Error in on_flush: " + flush.data.target_address) - print(e) \ No newline at end of file + print(e) diff --git a/registrydao/handlers/on_freeze.py b/registrydao/handlers/on_freeze.py index 66fdb5a..cd1e9a1 100644 --- a/registrydao/handlers/on_freeze.py +++ b/registrydao/handlers/on_freeze.py @@ -1,7 +1,7 @@ from registrydao.utils.ledger import update_ledger from typing import Optional -from dipdup.models import OperationData, Origination, Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models @@ -15,7 +15,8 @@ async def on_freeze( freeze: Transaction[FreezeParameter, RegistryStorage], ) -> None: try: - await update_ledger(freeze.data.target_address, freeze.data.diffs) + dao_address = freeze.data.target_address + await update_ledger(dao_address, freeze.data.diffs) except Exception as e: print("Error in on_freeze: " + str(freeze.data.target_address)) - print(e) \ No newline at end of file + print(e) diff --git a/registrydao/handlers/on_origination.py b/registrydao/handlers/on_origination.py index 1ab27a6..333b12b 100644 --- a/registrydao/handlers/on_origination.py +++ b/registrydao/handlers/on_origination.py @@ -8,7 +8,7 @@ from registrydao.utils.http import fetch from dipdup.context import HandlerContext -from dipdup.models import Origination +from dipdup.models.tezos_tzkt import TzktOrigination as Origination import registrydao.models as models from registrydao.types.registry.storage import RegistryStorage @@ -34,10 +34,12 @@ async def on_origination( registry_origination: Origination[RegistryStorage], ) -> None: try: - network = extract_network_from_ctx(ctx) - token_address = registry_origination.data.storage['governance_token']['address'] - token_id = registry_origination.data.storage['governance_token']['token_id'] dao_address = registry_origination.data.originated_contract_address + ctx.logger.info(f"on_origination called for {dao_address} at level {registry_origination.data.level}") + + network = extract_network_from_ctx(ctx) + token_address = registry_origination.storage.governance_token.address + token_id = registry_origination.storage.governance_token.token_id fetched_token_resp = (await fetch( f'https://api.{NETWORK_MAP[network]}.tzkt.io/v1/tokens?contract={token_address}&tokenId={token_id}')) @@ -76,7 +78,7 @@ async def on_origination( network=network, level=firstLevel, timestamp=datetime.strptime(fetched_token["firstTime"], '%Y-%m-%dT%H:%M:%SZ'), - token_id=token_id, + token_id=int(token_id), symbol=fetched_token_metadata["symbol"], name=fetched_token_metadata["name"], decimals=fetched_token_metadata["decimals"], @@ -86,23 +88,23 @@ async def on_origination( ) dao = await models.DAO.get_or_create( - admin=registry_origination.data.storage['admin'], + admin=registry_origination.storage.admin, address=dao_address, - frozen_token_id=registry_origination.data.storage['frozen_token_id'], - guardian=registry_origination.data.storage['guardian'], - # max_proposals=registry_origination.data.storage['max_proposals'], - max_quorum_change=registry_origination.data.storage['config']['max_quorum_change'], - max_quorum_threshold=registry_origination.data.storage['config']['max_quorum_threshold'], - min_quorum_threshold=registry_origination.data.storage['config']['min_quorum_threshold'], - period=registry_origination.data.storage['config']['period'], - proposal_expired_level=registry_origination.data.storage['config']['proposal_expired_level'], - proposal_flush_level=registry_origination.data.storage['config']['proposal_flush_level'], - quorum_change=registry_origination.data.storage['config']['quorum_change'], - fixed_proposal_fee_in_token=registry_origination.data.storage['config']['fixed_proposal_fee_in_token'], - last_updated_cycle=registry_origination.data.storage['quorum_threshold_at_cycle']['last_updated_cycle'], - quorum_threshold=round((int(registry_origination.data.storage['quorum_threshold_at_cycle']['quorum_threshold']) / 1000000) * int(fetched_token["totalSupply"])), - staked=registry_origination.data.storage['quorum_threshold_at_cycle']['staked'], - start_level=registry_origination.data.storage['start_level'], + frozen_token_id=registry_origination.storage.frozen_token_id, + guardian=registry_origination.storage.guardian, + # max_proposals=registry_origination.storage['max_proposals'], + max_quorum_change=registry_origination.storage.config.max_quorum_change, + max_quorum_threshold=registry_origination.storage.config.max_quorum_threshold, + min_quorum_threshold=registry_origination.storage.config.min_quorum_threshold, + period=registry_origination.storage.config.period, + proposal_expired_level=registry_origination.storage.config.proposal_expired_level, + proposal_flush_level=registry_origination.storage.config.proposal_flush_level, + quorum_change=registry_origination.storage.config.quorum_change, + fixed_proposal_fee_in_token=registry_origination.storage.config.fixed_proposal_fee_in_token, + last_updated_cycle=registry_origination.storage.quorum_threshold_at_cycle.last_updated_cycle, + quorum_threshold=round((int(registry_origination.storage.quorum_threshold_at_cycle.quorum_threshold) / 1000000) * int(fetched_token["totalSupply"])), + staked=registry_origination.storage.quorum_threshold_at_cycle.staked, + start_level=registry_origination.storage.start_level, network=network, name=fetched_metadata['name'], description=fetched_metadata['description'], @@ -111,44 +113,44 @@ async def on_origination( discourse=discourse ) - fetched_extra = registry_origination.data.storage["extra"] + fetched_extra = registry_origination.storage.extra if dao_type == 'registry': await models.RegistryExtra.get_or_create( - registry=fetched_extra['registry'], - registry_affected=fetched_extra['registry_affected'], - frozen_extra_value=fetched_extra['frozen_extra_value'], - frozen_scale_value=fetched_extra['frozen_scale_value'], - slash_division_value=fetched_extra['slash_division_value'], - min_xtz_amount=fetched_extra['min_xtz_amount'], - max_xtz_amount=fetched_extra['max_xtz_amount'], - slash_scale_value=fetched_extra['slash_scale_value'], + registry=fetched_extra.handler_storage.get('registry'), + registry_affected=fetched_extra.handler_storage.get('registry_affected'), + frozen_extra_value=fetched_extra.handler_storage.get('frozen_extra_value'), + frozen_scale_value=fetched_extra.handler_storage.get('frozen_scale_value'), + slash_division_value=fetched_extra.handler_storage.get('slash_division_value'), + min_xtz_amount=fetched_extra.handler_storage.get('min_xtz_amount'), + max_xtz_amount=fetched_extra.handler_storage.get('max_xtz_amount'), + slash_scale_value=fetched_extra.handler_storage.get('slash_scale_value'), dao=dao[0] ) else: if dao_type == 'lambda': await models.LambdaExtra.get_or_create( - registry=fetched_extra["handler_storage"]['registry'], - registry_affected=fetched_extra["handler_storage"]['registry_affected'], - frozen_extra_value=fetched_extra["handler_storage"]['frozen_extra_value'], - frozen_scale_value=fetched_extra["handler_storage"]['frozen_scale_value'], - slash_division_value=fetched_extra["handler_storage"]['slash_division_value'], - min_xtz_amount=fetched_extra["handler_storage"]['min_xtz_amount'], - max_xtz_amount=fetched_extra["handler_storage"]['max_xtz_amount'], - slash_scale_value=fetched_extra["handler_storage"]['slash_scale_value'], - max_proposal_size=fetched_extra["handler_storage"]['max_proposal_size'], + registry=fetched_extra.handler_storage.get('registry'), + registry_affected=fetched_extra.handler_storage.get('registry_affected'), + frozen_extra_value=fetched_extra.handler_storage.get('frozen_extra_value'), + frozen_scale_value=fetched_extra.handler_storage.get('frozen_scale_value'), + slash_division_value=fetched_extra.handler_storage.get('slash_division_value'), + min_xtz_amount=fetched_extra.handler_storage.get('min_xtz_amount'), + max_xtz_amount=fetched_extra.handler_storage.get('max_xtz_amount'), + slash_scale_value=fetched_extra.handler_storage.get('slash_scale_value'), + max_proposal_size=fetched_extra.handler_storage.get('max_proposal_size'), dao=dao[0] ) else: await models.TreasuryExtra.get_or_create( - frozen_extra_value=fetched_extra['frozen_extra_value'], - frozen_scale_value=fetched_extra['frozen_scale_value'], - slash_division_value=fetched_extra['slash_division_value'], - min_xtz_amount=fetched_extra['min_xtz_amount'], - max_xtz_amount=fetched_extra['max_xtz_amount'], - slash_scale_value=fetched_extra['slash_scale_value'], + frozen_extra_value=fetched_extra.handler_storage.get('frozen_extra_value'), + frozen_scale_value=fetched_extra.handler_storage.get('frozen_scale_value'), + slash_division_value=fetched_extra.handler_storage.get('slash_division_value'), + min_xtz_amount=fetched_extra.handler_storage.get('min_xtz_amount'), + max_xtz_amount=fetched_extra.handler_storage.get('max_xtz_amount'), + slash_scale_value=fetched_extra.handler_storage.get('slash_scale_value'), dao=dao[0] ) except Exception as e: - print("Error in Origination Handler: " + str(registry_origination.data.originated_contract_address)) + print(f"Error in Origination Handler: {registry_origination}") print(e) diff --git a/registrydao/handlers/on_propose.py b/registrydao/handlers/on_propose.py index a151c2d..71df74f 100644 --- a/registrydao/handlers/on_propose.py +++ b/registrydao/handlers/on_propose.py @@ -2,7 +2,7 @@ from typing import Optional from datetime import datetime -from dipdup.models import OperationData, Origination, Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models @@ -18,12 +18,7 @@ async def on_propose( try: dao_address = propose.data.target_address print("dao_address: " + dao_address) - diffs = propose.data.diffs or [] - proposals_content = next((d["content"] for d in diffs if d.get("path") == "proposals" and "content" in d), None) - if not proposals_content: - print("on_propose: proposals diff not found; skipping", dao_address) - return - proposal_diff = proposals_content + proposal_diff = propose.data.diffs[0]['content'] await update_ledger(dao_address, propose.data.diffs) diff --git a/registrydao/handlers/on_unfreeze.py b/registrydao/handlers/on_unfreeze.py index 65ba055..7c17f42 100644 --- a/registrydao/handlers/on_unfreeze.py +++ b/registrydao/handlers/on_unfreeze.py @@ -1,7 +1,7 @@ from registrydao.utils.ledger import update_ledger from typing import Optional -from dipdup.models import OperationData, Origination, Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models @@ -18,4 +18,4 @@ async def on_unfreeze( await update_ledger(unfreeze.data.target_address, unfreeze.data.diffs) except Exception as e: print("Error in on_unfreeze: " + str(unfreeze.data.target_address)) - print(e) \ No newline at end of file + print(e) diff --git a/registrydao/handlers/on_unstake_vote.py b/registrydao/handlers/on_unstake_vote.py index c766d9a..b34a808 100644 --- a/registrydao/handlers/on_unstake_vote.py +++ b/registrydao/handlers/on_unstake_vote.py @@ -1,7 +1,7 @@ from registrydao.utils.ledger import update_ledger from typing import Optional from datetime import datetime -from dipdup.models import OperationData, Origination, Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models diff --git a/registrydao/handlers/on_vote.py b/registrydao/handlers/on_vote.py index 0b256bc..091c25e 100644 --- a/registrydao/handlers/on_vote.py +++ b/registrydao/handlers/on_vote.py @@ -1,7 +1,7 @@ from registrydao.utils.ledger import update_ledger from typing import Optional -from dipdup.models import OperationData, Origination, Transaction +from dipdup.models.tezos_tzkt import TzktTransaction as Transaction from dipdup.context import HandlerContext import registrydao.models as models diff --git a/registrydao/hooks/on_rollback.py b/registrydao/hooks/on_rollback.py index 79baf70..a104587 100644 --- a/registrydao/hooks/on_rollback.py +++ b/registrydao/hooks/on_rollback.py @@ -1,18 +1,14 @@ -from cmath import exp from dipdup.context import HookContext -from dipdup.datasources.datasource import Datasource -from dipdup.enums import ReindexingReason async def on_rollback( ctx: HookContext, - datasource: Datasource, from_level: int, to_level: int, ) -> None: try: await ctx.execute_sql('on_rollback') - await ctx.reindex(ReindexingReason.ROLLBACK) + await ctx.reindex('rollback') except Exception as e: print("Error in on_rollback") - print(e) \ No newline at end of file + print(e) diff --git a/registrydao/hooks/on_startup.py b/registrydao/hooks/on_startup.py new file mode 100644 index 0000000..e71a295 --- /dev/null +++ b/registrydao/hooks/on_startup.py @@ -0,0 +1,306 @@ +from dipdup.context import HookContext +from registrydao.utils.http import fetch +from registrydao.constants import NETWORK_MAP +from typing import cast + + +def get_factory_address_from_config(ctx: HookContext, network: str) -> str: + """Get factory contract address from config for a given network""" + contract_name = f'registry_{network}' + if contract_name in ctx.config.contracts: + contract_config = ctx.config.contracts[contract_name] + if hasattr(contract_config, 'address'): + return contract_config.address + raise ValueError(f"Factory contract not found in config for network: {network}") + + +async def get_registry_code_hash(ctx: HookContext, network: str) -> str: + """Get the code hash of the registry contract""" + registry_address = get_factory_address_from_config(ctx, network) + + contract_info = await fetch( + f"https://api.{NETWORK_MAP[network]}.tzkt.io/v1/contracts/{registry_address}" + ) + return contract_info['codeHash'] + + +async def get_historical_daos(ctx: HookContext, network: str, code_hash: str, first_level: int) -> list: + """Query TzKT for all originations with the same code hash""" + all_daos = [] + offset = 0 + limit = 1000 + + ctx.logger.info(f"Fetching historical DAOs for {network} from level {first_level}...") + + while True: + url = ( + f"https://api.{NETWORK_MAP[network]}.tzkt.io/v1/operations/originations" + f"?codeHash.eq={code_hash}" + f"&level.ge={first_level}" + f"&limit={limit}" + f"&offset={offset}" + f"&status=applied" + f"&sort=level" + ) + + try: + originations = await fetch(url) + if not originations or len(originations) == 0: + break + + all_daos.extend(originations) + ctx.logger.info(f" Fetched {len(originations)} originations (total: {len(all_daos)})") + offset += limit + + if len(originations) < limit: + break + + # Small delay to avoid rate limiting + import asyncio + await asyncio.sleep(0.5) + except Exception as e: + ctx.logger.error(f"Error fetching originations: {e}") + break + + return all_daos + + +async def verify_dao_entrypoints(network: str, dao_address: str) -> bool: + """Verify that a DAO has the required entrypoints""" + try: + eps = await fetch( + f"https://api.{NETWORK_MAP[network]}.tzkt.io/v1/contracts/{dao_address}/entrypoints" + ) + if isinstance(eps, list): + ep_set = set(ep.get('name', ep) if isinstance(ep, dict) else ep for ep in eps) + else: + ep_set = set() + required = {"propose", "vote", "flush", "freeze", "unfreeze", "drop_proposal"} + return required.issubset(ep_set) + except Exception: + return False + + +def extract_network_configs(ctx: HookContext): + """Extract factory contract addresses, first levels, and datasources from dipdup.yml config""" + networks_config = {} + + for contract_name, contract_config in ctx.config.contracts.items(): + if contract_name.startswith('registry_'): + network = contract_name.replace('registry_', '') + if hasattr(contract_config, 'address'): + networks_config[network] = { + 'factory_address': contract_config.address, + 'contract_name': contract_name + } + + for index_name, index_config in ctx.config.indexes.items(): + if index_name.startswith('factory_'): + network = index_name.replace('factory_', '') + if network in networks_config: + if hasattr(index_config, 'first_level'): + networks_config[network]['first_level'] = index_config.first_level + if hasattr(index_config, 'datasource'): + datasource = index_config.datasource + if hasattr(datasource, 'name'): + networks_config[network]['datasource'] = datasource.name + elif isinstance(datasource, str): + networks_config[network]['datasource'] = datasource + else: + networks_config[network]['datasource'] = f'tzkt_{network}' + + return networks_config + + +async def on_startup(ctx: HookContext) -> None: + """Startup hook to discover and index historical DAOs""" + try: + print("=" * 60) + print("ON_STARTUP HOOK CALLED") + print("=" * 60) + ctx.logger.info("=" * 60) + ctx.logger.info("ON_STARTUP HOOK CALLED - Starting historical DAO discovery...") + ctx.logger.info("=" * 60) + + networks_config = extract_network_configs(ctx) + + for network, config in networks_config.items(): + if 'first_level' in config and 'datasource' in config: + await process_network( + ctx, + network, + config['first_level'], + config['factory_address'], + config['datasource'] + ) + + ctx.logger.info("Historical DAO discovery completed") + except Exception as e: + ctx.logger.error(f"Error in on_startup: {e}") + import traceback + ctx.logger.error(traceback.format_exc()) + + +async def process_network( + ctx: HookContext, + network: str, + first_level: int, + registry_address: str, + datasource_name: str +) -> None: + """Process historical DAOs for a specific network""" + try: + ctx.logger.info(f"Processing {network} network...") + + # Get registry code hash + code_hash = await get_registry_code_hash(ctx, network) + ctx.logger.info(f"Registry code hash: {code_hash}") + + # Get all historical DAOs + historical_daos = await get_historical_daos(ctx, network, code_hash, first_level) + ctx.logger.info(f"Found {len(historical_daos)} total originations") + + # Filter and create indexes for valid DAOs + valid_count = 0 + skipped_count = 0 + + # Factory contract address - skip this as it's already in config + factory_address = registry_address + + for dao_data in historical_daos: + dao_address = dao_data['originatedContract']['address'] + origination_level = dao_data['level'] + index_name = f'registry_dao_{dao_address}' + + # Skip factory contracts (already indexed) + if dao_address == factory_address: + ctx.logger.info(f" ⊙ Skipping factory contract {dao_address} (network: {network}, factory: {factory_address})") + skipped_count += 1 + continue + + ctx.logger.debug(f" Processing DAO {dao_address} (network: {network}, factory: {factory_address})") + + # Verify entrypoints + if not await verify_dao_entrypoints(network, dao_address): + ctx.logger.info(f" ✗ {dao_address} - missing required entrypoints") + skipped_count += 1 + continue + + # Check if index already exists + exists = False + try: + exists = index_name in ctx.config.indexes + except Exception: + pass + + if not exists: + try: + from dipdup.database import get_connection + conn = await get_connection() + try: + result = await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM dipdup_index WHERE name = $1)", + index_name + ) + exists = result if result else False + finally: + await conn.close() + except Exception: + exists = False + + if not exists: + try: + # Find the correct contract name to use + # 1. Check if contract is in config (use config name) + contract_name = None + for contract_alias, contract_config in ctx.config.contracts.items(): + if hasattr(contract_config, 'address') and contract_config.address == dao_address: + contract_name = contract_alias + ctx.logger.info(f" Found contract {contract_name} in config for {dao_address}") + break + + # 2. If not in config, check database + if not contract_name: + try: + from dipdup.database import get_connection + conn = await get_connection() + try: + result = await conn.fetchrow( + "SELECT name FROM dipdup_contract WHERE address = $1", + dao_address + ) + if result: + contract_name = result['name'] + ctx.logger.info(f" Found contract {contract_name} in database for {dao_address}") + finally: + await conn.close() + except Exception: + pass + + # 3. If still not found, add contract first, then use its name + if not contract_name: + contract_name = dao_address + try: + await ctx.add_contract( + name=contract_name, + address=dao_address, + typename='registry', + kind='tezos', + ) + ctx.logger.debug(f" Added contract {contract_name} for {dao_address}") + except Exception as contract_error: + # If contract already exists, try to find its name + error_str = str(contract_error) + if 'already in use' in error_str: + # Contract exists but we don't know the name - this shouldn't happen + # but let's try to find it + try: + from dipdup.database import get_connection + conn = await get_connection() + try: + result = await conn.fetchrow( + "SELECT name FROM dipdup_contract WHERE address = $1", + dao_address + ) + if result: + contract_name = result['name'] + else: + # Contract address conflict but not in DB/config + # This is likely a race condition or internal DipDup state + ctx.logger.warning(f" Contract {dao_address} address conflict but not found in DB/config") + raise contract_error + finally: + await conn.close() + except Exception: + raise contract_error + else: + raise + + # Add index with the contract name + await ctx.add_index( + name=index_name, + template='registry_dao', + values=dict(contract=contract_name, datasource=datasource_name), + first_level=origination_level, + ) + + ctx.logger.info(f" ✓ Created index for {dao_address} at level {origination_level}") + valid_count += 1 + except Exception as e: + ctx.logger.warning(f" ✗ Failed to create index for {dao_address}: {e}") + import traceback + ctx.logger.debug(traceback.format_exc()) + skipped_count += 1 + else: + ctx.logger.debug(f" ⊙ Index already exists for {dao_address}") + + ctx.logger.info(f"{network}: Created {valid_count} indexes, skipped {skipped_count}") + + except Exception as e: + ctx.logger.error(f"Error processing {network}: {e}") + import traceback + ctx.logger.error(traceback.format_exc()) + + + + diff --git a/registrydao/hooks/on_synchronized.py b/registrydao/hooks/on_synchronized.py index 2b4f0a4..2c45acb 100644 --- a/registrydao/hooks/on_synchronized.py +++ b/registrydao/hooks/on_synchronized.py @@ -1,4 +1,5 @@ from dipdup.context import HookContext +from registrydao.hooks.on_startup import process_network, extract_network_configs async def on_synchronized( @@ -6,6 +7,49 @@ async def on_synchronized( ) -> None: try: await ctx.execute_sql('on_synchronized') + + ctx.logger.info("on_synchronized hook called - checking for historical DAOs") + + dao_index_count = 0 + try: + for index_name in ctx.config.indexes.keys(): + if index_name.startswith('registry_dao_'): + dao_index_count += 1 + except Exception: + pass + + networks_config = extract_network_configs(ctx) + factory_addresses = {net: config['factory_address'] for net, config in networks_config.items()} + + actual_dao_count = 0 + try: + import registrydao.models as models + if factory_addresses: + addresses_list = list(factory_addresses.values()) + total_count = await models.DAO.all().count() + factory_count = await models.DAO.filter(address__in=addresses_list).count() + actual_dao_count = total_count - factory_count + except Exception as e: + ctx.logger.warning(f"Could not check actual DAO count: {e}") + actual_dao_count = 0 + + ctx.logger.info(f"Found {dao_index_count} DAO indexes in config, {actual_dao_count} actual DAOs in database") + + if dao_index_count == 0 or actual_dao_count < 10: + ctx.logger.info("Insufficient DAOs found - discovering historical DAOs...") + + for network, config in networks_config.items(): + if 'first_level' in config and 'datasource' in config: + await process_network( + ctx, + network, + config['first_level'], + config['factory_address'], + config['datasource'] + ) + else: + ctx.logger.info(f"Found {dao_index_count} DAO indexes and {actual_dao_count} DAOs - skipping historical discovery") except Exception as e: - print("Error in on_synchronized") - print(e) \ No newline at end of file + ctx.logger.error(f"Error in on_synchronized: {e}") + import traceback + ctx.logger.error(traceback.format_exc()) diff --git a/registrydao/types/registry/tezos_parameters/__init__.py b/registrydao/types/registry/tezos_parameters/__init__.py new file mode 100644 index 0000000..dc65f4c --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/__init__.py @@ -0,0 +1,24 @@ +# Re-export parameters for DipDup v7 +from registrydao.types.registry.parameter.propose import ProposeParameter +from registrydao.types.registry.parameter.vote import VoteParameter, VoteParameterItem, Argument, PermitItem +from registrydao.types.registry.parameter.drop_proposal import DropProposalParameter +from registrydao.types.registry.parameter.flush import FlushParameter +from registrydao.types.registry.parameter.freeze import FreezeParameter +from registrydao.types.registry.parameter.unfreeze import UnfreezeParameter +from registrydao.types.registry.parameter.unstake_vote import UnstakeVoteParameter +from registrydao.types.registry.parameter.call_custom import CallCustomParameter + +__all__ = [ + 'ProposeParameter', + 'VoteParameter', + 'VoteParameterItem', + 'Argument', + 'PermitItem', + 'DropProposalParameter', + 'FlushParameter', + 'FreezeParameter', + 'UnfreezeParameter', + 'UnstakeVoteParameter', + 'CallCustomParameter' +] + diff --git a/registrydao/types/registry/tezos_parameters/call_custom.py b/registrydao/types/registry/tezos_parameters/call_custom.py new file mode 100644 index 0000000..980f312 --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/call_custom.py @@ -0,0 +1,5 @@ +# Re-export call_custom parameter for DipDup v7 +from registrydao.types.registry.parameter.call_custom import CallCustomParameter + +__all__ = ['CallCustomParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/drop_proposal.py b/registrydao/types/registry/tezos_parameters/drop_proposal.py new file mode 100644 index 0000000..4fb8a2d --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/drop_proposal.py @@ -0,0 +1,5 @@ +# Re-export drop_proposal parameter for DipDup v7 +from registrydao.types.registry.parameter.drop_proposal import DropProposalParameter + +__all__ = ['DropProposalParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/flush.py b/registrydao/types/registry/tezos_parameters/flush.py new file mode 100644 index 0000000..c86a92a --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/flush.py @@ -0,0 +1,5 @@ +# Re-export flush parameter for DipDup v7 +from registrydao.types.registry.parameter.flush import FlushParameter + +__all__ = ['FlushParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/freeze.py b/registrydao/types/registry/tezos_parameters/freeze.py new file mode 100644 index 0000000..0ee0e2d --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/freeze.py @@ -0,0 +1,5 @@ +# Re-export freeze parameter for DipDup v7 +from registrydao.types.registry.parameter.freeze import FreezeParameter + +__all__ = ['FreezeParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/propose.py b/registrydao/types/registry/tezos_parameters/propose.py new file mode 100644 index 0000000..3c64742 --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/propose.py @@ -0,0 +1,5 @@ +# Re-export propose parameter for DipDup v7 +from registrydao.types.registry.parameter.propose import ProposeParameter + +__all__ = ['ProposeParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/unfreeze.py b/registrydao/types/registry/tezos_parameters/unfreeze.py new file mode 100644 index 0000000..ae2a18b --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/unfreeze.py @@ -0,0 +1,5 @@ +# Re-export unfreeze parameter for DipDup v7 +from registrydao.types.registry.parameter.unfreeze import UnfreezeParameter + +__all__ = ['UnfreezeParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/unstake_vote.py b/registrydao/types/registry/tezos_parameters/unstake_vote.py new file mode 100644 index 0000000..f209f8f --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/unstake_vote.py @@ -0,0 +1,5 @@ +# Re-export unstake_vote parameter for DipDup v7 +from registrydao.types.registry.parameter.unstake_vote import UnstakeVoteParameter + +__all__ = ['UnstakeVoteParameter'] + diff --git a/registrydao/types/registry/tezos_parameters/vote.py b/registrydao/types/registry/tezos_parameters/vote.py new file mode 100644 index 0000000..f7f597c --- /dev/null +++ b/registrydao/types/registry/tezos_parameters/vote.py @@ -0,0 +1,5 @@ +# Re-export vote parameter for DipDup v7 +from registrydao.types.registry.parameter.vote import VoteParameter, VoteParameterItem, Argument, PermitItem + +__all__ = ['VoteParameter', 'VoteParameterItem', 'Argument', 'PermitItem'] + diff --git a/registrydao/types/registry/tezos_storage.py b/registrydao/types/registry/tezos_storage.py new file mode 100644 index 0000000..3e64852 --- /dev/null +++ b/registrydao/types/registry/tezos_storage.py @@ -0,0 +1,40 @@ +# Re-export storage as tezos_storage for DipDup v7 compatibility +from registrydao.types.registry.storage import ( + Config, + Key, + Delegate, + Lambdas, + ExtraModel, + FreezeHistory, + GovernanceToken, + Key1, + MapItem, + OngoingProposalsDlistItem, + Proposals, + QuorumThresholdAtCycle, + Key2, + StakedVote, + RegistryStorage +) + +__all__ = [ + 'Config', + 'Key', + 'Delegate', + 'Lambdas', + 'ExtraModel', + 'FreezeHistory', + 'GovernanceToken', + 'Key1', + 'MapItem', + 'OngoingProposalsDlistItem', + 'Proposals', + 'QuorumThresholdAtCycle', + 'Key2', + 'StakedVote', + 'RegistryStorage' +] + + + + diff --git a/requirements.txt b/requirements.txt index 9b2785f..e69de29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,72 +0,0 @@ -aiohttp==3.8.1 -aiolimiter==1.0.0 -aiosignal==1.2.0 -aiosqlite==0.17.0 -anyio==3.5.0 -appdirs==1.4.4 -APScheduler==3.9.1 -argcomplete==2.0.0 -async-timeout==4.0.2 -asyncclick==8.0.3.2 -asyncpg==0.26.0 -attrs==21.4.0 -black==22.3.0 -certifi==2021.10.8 -chardet==4.0.0 -charset-normalizer==2.0.12 -click==8.1.2 -datamodel-code-generator==0.13.1 -dipdup==6.1.2 -dnspython==2.2.1 -email-validator==1.2.0 -fcache==0.4.7 -frozenlist==1.3.0 -genson==1.2.2 -idna==3.3 -inflect==5.5.2 -iso8601==1.0.2 -isodate==0.6.1 -isort==5.10.1 -Jinja2==3.1.1 -jsonschema==3.2.0 -MarkupSafe==2.1.1 -msgpack==1.0.3 -multidict==6.0.2 -mypy-extensions==0.4.3 -openapi-schema-validator==0.1.6 -openapi-spec-validator==0.3.3 -orjson==3.6.8 -pathspec==0.9.0 -platformdirs==2.5.2 -prance==0.21.8.0 -prometheus-client==0.14.1 -pydantic==1.9.2 -pyhumps==3.5.3 -pypika-tortoise==0.1.6 -pyrsistent==0.16.1 -pysignalr==0.1.2 -PySnooper==1.1.1 -python-dotenv==0.19.2 -pytz==2022.1 -pytz-deprecation-shim==0.1.0.post0 -PyYAML==6.0 -requests==2.27.1 -ruamel.yaml==0.17.21 -ruamel.yaml.clib==0.2.6 -semver==2.13.0 -sentry-sdk==1.5.10 -six==1.16.0 -sniffio==1.2.0 -sqlparse==0.4.2 -tabulate==0.8.9 -toml==0.10.2 -tomli==2.0.1 -tortoise-orm==0.19.2 -typed-ast==1.5.3 -typing-inspect==0.6.0 -typing_extensions==4.2.0 -tzdata==2022.1 -tzlocal==4.2 -urllib3==1.26.9 -websockets==10.3 -yarl==1.7.2