diff --git a/.env.IntegrationTest b/.env.IntegrationTest index 8cc98e009..3155fe607 100644 --- a/.env.IntegrationTest +++ b/.env.IntegrationTest @@ -4,8 +4,7 @@ NETWORK=devkit PROTOCOL_MAGIC=42 ## Postgres image -DB_IMAGE_NAME=postgres -DB_IMAGE_TAG=latest +PG_VERSION_TAG=REL_14_11 ## Yaci image YACI_VERSION=0.10.5 @@ -41,6 +40,8 @@ SNAPSHOT_DIGEST=latest AGGREGATOR_ENDPOINT= # if not set standard values will be used GENESIS_VERIFICATION_KEY= +ANCILLARY_VERIFICATION_KEY= + ## Api env API_DOCKER_IMAGE_TAG=main @@ -59,11 +60,9 @@ SEARCH_PAGE_SIZE=100 ## Yaci Indexer env INDEXER_DOCKER_IMAGE_TAG=main -PRUNING_ENABLED=false +REMOVE_SPENT_UTXOS=false #The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. -PRUNING_SAFE_BLOCKS=2160 -#Run transaction pruning every 600 seconds or 10 minutes. -PRUNING_INTERVAL=600 +REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160 YACI_SPRING_PROFILES=postgres,n2c-socat # database profiles: h2, h2-testdata, postgres @@ -138,4 +137,6 @@ DB_POSTGRES_MAX_PARALLEL_WORKERS=4 DB_POSTGRES_SEQ_PAGE_COST=1.0 DB_POSTGRES_JIT=off DB_POSTGRES_BGWRITER_LRU_MAXPAGES=50 -DB_POSTGRES_BGWRITER_DELAY=500ms \ No newline at end of file +DB_POSTGRES_BGWRITER_DELAY=500ms + +SYNC_GRACE_SLOTS_COUNT=100 \ No newline at end of file diff --git a/.env.docker-compose b/.env.docker-compose index 505d01088..6be1ce132 100644 --- a/.env.docker-compose +++ b/.env.docker-compose @@ -2,13 +2,12 @@ API_SPRING_PROFILES_ACTIVE=online LOG=INFO NETWORK=mainnet -# mainnet, preprod, preview, sanchonet, devkit +# mainnet, preprod, preview, devkit PROTOCOL_MAGIC=764824073 -# mainnet 764824073, preprod 1, preview 2, sanchonet 4, devkit 42 +# mainnet 764824073, preprod 1, preview 2, devkit 42 ## Postgres image -DB_IMAGE_NAME=postgres -DB_IMAGE_TAG=14.11-bullseye +PG_VERSION_TAG=REL_14_11 ## Postgres variables DB_NAME=rosetta-java @@ -35,11 +34,13 @@ CARDANO_CONFIG=./config/${NETWORK} ## Mithril MITHRIL_SYNC=true +MITHRIL_VERSION=2517.1 SNAPSHOT_DIGEST=latest # if not set standard values will be used AGGREGATOR_ENDPOINT= # if not set standard values will be used GENESIS_VERIFICATION_KEY= +ANCILLARY_VERIFICATION_KEY= ## Api env API_DOCKER_IMAGE_TAG=main @@ -49,6 +50,7 @@ PRINT_EXCEPTION=true ROSETTA_VERSION=1.4.13 TOPOLOGY_FILEPATH=/config/topology.json + GENESIS_SHELLEY_PATH=/config/shelley-genesis.json GENESIS_BYRON_PATH=/config/byron-genesis.json GENESIS_ALONZO_PATH=/config/alonzo-genesis.json @@ -57,11 +59,9 @@ SEARCH_PAGE_SIZE=100 ## Yaci Indexer env INDEXER_DOCKER_IMAGE_TAG=main -PRUNING_ENABLED=false +REMOVE_SPENT_UTXOS=false #The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. -PRUNING_SAFE_BLOCKS=2160 -#Run transaction pruning every 600 seconds or 10 minutes. -PRUNING_INTERVAL=600 +REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160 YACI_SPRING_PROFILES=postgres,n2c-socket # database profiles: h2, h2-testdata, postgres @@ -92,3 +92,5 @@ API_DB_POOL_MAX_LIFETIME_MS=2000000 API_DB_POOL_CONNECTION_TIMEOUT_MS=100000 API_DB_KEEP_ALIVE_MS=60000 API_DB_LEAK_CONNECTIONS_WARNING_MS=60000 + +SYNC_GRACE_SLOTS_COUNT=100 \ No newline at end of file diff --git a/.env.docker-compose-preprod b/.env.docker-compose-preprod new file mode 100644 index 000000000..d983621d6 --- /dev/null +++ b/.env.docker-compose-preprod @@ -0,0 +1,96 @@ +## Main variables +API_SPRING_PROFILES_ACTIVE=online +LOG=INFO +NETWORK=preprod +# mainnet, preprod, preview, devkit +PROTOCOL_MAGIC=1 +# mainnet 764824073, preprod 1, preview 2, devkit 42 + +## Postgres image +PG_VERSION_TAG=REL_14_11 + +## Postgres variables +DB_NAME=rosetta-java +DB_USER=rosetta_db_admin +DB_SECRET=weakpwd#123_d +DB_HOST=db +# Service name in docker-compose or local db +DB_PORT=5432 +DB_SCHEMA=${NETWORK} +DB_PATH=/opt/rosetta-java-preprod/sql_data + +## Cardano Node variables +CARDANO_NODE_HOST=cardano-node +# Service name in docker-compose or local cardano node +CARDANO_NODE_PORT=3001 +# Uncomment if you are using local cardano node +CARDANO_NODE_VERSION=10.3.1 +CARDANO_NODE_SUBMIT_HOST=cardano-submit-api +NODE_SUBMIT_API_PORT=8090 +CARDANO_NODE_DIR=/opt/rosetta-java-preprod/node_data +CARDANO_NODE_SOCKET_PATH=${CARDANO_NODE_DIR}/node.socket +CARDANO_NODE_DB=${CARDANO_NODE_DIR}/db +CARDANO_CONFIG=./config/${NETWORK} + +## Mithril +MITHRIL_SYNC=true +MITHRIL_VERSION=2517.1 +SNAPSHOT_DIGEST=latest +# if not set standard values will be used +AGGREGATOR_ENDPOINT= +# if not set standard values will be used +GENESIS_VERIFICATION_KEY= +ANCILLARY_VERIFICATION_KEY= + +## Api env +API_DOCKER_IMAGE_TAG=main +# staging, h2, test. Additional profiles: mempool (if mempool should be activated) +API_PORT=8082 +PRINT_EXCEPTION=true + +ROSETTA_VERSION=1.4.13 +TOPOLOGY_FILEPATH=/config/topology.json + +GENESIS_SHELLEY_PATH=/config/shelley-genesis.json +GENESIS_BYRON_PATH=/config/byron-genesis.json +GENESIS_ALONZO_PATH=/config/alonzo-genesis.json +GENESIS_CONWAY_PATH=/config/conway-genesis.json +SEARCH_PAGE_SIZE=100 + +## Yaci Indexer env +INDEXER_DOCKER_IMAGE_TAG=main +REMOVE_SPENT_UTXOS=false +#The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. +REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160 + +YACI_SPRING_PROFILES=postgres,n2c-socket +# database profiles: h2, h2-testdata, postgres +MEMPOOL_ENABLED=false + +## Devkit env +DEVKIT_ENABLED=false +DEVKIT_URL=yaci-cli +DEVKIT_PORT=3333 + +## Logger Config +LOG_FILE_PATH=/var/log/rosetta-java +LOG_FILE_NAME=/var/log/rosetta-java/rosetta-java.log +LOG_FILE_MAX_SIZE=10MB +LOG_FILE_MAX_HISTORY=10 + +YACI_HTTP_BASE_URL=http://yaci-indexer:9095/api/v1 +YACI_INDEXER_PORT=9095 +HTTP_CONNECT_TIMEOUT_SECONDS=5 +HTTP_REQUEST_TIMEOUT_SECONDS=5 + +## DB tuning / debugging +API_DB_SHOW_SQL=false +API_DB_MONITOR_PERFORMANCE=false #only needed for debugging and diagnostics + +## Basic db pool tuning, generally should not be changed but can be changed rarely if needed +API_DB_POOL_MAX_LIFETIME_MS=2000000 +API_DB_POOL_CONNECTION_TIMEOUT_MS=100000 +API_DB_KEEP_ALIVE_MS=60000 +API_DB_LEAK_CONNECTIONS_WARNING_MS=60000 + +SYNC_GRACE_SLOTS_COUNT=100 \ No newline at end of file diff --git a/.env.h2 b/.env.h2 index a3e48896b..1b85dec1a 100644 --- a/.env.h2 +++ b/.env.h2 @@ -1,9 +1,9 @@ ## Main variables LOG=INFO NETWORK=devkit -# mainnet, preprod, preview, sanchonet, devkit +# mainnet, preprod, preview, devkit PROTOCOL_MAGIC=42 -# mainnet 764824073, preprod 1, preview 2, sanchonet 4, devkit 42 +# mainnet 764824073, preprod 1, preview 2, devkit 42 ## H2 image DB_IMAGE_NAME=h2 @@ -19,7 +19,7 @@ CARDANO_NODE_HOST=localhost # Service name in docker-compose or local cardano node CARDANO_NODE_PORT=3001 # Uncomment if you are using local cardano node -CARDANO_NODE_VERSION=10.2.1 +CARDANO_NODE_VERSION=10.3.1 CARDANO_NODE_SUBMIT_HOST=localhost NODE_SUBMIT_API_PORT=8090 @@ -40,11 +40,8 @@ SEARCH_PAGE_SIZE=100 ## Yaci Indexer env INDEXER_DOCKER_IMAGE_TAG=main -PRUNING_ENABLED=false -#The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. -PRUNING_SAFE_BLOCKS=2160 -#Run transaction pruning every 600 seconds or 10 minutes. -PRUNING_INTERVAL=600 +REMOVE_SPENT_UTXOS=false +REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160 YACI_SPRING_PROFILES=h2,n2c-socket # database profiles: h2, h2-testdata, postgres MEMPOOL_ENABLED=false @@ -94,4 +91,6 @@ DB_POSTGRES_BGWRITER_LRU_MAXPAGES=50 DB_POSTGRES_BGWRITER_DELAY=500ms # Path to local cardano node socket (Yaci DevKit) -CARDANO_NODE_SOCKET_PATH=${HOME}/.yaci-cli/local-clusters/default/node/node.sock \ No newline at end of file +CARDANO_NODE_SOCKET_PATH=${HOME}/.yaci-cli/local-clusters/default/node/node.sock + +SYNC_GRACE_SLOTS_COUNT=100 \ No newline at end of file diff --git a/.env.h2-testdata b/.env.h2-testdata index 9b7cd71c9..dfe21e652 100644 --- a/.env.h2-testdata +++ b/.env.h2-testdata @@ -1,9 +1,9 @@ ## Main variables LOG=INFO NETWORK=devkit -# mainnet, preprod, preview, sanchonet, devkit +# mainnet, preprod, preview, devkit PROTOCOL_MAGIC=42 -# mainnet 764824073, preprod 1, preview 2, sanchonet 4, devkit 42 +# mainnet 764824073, preprod 1, preview 2, devkit 42 ## H2 image DB_IMAGE_NAME=h2 @@ -19,7 +19,7 @@ CARDANO_NODE_HOST=localhost # Service name in docker-compose or local cardano node CARDANO_NODE_PORT=3001 # Uncomment if you are using local cardano node -CARDANO_NODE_VERSION=10.2.1 +CARDANO_NODE_VERSION=10.3.1 CARDANO_NODE_SUBMIT_HOST=localhost NODE_SUBMIT_API_PORT=8090 @@ -40,11 +40,10 @@ SEARCH_PAGE_SIZE=100 ## Yaci Indexer env INDEXER_DOCKER_IMAGE_TAG=main -PRUNING_ENABLED=false +REMOVE_SPENT_UTXOS=false #The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. -PRUNING_SAFE_BLOCKS=2160 -#Run transaction pruning every 600 seconds or 10 minutes. -PRUNING_INTERVAL=600 +REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160 + YACI_SPRING_PROFILES=h2-testdata,n2c-socket # database profiles: h2, h2-testdata, postgres MEMPOOL_ENABLED=false @@ -94,4 +93,6 @@ DB_POSTGRES_BGWRITER_LRU_MAXPAGES=50 DB_POSTGRES_BGWRITER_DELAY=500ms # Path to local cardano node socket (Yaci DevKit) -CARDANO_NODE_SOCKET_PATH=${HOME}/.yaci-cli/local-clusters/default/node/node.sock \ No newline at end of file +CARDANO_NODE_SOCKET_PATH=${HOME}/.yaci-cli/local-clusters/default/node/node.sock + +SYNC_GRACE_SLOTS_COUNT=100 \ No newline at end of file diff --git a/.github/actions/build_docker_images/action.yml b/.github/actions/build_docker_images/action.yml index d2f51aa87..9bc8ba928 100644 --- a/.github/actions/build_docker_images/action.yml +++ b/.github/actions/build_docker_images/action.yml @@ -49,6 +49,51 @@ runs: file: ./yaci-indexer/Dockerfile tags: cardanofoundation/cardano-rosetta-java-indexer:latest push: true + - name: Cardano Node - Build and push Docker ${{ inputs.tag }} image + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/node/Dockerfile + tags: cardanofoundation/cardano-rosetta-java-cardano-node:${{ inputs.tag }} + push: true + - name: Cardano Node - Build and push Docker latest image + uses: docker/build-push-action@v4 + if: ${{ inputs.isRelease == 'true' }} + with: + context: . + file: ./docker/node/Dockerfile + tags: cardanofoundation/cardano-rosetta-java-cardano-node:latest + push: true + - name: Postgres - Build and push Docker ${{ inputs.tag }} image + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/postgres/Dockerfile + tags: cardanofoundation/cardano-rosetta-java-postgres:${{ inputs.tag }} + push: true + - name: Postgres - Build and push Docker latest image + uses: docker/build-push-action@v4 + if: ${{ inputs.isRelease == 'true' }} + with: + context: . + file: ./docker/postgres/Dockerfile + tags: cardanofoundation/cardano-rosetta-java-postgres:latest + push: true + - name: Mithril - Build and push Docker ${{ inputs.tag }} image + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/mithril/Dockerfile + tags: cardanofoundation/cardano-rosetta-java-mithril:${{ inputs.tag }} + push: true + - name: Mithril - Build and push Docker latest image + uses: docker/build-push-action@v4 + if: ${{ inputs.isRelease == 'true' }} + with: + context: . + file: ./docker/mithril/Dockerfile + tags: cardanofoundation/cardano-rosetta-java-mithril:latest + push: true - name: All-in-one - Build and push Docker image uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/feature-mvn-build.yaml b/.github/workflows/feature-mvn-build.yaml index 0a0a3e6b7..d70870a73 100644 --- a/.github/workflows/feature-mvn-build.yaml +++ b/.github/workflows/feature-mvn-build.yaml @@ -10,11 +10,18 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Set up JDK 21 - uses: actions/setup-java@v3 +# - name: Set up JDK 21 +# uses: actions/setup-java@v3 +# with: +# java-version: '21' +# distribution: 'temurin' +# cache: maven + + - name: Set up Amazon Corretto + uses: actions/setup-java@v4 with: - java-version: '21' - distribution: 'temurin' + distribution: 'corretto' + java-version: 24 cache: maven - name: Build project diff --git a/.github/workflows/sonar-cloud-report.yml b/.github/workflows/sonar-cloud-report.yml index 81a4e7012..3608c694a 100644 --- a/.github/workflows/sonar-cloud-report.yml +++ b/.github/workflows/sonar-cloud-report.yml @@ -13,12 +13,20 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Set up JDK 21 - uses: actions/setup-java@v3 +# - name: Set up JDK 21 +# uses: actions/setup-java@v3 +# with: +# java-version: '21' +# distribution: 'temurin' +# cache: maven + + - name: Set up Amazon Corretto + uses: actions/setup-java@v4 with: - java-version: '21' - distribution: 'temurin' + distribution: 'corretto' + java-version: 24 cache: maven + - name: Build env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.gitignore b/.gitignore index d0b5f0290..44368b664 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ settings.xml !.env.h2 !.env.h2-testdata !.env.docker-compose +!.env.docker-compose-preprod !.env.docker-compose-profile-entry-level !.env.docker-compose-profile-mid-level diff --git a/README.md b/README.md index a545036e7..aefe7e870 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![License](https://img.shields.io:/github/license/cardano-foundation/cardano-rosetta-java?label=license)](https://github.com/cardano-foundation/cardano-rosetta-java/blob/master/LICENSE) ![Discord](https://img.shields.io/discord/1022471509173882950) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=cardano-foundation_cardano-rosetta-java&metric=coverage)](https://sonarcloud.io/summary/overall?id=cardano-foundation_cardano-rosetta-java) -[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B45571%2Fgithub.com%2Fcardano-foundation%2Fcardano-rosetta-java.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B45571%2Fgithub.com%2Fcardano-foundation%2Fcardano-rosetta-java?ref=badge_shield&issueType=license) ## What the project is about? @@ -30,8 +29,8 @@ Since [Yaci-Store](https://github.com/bloxbean/yaci-store) is a comparatively li - 4CPU Cores - 32GB RAM -- 1TB of storage (PRUNING_ENABLED=false) [default] -- 400GB of storage (PRUNING_ENABLED=true) +- ca. > 1TB of storage (REMOVE_SPENT_UTXOS=false) [default] +- ca > 400GB of storage (REMOVE_SPENT_UTXOS=true) Better hardware will improve the performance of the indexer and the node, which will result in faster syncing times. @@ -100,7 +99,7 @@ For every Release we provide pre-built docker images stored in the DockerHub Rep To start it use the following command: ```bash - docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.2.8 + docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.2.9 ``` Changes to the configuration can be made by adjusting the `docker/.env.dockerfile` file. For more information on the environment variables, please refer to the [documentation](https://cardano-foundation.github.io/cardano-rosetta-java/docs/install-and-deploy/env-vars). @@ -108,7 +107,7 @@ Changes to the configuration can be made by adjusting the `docker/.env.dockerfil If you want to use the `cardano-submit-api` you can additionally expose port `8090`. It can then be used to submit raw cbor transaction (API documentation here: [Link](https://input-output-hk.github.io/cardano-rest/submit-api/)) ```bash - docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8090:8090 -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.2.8 + docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8090:8090 -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.2.9 ``` ### Docker compose @@ -142,3 +141,4 @@ Further adjustments can be made by changing `.env.docker-compose` file. For more --- Thanks for visiting us and enjoy :heart:! + diff --git a/api/Dockerfile b/api/Dockerfile index 3a095b612..6434429df 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,17 +1,29 @@ -FROM ubuntu:22.04 AS build-common +FROM ubuntu:24.04 AS build-common WORKDIR /build +# Install necessary tools but not OpenJDK from apt RUN apt update --fix-missing \ - && apt install -y --no-install-recommends openjdk-21-jdk maven curl \ + && apt install -y --no-install-recommends maven curl ca-certificates \ && apt clean +# Download and setup JDK 24.0.1 +RUN mkdir -p /opt/java \ + && curl -L https://download.java.net/java/GA/jdk24.0.1/24a58e0e276943138bf3e963e6291ac2/9/GPL/openjdk-24.0.1_linux-x64_bin.tar.gz -o /opt/jdk.tar.gz \ + && tar -xzf /opt/jdk.tar.gz -C /opt/java \ + && rm /opt/jdk.tar.gz + +# Set JAVA_HOME and update PATH +ENV JAVA_HOME=/opt/java/jdk-24.0.1 +ENV PATH="${JAVA_HOME}/bin:${PATH}" + COPY ./pom.xml /build/pom.xml COPY ./api /build/api COPY ./yaci-indexer /build/yaci-indexer COPY ./test-data-generator /build/test-data-generator COPY ./.git .git -RUN --mount=type=cache,target=/root/.m2 mvn clean package -DskipTests +RUN --mount=type=cache,target=/root/.m2 mvn -U clean package -DskipTests +RUN java --version WORKDIR /app RUN cp /build/api/target/*.jar /app/api.jar @@ -19,4 +31,4 @@ RUN rm -rf /build ENTRYPOINT ["java", "--enable-preview", "-jar", "/app/api.jar"] -CMD ["/bin/sh", "-c", "bash"] +CMD ["/bin/sh", "-c", "bash"] \ No newline at end of file diff --git a/api/pom.xml b/api/pom.xml index 87599728c..aadd92504 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -2,6 +2,7 @@ 4.0.0 + base org.cardanofoundation.rosetta-java @@ -30,6 +31,14 @@ junit-jupiter test + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-security + org.springframework.boot spring-boot-starter-web @@ -52,10 +61,6 @@ org.springframework.data spring-data-commons - - org.springframework.boot - spring-boot-starter-security - org.springframework.boot spring-boot-starter-validation @@ -131,6 +136,7 @@ org.springframework.boot spring-boot-starter-data-jpa + hibernate-jpamodelgen org.hibernate.orm @@ -138,17 +144,18 @@ provided - org.postgresql postgresql runtime + commons-validator commons-validator 1.9.0 + io.swagger.parser.v3 swagger-parser @@ -165,8 +172,6 @@ io.hypersistence hypersistence-utils-hibernate-63 - 3.7.0 - compile ${project.groupId} @@ -225,7 +230,7 @@ maven-compiler-plugin ${version.maven-compiler-plugin} - 21 + 24 org.projectlombok diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/account/model/entity/AddressUtxoEntity.java b/api/src/main/java/org/cardanofoundation/rosetta/api/account/model/entity/AddressUtxoEntity.java index b98b98bcb..eb58d580f 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/account/model/entity/AddressUtxoEntity.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/account/model/entity/AddressUtxoEntity.java @@ -44,4 +44,5 @@ public class AddressUtxoEntity { @Column(name = "block") private Long blockNumber; + } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/domain/NetworkStatus.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/domain/NetworkStatus.java index 4e3b6aa8f..a7a243a45 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/domain/NetworkStatus.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/domain/NetworkStatus.java @@ -8,6 +8,7 @@ import lombok.NoArgsConstructor; import org.openapitools.client.model.Peer; +import org.openapitools.client.model.SyncStatus; @Data @Builder @@ -16,6 +17,9 @@ public class NetworkStatus { private BlockIdentifierExtended latestBlock; + private BlockIdentifierExtended oldestBlock; private BlockIdentifierExtended genesisBlock; + private SyncStatus syncStatus; private List peers; + } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/BlockEntity.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/BlockEntity.java index b883bfe34..fd4523a1d 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/BlockEntity.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/BlockEntity.java @@ -1,16 +1,7 @@ package org.cardanofoundation.rosetta.api.block.model.entity; import java.util.List; -import jakarta.persistence.Column; -import jakarta.persistence.ConstraintMode; -import jakarta.persistence.Entity; -import jakarta.persistence.FetchType; -import jakarta.persistence.ForeignKey; -import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; -import jakarta.persistence.Table; +import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/projection/BlockIdentifierProjection.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/projection/BlockIdentifierProjection.java index 1f1a8c49a..100f3db29 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/projection/BlockIdentifierProjection.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/entity/projection/BlockIdentifierProjection.java @@ -1,6 +1,7 @@ package org.cardanofoundation.rosetta.api.block.model.entity.projection; public interface BlockIdentifierProjection { + String getHash(); Long getNumber(); Long getBlockTimeInSeconds(); diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/repository/BlockRepository.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/repository/BlockRepository.java index 873824074..a16eda04c 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/repository/BlockRepository.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/model/repository/BlockRepository.java @@ -4,6 +4,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.cardanofoundation.rosetta.api.block.model.entity.BlockEntity; @@ -15,6 +16,9 @@ public interface BlockRepository extends JpaRepository { @Query("FROM BlockEntity b WHERE b.number = 0 ORDER BY b.number ASC LIMIT 1") Optional findGenesisBlockIdentifier(); + @Query("FROM BlockEntity b WHERE b.number = :blockNumber ORDER BY b.number ASC LIMIT 1") + Optional findBlockProjectionByNumber(@Param("blockNumber") long blockNumber); + Optional findByNumber(Long blockNumber); Optional findByHash(String blockHash); @@ -32,6 +36,6 @@ public interface BlockRepository extends JpaRepository { Optional findBlockIdentifierByHash(String blockHash); Optional findBlockIdentifierByNumberAndHash( - Long blockNumber, - String blockHash); + Long blockNumber, + String blockHash); } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/BlockServiceImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/BlockServiceImpl.java index dc60b79bc..fe044d251 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/BlockServiceImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/BlockServiceImpl.java @@ -38,6 +38,7 @@ public Block findBlock(Long index, String hash) { } log.error("Block was not found"); + throw ExceptionFactory.blockNotFoundException(); } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockService.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockService.java index ddaf1ef80..135b7bba4 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockService.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockService.java @@ -46,6 +46,12 @@ public interface LedgerBlockService { */ BlockIdentifierExtended findLatestBlockIdentifier(); + /** + * Returns the oldest full block (block for which we have full data) + * @return the oldest block identifier + */ + BlockIdentifierExtended findOldestBlockIdentifier(BlockIdentifierExtended latestBlock); + /** * Returns the genesis block identifier. * @return the genesis block identifier diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImpl.java index 98570c874..991956bad 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImpl.java @@ -14,6 +14,7 @@ import jakarta.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -38,6 +39,7 @@ @Component @Transactional(readOnly = true) @RequiredArgsConstructor +@Setter public class LedgerBlockServiceImpl implements LedgerBlockService { private final BlockRepository blockRepository; @@ -57,9 +59,15 @@ public class LedgerBlockServiceImpl implements LedgerBlockService { private BlockIdentifierExtended cachedGenesisBlock; - @Value("${rosetta.blockFetchTimeoutInSeconds:5}") + @Value("${cardano.rosetta.BLOCK_FETCH_TIMEOUT_SECS:5}") private int blockFetchTimeoutInSeconds; + @Value("${cardano.rosetta.REMOVE_SPENT_UTXOS:false}") + private boolean isRemovalOfSpentUTxOsEnabled; + + @Value("${cardano.rosetta.REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT:2160}") + private int removeSpentUTxOsLastBlocksGraceCount; + @Override public Optional findBlock(Long blockNumber, String blockHash) { log.debug("Query blockNumber: {} , blockHash: {}", blockNumber, blockHash); @@ -179,6 +187,30 @@ public BlockIdentifierExtended findGenesisBlockIdentifier() { return cachedGenesisBlock; } + public BlockIdentifierExtended findOldestBlockIdentifier(BlockIdentifierExtended latestBlock) { + log.debug("About to run findOldestBlock query"); + + if (!isRemovalOfSpentUTxOsEnabled) { + throw ExceptionFactory.oldestBlockNotFound(); + } + + long targetBlockNo = latestBlock.getNumber() - removeSpentUTxOsLastBlocksGraceCount; // this could result in less than 0 + + if (targetBlockNo < 0) { + log.debug("Oldest block number is less than 0, returning genesis block"); + + return cachedGenesisBlock; + } + + BlockIdentifierExtended oldestBlock = blockRepository.findBlockProjectionByNumber(targetBlockNo) + .map(blockMapper::mapToBlockIdentifierExtended) + .orElseThrow(ExceptionFactory::oldestBlockNotFound); + + log.debug("Returning oldest block {}", oldestBlock); + + return oldestBlock; + } + private TransactionInfo findByTxHash(List transactions) { List txHashes = transactions.stream().map(BlockTx::getHash).toList(); List utxHashes = transactions.stream() @@ -300,7 +332,6 @@ record TransactionInfo(List utxos, } record UtxoKey(String txHash, Integer outputIndex) { - } } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImpl.java index 14d21decb..b0c289e15 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImpl.java @@ -126,7 +126,7 @@ public Array decodeTransaction(String encoded) { @Override public Long calculateTtl(Long ttlOffset) { - long absoluteSlot = offlineMode ? offlineSlotService.getCurrentSlotBasedOnTime() + long absoluteSlot = offlineMode ? offlineSlotService.getCurrentSlotBasedOnTimeWithFallback() : ledgerBlockService.findLatestBlockIdentifier().getSlot(); return absoluteSlot + ttlOffset; diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/ConstructionApiServiceImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/ConstructionApiServiceImpl.java index 10b4b10af..121ffbe3a 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/ConstructionApiServiceImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/construction/service/ConstructionApiServiceImpl.java @@ -91,7 +91,7 @@ public ConstructionPreprocessResponse constructionPreprocessService( int relativeTtl = calculateRelativeTtl(metadata); - long currentSlotBasedOnTime = offlineSlotService.getCurrentSlotBasedOnTime() + relativeTtl; + long currentSlotBasedOnTime = offlineSlotService.getCurrentSlotBasedOnTimeWithFallback() + relativeTtl; NetworkEnum networkEnum = NetworkEnum.findByName( networkIdentifier.getNetwork()) @@ -216,7 +216,7 @@ public ConstructionPayloadsResponse constructionPayloadsService( private long calculateTtl(ConstructionPayloadsRequestMetadata metadata) { return metadata != null ? cardanoConstructionService.checkOrReturnDefaultTtl(metadata.getTtl()) : - offlineSlotService.getCurrentSlotBasedOnTime() + Constants.DEFAULT_RELATIVE_TTL; + offlineSlotService.getCurrentSlotBasedOnTimeWithFallback() + Constants.DEFAULT_RELATIVE_TTL; } private ProtocolParams convertProtocolParams(@Nullable ConstructionPayloadsRequestMetadata metadata) { diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/controller/NetworkApiImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/controller/NetworkApiImpl.java index b11e7f976..5f4d27908 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/network/controller/NetworkApiImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/controller/NetworkApiImpl.java @@ -23,31 +23,32 @@ public class NetworkApiImpl implements NetworkApi { @Override public ResponseEntity networkList( - @RequestBody MetadataRequest metadataRequest) { + @RequestBody MetadataRequest metadataRequest) { final NetworkListResponse networkListResponse = networkService.getNetworkList(metadataRequest); return ResponseEntity.ok(networkListResponse); } @Override public ResponseEntity networkOptions( - @RequestBody NetworkRequest networkRequest) { + @RequestBody NetworkRequest networkRequest) { networkService.verifyNetworkRequest(networkRequest.getNetworkIdentifier()); final NetworkOptionsResponse networkOptionsResponse = networkService.getNetworkOptions( - networkRequest); + networkRequest); return ResponseEntity.ok(networkOptionsResponse); } @Override public ResponseEntity networkStatus( - @RequestBody NetworkRequest networkRequest) { + @RequestBody NetworkRequest networkRequest) { if (offlineMode) { throw ExceptionFactory.notSupportedInOfflineMode(); } networkService.verifyNetworkRequest(networkRequest.getNetworkIdentifier()); + final NetworkStatusResponse networkStatusResponse = networkService.getNetworkStatus( - networkRequest); + networkRequest); return ResponseEntity.ok(networkStatusResponse); } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/mapper/NetworkMapper.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/mapper/NetworkMapper.java index 7c9e808b0..099894770 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/network/mapper/NetworkMapper.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/mapper/NetworkMapper.java @@ -17,10 +17,16 @@ public interface NetworkMapper { @Mapping(target = "currentBlockIdentifier.index", source = "latestBlock.number") @Mapping(target = "currentBlockIdentifier.hash", source = "latestBlock.hash") + @Mapping(target = "oldestBlockIdentifier.index", source = "oldestBlock.number") + @Mapping(target = "oldestBlockIdentifier.hash", source = "oldestBlock.hash") + + @Mapping(target = "syncStatus", source = "syncStatus") @Mapping(target = "currentBlockTimestamp", - expression = "java(java.util.concurrent.TimeUnit.SECONDS.toMillis(networkStatus.getLatestBlock().getBlockTimeInSeconds()))") + expression = "java(java.util.concurrent.TimeUnit.SECONDS.toMillis(networkStatus.getLatestBlock().getBlockTimeInSeconds()))") + @Mapping(target = "genesisBlockIdentifier.index", source = "genesisBlock.number", defaultValue = "0L") @Mapping(target = "genesisBlockIdentifier.hash", source = "genesisBlock.hash") @Mapping(target = "peers", qualifiedByName = "getPeerWithoutMetadata") NetworkStatusResponse toNetworkStatusResponse(NetworkStatus networkStatus); + } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/GenesisDataProvider.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/GenesisDataProvider.java new file mode 100644 index 000000000..a19bc77e8 --- /dev/null +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/GenesisDataProvider.java @@ -0,0 +1,7 @@ +package org.cardanofoundation.rosetta.api.network.service; + +public interface GenesisDataProvider { + + int getProtocolMagic(); + +} diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/GenesisDataProviderImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/GenesisDataProviderImpl.java new file mode 100644 index 000000000..a1628ef46 --- /dev/null +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/GenesisDataProviderImpl.java @@ -0,0 +1,54 @@ +package org.cardanofoundation.rosetta.api.network.service; + +import java.io.IOException; +import java.util.Map; +import javax.annotation.PostConstruct; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.cardanofoundation.rosetta.common.exception.ExceptionFactory; +import org.cardanofoundation.rosetta.common.util.FileUtils; + +import static org.cardanofoundation.rosetta.common.util.Constants.NETWORK_MAGIC_NAME; + +@Service +@Slf4j +@RequiredArgsConstructor +public class GenesisDataProviderImpl implements GenesisDataProvider { + + @Value("${cardano.rosetta.GENESIS_SHELLEY_PATH}") + private String genesisShelleyPath; + + private int protocolMagic; + + @PostConstruct + public void init() { + Map jsonMap = loadGenesisShelleyConfig(); + protocolMagic = (Integer) jsonMap.get(NETWORK_MAGIC_NAME); + + log.info("Protocol magic number: {} loaded from genesis config json", protocolMagic); + } + + @Override + public int getProtocolMagic() { + return protocolMagic; + } + + private Map loadGenesisShelleyConfig() { + try { + String content = FileUtils.fileReader(genesisShelleyPath); + + return new ObjectMapper().readValue(content, new TypeReference<>() { + }); + } catch (IOException e) { + throw ExceptionFactory.configNotFoundException(genesisShelleyPath); + } + } + +} diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkService.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkService.java index 5a986a0ad..b0a91dc9d 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkService.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkService.java @@ -1,6 +1,5 @@ package org.cardanofoundation.rosetta.api.network.service; - import com.bloxbean.cardano.client.common.model.Network; import org.openapitools.client.model.MetadataRequest; import org.openapitools.client.model.NetworkIdentifier; @@ -10,6 +9,7 @@ import org.openapitools.client.model.NetworkStatusResponse; public interface NetworkService { + NetworkListResponse getNetworkList(final MetadataRequest metadataRequest); NetworkOptionsResponse getNetworkOptions(final NetworkRequest networkRequest); diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImpl.java index e24bbfd99..972367b74 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImpl.java @@ -2,12 +2,7 @@ import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; @@ -18,21 +13,10 @@ import org.springframework.stereotype.Service; import com.bloxbean.cardano.client.common.model.Network; import com.bloxbean.cardano.client.common.model.Networks; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.parser.OpenAPIV3Parser; -import org.openapitools.client.model.Allow; +import org.openapitools.client.model.*; import org.openapitools.client.model.Error; -import org.openapitools.client.model.MetadataRequest; -import org.openapitools.client.model.NetworkIdentifier; -import org.openapitools.client.model.NetworkListResponse; -import org.openapitools.client.model.NetworkOptionsResponse; -import org.openapitools.client.model.NetworkRequest; -import org.openapitools.client.model.NetworkStatusResponse; -import org.openapitools.client.model.OperationStatus; -import org.openapitools.client.model.Peer; -import org.openapitools.client.model.Version; import org.cardanofoundation.rosetta.api.block.model.domain.BlockIdentifierExtended; import org.cardanofoundation.rosetta.api.block.model.domain.NetworkStatus; @@ -41,42 +25,52 @@ import org.cardanofoundation.rosetta.common.enumeration.OperationType; import org.cardanofoundation.rosetta.common.enumeration.OperationTypeStatus; import org.cardanofoundation.rosetta.common.exception.ExceptionFactory; +import org.cardanofoundation.rosetta.common.time.OfflineSlotService; import org.cardanofoundation.rosetta.common.util.Constants; -import org.cardanofoundation.rosetta.common.util.FileUtils; import org.cardanofoundation.rosetta.common.util.RosettaConstants; -import org.cardanofoundation.rosetta.config.RosettaConfig; + +import static org.cardanofoundation.rosetta.common.util.Constants.*; @Service @Slf4j @RequiredArgsConstructor public class NetworkServiceImpl implements NetworkService { - private final RosettaConfig rosettaConfig; private final LedgerBlockService ledgerBlockService; + private final OfflineSlotService offlineSlotService; private final NetworkMapper networkMapper; private final TopologyConfigService topologyConfigService; private final ResourceLoader resourceLoader; - - private Integer cachedMagicNumber; + private final GenesisDataProvider genesisDataProvider; + private final SlotRangeChecker slotRangeChecker; @Value("${cardano.rosetta.GENESIS_SHELLEY_PATH}") private String genesisShelleyPath; + @Value("${cardano.rosetta.CARDANO_NODE_VERSION}") private String cardanoNodeVersion; + @Value("${cardano.rosetta.middleware-version}") private String revision; + @Value("${cardano.rosetta.SYNC_GRACE_SLOTS_COUNT:100}") + private int allowedSlotsDelta; + + @Value("${cardano.rosetta.REMOVE_SPENT_UTXOS:false}") + private boolean isRemovalOfSpentUTxOsEnabled; + @PostConstruct public void init() { - Map jsonMap = loadGenesisShelleyConfig(); - cachedMagicNumber = (Integer) jsonMap.get(Constants.NETWORK_MAGIC_NAME); - log.info("Magic number {} loaded from genesis config json", cachedMagicNumber); + log.info("NetworkServiceImpl initializing..."); + log.info("allowedSlotsDelta: {}", allowedSlotsDelta); + log.info("isRemovalOfSpentUTxOsEnabled: {}", isRemovalOfSpentUTxOsEnabled); } @Override public NetworkListResponse getNetworkList(MetadataRequest metadataRequest) { log.info("[networkList] Looking for all supported networks"); Network supportedNetwork = getSupportedNetwork(); + return networkMapper.toNetworkListResponse(supportedNetwork); } @@ -86,42 +80,42 @@ public NetworkOptionsResponse getNetworkOptions(NetworkRequest networkRequest) { String rosettaVersion = getRosettaVersion(); OperationStatus success = new OperationStatus().successful(true) - .status(OperationTypeStatus.SUCCESS.getValue()); + .status(OperationTypeStatus.SUCCESS.getValue()); OperationStatus invalid = new OperationStatus().successful(false) - .status(OperationTypeStatus.INVALID.getValue()); + .status(OperationTypeStatus.INVALID.getValue()); List operationStatuses = List.of(success, invalid); return NetworkOptionsResponse.builder() - .version(new Version().nodeVersion(cardanoNodeVersion) - .rosettaVersion(rosettaVersion) - .middlewareVersion(revision) - .metadata(new LinkedHashMap<>())) - .allow(new Allow().operationStatuses(operationStatuses) - .operationTypes( - Arrays.stream(OperationType.values()).map(OperationType::getValue).toList()) - .errors(RosettaConstants.ROSETTA_ERRORS.stream() - .map(error -> - new Error().code(error.getCode()) - .message(error.getMessage()) - .retriable(error.isRetriable()) - .description(error.getDescription()) - .code(error.getCode()) - ) - .sorted(Comparator.comparingInt(Error::getCode)) - .toList()) - .historicalBalanceLookup(true) - .callMethods(new ArrayList<>()) - .mempoolCoins(false)) - .build(); + .version(new Version().nodeVersion(cardanoNodeVersion) + .rosettaVersion(rosettaVersion) + .middlewareVersion(revision) + .metadata(new LinkedHashMap<>())) + .allow(new Allow().operationStatuses(operationStatuses) + .operationTypes( + Arrays.stream(OperationType.values()).map(OperationType::getValue).toList()) + .errors(RosettaConstants.ROSETTA_ERRORS.stream() + .map(error -> + new Error().code(error.getCode()) + .message(error.getMessage()) + .retriable(error.isRetriable()) + .description(error.getDescription()) + .code(error.getCode()) + ) + .sorted(Comparator.comparingInt(Error::getCode)) + .toList()) + .historicalBalanceLookup(true) + .callMethods(new ArrayList<>()) + .mempoolCoins(false)) + .build(); } private String getRosettaVersion() { try { InputStream openAPIStream = resourceLoader.getResource( - Constants.ROSETTA_API_PATH).getInputStream(); + Constants.ROSETTA_API_PATH).getInputStream(); OpenAPI openAPI = new OpenAPIV3Parser().readContents(new String(openAPIStream.readAllBytes()), - null, null) - .getOpenAPI(); + null, null) + .getOpenAPI(); return openAPI.getInfo().getVersion(); } catch (IOException e) { throw ExceptionFactory.configNotFoundException(Constants.ROSETTA_API_PATH); @@ -130,7 +124,7 @@ private String getRosettaVersion() { @Override public NetworkStatusResponse getNetworkStatus(NetworkRequest networkRequest) { - log.debug("[networkStatus] Request received: {}", networkRequest.toString()); + log.info("[networkStatus] Request received: {}", networkRequest.toString()); log.info("[networkStatus] Looking for latest block"); NetworkStatus networkStatus = networkStatus(); @@ -139,34 +133,59 @@ public NetworkStatusResponse getNetworkStatus(NetworkRequest networkRequest) { @Override public Network getSupportedNetwork() { - Integer networkMagic = getNetworkMagic(); + int networkMagic = genesisDataProvider.getProtocolMagic(); + return switch (networkMagic) { - case Constants.MAINNET_NETWORK_MAGIC -> Networks.mainnet(); - case Constants.PREPROD_NETWORK_MAGIC -> Networks.preprod(); - case Constants.PREVIEW_NETWORK_MAGIC -> Networks.preview(); - case Constants.SANCHONET_NETWORK_MAGIC -> new Network(0b0000, Constants.SANCHONET_NETWORK_MAGIC); - case Constants.DEVKIT_NETWORK_MAGIC -> new Network(0b0000, Constants.DEVKIT_NETWORK_MAGIC); + case MAINNET_NETWORK_MAGIC -> Networks.mainnet(); + case PREPROD_NETWORK_MAGIC -> Networks.preprod(); + case PREVIEW_NETWORK_MAGIC -> Networks.preview(); + case DEVKIT_NETWORK_MAGIC -> new Network(0b0000, DEVKIT_NETWORK_MAGIC); + default -> throw ExceptionFactory.invalidNetworkError(); }; } private NetworkStatus networkStatus() { - log.info("[networkStatus] Looking for latest block"); + log.info("[networkStatus] Looking for network status"); BlockIdentifierExtended latestBlock = ledgerBlockService.findLatestBlockIdentifier(); log.debug("[networkStatus] Latest block found {}", latestBlock); log.debug("[networkStatus] Looking for genesis block"); BlockIdentifierExtended genesisBlock = ledgerBlockService.findGenesisBlockIdentifier(); + log.debug("[networkStatus] Genesis block found {}", genesisBlock); List peers = topologyConfigService.getPeers(); - return NetworkStatus.builder() - .latestBlock(latestBlock) - .genesisBlock(genesisBlock) - .peers(peers) - .build(); + NetworkStatus.NetworkStatusBuilder networkStatusBuilder = NetworkStatus.builder() + .latestBlock(latestBlock) + .genesisBlock(genesisBlock) + .peers(peers); + + if (isRemovalOfSpentUTxOsEnabled) { + BlockIdentifierExtended oldestBlockIdentifier = ledgerBlockService.findOldestBlockIdentifier(latestBlock); + + networkStatusBuilder.oldestBlock(oldestBlockIdentifier); + } + + calculateSyncStatus(latestBlock).ifPresent(networkStatusBuilder::syncStatus); + + return networkStatusBuilder.build(); + } + + private Optional calculateSyncStatus(BlockIdentifierExtended latestBlock) { + return offlineSlotService.getCurrentSlotBasedOnTime().map(slotBasedOnTime -> { + long slotBasedOnLatestBlock = latestBlock.getSlot(); + + boolean isSynced = slotRangeChecker.isSlotWithinEpsilon(slotBasedOnTime, slotBasedOnLatestBlock, allowedSlotsDelta); + + return SyncStatus.builder() + .targetIndex(slotBasedOnTime) + .currentIndex(slotBasedOnLatestBlock) + .synced(isSynced) + .build(); + }); } @Override @@ -185,41 +204,17 @@ private boolean verifyBlockchain(String blockchain) { return blockchain.equals(Constants.CARDANO_BLOCKCHAIN); } - private boolean verifyNetwork(String network) { - Network supportedNetwork = getSupportedNetwork(); + private boolean verifyNetwork(String inputNetwork) { + Network currentNetwork = getSupportedNetwork(); - switch ((int) supportedNetwork.getProtocolMagic()) { - case Constants.MAINNET_NETWORK_MAGIC -> { - return network.equalsIgnoreCase(Constants.MAINNET); - } - case Constants.PREPROD_NETWORK_MAGIC -> { - return network.equals(Constants.PREPROD); - } - case Constants.PREVIEW_NETWORK_MAGIC -> { - return network.equals(Constants.PREVIEW); - } - case Constants.DEVKIT_NETWORK_MAGIC -> { - return network.equals(Constants.DEVKIT); - } - case Constants.SANCHONET_NETWORK_MAGIC -> { - return network.equals(Constants.SANCHONET); - } - default -> { - return false; - } - } - } + return switch ((int) currentNetwork.getProtocolMagic()) { + case MAINNET_NETWORK_MAGIC -> inputNetwork.equalsIgnoreCase(MAINNET); + case PREPROD_NETWORK_MAGIC -> inputNetwork.equals(PREPROD); + case PREVIEW_NETWORK_MAGIC -> inputNetwork.equals(PREVIEW); + case DEVKIT_NETWORK_MAGIC -> inputNetwork.equals(DEVKIT); - public Integer getNetworkMagic() { - return cachedMagicNumber; + default -> false; + }; } - private Map loadGenesisShelleyConfig() { - try { - String content = FileUtils.fileReader(genesisShelleyPath); - return new ObjectMapper().readValue(content, new TypeReference>() {}); - } catch (IOException e) { - throw ExceptionFactory.configNotFoundException(genesisShelleyPath); - } - } } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/SlotRangeChecker.java b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/SlotRangeChecker.java new file mode 100644 index 000000000..87ad4d78d --- /dev/null +++ b/api/src/main/java/org/cardanofoundation/rosetta/api/network/service/SlotRangeChecker.java @@ -0,0 +1,21 @@ +package org.cardanofoundation.rosetta.api.network.service; + +import org.springframework.stereotype.Service; + +@Service +public class SlotRangeChecker { + + public boolean isSlotWithinEpsilon(long slotBasedOnTime, + long slotBasedOnLatestBlock, + long epsilon) { + assert slotBasedOnTime >= 0; + assert slotBasedOnLatestBlock >= 0; + + if (epsilon < 0) { + throw new IllegalArgumentException("Epsilon must be non-negative"); + } + + return Math.abs(slotBasedOnTime - slotBasedOnLatestBlock) <= epsilon; + } + +} diff --git a/api/src/main/java/org/cardanofoundation/rosetta/common/enumeration/NetworkEnum.java b/api/src/main/java/org/cardanofoundation/rosetta/common/enumeration/NetworkEnum.java index 412dcd0ef..d7a1edc10 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/common/enumeration/NetworkEnum.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/common/enumeration/NetworkEnum.java @@ -16,8 +16,7 @@ public enum NetworkEnum { MAINNET(Constants.MAINNET, Networks.mainnet()), PREPROD(Constants.PREPROD, Networks.preprod()), DEVNET(Constants.DEVKIT, new Network(0b0000, 42)), - PREVIEW(Constants.PREVIEW, Networks.preview()), - SANCHONET(Constants.SANCHONET, new Network(0b0000, 4)); + PREVIEW(Constants.PREVIEW, Networks.preview()); private final String name; private final Network network; diff --git a/api/src/main/java/org/cardanofoundation/rosetta/common/exception/ExceptionFactory.java b/api/src/main/java/org/cardanofoundation/rosetta/common/exception/ExceptionFactory.java index bf0b8c0bf..ea1e698ec 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/common/exception/ExceptionFactory.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/common/exception/ExceptionFactory.java @@ -337,4 +337,8 @@ public static ApiException invalidBlockIdentifier(@NotNull long index) { return new ApiException(RosettaErrorType.INVALID_BLOCK_INDEX.toRosettaError(false, Details.builder().message("Invalid block index, must be greater than or equal to 0, supplied index:%d".formatted(index)).build())); } + public static ApiException oldestBlockNotFound() { + return new ApiException(RosettaErrorType.OLDEST_BLOCK_NOT_FOUND.toRosettaError(false)); + } + } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotService.java b/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotService.java index b97d554ec..23c4884eb 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotService.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotService.java @@ -1,7 +1,11 @@ package org.cardanofoundation.rosetta.common.time; +import java.util.Optional; + public interface OfflineSlotService { - long getCurrentSlotBasedOnTime(); + long getCurrentSlotBasedOnTimeWithFallback(); + + Optional getCurrentSlotBasedOnTime(); } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImpl.java b/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImpl.java index da59ee305..77e11881c 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImpl.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImpl.java @@ -1,6 +1,7 @@ package org.cardanofoundation.rosetta.common.time; import java.time.*; +import java.util.Optional; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; @@ -39,19 +40,33 @@ public class OfflineSlotServiceImpl implements OfflineSlotService { protected CardanoConverters cardanoConverters; @Override - public long getCurrentSlotBasedOnTime() { + public long getCurrentSlotBasedOnTimeWithFallback() { if (cardanoConverters != null) { - LocalDateTime nowDateTime = LocalDateTime.now(clock); - ZoneOffset zoneOffset = ZonedDateTime.now(zoneId).getOffset(); + return timeToAbsoluteSlotNo(); + } - if (nowDateTime.toInstant(zoneOffset).isBefore(shellyStartTime)) { - throw new ApiException(ExceptionFactory.misconfiguredTime(nowDateTime).getError()); - } + return shelleyStartSlot; + } - return cardanoConverters.time().toSlot(nowDateTime); + @Override + public Optional getCurrentSlotBasedOnTime() { + if (cardanoConverters != null) { + return Optional.of(timeToAbsoluteSlotNo()); } - return shelleyStartSlot; + return Optional.empty(); + } + + private Long timeToAbsoluteSlotNo() { + LocalDateTime nowDateTime = LocalDateTime.now(clock); + ZoneOffset zoneOffset = ZonedDateTime.now(zoneId).getOffset(); + + if (nowDateTime.toInstant(zoneOffset).isBefore(shellyStartTime)) { + throw new ApiException(ExceptionFactory.misconfiguredTime(nowDateTime).getError()); + + } + + return cardanoConverters.time().toSlot(nowDateTime); } } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/common/util/Constants.java b/api/src/main/java/org/cardanofoundation/rosetta/common/util/Constants.java index e6a509540..85c941155 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/common/util/Constants.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/common/util/Constants.java @@ -28,7 +28,6 @@ public class Constants { public static final int MAINNET_NETWORK_MAGIC = 764824073; public static final int PREPROD_NETWORK_MAGIC = 1; public static final int PREVIEW_NETWORK_MAGIC = 2; - public static final int SANCHONET_NETWORK_MAGIC = 4; public static final int DEVKIT_NETWORK_MAGIC = 42; public static final int OPERATION_TYPE_VOTE_DELEGATION = 43; @@ -208,7 +207,6 @@ private Constants() { public static final String LOVELACE = "lovelace"; public static final String DEVKIT = "devkit"; - public static final String SANCHONET = "sanchonet"; public static final String ROSETTA_API_PATH = "classpath:/rosetta-specifications-1.4.15/api.yaml"; } diff --git a/api/src/main/java/org/cardanofoundation/rosetta/common/util/RosettaConstants.java b/api/src/main/java/org/cardanofoundation/rosetta/common/util/RosettaConstants.java index 1eabf3986..8791164a9 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/common/util/RosettaConstants.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/common/util/RosettaConstants.java @@ -198,7 +198,8 @@ public enum RosettaErrorType { MISSING_DREP_ID("Missing drep id", 5039), MISSING_DREP_TYPE("Missing drep type", 5040), TIMEOUT("Downstream timeout", 5041), - INVALID_BLOCK_INDEX("Invalid block index", 5042); + INVALID_BLOCK_INDEX("Invalid block index", 5042), + OLDEST_BLOCK_NOT_FOUND("Oldest block not found", 5043); final String message; final int code; diff --git a/api/src/main/java/org/cardanofoundation/rosetta/config/CardanoConvertersConfig.java b/api/src/main/java/org/cardanofoundation/rosetta/config/CardanoConvertersConfig.java index 1a466236e..6a908a966 100644 --- a/api/src/main/java/org/cardanofoundation/rosetta/config/CardanoConvertersConfig.java +++ b/api/src/main/java/org/cardanofoundation/rosetta/config/CardanoConvertersConfig.java @@ -6,6 +6,7 @@ import java.time.ZonedDateTime; import jakarta.annotation.Nullable; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; @@ -14,49 +15,48 @@ import org.cardanofoundation.conversions.CardanoConverters; import org.cardanofoundation.conversions.ClasspathConversionsFactory; -import org.cardanofoundation.conversions.domain.NetworkType; -import org.cardanofoundation.rosetta.api.network.service.NetworkService; +import org.cardanofoundation.rosetta.api.network.service.GenesisDataProvider; import org.cardanofoundation.rosetta.common.util.Constants; +import static org.cardanofoundation.conversions.domain.NetworkType.*; import static org.cardanofoundation.rosetta.common.util.Constants.MAINNET_NETWORK_MAGIC; @Configuration @Slf4j +@RequiredArgsConstructor public class CardanoConvertersConfig { + private final GenesisDataProvider genesisDataProvider; + @Bean @Nullable - public CardanoConverters cardanoConverters(NetworkService networkService) { - if (networkService.getSupportedNetwork() == null) { // mostly helper for unit tests and integration tests - return null; - } - - long protocolMagic = networkService.getSupportedNetwork().getProtocolMagic(); + public CardanoConverters cardanoConverters() { + int protocolMagic = genesisDataProvider.getProtocolMagic(); log.info("Creating CardanoConverters for network magic: {}", protocolMagic); if (protocolMagic == MAINNET_NETWORK_MAGIC) { - return ClasspathConversionsFactory.createConverters(NetworkType.MAINNET); + log.info("Creating CardanoConverters for mainnet"); + return ClasspathConversionsFactory.createConverters(MAINNET); } if (protocolMagic == Constants.PREPROD_NETWORK_MAGIC) { - return ClasspathConversionsFactory.createConverters(NetworkType.PREPROD); + log.info("Creating CardanoConverters for preprod"); + return ClasspathConversionsFactory.createConverters(PREPROD); } if (protocolMagic == Constants.PREVIEW_NETWORK_MAGIC) { - return ClasspathConversionsFactory.createConverters(NetworkType.PREVIEW); - } - - if (protocolMagic == Constants.SANCHONET_NETWORK_MAGIC) { - return ClasspathConversionsFactory.createConverters(NetworkType.SANCHONET); + log.info("Creating CardanoConverters for preview"); + return ClasspathConversionsFactory.createConverters(PREVIEW); } if (protocolMagic == Constants.DEVKIT_NETWORK_MAGIC) { + log.info("Creating CardanoConverters for devkit, no converters available"); // Cardano Converters for DevKit is not supported but we still need to return something sensible return null; } - throw new IllegalArgumentException("Unsupported network magic: " + protocolMagic); + throw new IllegalArgumentException("Unsupported network magic: %d".formatted(protocolMagic)); } @Bean diff --git a/api/src/main/resources/config/application-offline.yaml b/api/src/main/resources/config/application-offline.yaml index 7d06fdcac..3b12803e2 100644 --- a/api/src/main/resources/config/application-offline.yaml +++ b/api/src/main/resources/config/application-offline.yaml @@ -12,6 +12,9 @@ spring: cardano: rosetta: OFFLINE_MODE: true + SYNC_GRACE_SLOTS_COUNT: ${SYNC_GRACE_SLOTS_COUNT:100} + REMOVE_SPENT_UTXOS: ${REMOVE_SPENT_UTXOS:false} + YACI_HTTP_BASE_URL: ${YACI_HTTP_BASE_URL:http://localhost:9095} HTTP_CONNECT_TIMEOUT_SECONDS: ${HTTP_CONNECT_TIMEOUT_SECONDS:5} HTTP_REQUEST_TIMEOUT_SECONDS: ${HTTP_REQUEST_TIMEOUT_SECONDS:5} \ No newline at end of file diff --git a/api/src/main/resources/config/application.yaml b/api/src/main/resources/config/application.yaml index 06b4db79b..376fa0e72 100644 --- a/api/src/main/resources/config/application.yaml +++ b/api/src/main/resources/config/application.yaml @@ -28,13 +28,15 @@ cardano: CARDANO_NODE_SUBMIT_HOST: ${CARDANO_NODE_SUBMIT_HOST} CARDANO_NODE_VERSION: ${CARDANO_NODE_VERSION} CARDANO_NODE_SOCKET_PATH: ${CARDANO_NODE_SOCKET_PATH} - # devkit variables DEVKIT_ENABLED: ${DEVKIT_ENABLED:false} DEVKIT_URL: ${DEVKIT_URL:yaci-cli} DEVKIT_PORT: ${DEVKIT_PORT:3333} SEARCH_PAGE_SIZE: ${SEARCH_PAGE_SIZE:10} OFFLINE_MODE: ${OFFLINE_MODE:false} + SYNC_GRACE_SLOTS_COUNT: ${SYNC_GRACE_SLOTS_COUNT:100} + REMOVE_SPENT_UTXOS: ${REMOVE_SPENT_UTXOS:false} + YACI_HTTP_BASE_URL: ${YACI_HTTP_BASE_URL:http://localhost:9095} HTTP_CONNECT_TIMEOUT_SECONDS: ${HTTP_CONNECT_TIMEOUT_SECONDS:5} HTTP_REQUEST_TIMEOUT_SECONDS: ${HTTP_REQUEST_TIMEOUT_SECONDS:5} diff --git a/api/src/test/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImplTest.java b/api/src/test/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImplTest.java index 2470a8089..6f490e92a 100644 --- a/api/src/test/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImplTest.java +++ b/api/src/test/java/org/cardanofoundation/rosetta/api/block/service/LedgerBlockServiceImplTest.java @@ -22,10 +22,15 @@ import org.cardanofoundation.rosetta.api.block.mapper.TransactionMapper; import org.cardanofoundation.rosetta.api.block.model.domain.*; import org.cardanofoundation.rosetta.api.block.model.entity.*; +import org.cardanofoundation.rosetta.api.block.model.entity.projection.BlockIdentifierProjection; import org.cardanofoundation.rosetta.api.block.model.repository.*; +import org.cardanofoundation.rosetta.common.exception.ApiException; import org.cardanofoundation.rosetta.common.services.ProtocolParamService; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.cardanofoundation.rosetta.common.util.RosettaConstants.RosettaErrorType.OLDEST_BLOCK_NOT_FOUND; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -277,4 +282,105 @@ void populateTransaction_populatesPoolRetirements() { assertThat(transaction.getPoolRetirements().size()).isEqualTo(1); } + @Test + void findOldestBlockIdentifier_throwsException_whenPruningIsEnabled() { + // Given + ledgerBlockService.setRemovalOfSpentUTxOsEnabled(true); + BlockIdentifierExtended latestBlock = new BlockIdentifierExtended(); + latestBlock.setNumber(5000L); + + // When & Then + ApiException exception = assertThrows(ApiException.class, () -> + ledgerBlockService.findOldestBlockIdentifier(latestBlock)); + + // Verify the exception is the correct type + assertThat(exception.getError().getCode()).isEqualTo(OLDEST_BLOCK_NOT_FOUND.getCode()); + } + + @Test + void findOldestBlockIdentifier_returnsGenesisBlock_whenTargetBlockNumberIsNegative() { + // Given + ledgerBlockService.setRemovalOfSpentUTxOsEnabled(true); + ledgerBlockService.setRemoveSpentUTxOsLastBlocksGraceCount(100); + + BlockIdentifierExtended latestBlock = new BlockIdentifierExtended(); + latestBlock.setNumber(50L); + + BlockIdentifierExtended genesisBlock = new BlockIdentifierExtended(); + genesisBlock.setNumber(0L); + genesisBlock.setHash("Genesis"); + ledgerBlockService.setCachedGenesisBlock(genesisBlock); + + // When + BlockIdentifierExtended result = ledgerBlockService.findOldestBlockIdentifier(latestBlock); + + // Then + assertThat(result).isSameAs(genesisBlock); + } + + @Test + void findOldestBlockIdentifier_returnsCorrectBlock_whenTargetBlockNumberIsValid() { + // Given + ledgerBlockService.setRemovalOfSpentUTxOsEnabled(true); + ledgerBlockService.setRemoveSpentUTxOsLastBlocksGraceCount(100); + + BlockIdentifierExtended latestBlock = new BlockIdentifierExtended(); + latestBlock.setNumber(500L); + + BlockIdentifierProjection blockProjection = new BlockIdentifierProjection() { + @Override + public String getHash() { + return "oldestHash"; + } + + @Override + public Long getNumber() { + return 400L; // 500 - 100 = 400 + } + + @Override + public Long getBlockTimeInSeconds() { + return 0L; + } + + @Override + public Long getSlot() { + return 0L; + } + }; + + BlockIdentifierExtended expectedBlock = new BlockIdentifierExtended(); + expectedBlock.setNumber(400L); + expectedBlock.setHash("oldestHash"); + + when(blockRepository.findBlockProjectionByNumber(400L)).thenReturn(Optional.of(blockProjection)); + when(blockMapper.mapToBlockIdentifierExtended(blockProjection)).thenReturn(expectedBlock); + + // When + BlockIdentifierExtended result = ledgerBlockService.findOldestBlockIdentifier(latestBlock); + + // Then + assertThat(result).isEqualTo(expectedBlock); + verify(blockRepository).findBlockProjectionByNumber(400L); + verify(blockMapper).mapToBlockIdentifierExtended(blockProjection); + } + + + @Test + void findOldestBlockIdentifier_throwsException_whenTargetBlockIsNotFound() { + // Given + ledgerBlockService.setRemovalOfSpentUTxOsEnabled(false); + ledgerBlockService.setRemoveSpentUTxOsLastBlocksGraceCount(100); + + BlockIdentifierExtended latestBlock = new BlockIdentifierExtended(); + latestBlock.setNumber(500L); + + // When & Then + ApiException exception = assertThrows(ApiException.class, () -> + ledgerBlockService.findOldestBlockIdentifier(latestBlock)); + + // Verify the exception is the correct type + assertThat(exception.getError().getCode()).isEqualTo(OLDEST_BLOCK_NOT_FOUND.getCode()); + } + } diff --git a/api/src/test/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImplTest.java b/api/src/test/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImplTest.java index e0ce4372c..723eb1f47 100644 --- a/api/src/test/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImplTest.java +++ b/api/src/test/java/org/cardanofoundation/rosetta/api/construction/service/CardanoConstructionServiceImplTest.java @@ -181,19 +181,6 @@ void parseTransactionSignedThrowTest() { assertFalse(actualException.getError().isRetriable()); } - @SuppressWarnings("java:S5778") - @Test - void parseTransactionSignedThrowSanchonetTest() { - ApiException actualException = assertThrows(ApiException.class, () -> - cardanoService.parseTransaction(NetworkEnum.SANCHONET.getNetwork(), TRANSACTION_NOT_SIGNED, true)); - - assertEquals(RosettaErrorType.CANT_CREATE_SIGNED_TRANSACTION_ERROR.getMessage(), - actualException.getError().getMessage()); - assertEquals(RosettaErrorType.CANT_CREATE_SIGNED_TRANSACTION_ERROR.getCode(), - actualException.getError().getCode()); - assertFalse(actualException.getError().isRetriable()); - } - @Test void parseTransactionNotSignedTest() { TransactionParsed actual = cardanoService.parseTransaction(NetworkEnum.PREVIEW.getNetwork(), TRANSACTION_NOT_SIGNED, false); diff --git a/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImplTest.java b/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImplTest.java index 54b4f6889..6fff8827d 100644 --- a/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImplTest.java +++ b/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/NetworkServiceImplTest.java @@ -116,4 +116,6 @@ private NetworkIdentifier createNetworkIdentifier(String blockchain, String netw .network(network) .build(); } + + } diff --git a/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/SlotRangeCheckerTest.java b/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/SlotRangeCheckerTest.java new file mode 100644 index 000000000..88b40df38 --- /dev/null +++ b/api/src/test/java/org/cardanofoundation/rosetta/api/network/service/SlotRangeCheckerTest.java @@ -0,0 +1,35 @@ +package org.cardanofoundation.rosetta.api.network.service; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class SlotRangeCheckerTest { + + @Test + void testIsSlotWithinEpsilonWhenWithinRange() { + SlotRangeChecker slotRangeChecker = new SlotRangeChecker(); + assertTrue(slotRangeChecker.isSlotWithinEpsilon(100, 105, 10)); + } + + @Test + void testIsSlotWithinEpsilonWhenOutOfRange() { + SlotRangeChecker slotRangeChecker = new SlotRangeChecker(); + assertFalse(slotRangeChecker.isSlotWithinEpsilon(100, 120, 10)); + } + + @Test + void testIsSlotWithinEpsilonWhenAtBoundary() { + SlotRangeChecker slotRangeChecker = new SlotRangeChecker(); + assertTrue(slotRangeChecker.isSlotWithinEpsilon(100, 110, 10)); + } + + @Test + void testIsSlotWithinEpsilonThrowsExceptionWhenEpsilonNegative() { + SlotRangeChecker slotRangeChecker = new SlotRangeChecker(); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> slotRangeChecker.isSlotWithinEpsilon(100, 105, -1)); + assertEquals("Epsilon must be non-negative", exception.getMessage()); + } + +} diff --git a/api/src/test/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImplTest.java b/api/src/test/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImplTest.java index ba5db3b85..9b500dc83 100644 --- a/api/src/test/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImplTest.java +++ b/api/src/test/java/org/cardanofoundation/rosetta/common/time/OfflineSlotServiceImplTest.java @@ -4,6 +4,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.Optional; import org.mockito.Mock; import org.mockito.Spy; @@ -35,15 +36,20 @@ class OfflineSlotServiceImplTest { private final Instant shellyStartTime = Instant.parse("2020-07-29T21:44:51Z"); private final long shelleyStartSlot = 1000L; // Example slot value - private OfflineSlotServiceImpl offlineSlotService; + private OfflineSlotServiceImpl offlineSlotServiceWithConverters; + private OfflineSlotServiceImpl offlineSlotServiceWithoutConverters; @BeforeEach @SuppressWarnings("java:S5786") public void setup() { - this.offlineSlotService = new OfflineSlotServiceImpl(clock, zoneOffset); - this.offlineSlotService.cardanoConverters = cardanoConverters; - this.offlineSlotService.shelleyStartSlot = shelleyStartSlot; - this.offlineSlotService.shellyStartTime = shellyStartTime; + this.offlineSlotServiceWithConverters = new OfflineSlotServiceImpl(clock, zoneOffset); + this.offlineSlotServiceWithConverters.cardanoConverters = cardanoConverters; + this.offlineSlotServiceWithConverters.shelleyStartSlot = shelleyStartSlot; + this.offlineSlotServiceWithConverters.shellyStartTime = shellyStartTime; + + this.offlineSlotServiceWithoutConverters = new OfflineSlotServiceImpl(clock, zoneOffset); + this.offlineSlotServiceWithoutConverters.shelleyStartSlot = shelleyStartSlot; + this.offlineSlotServiceWithoutConverters.shellyStartTime = shellyStartTime; } @Test @@ -54,7 +60,7 @@ void shouldReturnSlotBasedOnCurrentTime_WhenCardanoConvertersAvailable() { when(clock.getZone()).thenReturn(zoneOffset); // When - long slot = offlineSlotService.getCurrentSlotBasedOnTime(); + long slot = offlineSlotServiceWithConverters.getCurrentSlotBasedOnTimeWithFallback(); // Then assertThat(slot).isEqualTo(105366319L); @@ -68,7 +74,7 @@ void shouldThrowException_WhenTimeIsBeforeShellyStartTime() { when(clock.getZone()).thenReturn(zoneOffset); // When / Then - assertThatThrownBy(() -> offlineSlotService.getCurrentSlotBasedOnTime()) + assertThatThrownBy(() -> offlineSlotServiceWithConverters.getCurrentSlotBasedOnTimeWithFallback()) .isInstanceOf(ApiException.class); } @@ -80,11 +86,37 @@ void shouldVerifyCardanoConvertersSlotMethodIsCalled() { when(clock.getZone()).thenReturn(zoneOffset); // When - long slot = offlineSlotService.getCurrentSlotBasedOnTime(); + long slot = offlineSlotServiceWithConverters.getCurrentSlotBasedOnTimeWithFallback(); // Then assertThat(slot).isEqualTo(105366319L); verify(cardanoConverters, atLeastOnce()).time(); } + @Test + void shouldVerifyCardanoConvertersSlotMethodIsCalledWithoutFallback() { + // Given + LocalDateTime now = LocalDateTime.of(2023, 10, 10, 10, 10, 10); + when(clock.instant()).thenReturn(now.toInstant(zoneOffset)); + when(clock.getZone()).thenReturn(zoneOffset); + + // When + Optional slotM = offlineSlotServiceWithConverters.getCurrentSlotBasedOnTime(); + + // Then + assertThat(slotM).isPresent(); + assertThat(slotM.orElseThrow()).isEqualTo(105366319L); + verify(cardanoConverters, atLeastOnce()).time(); + } + + @Test + public void cardanoConvertersNotAvailableNoCurrentSlot() { + assertThat(offlineSlotServiceWithoutConverters.getCurrentSlotBasedOnTime()).isNotPresent(); + } + + @Test + public void cardanoConvertersNotAvailableCurrentSlotFallback() { + assertThat(offlineSlotServiceWithoutConverters.getCurrentSlotBasedOnTimeWithFallback()).isEqualTo(shelleyStartSlot); + } + } diff --git a/api/src/test/resources/config/application-test-integration.yaml b/api/src/test/resources/config/application-test-integration.yaml index fbd047230..836d5c0dd 100644 --- a/api/src/test/resources/config/application-test-integration.yaml +++ b/api/src/test/resources/config/application-test-integration.yaml @@ -43,7 +43,7 @@ cardano: GENESIS_SHELLEY_PATH: ${GENESIS_SHELLEY_PATH:../config/devkit/shelley-genesis.json} GENESIS_ALONZO_PATH: ${GENESIS_ALONZO_PATH:../config/devkit/alonzo-genesis.json} GENESIS_CONWAY_PATH: ${GENESIS_CONWAY_PATH:../config/devkit/conway-genesis.json} - CARDANO_NODE_VERSION: ${CARDANO_NODE_VERSION:10.1.2} + CARDANO_NODE_VERSION: ${CARDANO_NODE_VERSION:10.3.1} CARDANO_NODE_SUBMIT_HOST: ${CARDANO_NODE_SUBMIT_HOST:localhost} NODE_SUBMIT_API_PORT: ${NODE_SUBMIT_API_PORT:8090} CARDANO_NODE_SOCKET_PATH: ${CARDANO_NODE_SOCKET_PATH:""} @@ -53,10 +53,9 @@ cardano: DEVKIT_PORT: ${DEVKIT_PORT:3333} SEARCH_PAGE_SIZE: ${SEARCH_PAGE_SIZE:10} OFFLINE_MODE: ${OFFLINE_MODE:false} - #The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. - PRUNING_SAFE_BLOCKS: ${PRUNING_SAFE_BLOCKS} - #Run transaction pruning every 600 seconds or 10 minutes. - PRUNING_INTERVAL: ${PRUNING_INTERVAL} + SYNC_GRACE_SLOTS_COUNT: ${SYNC_GRACE_SLOTS_COUNT:100} + BLOCK_FETCH_TIMEOUT_SECS: ${BLOCK_FETCH_TIMEOUT_SECS:5} + REMOVE_SPENT_UTXOS: ${REMOVE_SPENT_UTXOS:false} YACI_HTTP_BASE_URL: http://localhost:9095/api/v1 HTTP_CONNECT_TIMEOUT_SECONDS: 5 diff --git a/config/mainnet/topology.json b/config/mainnet/topology.json index 0433be94d..e0d29cc65 100644 --- a/config/mainnet/topology.json +++ b/config/mainnet/topology.json @@ -28,4 +28,4 @@ } ], "useLedgerAfterSlot": 148350000 -} +} \ No newline at end of file diff --git a/config/preprod/topology.json b/config/preprod/topology.json index fc0fe0305..0a47930da 100644 --- a/config/preprod/topology.json +++ b/config/preprod/topology.json @@ -20,4 +20,4 @@ } ], "useLedgerAfterSlot": 83894000 -} +} \ No newline at end of file diff --git a/docker-compose-api.yaml b/docker-compose-api.yaml index 30b2c0bd3..9f87308e7 100644 --- a/docker-compose-api.yaml +++ b/docker-compose-api.yaml @@ -32,6 +32,9 @@ services: HTTP_CONNECT_TIMEOUT_SECONDS: ${HTTP_CONNECT_TIMEOUT_SECONDS} HTTP_REQUEST_TIMEOUT_SECONDS: ${HTTP_REQUEST_TIMEOUT_SECONDS} + SYNC_GRACE_SLOTS_COUNT: ${SYNC_GRACE_SLOTS_COUNT} + REMOVE_SPENT_UTXOS: ${REMOVE_SPENT_UTXOS} + # DB performance tuning API_DB_POOL_MIN_COUNT: ${API_DB_POOL_MIN_COUNT} API_DB_POOL_MAX_COUNT: ${API_DB_POOL_MAX_COUNT} diff --git a/docker-compose-indexer.yaml b/docker-compose-indexer.yaml index 4cf59eab6..d6b13783e 100644 --- a/docker-compose-indexer.yaml +++ b/docker-compose-indexer.yaml @@ -21,16 +21,12 @@ services: GENESIS_BYRON_PATH: ${GENESIS_BYRON_PATH} GENESIS_ALONZO_PATH: ${GENESIS_ALONZO_PATH} GENESIS_CONWAY_PATH: ${GENESIS_CONWAY_PATH} - PRUNING_ENABLED: ${PRUNING_ENABLED} + REMOVE_SPENT_UTXOS: ${REMOVE_SPENT_UTXOS} CARDANO_NODE_SOCKET_PATH: ${CARDANO_NODE_SOCKET_PATH} HOST_N2C_SOCAT_HOST: ${HOST_N2C_SOCAT_HOST} HOST_N2C_SOCAT_PORT: ${HOST_N2C_SOCAT_PORT} SEARCH_PAGE_SIZE: ${SEARCH_PAGE_SIZE} - #The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours. - PRUNING_INTERVAL: ${PRUNING_INTERVAL} - #Run transaction pruning every 600 seconds or 10 minutes. - PRUNING_SAFE_BLOCKS: ${PRUNING_SAFE_BLOCKS} - + REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT: ${REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT} volumes: - ${CARDANO_CONFIG}:/config - ${CARDANO_NODE_DIR}:${CARDANO_NODE_DIR} @@ -42,42 +38,45 @@ services: condition: service_healthy db: - image: ${DB_IMAGE_NAME}:${DB_IMAGE_TAG} + image: cardanofoundation/cardano-rosetta-java-postgres:${PG_VERSION_TAG} + build: + context: ./ + dockerfile: ./docker/dockerfiles/postgres/Dockerfile shm_size: 4g ports: - ${DB_PORT}:${DB_PORT} - command: [ - # "postgres", - "-p", "${DB_PORT}", - "-c", "max_connections=${DB_POSTGRES_MAX_CONNECTIONS}", - "-c", "shared_buffers=${DB_POSTGRES_SHARED_BUFFERS}", - "-c", "effective_cache_size=${DB_POSTGRES_EFFECTIVE_CACHE_SIZE}", - "-c", "work_mem=${DB_POSTGRES_WORK_MEM}", - "-c", "maintenance_work_mem=${DB_POSTGRES_MAINTENANCE_WORK_MEM}", - "-c", "wal_buffers=${DB_POSTGRES_WAL_BUFFERS}", - "-c", "checkpoint_completion_target=${DB_POSTGRES_CHECKPOINT_COMPLETION_TARGET}", - "-c", "random_page_cost=${DB_POSTGRES_RANDOM_PAGE_COST}", + environment: + DB_SECRET: ${DB_SECRET} + DB_USER: ${DB_USER} + DB_NAME: ${DB_NAME} + DB_PORT: ${DB_PORT} + NETWORK: ${NETWORK} + PGPASSWORD: postgres + + DB_POSTGRES_MAX_CONNECTIONS: ${DB_POSTGRES_MAX_CONNECTIONS} + DB_POSTGRES_SHARED_BUFFERS: ${DB_POSTGRES_SHARED_BUFFERS} + DB_POSTGRES_EFFECTIVE_CACHE_SIZE: ${DB_POSTGRES_EFFECTIVE_CACHE_SIZE} + DB_POSTGRES_WORK_MEM: ${DB_POSTGRES_WORK_MEM} + DB_POSTGRES_MAINTENANCE_WORK_MEM: ${DB_POSTGRES_MAINTENANCE_WORK_MEM} + DB_POSTGRES_WAL_BUFFERS: ${DB_POSTGRES_WAL_BUFFERS} + DB_POSTGRES_CHECKPOINT_COMPLETION_TARGET: ${DB_POSTGRES_CHECKPOINT_COMPLETION_TARGET} + DB_POSTGRES_RANDOM_PAGE_COST: ${DB_POSTGRES_RANDOM_PAGE_COST} # Additional settings for advanced PostgreSQL tuning - "-c", "effective_io_concurrency=${DB_POSTGRES_EFFECTIVE_IO_CONCURRENCY}", - "-c", "parallel_tuple_cost=${DB_POSTGRES_PARALLEL_TUPLE_COST}", - "-c", "parallel_setup_cost=${DB_POSTGRES_PARALLEL_SETUP_COST}", - "-c", "max_parallel_workers_per_gather=${DB_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER}", - "-c", "max_parallel_workers=${DB_POSTGRES_MAX_PARALLEL_WORKERS}", - "-c", "seq_page_cost=${DB_POSTGRES_SEQ_PAGE_COST}", - "-c", "jit=${DB_POSTGRES_JIT}", - "-c", "bgwriter_lru_maxpages=${DB_POSTGRES_BGWRITER_LRU_MAXPAGES}", - "-c", "bgwriter_delay=${DB_POSTGRES_BGWRITER_DELAY}" - ] - environment: - POSTGRES_PASSWORD: ${DB_SECRET} - POSTGRES_USER: ${DB_USER} - POSTGRES_DB: ${DB_NAME} + DB_POSTGRES_EFFECTIVE_IO_CONCURRENCY: ${DB_POSTGRES_EFFECTIVE_IO_CONCURRENCY} + DB_POSTGRES_PARALLEL_TUPLE_COST: ${DB_POSTGRES_PARALLEL_TUPLE_COST} + DB_POSTGRES_PARALLEL_SETUP_COST: ${DB_POSTGRES_PARALLEL_SETUP_COST} + DB_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER: ${DB_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER} + DB_POSTGRES_MAX_PARALLEL_WORKERS: ${DB_POSTGRES_MAX_PARALLEL_WORKERS} + DB_POSTGRES_SEQ_PAGE_COST: ${DB_POSTGRES_SEQ_PAGE_COST} + DB_POSTGRES_JIT: ${DB_POSTGRES_JIT} + DB_POSTGRES_BGWRITER_LRU_MAXPAGES: ${DB_POSTGRES_BGWRITER_LRU_MAXPAGES} + DB_POSTGRES_BGWRITER_DELAY: ${DB_POSTGRES_BGWRITER_DELAY} restart: on-failure volumes: - ${DB_PATH}:/var/lib/postgresql/data healthcheck: - test: [ "CMD-SHELL", "pg_isready -U ${DB_USER} -d ${DB_NAME} -p ${DB_PORT} -h localhost"] + test: [ "CMD-SHELL", "PGPASSWORD=${DB_SECRET} pg_isready -U ${DB_USER} -d ${DB_NAME} -p ${DB_PORT} -h localhost"] interval: 10s timeout: 3s retries: 10 diff --git a/docker-compose-node.yaml b/docker-compose-node.yaml index 56029f002..383453def 100644 --- a/docker-compose-node.yaml +++ b/docker-compose-node.yaml @@ -1,49 +1,67 @@ version: '3.8' services: mithril: + image: cardanofoundation/cardano-rosetta-java-mithril:${MITHRIL_VERSION} build: - context: ./docker/dockerfiles/mithril + context: ./ + dockerfile: ./docker/dockerfiles/mithril/Dockerfile + args: + MITHRIL_VERSION: ${MITHRIL_VERSION} environment: - NETWORK=${NETWORK} - MITHRIL_SYNC=${MITHRIL_SYNC} - SNAPSHOT_DIGEST=${SNAPSHOT_DIGEST} - AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT} - GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY} + - ANCILLARY_VERIFICATION_KEY=${ANCILLARY_VERIFICATION_KEY} volumes: - ${CARDANO_NODE_DIR}:/node cardano-node: - image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_VERSION} + image: cardanofoundation/cardano-rosetta-java-cardano-node:${CARDANO_NODE_VERSION} + build: + context: ./ + dockerfile: ./docker/dockerfiles/node/Dockerfile + args: + CARDANO_NODE_VERSION: ${CARDANO_NODE_VERSION} environment: - - NETWORK=${NETWORK} + - CARDANO_NODE_SOCKET_PATH=${CARDANO_NODE_SOCKET_PATH} + - CARDANO_NODE_PORT=${CARDANO_NODE_PORT} + - CARDANO_NODE_DB=${CARDANO_NODE_DB} volumes: - - ${CARDANO_NODE_DIR}:/node/ - - ${CARDANO_NODE_DB}:/node/db + - ${CARDANO_NODE_DIR}:${CARDANO_NODE_DIR} + - ${CARDANO_NODE_DB}:${CARDANO_NODE_DB} - ${CARDANO_CONFIG}:/config restart: unless-stopped ports: - ${CARDANO_NODE_PORT}:${CARDANO_NODE_PORT} - entrypoint: cardano-node run --database-path /node/db --port ${CARDANO_NODE_PORT} --socket-path /node/node.socket --topology /config/topology.json --config /config/config.json + entrypoint: ["/sbin/entrypoint.sh", "cardano-node"] depends_on: mithril: condition: service_completed_successfully + cardano-submit-api: - image: ghcr.io/intersectmbo/cardano-submit-api:${CARDANO_NODE_VERSION} + image: cardanofoundation/cardano-rosetta-java-cardano-node:${CARDANO_NODE_VERSION} + build: + context: ./ + dockerfile: ./docker/dockerfiles/node/Dockerfile + args: + CARDANO_NODE_VERSION: ${CARDANO_NODE_VERSION} environment: - NETWORK=${NETWORK} - depends_on: - - cardano-node + - PROTOCOL_MAGIC=${PROTOCOL_MAGIC} + - CARDANO_NODE_SOCKET_PATH=${CARDANO_NODE_SOCKET_PATH} + - NODE_SUBMIT_API_PORT=${NODE_SUBMIT_API_PORT} volumes: - - ${CARDANO_NODE_DIR}:/node-ipc + - ${CARDANO_NODE_DIR}:${CARDANO_NODE_DIR} + - ${CARDANO_CONFIG}:/config + restart: unless-stopped ports: - - ${NODE_SUBMIT_API_PORT}:8090 - restart: on-failure - logging: - driver: "json-file" - options: - max-size: "200k" - max-file: "10" - + - ${NODE_SUBMIT_API_PORT}:${NODE_SUBMIT_API_PORT} + entrypoint: ["/sbin/entrypoint.sh", "cardano-submit-api"] + depends_on: + mithril: + condition: service_completed_successfully networks: default: diff --git a/docker/.env.dockerfile b/docker/.env.dockerfile index 9e5cbc286..52025ceeb 100644 --- a/docker/.env.dockerfile +++ b/docker/.env.dockerfile @@ -7,9 +7,9 @@ API_SPRING_PROFILES_ACTIVE=online ## Main variables LOG=INFO NETWORK=mainnet -# mainnet, preprod, preview, sanchonet, devkit +# mainnet, preprod, preview, devkit PROTOCOL_MAGIC=764824073 -# mainnet 764824073, preprod 1, preview 2, sanchonet 4, devkit 42 +# mainnet 764824073, preprod 1, preview 2, devkit 42 # Node synchronization SYNC=true @@ -52,9 +52,10 @@ PRINT_EXCEPTION=true ## Yaci Indexer env YACI_SPRING_PROFILES=postgres,n2c-socket -PRUNING_ENABLED=false -PRUNING_INTERVAL=600 -PRUNING_SAFE_BLOCKS=2160 +REMOVE_SPENT_UTXOS=false + +REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160 + # database profiles: h2, h2-testdata, postgres MEMPOOL_ENABLED=false # Haven't implemented yet @@ -74,3 +75,5 @@ API_DB_POOL_CONNECTION_TIMEOUT_MS=100000 API_DB_KEEP_ALIVE_MS=60000 API_DB_LEAK_CONNECTIONS_WARNING_MS=60000 API_DB_MONITOR_PERFORMANCE=false + +SYNC_GRACE_SLOTS_COUNT=100 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index e72248baf..457675147 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 AS cardano-builder +FROM ubuntu:24.04 AS cardano-builder SHELL ["/bin/bash", "-c"] @@ -19,28 +19,30 @@ RUN bash -c "curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org ENV PATH=/root/.local/bin:/root/.ghcup/bin:/root/.cabal/bin:${PATH} -# Install cabal -ARG CABAL_VERSION=3.12.1.0 +ARG BLST_VERSION=0.3.11 +ARG LIBSODIUM_VERSION=dbb48cc +ARG SECP256K1_VERSION=ac83be33 + +ARG CABAL_VERSION=${CABAL_VERSION:-3.12.1.0} +ARG GHC_VERSION=${GHC_VERSION:-9.6.7} + +# Cardano node version +ARG CARDANO_NODE_VERSION=${CARDANO_NODE_VERSION:-10.3.1} + +ARG MITHRIL_VERSION=${MITHRIL_VERSION:-2517.1} RUN bash -c "ghcup install cabal ${CABAL_VERSION}" RUN bash -c "ghcup set cabal ${CABAL_VERSION}" -# Install GHC -ARG GHC_VERSION=9.6.7 - RUN bash -c "ghcup install ghc ${GHC_VERSION}" RUN bash -c "ghcup set ghc ${GHC_VERSION}" - -# Mithril setup -ARG MITHRIL_VERSION=2517.1 -# Install dependencies - +# Install mithril RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y \ && export PATH="$HOME/.cargo/bin:$PATH" \ && apt update --fix-missing \ && apt install -y --no-install-recommends \ - build-essential m4 libssl-dev docker jq git \ + build-essential m4 libssl-dev docker.io jq git \ && rustup update stable \ && apt-get clean @@ -53,58 +55,85 @@ RUN git clone https://github.com/input-output-hk/mithril.git \ && mkdir -p /root/.local/bin \ && cp mithril-client /root/.local/bin/ -# Cardano node version -ARG CARDANO_NODE_VERSION=10.3.1 +WORKDIR /usr/local/src # Install sodium -RUN export IOHKNIX_VERSION=$(curl https://raw.githubusercontent.com/IntersectMBO/cardano-node/${CARDANO_NODE_VERSION}/flake.lock | jq -r '.nodes.iohkNix.locked.rev') \ - && echo "iohk-nix version: $IOHKNIX_VERSION" \ - && export SODIUM_VERSION=$(curl https://raw.githubusercontent.com/input-output-hk/iohk-nix/${IOHKNIX_VERSION}/flake.lock | jq -r '.nodes.sodium.original.rev') \ - && echo "Using sodium version: $SODIUM_VERSION" \ - && git clone https://github.com/intersectmbo/libsodium \ - && cd libsodium \ - && git checkout ${SODIUM_VERSION} \ - && ./autogen.sh \ - && ./configure \ - && make \ - && make check \ - && make install +RUN git clone --branch master https://github.com/IntersectMBO/libsodium.git \ + && cd libsodium \ + && git checkout ${LIBSODIUM_VERSION} \ + && \ + for i in $(seq 1 10); do \ + echo "Attempt $i/10: building libsodium…"; \ + if ./autogen.sh && ./configure && make && make check && make install; then \ + echo "Success on attempt $i"; \ + break; \ + elif [ "$i" -eq 10 ]; then \ + echo "All 10 attempts failed, aborting" >&2; \ + exit 1; \ + else \ + echo "Build failed, retrying in 10s…"; \ + sleep 10; \ + fi; \ + done # Install secp256k1 -RUN export IOHKNIX_VERSION=$(curl https://raw.githubusercontent.com/IntersectMBO/cardano-node/${CARDANO_NODE_VERSION}/flake.lock | jq -r '.nodes.iohkNix.locked.rev') \ - && echo "iohk-nix version: ${IOHKNIX_VERSION}" \ - && export SECP256K1_VERSION=$(curl https://raw.githubusercontent.com/input-output-hk/iohk-nix/${IOHKNIX_VERSION}/flake.lock | jq -r '.nodes.secp256k1.original.ref') \ - && echo "Using secp256k1 version:${SECP256K1_VERSION}" \ - && git clone --depth 1 --branch ${SECP256K1_VERSION} https://github.com/bitcoin-core/secp256k1 \ - && cd secp256k1 \ - && ./autogen.sh \ - && ./configure --enable-module-schnorrsig --enable-experimental \ - && make \ - && make check \ - && make install +RUN git clone --branch master https://github.com/bitcoin-core/secp256k1.git \ + && cd secp256k1 \ + && git checkout ${SECP256K1_VERSION} \ + && \ + for i in $(seq 1 10); do \ + echo "Attempt $i/10: building secp256k1…"; \ + if ./autogen.sh \ + && ./configure --prefix=/usr --enable-module-schnorrsig --enable-experimental \ + && make \ + && make check \ + && make install; then \ + echo "secp256k1 built successfully on attempt $i"; \ + break; \ + elif [ "$i" -eq 10 ]; then \ + echo "All 10 attempts failed for secp256k1, aborting." >&2; \ + exit 1; \ + else \ + echo "Build failed on attempt $i — retrying in 10s…"; \ + sleep 10; \ + fi; \ + done # Install blst -RUN export BLST_VERSION=$(curl https://raw.githubusercontent.com/input-output-hk/iohk-nix/master/flake.lock | jq -r '.nodes.blst.original.ref') \ - && git clone --depth 1 --branch ${BLST_VERSION} https://github.com/supranational/blst \ - && cd blst \ - && ./build.sh \ - && echo "prefix=/usr/local" >> libblst.pc \ - && echo "exec_prefix=\${prefix}" >> libblst.pc \ - && echo "libdir=\${exec_prefix}/lib" >> libblst.pc \ - && echo "includedir=\${prefix}/include" >> libblst.pc \ - && echo "" >> libblst.pc \ - && echo "Name: libblst" >> libblst.pc \ - && echo "Description: Multilingual BLS12-381 signature library" >> libblst.pc \ - && echo "URL: https://github.com/supranational/blst" >> libblst.pc \ - && echo "Version: ${BLST_VERSION#v}" >> libblst.pc \ - && echo "Cflags: -I\${includedir}" >> libblst.pc \ - && echo "Libs: -L\${libdir} -lblst" >> libblst.pc \ - && cp libblst.pc /usr/local/lib/pkgconfig/ \ - && cp bindings/blst_aux.h bindings/blst.h bindings/blst.hpp /usr/local/include/ \ - && cp libblst.a /usr/local/lib \ - && bash -c "chmod u=rw,go=r /usr/local/{lib/{libblst.a,pkgconfig/libblst.pc},include/{blst.{h,hpp},blst_aux.h}}" - -# Install node +RUN git clone --branch master https://github.com/supranational/blst.git \ + && cd blst \ + && git checkout v${BLST_VERSION} \ + && \ + for i in $(seq 1 10); do \ + echo "Attempt $i/10: building blst…"; \ + if ./build.sh \ + && echo "prefix=/usr/local" >> libblst.pc \ + && echo "exec_prefix=\${prefix}" >> libblst.pc \ + && echo "libdir=\${exec_prefix}/lib" >> libblst.pc \ + && echo "includedir=\${prefix}/include" >> libblst.pc \ + && echo "" >> libblst.pc \ + && echo "Name: libblst" >> libblst.pc \ + && echo "Description: Multilingual BLS12-381 signature library" >> libblst.pc \ + && echo "URL: https://github.com/supranational/blst" >> libblst.pc \ + && echo "Version: ${BLST_VERSION#v}" >> libblst.pc \ + && echo "Cflags: -I\${includedir}" >> libblst.pc \ + && echo "Libs: -L\${libdir} -lblst" >> libblst.pc \ + && cp libblst.pc /usr/local/lib/pkgconfig/ \ + && cp bindings/blst_aux.h bindings/blst.h bindings/blst.hpp /usr/local/include/ \ + && cp libblst.a /usr/local/lib \ + && bash -c "chmod u=rw,go=r /usr/local/{lib/{libblst.a,pkgconfig/libblst.pc},include/{blst.{h,hpp},blst_aux.h}}"; then \ + echo "blst built successfully on attempt $i"; \ + break; \ + elif [ "$i" -eq 10 ]; then \ + echo "All 10 attempts failed for blst, aborting." >&2; \ + exit 1; \ + else \ + echo "blst build failed on attempt $i — retrying in 10s…"; \ + sleep 10; \ + fi; \ + done + +# Install cardano-node RUN git clone https://github.com/intersectmbo/cardano-node.git \ && cd cardano-node \ && git checkout tags/${CARDANO_NODE_VERSION} \ @@ -123,7 +152,7 @@ RUN git clone https://github.com/intersectmbo/cardano-node.git \ && ln -s /opt/homebrew/opt/openssl@3/lib /usr/local/opt/openssl/lib \ && ln -s /opt/homebrew/opt/openssl@3/include /usr/local/opt/openssl/include -WORKDIR /root/src/cardano-node +WORKDIR /usr/local/src/cardano-node RUN bash -c "cabal update" RUN bash -c "cabal build all" @@ -137,14 +166,26 @@ RUN mkdir -p /root/.local/bin \ # Compile java applications -FROM ubuntu:22.04 AS java-builder - -WORKDIR /root/app +FROM ubuntu:24.04 AS java-builder +# Install dependencies RUN apt update --fix-missing \ - && apt install -y --no-install-recommends openjdk-21-jdk maven \ + && apt install -y --no-install-recommends \ + maven curl \ && apt-get clean +# Download and setup JDK 24.0.1 +RUN mkdir -p /opt/java \ + && curl -L https://download.java.net/java/GA/jdk24.0.1/24a58e0e276943138bf3e963e6291ac2/9/GPL/openjdk-24.0.1_linux-x64_bin.tar.gz -o /opt/jdk.tar.gz \ + && tar -xzf /opt/jdk.tar.gz -C /opt/java \ + && rm /opt/jdk.tar.gz + +# Set JAVA_HOME and update PATH +ENV JAVA_HOME=/opt/java/jdk-24.0.1 +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +WORKDIR /root/app + COPY ./pom.xml /root/app/pom.xml COPY ./api /root/app/api @@ -154,55 +195,70 @@ COPY ./.git /root/app/.git RUN --mount=type=cache,target=/root/.m2 mvn -U clean package -DskipTests -# Main -FROM ubuntu:22.04 +# Build postgres +FROM ubuntu:24.04 AS postgres-builder -WORKDIR / +RUN apt update && apt install -y \ + build-essential \ + libreadline-dev \ + zlib1g-dev \ + flex \ + bison \ + wget \ + git \ + ca-certificates \ + sudo \ + locales \ + && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ + && locale-gen \ + && update-locale LANG=en_US.UTF-8 \ + && apt clean -# Install postgres -ARG PG_VERSION=14 -ARG PG_VERSION_TEMP=$PG_VERSION -ENV PG_VERSION=$PG_VERSION_TEMP +ENV PG_VERSION_TAG=REL_14_11 -RUN apt update --fix-missing \ - && DEBIAN_FRONTEND=noninteractive apt install -y wget sudo gnupg \ - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ - && echo 'deb http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main' >> /etc/apt/sources.list \ - && apt-get clean +WORKDIR /usr/src -RUN apt update --fix-missing \ - && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ - postgresql-${PG_VERSION} \ - && apt-get clean +RUN git clone --branch $PG_VERSION_TAG https://github.com/postgres/postgres.git \ + && cd postgres \ + && ./configure --prefix=/usr/local/pgsql \ + && make -j$(nproc) \ + && make install -# Allow remote connections -RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/${PG_VERSION}/main/pg_hba.conf -# Listen on all interfaces -RUN echo "listen_addresses='*'" >> /etc/postgresql/${PG_VERSION}/main/postgresql.conf -# Change data directory path -RUN sed -i "s|var/lib/postgresql/${PG_VERSION}/main|node/postgres|" /etc/postgresql/${PG_VERSION}/main/postgresql.conf -RUN mkdir -p /node/postgres -RUN rm -rf /var/lib/postgresql/${PG_VERSION}/main -# Install jdk 21 -RUN apt install -y --no-install-recommends openjdk-21-jdk jq bc sudo curl \ - && apt clean +# Main +FROM ubuntu:24.04 + +RUN apt update && apt install -y --no-install-recommends jq bc sudo curl \ + libreadline-dev \ + sudo \ + locales \ + && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ + && locale-gen \ + && update-locale LANG=en_US.UTF-8 \ + && apt clean + +WORKDIR / # Copy cardano node COPY --from=cardano-builder /usr/local/lib /usr/local/lib COPY --from=cardano-builder /root/.local/bin/cardano-* /usr/local/bin/ COPY --from=cardano-builder /root/.local/bin/mithril-client /usr/local/bin/ -COPY --from=cardano-builder /root/src/cardano-node/cardano-submit-api/config/tx-submit-mainnet-config.yaml /cardano-submit-api-config/cardano-submit-api.yaml +COPY --from=cardano-builder /usr/local/src/cardano-node/cardano-submit-api/config/tx-submit-mainnet-config.yaml /cardano-submit-api-config/cardano-submit-api.yaml +ENV NODE_DATA="/node/db" +RUN mkdir -p $NODE_DATA +# Link libsodium library +COPY --from=cardano-builder --chown=root:root /usr/local/lib/libsodium.so /usr/local/lib RUN ln -snf /usr/local/lib/libsodium.so /usr/local/lib/libsodium.so.23 \ - || true && \ - ln -snf /usr/local/lib/libsodium.so /usr/local/lib/libsodium.so.23.3.0 \ - || true + && ln -snf /usr/local/lib/libsodium.so /usr/local/lib/libsodium.so.23.3.0 +RUN ldconfig -RUN ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.1 \ - || true && \ - ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.1.0.1 \ - || true +# Link secp256k1 library +COPY --from=cardano-builder --chown=root:root /usr/local/src/secp256k1/.libs/libsecp256k1.so /usr/local/lib +RUN ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.0 \ + && ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.1 \ + && ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.1.0.1 +RUN ldconfig ENV LD_LIBRARY_PATH=/usr/local/lib ENV PATH=/usr/local/lib/:$PATH @@ -210,12 +266,21 @@ ENV PATH=/usr/local/lib/:$PATH COPY ./config /networks RUN mkdir /config -RUN mkdir -p /node/db -RUN mkdir -p /node/postgres - # Copy jars COPY --from=java-builder /root/app/api/target/*.jar /api/app.jar COPY --from=java-builder /root/app/yaci-indexer/target/*.jar /yaci-indexer/app.jar +# Copy JDK +COPY --from=java-builder /opt/java/jdk-24.0.1 /opt/java/jdk-24.0.1 +ENV JAVA_HOME=/opt/java/jdk-24.0.1 +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# Copy and configure postgres +COPY --from=postgres-builder /usr/local/pgsql /usr/local/pgsql +ENV PG_BIN="/usr/local/pgsql/bin" +ENV PG_DATA="/var/lib/postgresql/data" +ENV PATH="$PG_BIN:$PATH" +RUN mkdir -p $PG_DATA +RUN useradd -m -U -s /bin/bash postgres # Run RUN mkdir /logs diff --git a/docker/dockerfiles/mithril/Dockerfile b/docker/dockerfiles/mithril/Dockerfile index e36abd39e..e73ef33c9 100644 --- a/docker/dockerfiles/mithril/Dockerfile +++ b/docker/dockerfiles/mithril/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 AS cardano-builder +FROM ubuntu:24.04 AS cardano-builder SHELL ["/bin/bash", "-c"] @@ -16,7 +16,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y \ && export PATH="$HOME/.cargo/bin:$PATH" \ && apt update --fix-missing \ && apt install -y --no-install-recommends \ - build-essential m4 libssl-dev docker jq git pkg-config \ + build-essential m4 libssl-dev docker.io jq git pkg-config \ && apt-get clean RUN git clone https://github.com/input-output-hk/mithril.git \ @@ -28,12 +28,12 @@ RUN git clone https://github.com/input-output-hk/mithril.git \ && mkdir -p /root/.local/bin \ && cp mithril-client /root/.local/bin/ -FROM ubuntu:22.04 AS mithril-runner +FROM ubuntu:24.04 AS mithril-runner COPY --from=cardano-builder /root/.local/bin/mithril-client /usr/local/bin RUN apt update --fix-missing \ && apt install -y wget -COPY entrypoint.sh /sbin/entrypoint.sh +COPY ./docker/dockerfiles/mithril/entrypoint.sh /sbin/entrypoint.sh RUN chmod +x /sbin/entrypoint.sh -CMD ["/sbin/entrypoint.sh"] \ No newline at end of file +CMD ["/sbin/entrypoint.sh"] diff --git a/docker/dockerfiles/mithril/entrypoint.sh b/docker/dockerfiles/mithril/entrypoint.sh index 82fbb8ccf..344708852 100644 --- a/docker/dockerfiles/mithril/entrypoint.sh +++ b/docker/dockerfiles/mithril/entrypoint.sh @@ -7,27 +7,27 @@ download_mithril_snapshot() { mainnet) AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT:-https://aggregator.release-mainnet.api.mithril.network/aggregator} GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/release-mainnet/genesis.vkey)} + ANCILLARY_VERIFICATION_KEY=${ANCILLARY_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/release-mainnet/ancillary.vkey)} ;; preprod) AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT:-https://aggregator.release-preprod.api.mithril.network/aggregator} GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/release-preprod/genesis.vkey)} + ANCILLARY_VERIFICATION_KEY=${ANCILLARY_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/release-preprod/ancillary.vkey)} ;; preview) AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT:-https://aggregator.pre-release-preview.api.mithril.network/aggregator} GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/pre-release-preview/genesis.vkey)} - ;; - sanchonet) - AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT:-https://aggregator.testing-sanchonet.api.mithril.network/aggregator} - GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/testing-sanchonet/genesis.vkey)} + ANCILLARY_VERIFICATION_KEY=${ANCILLARY_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/pre-release-preview/ancillary.vkey)} ;; esac echo "Listing content of /node dir:" ls -la /node - mithril-client cardano-db download latest --download-dir /node & + mithril-client cardano-db download latest --include-ancillary --ancillary-verification-key $ANCILLARY_VERIFICATION_KEY --download-dir /node & MITHRIL_PID=$! wait $MITHRIL_PID echo "Done downloading Mithril Snapshot" } + echo $NETWORK if [ "${MITHRIL_SYNC}" == "true" ]; then download_mithril_snapshot diff --git a/docker/dockerfiles/node/Dockerfile b/docker/dockerfiles/node/Dockerfile index 764e641e3..e79f44281 100644 --- a/docker/dockerfiles/node/Dockerfile +++ b/docker/dockerfiles/node/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 AS cardano-builder +FROM ubuntu:24.04 AS cardano-builder WORKDIR /root/src @@ -9,87 +9,106 @@ RUN apt update --fix-missing \ g++ tmux git jq wget libncursesw5-dev libtool autoconf liblmdb-dev curl ca-certificates \ && apt-get clean - # Install ghcup ENV BOOTSTRAP_HASKELL_NONINTERACTIVE=1 RUN bash -c "curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh" ENV PATH=/root/.local/bin:/root/.ghcup/bin:/root/.cabal/bin:${PATH} -# Install cabal -ARG CABAL_VERSION=3.8.1.0 +ARG BLST_VERSION="0.3.11" +ARG LIBSODIUM_VERSION="dbb48cc" +ARG SECP256K1_VERSION="ac83be33" + +ARG CABAL_VERSION=${CABAL_VERSION:-3.12.1.0} +ARG GHC_VERSION=${GHC_VERSION:-9.6.7} + +ARG CARDANO_NODE_VERSION=${CARDANO_NODE_VERSION:-10.3.1} RUN bash -c "ghcup install cabal ${CABAL_VERSION}" RUN bash -c "ghcup set cabal ${CABAL_VERSION}" -# Install GHC -ARG GHC_VERSION=8.10.7 - RUN bash -c "ghcup install ghc ${GHC_VERSION}" RUN bash -c "ghcup set ghc ${GHC_VERSION}" -# Cardano node version -ARG CARDANO_NODE_VERSION=10.2.1 +WORKDIR /usr/local/src # Install sodium -RUN export IOHKNIX_VERSION=$(curl https://raw.githubusercontent.com/IntersectMBO/cardano-node/${CARDANO_NODE_VERSION}/flake.lock | jq -r '.nodes.iohkNix.locked.rev') \ - && echo "iohk-nix version: $IOHKNIX_VERSION" \ - && export SODIUM_VERSION=$(curl https://raw.githubusercontent.com/input-output-hk/iohk-nix/${IOHKNIX_VERSION}/flake.lock | jq -r '.nodes.sodium.original.rev') \ - && echo "Using sodium version: $SODIUM_VERSION" \ - && git clone https://github.com/intersectmbo/libsodium \ - && cd libsodium \ - && git checkout ${SODIUM_VERSION} \ - && ./autogen.sh \ - && ./configure \ - && make \ - && make check \ - && make install - -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH - -# Install libsodium -RUN apt install -y --no-install-recommends libsodium-dev \ - && apt-get clean +RUN git clone --branch master https://github.com/IntersectMBO/libsodium.git \ + && cd libsodium \ + && git checkout ${LIBSODIUM_VERSION} \ + && \ + for i in $(seq 1 10); do \ + echo "Attempt $i/10: building libsodium…"; \ + if ./autogen.sh && ./configure && make && make check && make install; then \ + echo "Success on attempt $i"; \ + break; \ + elif [ "$i" -eq 10 ]; then \ + echo "All 10 attempts failed, aborting" >&2; \ + exit 1; \ + else \ + echo "Build failed, retrying in 10s…"; \ + sleep 10; \ + fi; \ + done # Install secp256k1 -RUN export IOHKNIX_VERSION=$(curl https://raw.githubusercontent.com/IntersectMBO/cardano-node/${CARDANO_NODE_VERSION}/flake.lock | jq -r '.nodes.iohkNix.locked.rev') \ - && echo "iohk-nix version: ${IOHKNIX_VERSION}" \ - && export SECP256K1_VERSION=$(curl https://raw.githubusercontent.com/input-output-hk/iohk-nix/${IOHKNIX_VERSION}/flake.lock | jq -r '.nodes.secp256k1.original.ref') \ - && echo "Using secp256k1 version:${SECP256K1_VERSION}" \ - && git clone --depth 1 --branch ${SECP256K1_VERSION} https://github.com/bitcoin-core/secp256k1 \ - && cd secp256k1 \ - && ./autogen.sh \ - && ./configure --enable-module-schnorrsig --enable-experimental \ - && make \ - && make check \ - && make install +RUN git clone --branch master https://github.com/bitcoin-core/secp256k1.git \ + && cd secp256k1 \ + && git checkout ${SECP256K1_VERSION} \ + && \ + for i in $(seq 1 10); do \ + echo "Attempt $i/10: building secp256k1…"; \ + if ./autogen.sh \ + && ./configure --prefix=/usr --enable-module-schnorrsig --enable-experimental \ + && make \ + && make check \ + && make install; then \ + echo "secp256k1 built successfully on attempt $i"; \ + break; \ + elif [ "$i" -eq 10 ]; then \ + echo "All 10 attempts failed for secp256k1, aborting." >&2; \ + exit 1; \ + else \ + echo "Build failed on attempt $i — retrying in 10s…"; \ + sleep 10; \ + fi; \ + done # Install blst -RUN export BLST_VERSION=$(curl https://raw.githubusercontent.com/input-output-hk/iohk-nix/master/flake.lock | jq -r '.nodes.blst.original.ref') \ - && git clone --depth 1 --branch ${BLST_VERSION} https://github.com/supranational/blst \ - && cd blst \ - && ./build.sh \ - && echo "prefix=/usr/local" >> libblst.pc \ - && echo "exec_prefix=\${prefix}" >> libblst.pc \ - && echo "libdir=\${exec_prefix}/lib" >> libblst.pc \ - && echo "includedir=\${prefix}/include" >> libblst.pc \ - && echo "" >> libblst.pc \ - && echo "Name: libblst" >> libblst.pc \ - && echo "Description: Multilingual BLS12-381 signature library" >> libblst.pc \ - && echo "URL: https://github.com/supranational/blst" >> libblst.pc \ - && echo "Version: ${BLST_VERSION#v}" >> libblst.pc \ - && echo "Cflags: -I\${includedir}" >> libblst.pc \ - && echo "Libs: -L\${libdir} -lblst" >> libblst.pc \ - && cp libblst.pc /usr/local/lib/pkgconfig/ \ - && cp bindings/blst_aux.h bindings/blst.h bindings/blst.hpp /usr/local/include/ \ - && cp libblst.a /usr/local/lib \ - && bash -c "chmod u=rw,go=r /usr/local/{lib/{libblst.a,pkgconfig/libblst.pc},include/{blst.{h,hpp},blst_aux.h}}" - -RUN apt install -y --no-install-recommends libsecp256k1-dev \ - && apt-get clean - -# Install node +RUN git clone --branch master https://github.com/supranational/blst.git \ + && cd blst \ + && git checkout v${BLST_VERSION} \ + && \ + for i in $(seq 1 10); do \ + echo "Attempt $i/10: building blst…"; \ + if ./build.sh \ + && echo "prefix=/usr/local" >> libblst.pc \ + && echo "exec_prefix=\${prefix}" >> libblst.pc \ + && echo "libdir=\${exec_prefix}/lib" >> libblst.pc \ + && echo "includedir=\${prefix}/include" >> libblst.pc \ + && echo "" >> libblst.pc \ + && echo "Name: libblst" >> libblst.pc \ + && echo "Description: Multilingual BLS12-381 signature library" >> libblst.pc \ + && echo "URL: https://github.com/supranational/blst" >> libblst.pc \ + && echo "Version: ${BLST_VERSION#v}" >> libblst.pc \ + && echo "Cflags: -I\${includedir}" >> libblst.pc \ + && echo "Libs: -L\${libdir} -lblst" >> libblst.pc \ + && cp libblst.pc /usr/local/lib/pkgconfig/ \ + && cp bindings/blst_aux.h bindings/blst.h bindings/blst.hpp /usr/local/include/ \ + && cp libblst.a /usr/local/lib \ + && bash -c "chmod u=rw,go=r /usr/local/{lib/{libblst.a,pkgconfig/libblst.pc},include/{blst.{h,hpp},blst_aux.h}}"; then \ + echo "blst built successfully on attempt $i"; \ + break; \ + elif [ "$i" -eq 10 ]; then \ + echo "All 10 attempts failed for blst, aborting." >&2; \ + exit 1; \ + else \ + echo "blst build failed on attempt $i — retrying in 10s…"; \ + sleep 10; \ + fi; \ + done + +# Install cardano-node RUN git clone https://github.com/intersectmbo/cardano-node.git \ && cd cardano-node \ && git checkout tags/${CARDANO_NODE_VERSION} \ @@ -108,7 +127,7 @@ RUN git clone https://github.com/intersectmbo/cardano-node.git \ && ln -s /opt/homebrew/opt/openssl@3/lib /usr/local/opt/openssl/lib \ && ln -s /opt/homebrew/opt/openssl@3/include /usr/local/opt/openssl/include -WORKDIR /root/src/cardano-node +WORKDIR /usr/local/src/cardano-node RUN bash -c "cabal update" RUN bash -c "cabal build all" @@ -121,5 +140,30 @@ RUN mkdir -p /root/.local/bin \ && cp -p "$(./scripts/bin-path.sh cardano-submit-api)" /root/.local/bin/ -FROM ubuntu:22.04 AS node-runner +FROM ubuntu:24.04 AS node-runner +COPY --from=cardano-builder /usr/local/lib /usr/local/lib COPY --from=cardano-builder /root/.local/bin/cardano-* /usr/local/bin/ +COPY --from=cardano-builder /usr/local/src/cardano-node/cardano-submit-api/config/tx-submit-mainnet-config.yaml /cardano-submit-api-config/cardano-submit-api.yaml + +# Link libsodium library +COPY --from=cardano-builder --chown=root:root /usr/local/lib/libsodium.so /usr/local/lib +RUN ln -snf /usr/local/lib/libsodium.so /usr/local/lib/libsodium.so.23 \ + && ln -snf /usr/local/lib/libsodium.so /usr/local/lib/libsodium.so.23.3.0 +RUN ldconfig + +# Link secp256k1 library +COPY --from=cardano-builder --chown=root:root /usr/local/src/secp256k1/.libs/libsecp256k1.so /usr/local/lib +RUN ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.0 \ + && ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.1 \ + && ln -snf /usr/local/lib/libsecp256k1.so /usr/local/lib/libsecp256k1.so.1.0.1 +RUN ldconfig + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH + +RUN mkdir /logs + +COPY ./docker/dockerfiles/node/entrypoint.sh /sbin/entrypoint.sh +RUN chmod +x /sbin/entrypoint.sh + +ENTRYPOINT ["/sbin/entrypoint.sh"] diff --git a/docker/dockerfiles/node/entrypoint.sh b/docker/dockerfiles/node/entrypoint.sh new file mode 100644 index 000000000..fd667628f --- /dev/null +++ b/docker/dockerfiles/node/entrypoint.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ "$NETWORK" == "mainnet" ]; then + NETWORK_STR="--mainnet" +else + NETWORK_STR="--testnet-magic $PROTOCOL_MAGIC" +fi + +cmd="$1"; shift +case "$cmd" in + cardano-node) + echo "Starting Cardano node..." + exec cardano-node run \ + --socket-path "$CARDANO_NODE_SOCKET_PATH" \ + --port "$CARDANO_NODE_PORT" \ + --database-path "$CARDANO_NODE_DB" \ + --config /config/config.json \ + --topology /config/topology.json + ;; + cardano-submit-api) + echo "Starting Cardano submit api..." + exec cardano-submit-api \ + --listen-address 0.0.0.0 \ + --socket-path "$CARDANO_NODE_SOCKET_PATH" \ + --port "$NODE_SUBMIT_API_PORT" \ + $NETWORK_STR \ + --config /cardano-submit-api-config/cardano-submit-api.yaml + ;; + *) + exit 1 + ;; +esac diff --git a/docker/dockerfiles/postgres/Dockerfile b/docker/dockerfiles/postgres/Dockerfile new file mode 100644 index 000000000..50465529c --- /dev/null +++ b/docker/dockerfiles/postgres/Dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:24.04 + +RUN apt update && apt install -y \ + build-essential \ + libreadline-dev \ + zlib1g-dev \ + flex \ + bison \ + wget \ + git \ + ca-certificates \ + sudo \ + locales \ + && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ + && locale-gen \ + && update-locale LANG=en_US.UTF-8 \ + && apt clean + +ENV PG_VERSION_TAG=REL_14_11 +ENV PG_DATA="/var/lib/postgresql/data" +ENV PG_BIN="/usr/local/pgsql/bin" +ENV PATH="$PG_BIN:$PATH" + +WORKDIR /usr/src + +RUN git clone --branch $PG_VERSION_TAG https://github.com/postgres/postgres.git \ + && cd postgres \ + && ./configure --prefix=/usr/local/pgsql \ + && make -j$(nproc) \ + && make install + +RUN useradd -m -U -s /bin/bash postgres + +RUN mkdir -p $PG_DATA + +VOLUME ["$PG_DATA"] + +COPY ./docker/dockerfiles/postgres/entrypoint.sh /sbin/entrypoint.sh +RUN chmod +x /sbin/entrypoint.sh + +WORKDIR / + +ENTRYPOINT ["/sbin/entrypoint.sh"] + diff --git a/docker/dockerfiles/postgres/entrypoint.sh b/docker/dockerfiles/postgres/entrypoint.sh new file mode 100644 index 000000000..bd731cba8 --- /dev/null +++ b/docker/dockerfiles/postgres/entrypoint.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +database_initialization() { + echo "$PGPASSWORD" > /tmp/password + echo "*:*:*:postgres:$PGPASSWORD" > /home/postgres/.pgpass + + chmod 600 /home/postgres/.pgpass + chown postgres:postgres /home/postgres/.pgpass + + if [ -z "$(ls -A "$PG_DATA")" ]; then + sudo -u postgres "$PG_BIN/initdb" --pgdata="$PG_DATA" --auth=md5 --auth-local=md5 --auth-host=md5 --username=postgres --pwfile=/tmp/password + else + echo "Database already initialized, skipping initdb." + fi + + rm -f /tmp/password +} + +configure_postgres() { + local config_file="$PG_DATA/postgresql.conf" + + # List of parameter names and their corresponding environment variables + declare -A param_vars=( + ["max_connections"]="DB_POSTGRES_MAX_CONNECTIONS" + ["shared_buffers"]="DB_POSTGRES_SHARED_BUFFERS" + ["effective_cache_size"]="DB_POSTGRES_EFFECTIVE_CACHE_SIZE" + ["work_mem"]="DB_POSTGRES_WORK_MEM" + ["maintenance_work_mem"]="DB_POSTGRES_MAINTENANCE_WORK_MEM" + ["wal_buffers"]="DB_POSTGRES_WAL_BUFFERS" + ["checkpoint_completion_target"]="DB_POSTGRES_CHECKPOINT_COMPLETION_TARGET" + ["random_page_cost"]="DB_POSTGRES_RANDOM_PAGE_COST" + ["effective_io_concurrency"]="DB_POSTGRES_EFFECTIVE_IO_CONCURRENCY" + ["parallel_tuple_cost"]="DB_POSTGRES_PARALLEL_TUPLE_COST" + ["parallel_setup_cost"]="DB_POSTGRES_PARALLEL_SETUP_COST" + ["max_parallel_workers_per_gather"]="DB_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER" + ["max_parallel_workers"]="DB_POSTGRES_MAX_PARALLEL_WORKERS" + ["seq_page_cost"]="DB_POSTGRES_SEQ_PAGE_COST" + ["jit"]="DB_POSTGRES_JIT" + ["bgwriter_lru_maxpages"]="DB_POSTGRES_BGWRITER_LRU_MAXPAGES" + ["bgwriter_delay"]="DB_POSTGRES_BGWRITER_DELAY" + ) + + # Check for missing required environment variables + local missing_vars=() + for param in "${!param_vars[@]}"; do + local env_var="${param_vars[$param]}" + if [ -z "${!env_var}" ]; then + missing_vars+=("$env_var") + fi + done + + # If there are missing variables, print an error and exit + if [ ${#missing_vars[@]} -gt 0 ]; then + echo "Error: The following required environment variables are missing or empty:" + for var in "${missing_vars[@]}"; do + echo " - $var" + done + echo "" + echo "Most likely, you are missing a hardware profile in your environment configuration." + echo "Make sure to pass an additional --env-file parameter when running the container." + echo "" + echo "Example Docker run command for an 'entry_level' hardware profile:" + echo "docker run --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:latest" + echo "" + exit 1 + fi + + # Remove all commented-out lines for the parameters we are about to update + for param in "${!param_vars[@]}"; do + sed -i "/^#$param /d" "$config_file" + done + + # Update or add parameters + for param in "${!param_vars[@]}"; do + local env_var="${param_vars[$param]}" + local value="${!env_var}" + + # Try to replace existing parameter + sed -i "s/^$param = .*/$param = $value/" "$config_file" + + # Parameter not found, append it + if ! grep -q "^$param =" "$config_file"; then + echo "$param = $value" >> "$config_file" + fi + done + + if ! grep -q "^host all all 0.0.0.0/0 md5\$" "$PG_DATA/pg_hba.conf"; then + echo "host all all 0.0.0.0/0 md5" >> "$PG_DATA/pg_hba.conf" + fi + + if ! grep -q "^listen_addresses *= *'\*'\$" "$PG_DATA/postgresql.conf"; then + echo "listen_addresses='*'" >> "$PG_DATA/postgresql.conf" + fi + + echo "PostgreSQL configuration updated successfully!" +} + +start_postgres() { + sudo -u postgres "$PG_BIN/postgres" -D "$PG_DATA" -p "$DB_PORT" -c config_file="$PG_DATA/postgresql.conf" & + PG_PID=$! + + until "$PG_BIN/pg_isready" -U postgres > /dev/null; do sleep 1; done +} + +create_database_and_user() { + export DB_SCHEMA="$NETWORK" + + flag=true + while [ $(sudo -u postgres "$PG_BIN/psql" -U postgres -Atc "SELECT pg_is_in_recovery()";) == "t" ]; do + if $flag ; then + echo "Waiting for database recovery..." + flag=false + fi + sleep 1 + done + + if [[ -z $(sudo -u postgres "$PG_BIN/psql" -U postgres -Atc "SELECT 1 FROM pg_catalog.pg_user WHERE usename = '$DB_USER'";) ]]; then + echo "Creating database..." + sudo -u postgres "$PG_BIN/psql" -U postgres -c "CREATE ROLE \"$DB_USER\" with LOGIN CREATEDB PASSWORD '$DB_SECRET';" > /dev/null + fi + + if [[ -z $(sudo -u postgres "$PG_BIN/psql" -U postgres -Atc "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '$DB_NAME'";) ]]; then + echo "Creating user..." + sudo -u postgres "$PG_BIN/psql" -U postgres -c "CREATE DATABASE \"$DB_NAME\";" >/dev/null + sudo -u postgres "$PG_BIN/psql" -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$DB_NAME\" to \"$DB_USER\";" > /dev/null + fi + + echo "User configured" +} + +chown -R postgres:postgres "$PG_DATA" +chmod -R 0700 "$PG_DATA" + +echo "Initializing Database..." +database_initialization + +echo "Configuring the Database..." +configure_postgres + +echo "Starting Postgres..." +start_postgres + +echo "Creating database and user..." +create_database_and_user + +wait "$PG_PID" + diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b30a71236..11f5fa3ab 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,6 +9,7 @@ function clean_up() { trap clean_up SIGHUP SIGINT SIGTERM +# NODE FUNCTIONS show_progress() { message="$1"; percent="$2" done=$(bc <<< "scale=0; 40 * ${percent%.*} / 100" ) @@ -68,15 +69,25 @@ node_synchronization() { echo "Node synchronization: DONE" } +# POSTGRES FUNCTIONS database_initialization() { - echo "Starting database initialization..." - echo "postgres" >> /tmp/password - initdb_command="/usr/lib/postgresql/$PG_VERSION/bin/initdb --pgdata=/node/postgres --auth=md5 --auth-local=md5 --auth-host=md5 --username=postgres --pwfile=/tmp/password" - sudo -H -u postgres bash -c "$initdb_command" + echo "postgres" > /tmp/password + echo "*:*:*:postgres:postgres" > /home/postgres/.pgpass + + chmod 600 /home/postgres/.pgpass + chown postgres:postgres /home/postgres/.pgpass + + if [ -z "$(ls -A "$PG_DATA")" ]; then + sudo -u postgres "$PG_BIN/initdb" --pgdata="$PG_DATA" --auth=md5 --auth-local=md5 --auth-host=md5 --username=postgres --pwfile=/tmp/password + else + echo "Database already initialized, skipping initdb." + fi + + rm -f /tmp/password } configure_postgres() { - local config_file="/etc/postgresql/${PG_VERSION}/main/postgresql.conf" + local config_file="$PG_DATA/postgresql.conf" # List of parameter names and their corresponding environment variables declare -A param_vars=( @@ -143,14 +154,29 @@ configure_postgres() { fi done + if ! grep -q "^host all all 0.0.0.0/0 md5\$" "$PG_DATA/pg_hba.conf"; then + echo "host all all 0.0.0.0/0 md5" >> "$PG_DATA/pg_hba.conf" + fi + + if ! grep -q "^listen_addresses *= *'\*'\$" "$PG_DATA/postgresql.conf"; then + echo "listen_addresses='*'" >> "$PG_DATA/postgresql.conf" + fi + echo "PostgreSQL configuration updated successfully!" } +start_postgres() { + sudo -u postgres "$PG_BIN/postgres" -D "$PG_DATA" -p "$DB_PORT" -c config_file="$PG_DATA/postgresql.conf" > /logs/postgres.log 2>&1 & + POSTGRES_PID=$! + + until "$PG_BIN/pg_isready" -U postgres > /dev/null; do sleep 1; done +} + create_database_and_user() { - export DB_SCHEMA=$NETWORK + export DB_SCHEMA="$NETWORK" flag=true - while [ $(sudo -u postgres psql -U postgres -Atc "SELECT pg_is_in_recovery()";) == "t" ]; do + while [ $(sudo -u postgres "$PG_BIN/psql" -U postgres -Atc "SELECT pg_is_in_recovery()";) == "t" ]; do if $flag ; then echo "Waiting for database recovery..." flag=false @@ -158,18 +184,21 @@ create_database_and_user() { sleep 1 done - if [[ -z $(sudo -u postgres psql -U postgres -Atc "SELECT 1 FROM pg_catalog.pg_user WHERE usename = '$DB_USER'";) ]]; then - echo "Creating database..." - sudo -u postgres psql -U postgres -c "CREATE ROLE \"$DB_USER\" with LOGIN CREATEDB PASSWORD '$DB_SECRET';" > /dev/null + if [[ -z $(sudo -u postgres "$PG_BIN/psql" -U postgres -Atc "SELECT 1 FROM pg_catalog.pg_user WHERE usename = '$DB_USER'";) ]]; then + echo "Creating database..." + sudo -u postgres "$PG_BIN/psql" -U postgres -c "CREATE ROLE \"$DB_USER\" with LOGIN CREATEDB PASSWORD '$DB_SECRET';" > /dev/null fi - if [[ -z $(sudo -u postgres psql -U postgres -Atc "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '$DB_NAME'";) ]]; then - echo "Creating user..." - sudo -u postgres psql -U postgres -c "CREATE DATABASE \"$DB_NAME\";" >/dev/null - sudo -u postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$DB_NAME\" to \"$DB_USER\";" > /dev/null + if [[ -z $(sudo -u postgres "$PG_BIN/psql" -U postgres -Atc "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '$DB_NAME'";) ]]; then + echo "Creating user..." + sudo -u postgres "$PG_BIN/psql" -U postgres -c "CREATE DATABASE \"$DB_NAME\";" >/dev/null + sudo -u postgres "$PG_BIN/psql" -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$DB_NAME\" to \"$DB_USER\";" > /dev/null fi + + echo "User configured" } +# API FUNCTIONS get_current_index() { json="{\"network_identifier\":{\"blockchain\":\"cardano\",\"network\":\"${NETWORK}\"},\"metadata\":{}}" response=$(curl -s -X POST -H "Content-Type: application/json" -H "Content-length: 1000" -H "Host: localhost.com" --data "$json" "localhost:{$API_PORT}/network/status") @@ -177,6 +206,7 @@ get_current_index() { if [[ -z "$current_index" || "$current_index" == "null" ]]; then current_index=0; fi } +# MITHRIL FUNCTIONS download_mithril_snapshot() { echo "Downloading Mithril Snapshot..." export CARDANO_NETWORK=$NETWORK @@ -194,11 +224,7 @@ download_mithril_snapshot() { preview) AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT:-https://aggregator.pre-release-preview.api.mithril.network/aggregator} GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/pre-release-preview/genesis.vkey)} - ANCILLARY_VERIFICATION_KEY=${ANCILLARY_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/pre-release-preview/ancillary.vkey)} - ;; - sanchonet) - AGGREGATOR_ENDPOINT=${AGGREGATOR_ENDPOINT:-https://aggregator.testing-sanchonet.api.mithril.network/aggregator} - GENESIS_VERIFICATION_KEY=${GENESIS_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/testing-sanchonet/genesis.vkey)} + ANCILLARY_VERIFICATION_KEY=${ANCILLARY_VERIFICATION_KEY:-$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/pre-release-testing-preview/ancillary.vkey)} ;; esac echo "Listing content of /node dir:" @@ -252,17 +278,18 @@ else cardano-submit-api --listen-address 0.0.0.0 --socket-path "$CARDANO_NODE_SOCKET_PATH" --port $NODE_SUBMIT_API_PORT $NETWORK_STR --config /cardano-submit-api-config/cardano-submit-api.yaml > /logs/submit-api.log & CARDANO_SUBMIT_PID=$! - mkdir -p /node/postgres - chown -R postgres:postgres /node/postgres - chmod -R 0700 /node/postgres - if [ ! -f "/node/postgres/PG_VERSION" ]; then - database_initialization - fi + echo "Initializing Database..." + chown -R postgres:postgres "$PG_DATA" + chmod -R 0700 "$PG_DATA" + database_initialization + echo "Configuring the Database..." configure_postgres echo "Starting Postgres..." - /etc/init.d/postgresql start + start_postgres + + echo "Creating database and user..." create_database_and_user echo "Starting Yaci indexer..." @@ -294,4 +321,4 @@ else tail -f -n +1 /logs/*.log > >(tee $logf) & tail_pid=$! wait $CARDANO_NODE_PID -fi +fi \ No newline at end of file diff --git a/docker/readme.md b/docker/readme.md index 2d422d503..7d4518d2b 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -42,13 +42,13 @@ You can specify Cabal, GHC, Cardano node, and Postgres versions when building an The default values: `` -CABAL_VERSION=3.8.1.0 +CABAL_VERSION=3.12.1.0 `` `` -GHC_VERSION=8.10.7 +GHC_VERSION=9.6.7 `` `` -CARDANO_NODE_VERSION=10.2.1 +CARDANO_NODE_VERSION=10.3.1 `` `` PG_VERSION=14 diff --git a/docs/docs/advanced-configuration/pruning.md b/docs/docs/advanced-configuration/pruning.md index b0e6e9ff1..482e9ef7d 100644 --- a/docs/docs/advanced-configuration/pruning.md +++ b/docs/docs/advanced-configuration/pruning.md @@ -1,28 +1,31 @@ --- sidebar_position: 1 -title: Pruning UTXOs -description: Optimizing disk usage with pruning +title: Remove Spent UTXOs +description: Optimizing disk usage with removal of spent UTxOs --- -# Pruning UTXOs +# Remove Spent UTXOs -This guide explains how to optimize disk usage in **cardano-rosetta-java** through pruning. +This guide explains how to optimize disk usage in **cardano-rosetta-java** through removal of spent utxo (a special rosetta-java specific form of pruning). -## What is Pruning? +## What is UTxO Spent Pruning (removal of spent UTxOs)? -Pruning removes spent (consumed) UTXOs from local storage, keeping only unspent UTXOs. This can reduce on-disk storage from ~1TB down to ~400GB, but discards historical transaction data. +UTxO Spent Pruning removes spent (consumed) UTXOs from local storage, keeping only unspent UTXOs. This can reduce on-disk storage from ~1TB down to ~400GB, but discards historical transaction data. - Only unspent outputs are preserved. - You can still validate the chain's current state (and spend tokens), since active UTXOs remain. -- **Enable Pruning**: Set `PRUNING_ENABLED=true` in your environment (e.g., in `.env.dockerfile` or `.env.docker-compose`). -- **Disable Pruning** (default): Set `PRUNING_ENABLED=false`. +- You should be able to build transaction because only spent UTxOs are removed (unspent stay) -## When to Enable Pruning +**Enable Spent UTxO removal **: Set `REMOVE_SPENT_UTXOS=true` in your environment (e.g., in `.env.dockerfile` or `.env.docker-compose`). +**Disable Spent UTxO removal ** (default): Set `REMOVE_SPENT_UTXOS=false`. + +## When Spent UTxO Removal should be enabled? - **Low Disk Environments**: If you need to minimize disk usage and only require UTXO data for current balances. - **Exploratory / Dev Environments**: If historical queries are not critical. +- **Performance**: if you are running into performance / scalability issues, i.e. especially on /account/balance when working with large addresses -## When to Avoid Pruning +## When to avoid setting UtxO Removal feature? - **Full Historical Data Requirements**: If you need the complete transaction history—whether for exchange operations, audit trails, or compliance mandates—do not enable pruning. Pruning discards spent UTXOs, which removes older transaction data and prevents certain types of historical lookups or reporting. @@ -31,9 +34,8 @@ Pruning removes spent (consumed) UTXOs from local storage, keeping only unspent Below is a snippet of how you might configure `.env.dockerfile` or `.env.docker-compose` for pruning: ```bash -# --- Pruning Toggle --- -PRUNING_ENABLED=true -# Enables pruning to reduce disk space requirements +# --- Remove Spent UTxOs Toggle --- +REMOVE_SPENT_UTXOS=true ``` ## Further Reading diff --git a/docs/docs/core-concepts/architecture.md b/docs/docs/core-concepts/architecture.md index ec824d7b8..594d68b2a 100644 --- a/docs/docs/core-concepts/architecture.md +++ b/docs/docs/core-concepts/architecture.md @@ -53,11 +53,11 @@ _Figure 2: Component Diagram showing internal architecture_ The Cardano Node is a full implementation of the Cardano blockchain protocol that connects to the Cardano network, validates transactions and blocks, and maintains the blockchain state. :::info -**Version**: 10.2.1 (configurable via build args) -**Built with**: GHC 8.10.7 and Cabal 3.8.1.0 +**Version**: 10.3.1 (configurable via build args) +**Built with**: GHC 9.6.7 and Cabal 3.12.1.0 **Runtime socket path**: `/node/node.socket` **Data directory**: `/node/db` -**Network options**: mainnet, preprod, preview, sanchonet, devkit +**Network options**: mainnet, preprod, preview, devkit **Configuration files**: stored in `/config` directory ::: diff --git a/docs/docs/install-and-deploy/docker.md b/docs/docs/install-and-deploy/docker.md index 7f8d0714a..e7b3deb6f 100644 --- a/docs/docs/install-and-deploy/docker.md +++ b/docs/docs/install-and-deploy/docker.md @@ -64,7 +64,7 @@ docker compose --env-file .env.docker-compose -f docker-compose.yaml up :::tip Node Synchronization To speed up the initial synchronization process, you can use [Mithril](https://mithril.network/doc/) snapshots by setting `MITHRIL_SYNC=true` in your environment file. -Mithril provides cryptographically certified blockchain snapshots for multiple Cardano networks (mainnet, preprod, preview, sanchonet) and is integrated directly into the Docker setup. This can reduce synchronization time from days to hours. +Mithril provides cryptographically certified blockchain snapshots for multiple Cardano networks (mainnet, preprod, preview) and is integrated directly into the Docker setup. This can reduce synchronization time from days to hours. ::: @@ -128,9 +128,9 @@ docker build -t {image_name} --build-arg PG_VERSION=14 -f ./docker/Dockerfile . Default values: -- `CABAL_VERSION=3.8.1.0` -- `GHC_VERSION=8.10.7` -- `CARDANO_NODE_VERSION=8.9.2` +- `CABAL_VERSION=3.12.1.0` +- `GHC_VERSION=9.6.7` +- `CARDANO_NODE_VERSION=10.2.1` - `PG_VERSION=14` #### Configuration diff --git a/docs/docs/install-and-deploy/env-vars.md b/docs/docs/install-and-deploy/env-vars.md index c6189e34d..39a59db30 100644 --- a/docs/docs/install-and-deploy/env-vars.md +++ b/docs/docs/install-and-deploy/env-vars.md @@ -13,43 +13,43 @@ Within root folder of the project there are example `.env` files, which can be c - `.env.IntegrationTest` - Is used for integration tests with yaci devkit - `.env.docker-compose` - Is used for standard docker-compose setup (Copy this file and adjusted it to your needs) -| Variable | Description | Default | Notes | -|-----------------------------------------|-----------------------------------------------------|---------------------------------------|----------------------------| -| `LOG` | Log level | INFO | added in release 1.0.0 | -| `NETWORK` | Network | mainnet | added in release 1.0.0 | -| `MITHRIL_SYNC` | Sync from Mithril snapshot | true | added in release 1.0.0 | -| `PROTOCOL_MAGIC` | Cardano protocol magic | 764824073 | added in release 1.0.0 | -| `DB_IMAGE_NAME` | Postgres docker image name | postgres | added in release 1.0.0 | -| `DB_IMAGE_TAG` | Postgres docker image tag | 14.11-bullseye | added in release 1.0.0 | -| `DB_NAME` | Postgres database | rosetta-java | added in release 1.0.0 | -| `DB_USER` | Postgres admin user | rosetta_db_admin | added in release 1.0.0 | -| `DB_SECRET` | Postgres admin secret | weakpwd#123_d | added in release 1.0.0 | -| `DB_HOST` | Postgres host | db | added in release 1.0.0 | -| `DB_PORT` | Postgres port | 5432 | added in release 1.0.0 | -| `DB_SCHEMA` | Database schema | mainnet | added in release 1.0.0 | -| `DB_PATH` | Database path | /data | added in release 1.0.0 | -| `CARDANO_NODE_HOST` | Cardano node host | cardano-node | added in release 1.0.0 | -| `CARDANO_NODE_PORT` | Cardano node port | 3001 | added in release 1.0.0 | -| `CARDANO_NODE_VERSION` | Cardano node version | 8.9.2 | added in release 1.0.0 | -| `CARDANO_NODE_SUBMIT_HOST` | Cardano node submit API host | cardano-submit-api | added in release 1.0.0 | -| `NODE_SUBMIT_API_PORT` | Cardano node submit API port | 8090 | added in release 1.0.0 | -| `CARDANO_NODE_SOCKET_PATH` | Cardano node socket path | /node | added in release 1.0.0 | -| `CARDANO_NODE_SOCKET` | Cardano node socket file | /node/node.socket | added in release 1.0.0 | -| `CARDANO_NODE_DB` | Cardano node db path | /node/db | added in release 1.0.0 | -| `CARDANO_CONFIG` | Cardano node config path | /config/mainnet | added in release 1.0.0 | -| `API_DOCKER_IMAGE_TAG` | Docker Tag for API Image | main | added in release 1.0.0 | -| `API_SPRING_PROFILES_ACTIVE` | API spring profile | staging | added in release 1.0.0 | -| `API_PORT` | Rosetta API exposed port | 8082 | added in release 1.0.0 | -| `ROSETTA_VERSION` | Rosetta version | 1.4.13 | added in release 1.0.0 | -| `TOPOLOGY_FILEPATH` | Topology file path | ./config/mainnet/topology.json | added in release 1.0.0 | -| `GENESIS_SHELLEY_PATH` | Genesis file path | ./config/mainnet/shelley-genesis.json | added in release 1.0.0 | -| `GENESIS_BYRON_PATH` | Genesis file path | ./config/mainnet/byron-genesis.json | added in release 1.0.0 | -| `GENESIS_ALONZO_PATH` | Genesis file path | ./config/mainnet/alonzo-genesis.json | added in release 1.0.0 | -| `GENESIS_CONWAY_PATH` | Genesis file path | ./config/mainnet/conway-genesis.json | added in release 1.0.0 | -| `INDEXER_DOCKER_IMAGE_TAG` | Yaci indexer Docker version | main | added in release 1.0.0 | -| `PRUNING_ENABLED` | If pruning should be enabled | false | added in release 1.0.0 | -| `PRUNING_INTERVAL` | The pruning interval in seconds | 600 | added in release 1.2.4 | -| `PRUNING_SAFE_BLOCKS` | Number of safe blocks to keep in the store | 2160 | added in release 1.2.4 | +| Variable | Description | Default | Notes | +|--------------------------------------|-----------------------------------------------------|---------------------------------------|----------------------------| +| `LOG` | Log level | INFO | added in release 1.0.0 | +| `NETWORK` | Network | mainnet | added in release 1.0.0 | +| `MITHRIL_SYNC` | Sync from Mithril snapshot | true | added in release 1.0.0 | +| `PROTOCOL_MAGIC` | Cardano protocol magic | 764824073 | added in release 1.0.0 | +| `DB_IMAGE_NAME` | Postgres docker image name | postgres | added in release 1.0.0 | +| `DB_IMAGE_TAG` | Postgres docker image tag | 14.11-bullseye | added in release 1.0.0 | +| `DB_NAME` | Postgres database | rosetta-java | added in release 1.0.0 | +| `DB_USER` | Postgres admin user | rosetta_db_admin | added in release 1.0.0 | +| `DB_SECRET` | Postgres admin secret | weakpwd#123_d | added in release 1.0.0 | +| `DB_HOST` | Postgres host | db | added in release 1.0.0 | +| `DB_PORT` | Postgres port | 5432 | added in release 1.0.0 | +| `DB_SCHEMA` | Database schema | mainnet | added in release 1.0.0 | +| `DB_PATH` | Database path | /data | added in release 1.0.0 | +| `CARDANO_NODE_HOST` | Cardano node host | cardano-node | added in release 1.0.0 | +| `CARDANO_NODE_PORT` | Cardano node port | 3001 | added in release 1.0.0 | +| `CARDANO_NODE_VERSION` | Cardano node version | 8.9.2 | added in release 1.0.0 | +| `CARDANO_NODE_SUBMIT_HOST` | Cardano node submit API host | cardano-submit-api | added in release 1.0.0 | +| `NODE_SUBMIT_API_PORT` | Cardano node submit API port | 8090 | added in release 1.0.0 | +| `CARDANO_NODE_SOCKET_PATH` | Cardano node socket path | /node | added in release 1.0.0 | +| `CARDANO_NODE_SOCKET` | Cardano node socket file | /node/node.socket | added in release 1.0.0 | +| `CARDANO_NODE_DB` | Cardano node db path | /node/db | added in release 1.0.0 | +| `CARDANO_CONFIG` | Cardano node config path | /config/mainnet | added in release 1.0.0 | +| `API_DOCKER_IMAGE_TAG` | Docker Tag for API Image | main | added in release 1.0.0 | +| `API_SPRING_PROFILES_ACTIVE` | API spring profile | staging | added in release 1.0.0 | +| `API_PORT` | Rosetta API exposed port | 8082 | added in release 1.0.0 | +| `ROSETTA_VERSION` | Rosetta version | 1.4.13 | added in release 1.0.0 | +| `TOPOLOGY_FILEPATH` | Topology file path | ./config/mainnet/topology.json | added in release 1.0.0 | +| `GENESIS_SHELLEY_PATH` | Genesis file path | ./config/mainnet/shelley-genesis.json | added in release 1.0.0 | +| `GENESIS_BYRON_PATH` | Genesis file path | ./config/mainnet/byron-genesis.json | added in release 1.0.0 | +| `GENESIS_ALONZO_PATH` | Genesis file path | ./config/mainnet/alonzo-genesis.json | added in release 1.0.0 | +| `GENESIS_CONWAY_PATH` | Genesis file path | ./config/mainnet/conway-genesis.json | added in release 1.0.0 | +| `INDEXER_DOCKER_IMAGE_TAG` | Yaci indexer Docker version | main | added in release 1.0.0 | +| `REMOVE_SPENT_UTXOS` | If pruning should be enabled | false | added in release 1.0.0 | +| `PRUNING_INTERVAL` | The pruning interval in seconds | 600 | added in release 1.2.4 | +| `PRUNING_SAFE_BLOCKS` | Number of safe blocks to keep in the store | 2160 | added in release 1.2.4 | | `YACI_SPRING_PROFILES` | Yaci indexer spring profile | postgres | added in release 1.0.0 | | `DEVKIT_ENABLED` | Devkit enabled | false | added in release 1.0.0 | | `YACI_HTTP_BASE_URL` | Yaci Indexer's URL | http://yaci-indexer:9095/api/v1 | added in release 1.2.1 | diff --git a/docs/docs/intro.md b/docs/docs/intro.md index 186a8aefe..44ac92104 100644 --- a/docs/docs/intro.md +++ b/docs/docs/intro.md @@ -23,12 +23,12 @@ For a more detailed architecture overview, see the [Architecture](core-concepts/ Running Cardano Rosetta Java requires the following minimum resources: -| Component | Requirement | Notes | -| :---------- | :---------- | :----------------------- | -| **CPU** | 4 Cores | | -| **RAM** | 32 GB | | -| **Storage** | 1 TB | When pruning is disabled | -| | 400 GB | When pruning is enabled | +| Component | Requirement | Notes | +| :---------- | :---------- |:------------------------------------| +| **CPU** | 4 Cores | | +| **RAM** | 32 GB | | +| **Storage** | 1 TB | When remove spent UTxOs is disabled | +| | 400 GB | When remove spent UTxOs is enabled | :::tip Performance Note Better hardware resources (more CPU cores, RAM) will improve indexer and node performance, resulting in faster chain synchronization and better API response times. diff --git a/pom.xml b/pom.xml index 298e55cd4..bf32e4854 100644 --- a/pom.xml +++ b/pom.xml @@ -26,17 +26,19 @@ - 1.2.8 - 21 + full + 1.2.9 + 24 UTF-8 - 3.4.3 + 3.4.4 1.6.15 6.4.0 3.13.0 3.2.5 + 3.7.1 0.8.11 1.3.0 - 1.18.30 + 1.18.38 2.2.8 2.20.0 0.6.4 @@ -90,6 +92,18 @@ + + org.springframework.boot + spring-boot-starter + ${version.spring-boot} + + + org.springframework.boot + spring-boot-dependencies + pom + ${version.spring-boot} + import + org.cardanofoundation cf-cardano-conversions-java @@ -100,13 +114,6 @@ vavr ${version.vavr} - - org.springframework.boot - spring-boot-dependencies - pom - ${version.spring-boot} - import - org.springdoc springdoc-openapi-ui @@ -143,6 +150,11 @@ yaci ${version.yaci} + + io.hypersistence + hypersistence-utils-hibernate-63 + ${version.hibernate-types} + com.h2database h2 diff --git a/test-data-generator/pom.xml b/test-data-generator/pom.xml index d4f75861d..80f2c0670 100644 --- a/test-data-generator/pom.xml +++ b/test-data-generator/pom.xml @@ -62,7 +62,14 @@ maven-compiler-plugin ${version.maven-compiler-plugin} - 21 + 24 + + + org.projectlombok + lombok + ${version.lombok} + + diff --git a/yaci-indexer/Dockerfile b/yaci-indexer/Dockerfile index 504b98e52..7af17b4a2 100644 --- a/yaci-indexer/Dockerfile +++ b/yaci-indexer/Dockerfile @@ -1,17 +1,28 @@ -FROM ubuntu:22.04 AS build-common +FROM ubuntu:24.04 AS build-common WORKDIR /build RUN apt update --fix-missing \ - && apt install -y --no-install-recommends openjdk-21-jdk maven curl \ + && apt install -y --no-install-recommends maven curl \ && apt clean +# Download and setup JDK 24.0.1 +RUN mkdir -p /opt/java \ + && curl -L https://download.java.net/java/GA/jdk24.0.1/24a58e0e276943138bf3e963e6291ac2/9/GPL/openjdk-24.0.1_linux-x64_bin.tar.gz -o /opt/jdk.tar.gz \ + && tar -xzf /opt/jdk.tar.gz -C /opt/java \ + && rm /opt/jdk.tar.gz + +# Set JAVA_HOME and update PATH +ENV JAVA_HOME=/opt/java/jdk-24.0.1 +ENV PATH="${JAVA_HOME}/bin:${PATH}" + COPY ./pom.xml /build/pom.xml COPY ./api /build/api COPY ./yaci-indexer /build/yaci-indexer COPY ./test-data-generator /build/test-data-generator COPY ./.git .git -RUN --mount=type=cache,target=/root/.m2 mvn clean package -DskipTests +RUN --mount=type=cache,target=/root/.m2 mvn -U clean package -DskipTests +RUN java --version WORKDIR /app RUN cp /build/yaci-indexer/target/*.jar /app/yaci-indexer.jar diff --git a/yaci-indexer/pom.xml b/yaci-indexer/pom.xml index b14b83a94..0129e559e 100644 --- a/yaci-indexer/pom.xml +++ b/yaci-indexer/pom.xml @@ -2,20 +2,23 @@ 4.0.0 + - org.springframework.boot - spring-boot-starter-parent - 3.2.4 - + base + org.cardanofoundation.rosetta-java + ${revision} - org.cardanofoundation.rosetta-java yaci-indexer 1.0.0 yaci-indexer + yaci-indexer + jar + - 21 + 24 + full 0.1.2 src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/model/* 2.43.0 @@ -37,7 +40,6 @@ org.springframework.boot spring-boot-starter - org.springframework.boot spring-boot-starter-test @@ -87,6 +89,10 @@ yaci-store-epoch-spring-boot-starter ${yaci-store.version} + + io.hypersistence + hypersistence-utils-hibernate-63 + org.postgresql @@ -99,6 +105,31 @@ org.springframework.boot spring-boot-maven-plugin + ${version.spring-boot} + + + + repackage + build-info + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${version.maven-compiler-plugin} + + 24 + + + org.projectlombok + lombok + ${version.lombok} + + + com.diffplug.spotless diff --git a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/AccountServiceImpl.java b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/AccountServiceImpl.java index 86b5771ef..01ce953ae 100644 --- a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/AccountServiceImpl.java +++ b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/AccountServiceImpl.java @@ -1,22 +1,24 @@ package org.cardanofoundation.rosetta.yaciindexer.service; +import java.math.BigInteger; +import java.time.Duration; +import java.util.Optional; +import java.util.Set; +import jakarta.annotation.Nullable; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; import com.bloxbean.cardano.client.address.Address; import com.bloxbean.cardano.yaci.core.protocol.localstate.api.Era; import com.bloxbean.cardano.yaci.core.protocol.localstate.queries.DelegationsAndRewardAccountsQuery; import com.bloxbean.cardano.yaci.core.protocol.localstate.queries.DelegationsAndRewardAccountsResult; import com.bloxbean.cardano.yaci.helper.LocalClientProvider; import com.bloxbean.cardano.yaci.store.core.service.local.LocalClientProviderManager; -import jakarta.annotation.Nullable; -import lombok.extern.slf4j.Slf4j; -import org.cardanofoundation.rosetta.yaciindexer.domain.model.StakeAccountRewardInfo; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; -import java.math.BigInteger; -import java.time.Duration; -import java.util.Optional; -import java.util.Set; +import org.cardanofoundation.rosetta.yaciindexer.domain.model.StakeAccountRewardInfo; @Service @Slf4j diff --git a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionScriptSizeCalculatorImpl.java b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionScriptSizeCalculatorImpl.java index 6829c1573..ad8934e67 100644 --- a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionScriptSizeCalculatorImpl.java +++ b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionScriptSizeCalculatorImpl.java @@ -1,5 +1,10 @@ package org.cardanofoundation.rosetta.yaciindexer.service; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Service; import co.nstant.in.cbor.model.Array; import co.nstant.in.cbor.model.ByteString; import co.nstant.in.cbor.model.Map; @@ -10,11 +15,8 @@ import com.bloxbean.cardano.yaci.core.model.PlutusScript; import com.bloxbean.cardano.yaci.helper.model.Transaction; import com.fasterxml.jackson.core.JsonProcessingException; -import lombok.extern.slf4j.Slf4j; -import org.cardanofoundation.rosetta.yaciindexer.domain.model.TransactionBuildingConstants; -import org.springframework.stereotype.Service; -import java.util.List; +import org.cardanofoundation.rosetta.yaciindexer.domain.model.TransactionBuildingConstants; import static com.bloxbean.cardano.client.util.HexUtil.decodeHexString; import static org.cardanofoundation.rosetta.yaciindexer.domain.model.TransactionBuildingConstants.*; diff --git a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionSizeCalculatorImpl.java b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionSizeCalculatorImpl.java index 7cc7d0578..ed20bf15e 100644 --- a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionSizeCalculatorImpl.java +++ b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/service/TransactionSizeCalculatorImpl.java @@ -1,9 +1,10 @@ package org.cardanofoundation.rosetta.yaciindexer.service; -import co.nstant.in.cbor.model.Map; -import com.bloxbean.cardano.yaci.core.util.CborSerializationUtil; import lombok.extern.slf4j.Slf4j; + import org.springframework.stereotype.Service; +import co.nstant.in.cbor.model.Map; +import com.bloxbean.cardano.yaci.core.util.CborSerializationUtil; @Service @Slf4j diff --git a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStore.java b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStore.java index 176a24c52..b0cbcf668 100644 --- a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStore.java +++ b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStore.java @@ -1,5 +1,14 @@ package org.cardanofoundation.rosetta.yaciindexer.stores.txsize; +import java.util.List; +import java.util.Optional; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import co.nstant.in.cbor.model.Array; import co.nstant.in.cbor.model.DataItem; import co.nstant.in.cbor.model.Map; @@ -8,19 +17,12 @@ import com.bloxbean.cardano.yaci.core.util.CborSerializationUtil; import com.bloxbean.cardano.yaci.helper.model.Transaction; import com.bloxbean.cardano.yaci.store.events.TransactionEvent; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; + import org.cardanofoundation.rosetta.yaciindexer.domain.model.TransactionBuildingConstants; import org.cardanofoundation.rosetta.yaciindexer.service.TransactionScriptSizeCalculator; import org.cardanofoundation.rosetta.yaciindexer.service.TransactionSizeCalculator; import org.cardanofoundation.rosetta.yaciindexer.stores.txsize.model.TransactionSizeEntity; import org.cardanofoundation.rosetta.yaciindexer.stores.txsize.model.TransactionSizeRepository; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.List; -import java.util.Optional; import static co.nstant.in.cbor.model.SimpleValue.TRUE; import static org.cardanofoundation.rosetta.yaciindexer.domain.model.TransactionBuildingConstants.*; diff --git a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/model/TransactionSizeEntity.java b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/model/TransactionSizeEntity.java index 0b1c2f25c..7000f36be 100644 --- a/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/model/TransactionSizeEntity.java +++ b/yaci-indexer/src/main/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/model/TransactionSizeEntity.java @@ -4,6 +4,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Id; import jakarta.persistence.Table; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/yaci-indexer/src/main/resources/application-h2-testdata.properties b/yaci-indexer/src/main/resources/application-h2-testdata.properties index cf067f902..f29b64443 100644 --- a/yaci-indexer/src/main/resources/application-h2-testdata.properties +++ b/yaci-indexer/src/main/resources/application-h2-testdata.properties @@ -7,4 +7,6 @@ spring.datasource.url=jdbc:h2:file:./testData/devkit.db spring.datasource.username=rosetta_db_admin spring.datasource.password=weakpwd#123_d -spring.h2.console.enabled=true \ No newline at end of file +spring.h2.console.enabled=true + +spring.jpa.hibernate.ddl-auto=update diff --git a/yaci-indexer/src/main/resources/application-h2.properties b/yaci-indexer/src/main/resources/application-h2.properties index 7c1db7c03..4165442a0 100644 --- a/yaci-indexer/src/main/resources/application-h2.properties +++ b/yaci-indexer/src/main/resources/application-h2.properties @@ -7,4 +7,6 @@ spring.datasource.url=jdbc:h2:mem:${DB_NAME:rosetta-java} spring.datasource.username=${DB_USER:rosetta_db_admin} spring.datasource.password=${DB_SECRET:weakpwd#123_d} -spring.h2.console.enabled=true \ No newline at end of file +spring.h2.console.enabled=true + +spring.jpa.hibernate.ddl-auto=update diff --git a/yaci-indexer/src/main/resources/application-postgres.properties b/yaci-indexer/src/main/resources/application-postgres.properties index 065da48af..839fdc4e7 100644 --- a/yaci-indexer/src/main/resources/application-postgres.properties +++ b/yaci-indexer/src/main/resources/application-postgres.properties @@ -6,4 +6,6 @@ ##################### Postgres DB ####################### spring.datasource.url=jdbc:postgresql://${DB_HOST:localhost}:${DB_PORT:5433}/${DB_NAME:rosetta-java}?currentSchema=${DB_SCHEMA:preprod} spring.datasource.username=${DB_USER:rosetta_db_admin} -spring.datasource.password=${DB_SECRET:weakpwd#123_d} \ No newline at end of file +spring.datasource.password=${DB_SECRET:weakpwd#123_d} + +spring.jpa.hibernate.ddl-auto=none \ No newline at end of file diff --git a/yaci-indexer/src/main/resources/application.properties b/yaci-indexer/src/main/resources/application.properties index 3a79b271c..970051325 100644 --- a/yaci-indexer/src/main/resources/application.properties +++ b/yaci-indexer/src/main/resources/application.properties @@ -2,11 +2,10 @@ spring.profiles.active=${YACI_SPRING_PROFILES: h2} server.port=9095 #The following flyway section is mandatory spring.flyway.out-of-order=true -spring.jpa.hibernate.ddl-auto=update spring.flyway.locations=classpath:db/store/{vendor} #Api prefix for out-of-box REST API -apiPrefix: /api/v1 +apiPrefix=/api/v1 ## Other Yaci Store properties can be configured here or in application properties under config folder. ## ################## Network Configuration ########### @@ -16,6 +15,8 @@ apiPrefix: /api/v1 store.cardano.host=${CARDANO_NODE_HOST:preprod-node.world.dev.cardano.org} store.cardano.port=${CARDANO_NODE_PORT:30000} store.cardano.protocol-magic=${PROTOCOL_MAGIC:1} +# 3 seconds for keep alive +store.cardano.keep-alive-interval=3000 ############### Genesis files ############################# # The application reads the below genesis files to get data like slotLength, maxLovelaceSupply @@ -104,9 +105,8 @@ logging.level.com.bloxbean.cardano.yaci.core.protocol.keepalive=debug store.admin.auto-recovery-enabled=true store.admin.health-check-interval=20 - -store.utxo.pruning-enabled=${PRUNING_ENABLED:false} -store.utxo.pruning-interval=${PRUNING_INTERVAL:600} -store.utxo.pruning-safe-blocks=${PRUNING_SAFE_BLOCKS:2160} +store.utxo.pruning-enabled=${REMOVE_SPENT_UTXOS:false} +store.utxo.pruning-safe-blocks=${REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT:2160} +store.utxo.pruning-interval=600 store.epoch.endpoints.epoch.local.enabled=true diff --git a/yaci-indexer/src/main/resources/db/store/postgresql/V1.0_1000_0__rosetta_app_transaction_size.sql b/yaci-indexer/src/main/resources/db/store/postgresql/V1.0_1000_0__rosetta_app_transaction_size.sql new file mode 100644 index 000000000..53e78b069 --- /dev/null +++ b/yaci-indexer/src/main/resources/db/store/postgresql/V1.0_1000_0__rosetta_app_transaction_size.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS transaction_size ( + tx_hash TEXT PRIMARY KEY, + block_number BIGINT NOT NULL, + size INTEGER NOT NULL, + script_size INTEGER NOT NULL +); \ No newline at end of file diff --git a/yaci-indexer/src/test/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStoreTest.java b/yaci-indexer/src/test/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStoreTest.java index 0267c7e16..d80f0e632 100644 --- a/yaci-indexer/src/test/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStoreTest.java +++ b/yaci-indexer/src/test/java/org/cardanofoundation/rosetta/yaciindexer/stores/txsize/CustomTransactionSizeStoreTest.java @@ -1,20 +1,22 @@ package org.cardanofoundation.rosetta.yaciindexer.stores.txsize; +import java.util.List; + import com.bloxbean.cardano.yaci.store.events.TransactionEvent; import org.assertj.core.api.Assertions; +import org.mockito.*; +import org.mockito.junit.jupiter.MockitoExtension; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + import org.cardanofoundation.rosetta.yaciindexer.TestDataGenerator; import org.cardanofoundation.rosetta.yaciindexer.service.TransactionScriptSizeCalculator; import org.cardanofoundation.rosetta.yaciindexer.service.TransactionScriptSizeCalculatorImpl; import org.cardanofoundation.rosetta.yaciindexer.service.TransactionSizeCalculatorImpl; import org.cardanofoundation.rosetta.yaciindexer.stores.txsize.model.TransactionSizeEntity; import org.cardanofoundation.rosetta.yaciindexer.stores.txsize.model.TransactionSizeRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.*; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.List; @ExtendWith(MockitoExtension.class) class CustomTransactionSizeStoreTest { diff --git a/yaci-indexer/src/test/resources/application-test-integration.properties b/yaci-indexer/src/test/resources/application-test-integration.properties index fc25f73a9..ccdca2d9f 100644 --- a/yaci-indexer/src/test/resources/application-test-integration.properties +++ b/yaci-indexer/src/test/resources/application-test-integration.properties @@ -13,5 +13,5 @@ spring.h2.console.enabled=true store.cardano.host=${CARDANO_NODE_HOST:localhost} store.cardano.port=${CARDANO_NODE_PORT:8090} -store.utxo.pruning-interval=${PRUNING_INTERVAL:600} -store.utxo.pruning-safe-blocks=${PRUNING_SAFE_BLOCKS:2160} \ No newline at end of file +store.utxo.pruning-interval=600 +store.utxo.pruning-safe-blocks=${REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT:2160} \ No newline at end of file