diff --git a/apps/s03-indexer/.env.develop b/apps/s03-indexer/.env.develop index fe18b1e..6b14c77 100644 --- a/apps/s03-indexer/.env.develop +++ b/apps/s03-indexer/.env.develop @@ -1,2 +1,3 @@ ENDPOINT=https://horizon-testnet.stellar.org -CHAIN_ID=Test SDF Network ; September 2015 \ No newline at end of file +CHAIN_ID="Test SDF Network ; September 2015" +SOROBAN_ENDPOINT=https://soroban-testnet.stellar.org diff --git a/apps/s03-indexer/.env.example b/apps/s03-indexer/.env.example new file mode 100644 index 0000000..3872c01 --- /dev/null +++ b/apps/s03-indexer/.env.example @@ -0,0 +1,10 @@ +# Stellar testnet defaults. Use these when indexing public SDF testnet data. +ENDPOINT=https://horizon-testnet.stellar.org +CHAIN_ID="Test SDF Network ; September 2015" +SOROBAN_ENDPOINT=https://soroban-testnet.stellar.org + +# Local standalone defaults. Uncomment these when running against a local +# stellar/quickstart or standalone network instead of public testnet. +# ENDPOINT=http://host.docker.internal:8000 +# CHAIN_ID="Standalone Network ; February 2017" +# SOROBAN_ENDPOINT=http://host.docker.internal:8000/soroban/rpc diff --git a/apps/s03-indexer/.gitignore b/apps/s03-indexer/.gitignore index 36045d7..9e630ec 100644 --- a/apps/s03-indexer/.gitignore +++ b/apps/s03-indexer/.gitignore @@ -5,11 +5,11 @@ # Node artifact files node_modules/ -dist/ # lock files yarn.lock package-lock.json +pnpm-lock.yaml # Compiled Java class files *.class @@ -23,7 +23,8 @@ package-lock.json # Package files *.jar -# Generated files +# SubQuery generated files. Regenerate these with Bun scripts instead of +# committing template output. target/ dist/ src/types @@ -59,4 +60,4 @@ Thumbs.db # ENV local files .env.local -.env.develop.local \ No newline at end of file +.env.develop.local diff --git a/apps/s03-indexer/README.md b/apps/s03-indexer/README.md index a73c3ed..a4ee9f6 100644 --- a/apps/s03-indexer/README.md +++ b/apps/s03-indexer/README.md @@ -1,42 +1,110 @@ -# SubQuery - Example Project for Stellar Soroban +# Stellar Testnet Indexer -[SubQuery](https://subquery.network) is a fast, flexible, and reliable open-source data indexer that provides you with custom APIs for your web3 project across all of our supported networks. To learn about how to get started with SubQuery, [visit our docs](https://academy.subquery.network). +This Bun workspace package contains the SubQuery Stellar indexer scaffold for +the S03 app. It indexes Stellar testnet payments, account credit/debit effects, +and Soroban `transfer` events. -**The example project indexes all soroban transfer events on Stellar's Testnet. It also indexes all account payments including credits and debits** +## Workspace Commands -## Start +Run these commands from the repository root: -First, install SubQuery CLI globally on your terminal by using NPM `npm install -g @subql/cli` +```bash +bun install +bun run indexer:codegen +bun run indexer:build +bun run indexer:dev +``` + +The same commands are available inside the workspace package: + +```bash +bun run --cwd apps/s03-indexer codegen +bun run --cwd apps/s03-indexer build +bun run --cwd apps/s03-indexer dev +bun run --cwd apps/s03-indexer start +``` + +`indexer:dev` is the clean-checkout local path. It generates SubQuery artifacts, +builds the mappings, pulls the required Docker images, and starts the local +stack. -You can either clone this GitHub repo, or use the `subql` CLI to bootstrap a clean project in the network of your choosing by running `subql init` and following the prompts. +## Local Services -Don't forget to install dependencies with `npm install` or `yarn install`! +The Docker stack in `docker-compose.yml` starts three services: -## Editing your SubQuery project +| Service | Image/build | Port | Purpose | +| --- | --- | --- | --- | +| `postgres` | local `docker/pg-Dockerfile` | `localhost:5432` | SubQuery metadata and indexed entities | +| `subquery-node` | `subquerynetwork/subql-node-stellar:latest` | internal `3000` | Stellar/SubQuery indexing node | +| `graphql-engine` | `subquerynetwork/subql-query:latest` | `localhost:3000` | GraphQL API and playground | -Although this is a working example SubQuery project, you can edit the SubQuery project by changing the following files: +Requirements: -- The project manifest in `project.yaml` defines the key project configuration and mapping handler filters -- The GraphQL Schema (`schema.graphql`) defines the shape of the resulting data that you are using SubQuery to index -- The Mapping functions in `src/mappings/` directory are typescript functions that handle transformation logic +- Bun 1.3.x or newer, matching the root `packageManager` field. +- Docker Engine with Docker Compose v2. +- Network access for the first image pull and for public Stellar testnet + endpoints when using the default config. -SubQuery supports various layer-1 blockchain networks and provides [dedicated quick start guides](https://academy.subquery.network/quickstart/quickstart.html) as well as [detailed technical documentation](https://academy.subquery.network/build/introduction.html) for each of them. +Open `http://localhost:3000` after the services become healthy to query the +GraphQL playground. -## Run your project +## Environment -_If you get stuck, find out how to get help below._ +Copy the example file when you need to customize the runtime: + +```bash +cp apps/s03-indexer/.env.example apps/s03-indexer/.env +``` + +The default configuration targets public SDF testnet: + +```dotenv +ENDPOINT=https://horizon-testnet.stellar.org +CHAIN_ID=Test SDF Network ; September 2015 +SOROBAN_ENDPOINT=https://soroban-testnet.stellar.org +``` -The simplest way to run your project is by running `yarn dev` or `npm run-script dev`. This does all of the following: +Use these defaults when indexing public testnet data or when contributors need +the same deterministic remote network. -1. `yarn codegen` - Generates types from the GraphQL schema definition and contract ABIs and saves them in the `/src/types` directory. This must be done after each change to the `schema.graphql` file or the contract ABIs -2. `yarn build` - Builds and packages the SubQuery project into the `/dist` directory -3. `docker compose pull && docker compose up` - Runs a Docker container with an indexer, PostgeSQL DB, and a query service. This requires [Docker to be installed](https://docs.docker.com/engine/install) and running locally. The configuration for this container is set from your `docker-compose.yml` +For local standalone or `stellar/quickstart` development, switch the same +variables to the local values shown in `.env.example`: + +```dotenv +ENDPOINT=http://host.docker.internal:8000 +CHAIN_ID=Standalone Network ; February 2017 +SOROBAN_ENDPOINT=http://host.docker.internal:8000/soroban/rpc +``` + +Use local endpoints when you are testing against a local Stellar network, +contract deployment, or fixture data. The compose file maps +`host.docker.internal` to the host gateway so the indexer container can reach a +host-run standalone node. + +Endpoint and chain settings are read while SubQuery generates/builds +`project.yaml`; after changing `.env`, run `bun run indexer:codegen` and +`bun run indexer:build` before `bun run indexer:start`. `bun run indexer:dev` +does all three steps for you. + +## Generated Artifacts + +`project.ts`, `schema.graphql`, and the TypeScript mapping sources are the +committed source of truth. `project.yaml`, `dist/`, and `src/types/` are +generated artifacts and stay ignored so they cannot drift from the source. + +Regenerate them with: + +```bash +bun run indexer:codegen +bun run indexer:build +``` -You can observe the three services start, and once all are running (it may take a few minutes on your first start), please open your browser and head to [http://localhost:3000](http://localhost:3000) - you should see a GraphQL playground showing with the schemas ready to query. [Read the docs for more information](https://academy.subquery.network/run_publish/run.html) or [explore the possible service configuration for running SubQuery](https://academy.subquery.network/run_publish/references.html). +The repository uses the root Bun lockfile only. Do not add `package-lock.json`, +`yarn.lock`, or `pnpm-lock.yaml` files. -## Query your project +## Useful Query -For this project, you can try to query with the following GraphQL code to get a taste of how it works. +After the stack is running, try this query in the GraphQL playground: ```graphql { @@ -70,81 +138,3 @@ For this project, you can try to query with the following GraphQL code to get a } } ``` - -Results: - -``` -{ - "data": { - "query": { - "transfers": { - "totalCount": 0, - "nodes": [] - }, - "accounts": { - "nodes": [ - { - "id": "gbtbdklzbabdgnpvdygchqzxqxkjfu73ayhls44j7u26ti6lzvlzfg5a", - "sentTransfers": { - "totalCount": 0, - "nodes": [] - }, - "firstSeenLedger": 1700000, - "lastSeenLedger": 1700000 - }, - { - "id": "gdea4efymx2vcx7hduurwvuy6de36qihe6faouhuzaldgul4ooq5euvu", - "sentTransfers": { - "totalCount": 0, - "nodes": [] - }, - "firstSeenLedger": 1700000, - "lastSeenLedger": 1700000 - }, - { - "id": "gcbwgcat2nhokpnr2toy6o3qdky22lzzeydwh4azhdyyl57qfu53ugzr", - "sentTransfers": { - "totalCount": 0, - "nodes": [] - }, - "firstSeenLedger": 1700000, - "lastSeenLedger": 1700000 - }, - { - "id": "gb7yga2xmamr6wqb2z5c4l6s2imudrqzu7zxdpr4df2dewcxpxhmzjy5", - "sentTransfers": { - "totalCount": 0, - "nodes": [] - }, - "firstSeenLedger": 1700000, - "lastSeenLedger": 1700000 - } - ] - } - } - } -} -``` - -You can explore the different possible queries and entities to help you with GraphQL using the documentation draw on the right. - -## Publish your project - -SubQuery is open-source and designed to be easy to run, meaning you have the freedom to run it a variety of ways: - -- [Locally on your own computer or on your cloud provider of choice.](https://academy.subquery.network/indexer/run_publish/introduction.html#locally-run-it-yourself) -- [By publishing it to the decentralised SubQuery Network](https://academy.subquery.network/indexer/run_publish/introduction.html#publish-to-the-subquery-network), the most open, performant, reliable, and scalable data service for dApp developers. -- [Leveraging a centralised hosting partner in the SubQuery community](https://academy.subquery.network/indexer/run_publish/introduction.html#other-hosting-providers-in-the-subquery-community), like OnFinality or Traceye. - -## What Next? - -Take a look at some of our advanced features to take your project to the next level! - -- [**Multi-chain indexing support**](https://academy.subquery.network/build/multi-chain.html) - SubQuery allows you to index data from across different layer-1 networks into the same database, this allows you to query a single endpoint to get data for all supported networks. -- [**Dynamic Data Sources**](https://academy.subquery.network/build/dynamicdatasources.html) - When you want to index factory contracts, for example on a DEX or generative NFT project. -- [**Project Optimisation Advice**](https://academy.subquery.network/build/optimisation.html) - Some common tips on how to tweak your project to maximise performance. -- [**GraphQL Subscriptions**](https://academy.subquery.network/run_publish/subscription.html) - Build more reactive front end applications that subscribe to changes in your SubQuery project. - -## Need Help? - -The fastest way to get support is by [searching our documentation](https://academy.subquery.network), or by [joining our discord](https://discord.com/invite/subquery) and messaging us in the `#technical-support` channel. diff --git a/apps/s03-indexer/docker-compose.yml b/apps/s03-indexer/docker-compose.yml index 31f9912..e6cb227 100644 --- a/apps/s03-indexer/docker-compose.yml +++ b/apps/s03-indexer/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: postgres: build: @@ -31,6 +29,8 @@ services: DB_PORT: 5432 volumes: - ./:/app + extra_hosts: + - "host.docker.internal:host-gateway" command: - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests - -f=/app diff --git a/apps/s03-indexer/package.json b/apps/s03-indexer/package.json index c4b7f4a..8160c90 100644 --- a/apps/s03-indexer/package.json +++ b/apps/s03-indexer/package.json @@ -4,13 +4,13 @@ "description": "This project can be use as a starting point for developing your new Stellar Soroban Test Network SubQuery project", "main": "dist/index.js", "scripts": { - "build": "subql codegen && subql build", + "build": "subql build", "codegen": "subql codegen", - "start:docker": "docker compose pull && docker compose up --remove-orphans", - "dev": "subql codegen && subql build && docker compose pull && docker compose up --remove-orphans", + "start": "docker compose pull && docker compose up --remove-orphans", + "start:docker": "bun run start", + "dev": "bun run codegen && bun run build && bun run start", "prepack": "rm -rf dist && bun run build", - "test": "subql build && subql-node-stellar test", - "build:develop": "NODE_ENV=develop subql codegen && NODE_ENV=develop subql build" + "test": "subql build && subql-node-stellar test" }, "homepage": "https://github.com/subquery/stellar-subql-starter", "repository": "github:subquery/stellar-subql-starter", diff --git a/apps/s03-indexer/project.ts b/apps/s03-indexer/project.ts index 42b76e6..a010152 100644 --- a/apps/s03-indexer/project.ts +++ b/apps/s03-indexer/project.ts @@ -14,6 +14,13 @@ const mode = process.env.NODE_ENV || 'production'; const dotenvPath = path.resolve(__dirname, `.env${mode !== 'production' ? `.${mode}` : ''}`); dotenv.config({ path: dotenvPath, quiet: true }); +const endpoint = + process.env.ENDPOINT ?? "https://horizon-testnet.stellar.org"; +const chainId = + process.env.CHAIN_ID ?? "Test SDF Network ; September 2015"; +const sorobanEndpoint = + process.env.SOROBAN_ENDPOINT ?? "https://soroban-testnet.stellar.org"; + /* This is your project configuration */ const project: StellarProject = { specVersion: "1.0.0", @@ -40,7 +47,7 @@ const project: StellarProject = { 'Test SDF Network ; September 2015' for testnet 'Public Global Stellar Network ; September 2015' for mainnet 'Test SDF Future Network ; October 2022' for Future Network */ - chainId: process.env.CHAIN_ID!, + chainId, /** * These endpoint(s) should be public non-pruned archive node * We recommend providing more than one endpoint for improved reliability, performance, and uptime @@ -49,10 +56,10 @@ const project: StellarProject = { * If you use a rate limited endpoint, adjust the --batch-size and --workers parameters * These settings can be found in your docker-compose.yaml, they will slow indexing but prevent your project being rate limited */ - endpoint: process.env.ENDPOINT!?.split(',') as string[] | string, + endpoint: endpoint.split(',').map((value) => value.trim()), /* This is a specific Soroban endpoint It is only required when you are using a soroban/EventHandler */ - sorobanEndpoint: "https://soroban-testnet.stellar.org", + sorobanEndpoint, }, dataSources: [ { diff --git a/package.json b/package.json index 4138ab8..243018f 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,11 @@ "lint": "turbo lint", "format": "turbo format", "typecheck": "turbo typecheck", - "contracts:gen:all": "bun run scripts/generate-bindings.ts" + "contracts:gen:all": "bun run scripts/generate-bindings.ts", + "indexer:codegen": "bun run --cwd apps/s03-indexer codegen", + "indexer:build": "bun run --cwd apps/s03-indexer build", + "indexer:dev": "bun run --cwd apps/s03-indexer dev", + "indexer:start": "bun run --cwd apps/s03-indexer start" }, "devDependencies": { "prettier": "^3.8.1",