Skip to content

Commit

Permalink
Merge pull request #1051 from autonomys/feat/keep-improving-consensus…
Browse files Browse the repository at this point in the history
…-indexer-speed

Keep improving consensus indexer speed
marc-aurele-besner authored Jan 7, 2025
2 parents 1bde0b7 + 42d9f08 commit cf5ece1
Showing 10 changed files with 128 additions and 108,584 deletions.
92 changes: 86 additions & 6 deletions indexers/db/docker-entrypoint-initdb.d/init-db.sql
Original file line number Diff line number Diff line change
@@ -145,6 +145,14 @@ CREATE TABLE consensus.blocks (
reward_value numeric NOT NULL,
block_reward_value numeric NOT NULL,
vote_reward_value numeric NOT NULL,
author_id text NOT NULL,
_id uuid NOT NULL,
_block_range int8range NOT NULL
);
ALTER TABLE consensus.blocks OWNER TO postgres;

CREATE TABLE consensus.cumulative_blocks (
id text NOT NULL,
cumulative_extrinsics_count numeric NOT NULL,
cumulative_events_count numeric NOT NULL,
cumulative_transfers_count numeric NOT NULL,
@@ -154,12 +162,9 @@ CREATE TABLE consensus.blocks (
cumulative_transfer_value numeric NOT NULL,
cumulative_reward_value numeric NOT NULL,
cumulative_block_reward_value numeric NOT NULL,
cumulative_vote_reward_value numeric NOT NULL,
author_id text NOT NULL,
_id uuid NOT NULL,
_block_range int8range NOT NULL
cumulative_vote_reward_value numeric NOT NULL
);
ALTER TABLE consensus.blocks OWNER TO postgres;
ALTER TABLE consensus.cumulative_blocks OWNER TO postgres;

CREATE TABLE consensus.event_modules (
id text NOT NULL,
@@ -883,6 +888,9 @@ ALTER TABLE ONLY consensus.accounts
ALTER TABLE ONLY consensus.blocks
ADD CONSTRAINT blocks_pkey PRIMARY KEY (_id);

ALTER TABLE ONLY consensus.cumulative_blocks
ADD CONSTRAINT cumulative_blocks_pkey PRIMARY KEY (id);

ALTER TABLE ONLY consensus.event_modules
ADD CONSTRAINT event_modules_id_key PRIMARY KEY (id);

@@ -1445,4 +1453,76 @@ ALTER FUNCTION consensus.update_account() OWNER TO postgres;
CREATE TRIGGER ensure_account_updated
BEFORE INSERT ON consensus.account_histories
FOR EACH ROW
EXECUTE FUNCTION consensus.update_account();
EXECUTE FUNCTION consensus.update_account();

CREATE FUNCTION consensus.update_cumulative_blocks() RETURNS trigger
LANGUAGE plpgsql
AS $$
DECLARE
prev_cumulative consensus.cumulative_blocks%ROWTYPE;
BEGIN
SELECT *
INTO prev_cumulative
FROM consensus.cumulative_blocks
WHERE id = text(NEW.height - 1);

IF prev_cumulative IS NULL THEN
prev_cumulative.cumulative_extrinsics_count := 0;
prev_cumulative.cumulative_events_count := 0;
prev_cumulative.cumulative_transfers_count := 0;
prev_cumulative.cumulative_rewards_count := 0;
prev_cumulative.cumulative_block_rewards_count := 0;
prev_cumulative.cumulative_vote_rewards_count := 0;
prev_cumulative.cumulative_transfer_value := 0;
prev_cumulative.cumulative_reward_value := 0;
prev_cumulative.cumulative_block_reward_value := 0;
prev_cumulative.cumulative_vote_reward_value := 0;
END IF;

INSERT INTO consensus.cumulative_blocks (
id,
cumulative_extrinsics_count,
cumulative_events_count,
cumulative_transfers_count,
cumulative_rewards_count,
cumulative_block_rewards_count,
cumulative_vote_rewards_count,
cumulative_transfer_value,
cumulative_reward_value,
cumulative_block_reward_value,
cumulative_vote_reward_value
)
VALUES (
NEW.id,
prev_cumulative.cumulative_extrinsics_count + NEW.extrinsics_count,
prev_cumulative.cumulative_events_count + NEW.events_count,
prev_cumulative.cumulative_transfers_count + NEW.transfers_count,
prev_cumulative.cumulative_rewards_count + NEW.rewards_count,
prev_cumulative.cumulative_block_rewards_count + NEW.block_rewards_count,
prev_cumulative.cumulative_vote_rewards_count + NEW.vote_rewards_count,
prev_cumulative.cumulative_transfer_value + NEW.transfer_value,
prev_cumulative.cumulative_reward_value + NEW.reward_value,
prev_cumulative.cumulative_block_reward_value + NEW.block_reward_value,
prev_cumulative.cumulative_vote_reward_value + NEW.vote_reward_value
)
ON CONFLICT (id) DO UPDATE SET
cumulative_extrinsics_count = prev_cumulative.cumulative_extrinsics_count + NEW.extrinsics_count,
cumulative_events_count = prev_cumulative.cumulative_events_count + NEW.events_count,
cumulative_transfers_count = prev_cumulative.cumulative_transfers_count + NEW.transfers_count,
cumulative_rewards_count = prev_cumulative.cumulative_rewards_count + NEW.rewards_count,
cumulative_block_rewards_count = prev_cumulative.cumulative_block_rewards_count + NEW.block_rewards_count,
cumulative_vote_rewards_count = prev_cumulative.cumulative_vote_rewards_count + NEW.vote_rewards_count,
cumulative_transfer_value = prev_cumulative.cumulative_transfer_value + NEW.transfer_value,
cumulative_reward_value = prev_cumulative.cumulative_reward_value + NEW.reward_value,
cumulative_block_reward_value = prev_cumulative.cumulative_block_reward_value + NEW.block_reward_value,
cumulative_vote_reward_value = prev_cumulative.cumulative_vote_reward_value + NEW.vote_reward_value;

RETURN NEW;
END;
$$;
ALTER FUNCTION consensus.update_cumulative_blocks() OWNER TO postgres;

CREATE TRIGGER ensure_cumulative_blocks
BEFORE INSERT ON consensus.blocks
FOR EACH ROW
EXECUTE FUNCTION consensus.update_cumulative_blocks();
20 changes: 10 additions & 10 deletions indexers/db/metadata/databases/default/tables/consensus_blocks.yaml
Original file line number Diff line number Diff line change
@@ -8,6 +8,16 @@ configuration:
custom_column_names:
_id: uuid
custom_root_fields: {}
object_relationships:
- name: cumulative
using:
manual_configuration:
column_mapping:
id: id
insertion_order: null
remote_table:
name: cumulative_blocks
schema: consensus
array_relationships:
- name: cids
using:
@@ -58,16 +68,6 @@ select_permissions:
- vote_rewards_count
- block_reward_value
- blockchain_size
- cumulative_block_reward_value
- cumulative_block_rewards_count
- cumulative_events_count
- cumulative_extrinsics_count
- cumulative_reward_value
- cumulative_rewards_count
- cumulative_transfer_value
- cumulative_transfers_count
- cumulative_vote_reward_value
- cumulative_vote_rewards_count
- height
- reward_value
- space_pledged
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
table:
name: cumulative_blocks
schema: consensus
object_relationships:
- name: block
using:
manual_configuration:
column_mapping:
id: id
insertion_order: null
remote_table:
name: blocks
schema: consensus
select_permissions:
- role: user
permission:
columns:
- cumulative_block_reward_value
- cumulative_block_rewards_count
- cumulative_events_count
- cumulative_extrinsics_count
- cumulative_reward_value
- cumulative_rewards_count
- cumulative_transfer_value
- cumulative_transfers_count
- cumulative_vote_reward_value
- cumulative_vote_rewards_count
- id
filter: {}
comment: ""
1 change: 1 addition & 0 deletions indexers/db/metadata/databases/default/tables/tables.yaml
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
- "!include consensus_account_rewards.yaml"
- "!include consensus_accounts.yaml"
- "!include consensus_blocks.yaml"
- "!include consensus_cumulative_blocks.yaml"
- "!include consensus_event_modules.yaml"
- "!include consensus_events.yaml"
- "!include consensus_extrinsic_modules.yaml"
12 changes: 0 additions & 12 deletions indexers/db/scripts/genisis-allocation-to-seeds/main.ts
Original file line number Diff line number Diff line change
@@ -11,11 +11,6 @@ const data = await response.json();
await Deno.mkdir("seeds", { recursive: true });

// Open a file to write the SQL output
const accountsFile = await Deno.open("seeds/accounts.sql", {
write: true,
create: true,
truncate: true,
});
const accountHistoriesFile = await Deno.open("seeds/account_histories.sql", {
write: true,
create: true,
@@ -32,16 +27,10 @@ for (const entry of data) {
const reservedBalance = "0";
const totalBalance = freeBalance;
const createdAt = 0;
const updatedAt = 0;
const uniqueNamespace = await v1.generate();
const _id = await v5.generate(uniqueNamespace, accountId);
const _blockRange = "[0,)";

await accountsFile.write(
new TextEncoder().encode(
`INSERT INTO consensus.accounts (id, account_id, nonce, free, reserved, total, created_at, updated_at, _id, _block_range) VALUES ('${accountId}', '${accountId}', 0, ${freeBalance}, ${reservedBalance}, ${totalBalance}, ${createdAt}, ${updatedAt}, '${_id}', '${_blockRange}');\n`
)
);
await accountHistoriesFile.write(
new TextEncoder().encode(
`INSERT INTO consensus.account_histories (id, nonce, free, reserved, total, created_at, _id, _block_range) VALUES ('${accountId}', 0, ${freeBalance}, ${reservedBalance}, ${totalBalance}, ${createdAt}, '${_id}', '${_blockRange}');\n`
@@ -50,5 +39,4 @@ for (const entry of data) {
}

// Close the file
accountsFile.close();
accountHistoriesFile.close();
Loading

0 comments on commit cf5ece1

Please sign in to comment.