diff --git a/docker/polkastats-backend/docker-compose.hasura.yml b/docker/polkastats-backend/docker-compose.hasura.yml new file mode 100644 index 00000000..bb10c6d4 --- /dev/null +++ b/docker/polkastats-backend/docker-compose.hasura.yml @@ -0,0 +1,20 @@ +version: '3.7' + +services: + # + # Hasura + # + graphql-engine: + image: hasura/graphql-engine:v2.2.0 + ports: + - '8080:8080' + environment: + HASURA_GRAPHQL_DATABASE_URL: '${GRAPHQL_DATABASE_URL}' # postgres://polkastats:polkastats@host.docker.internal:5432/polkastats + HASURA_GRAPHQL_ENABLE_CONSOLE: '${GRAPHQL_ENABLE_CONSOLE}' # set to "false" to disable console + HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log + HASURA_GRAPHQL_DISABLE_CORS: '${GRAPHQL_DISABLE_CORS}' + HASURA_GRAPHQL_UNAUTHORIZED_ROLE: 'anonymous' + HASURA_GRAPHQL_JWT_SECRET: '${GRAPHQL_JWT_SECRET}' + ## uncomment next line to set an admin secret + HASURA_GRAPHQL_ADMIN_SECRET: '${GRAPHQL_ADMIN_SECRET}' + HASURA_GRAPHQL_PG_CONNECTIONS: '${GRAPHQL_PG_CONNECTIONS}' diff --git a/hasura/hasura_metadata.json b/hasura/hasura_metadata.json index 8962ecf2..362b3f48 100644 --- a/hasura/hasura_metadata.json +++ b/hasura/hasura_metadata.json @@ -1 +1 @@ -{"version":3,"sources":[{"name":"default","kind":"postgres","tables":[{"table":{"schema":"public","name":"SequelizeMeta"}},{"table":{"schema":"public","name":"account"},"select_permissions":[{"role":"anonymous","permission":{"columns":["account_id","balances","available_balance","free_balance","locked_balance","nonce","timestamp","block_height","account_id_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","session_length","timestamp","need_rescan","new_accounts","num_transfers","spec_version","total_events","block_hash","extrinsics_root","parent_hash","spec_name","state_root","total_issuance"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"collections"},"array_relationships":[{"name":"tokens","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"tokens"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes_schema","collection_cover","collection_id","const_chain_schema","date_of_creation","description","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","properties","schema_version","sponsorship","token_limit","token_prefix","variable_on_chain_schema"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"event"}},{"table":{"schema":"public","name":"extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","args","block_index","block_number","extrinsic_index","fee","hash","is_signed","method","section","signer","signer_normalized","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"system"}},{"table":{"schema":"public","name":"tokens"},"object_relationships":[{"name":"collection","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"collections"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["date_of_creation","id","owner","token_id","data","owner_normalized","collection_id","parent_id"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"total"},"select_permissions":[{"role":"anonymous","permission":{"columns":["count","name"],"filter":{}}}]},{"table":{"schema":"public","name":"view_collections"},"select_permissions":[{"role":"anonymous","permission":{"columns":["actions_count","collection_cover","collection_id","const_chain_schema","date_of_creation","description","holders_count","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","schema_version","sponsorship","token_limit","token_prefix","tokens_count","type"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","block_index","block_number","fee","from_owner","from_owner_normalized","hash","method","section","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_holders"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_id","count","owner","owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_last_block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","event_count","extrinsic_count","timestamp"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_tokens"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_cover","collection_description","collection_id","collection_name","collection_owner","collection_owner_normalized","data","date_of_creation","image_path","is_sold","owner","owner_normalized","token_id","token_name","token_prefix"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_transfer"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_index","section","method","data"],"filter":{},"allow_aggregations":true}}]}],"configuration":{"connection_info":{"use_prepared_statements":true,"database_url":{"from_env":"HASURA_GRAPHQL_DATABASE_URL"},"isolation_level":"read-committed","pool_settings":{"connection_lifetime":600,"retries":1,"idle_timeout":180,"max_connections":2}}}}]} \ No newline at end of file +{"version":3,"sources":[{"name":"default","kind":"postgres","tables":[{"table":{"schema":"public","name":"SequelizeMeta"}},{"table":{"schema":"public","name":"account"},"select_permissions":[{"role":"anonymous","permission":{"columns":["account_id","balances","available_balance","free_balance","locked_balance","nonce","timestamp","block_height","account_id_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","session_length","timestamp","need_rescan","new_accounts","num_transfers","spec_version","total_events","block_hash","extrinsics_root","parent_hash","spec_name","state_root","total_issuance"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"collections"},"array_relationships":[{"name":"tokens","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"tokens"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes_schema","collection_cover","collection_id","const_chain_schema","date_of_creation","description","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","properties","schema_version","sponsorship","token_limit","token_prefix","variable_on_chain_schema"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"event"}},{"table":{"schema":"public","name":"extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","args","block_index","block_number","extrinsic_index","fee","hash","is_signed","method","section","signer","signer_normalized","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"system"}},{"table":{"schema":"public","name":"tokens"},"object_relationships":[{"name":"collection","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"collections"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes","collection_id","data","date_of_creation","id","owner","owner_normalized","parent_id","properties","token_id"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"total"},"select_permissions":[{"role":"anonymous","permission":{"columns":["count","name"],"filter":{}}}]},{"table":{"schema":"public","name":"view_collections"},"select_permissions":[{"role":"anonymous","permission":{"columns":["actions_count","collection_cover","collection_id","const_chain_schema","date_of_creation","description","holders_count","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","schema_version","sponsorship","token_limit","token_prefix","tokens_count","type"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","block_index","block_number","fee","from_owner","from_owner_normalized","hash","method","section","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_holders"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_id","count","owner","owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_last_block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","event_count","extrinsic_count","timestamp"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_tokens"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_cover","collection_description","collection_id","collection_name","collection_owner","collection_owner_normalized","data","date_of_creation","image_path","is_sold","owner","owner_normalized","token_id","token_name","token_prefix"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_transfer"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_index","section","method","data"],"filter":{},"allow_aggregations":true}}]}],"configuration":{"connection_info":{"use_prepared_statements":true,"database_url":{"from_env":"HASURA_GRAPHQL_DATABASE_URL"},"isolation_level":"read-committed","pool_settings":{"connection_lifetime":600,"retries":1,"idle_timeout":180,"max_connections":2}}}}]} \ No newline at end of file diff --git a/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts b/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts index d8096e05..f630a0ab 100644 --- a/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts @@ -1,5 +1,9 @@ -import { CollectionInfoWithSchema } from '@unique-nft/sdk/tokens'; -import { UpDataStructsCollectionLimits, UpDataStructsRpcCollection } from '@unique-nft/unique-mainnet-types'; +import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; +import { + UpDataStructsCollectionLimits, + UpDataStructsRpcCollection, + UpDataStructsTokenData, +} from '@unique-nft/unique-mainnet-types'; import { ImplementOpalAPI } from '../implement/implementOpalAPI'; import AbstractAPI from './abstractAPI'; @@ -26,9 +30,22 @@ export class OpalAPI extends AbstractAPI { }; } - async getToken(collectionId, tokenId) { - const token = await this.impl.impGetToken(collectionId, tokenId); - return token || null; + async getToken(collectionId, tokenId): Promise<{ + rawToken: UpDataStructsTokenData | null, + tokenDecoded: UniqueTokenDecoded | null, + tokenProperties: TokenPropertiesResult | null + }> { + const [rawToken, tokenDecoded, tokenProperties] = await Promise.all([ + this.impl.impGetToken(collectionId, tokenId), + this.impl.impGetTokenSdk(collectionId, tokenId), + this.impl.impGetTokenPropertiesSdk(collectionId, tokenId) + ]); + + return { + rawToken, + tokenDecoded, + tokenProperties + }; } getCollectionCount() { diff --git a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts index 97fa61ac..417b61b7 100644 --- a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts @@ -2,10 +2,10 @@ import { UpDataStructsCollectionLimits, UpDataStructsRpcCollection, - UpDataStructsTokenData, + UpDataStructsTokenData } from '@unique-nft/unique-mainnet-types'; -import { CollectionInfoWithSchema } from '@unique-nft/sdk/tokens'; -import '@unique-nft/sdk/tokens'; // need this to get sdk.collections +import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; +import '@unique-nft/sdk/tokens'; // need this to get sdk.collections and sdk.tokens declarations import ImplementorAPI from './implementorAPI'; export class ImplementOpalAPI extends ImplementorAPI { @@ -20,9 +20,7 @@ export class ImplementOpalAPI extends ImplementorAPI { } async impGetCollectionSdk(collectionId): Promise { - const result = await this.sdk.collections.get_new({ collectionId }); - - return result; + return this.sdk.collections.get_new({ collectionId }); } async impGetCollectionCount() { @@ -30,12 +28,20 @@ export class ImplementOpalAPI extends ImplementorAPI { return collectionStats?.created.toNumber(); } - async impGetToken(collectionId, tokenId): Promise { + async impGetToken(collectionId, tokenId): Promise { const tokenData = await this.api.rpc.unique.tokenData(collectionId, tokenId); return tokenData || null; } + async impGetTokenSdk(collectionId, tokenId): Promise { + return this.sdk.tokens.get_new({ collectionId, tokenId }); + } + async impGetTokenCount(collectionId) { return (await this.api.rpc.unique.lastTokenId(collectionId)).toNumber(); } + + async impGetTokenPropertiesSdk(collectionId, tokenId): Promise { + return this.sdk.tokens.properties({ collectionId, tokenId }); + } } diff --git a/lib/token/tokenData.ts b/lib/token/tokenData.ts index 2456afd5..9e2dfdb1 100644 --- a/lib/token/tokenData.ts +++ b/lib/token/tokenData.ts @@ -1,7 +1,12 @@ +/* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable no-underscore-dangle */ -import { ICollectionSchemaInfo } from 'crawlers/crawlers.interfaces'; import { UpDataStructsTokenData } from '@unique-nft/unique-mainnet-types'; -import { normalizeSubstrateAddress } from '../../utils/utils'; +import { ICollectionSchemaInfo } from 'crawlers/crawlers.interfaces'; +import { TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; +import { + normalizeSubstrateAddress, + sanitizePropertiesValues +} from '../../utils/utils'; import protobuf from '../../utils/protobuf'; import { ITokenDbEntity } from './tokenDbEntity.interface'; import { OpalAPI } from '../providerAPI/bridgeProviderAPI/concreate/opalAPI'; @@ -51,13 +56,11 @@ function processConstData(constData, schema) { return getDeserializeConstData(statement); } -function processProperties(schema: any, rawToken: UpDataStructsTokenData) - : { data: Object, properties: Object } { +function processOldProperties(schema: any, rawToken: UpDataStructsTokenData) + : { data: Object } { const rawProperties = rawToken.properties; - const properties : { - _old_constData?: Object | string, - } = {}; + let oldConstData: Object | string | null = null; rawProperties.forEach(({ key, value }) => { const strKey = key.toUtf8(); @@ -66,38 +69,71 @@ function processProperties(schema: any, rawToken: UpDataStructsTokenData) if (['_old_constData'].includes(strKey)) { try { processedValue = value.toHex(); } catch (err) { /* */ } - } - properties[strKey] = processedValue || strValue; + oldConstData = processedValue || strValue; + } }); return { - data: processConstData(properties._old_constData, schema), - properties, + data: processConstData(oldConstData, schema), }; } -function formatTokenData(tokenId: number, collectionInfo: ICollectionSchemaInfo, rawToken: UpDataStructsTokenData) - : ITokenDbEntity { - const { collectionId, schema } = collectionInfo; - - const rawOwnerJson = rawToken.owner.toJSON() as { substrate?: string, ethereum?: string }; - - const owner = rawOwnerJson?.substrate || rawOwnerJson?.ethereum; +function formatTokenData({ + rawToken, + tokenDecoded, + tokenProperties, + collectionInfo +}: { + rawToken: UpDataStructsTokenData, + tokenDecoded: UniqueTokenDecoded, + tokenProperties: TokenPropertiesResult, + collectionInfo: ICollectionSchemaInfo +}) : ITokenDbEntity { + const { schema } = collectionInfo; + + const { + tokenId: token_id, + collectionId: collection_id, + attributes, + nestingParentToken, + } = tokenDecoded; + + const { + owner: rawOwner, + }: { owner: { Ethereum?: string; Substrate?: string } } = tokenDecoded; + + const owner = rawOwner?.Ethereum || rawOwner?.Substrate; + + let parentId = null; + if (nestingParentToken) { + const { collectionId, tokenId } = nestingParentToken as { collectionId: number; tokenId: number }; + parentId = `${collectionId}_${tokenId}`; + } return { - token_id: tokenId, - collection_id: collectionId, + token_id, + collection_id, owner, owner_normalized: normalizeSubstrateAddress(owner), - ...processProperties(schema, rawToken), + attributes: JSON.stringify(attributes), + properties: tokenProperties + ? JSON.stringify(sanitizePropertiesValues(tokenProperties)) + : '[]', + parent_id: parentId, + ...processOldProperties(schema, rawToken), }; } export async function getFormattedToken(tokenId: number, collectionInfo: ICollectionSchemaInfo, bridgeAPI: OpalAPI) : Promise { const { collectionId } = collectionInfo; - const rawToken = await bridgeAPI.getToken(collectionId, tokenId); - - return rawToken ? formatTokenData(tokenId, collectionInfo, rawToken) : null; + const { rawToken, tokenDecoded, tokenProperties } = await bridgeAPI.getToken(collectionId, tokenId); + + return tokenDecoded ? formatTokenData({ + rawToken, + tokenDecoded, + tokenProperties, + collectionInfo + }) : null; } diff --git a/lib/token/tokenDb.ts b/lib/token/tokenDb.ts index 055ae0ea..15f280b4 100644 --- a/lib/token/tokenDb.ts +++ b/lib/token/tokenDb.ts @@ -17,6 +17,8 @@ const TOKEN_FIELDS = [ 'data', 'date_of_creation', 'parent_id', + 'properties', + 'attributes' ]; function prepareQueryReplacements(token: ITokenDbEntity) { diff --git a/lib/token/tokenDbEntity.interface.ts b/lib/token/tokenDbEntity.interface.ts index df212713..a30879de 100644 --- a/lib/token/tokenDbEntity.interface.ts +++ b/lib/token/tokenDbEntity.interface.ts @@ -6,4 +6,7 @@ export interface ITokenDbEntity { owner_normalized: string, data: Object, date_of_creation?: number, + properties: string, + attributes: string, + parent_id: string | null } diff --git a/sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js b/sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js new file mode 100644 index 00000000..019801a1 --- /dev/null +++ b/sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js @@ -0,0 +1,23 @@ +'use strict'; + +module.exports = { + async up (queryInterface, Sequelize) { + return Promise.all([ + queryInterface.addColumn('tokens', 'properties', { + type: Sequelize.DataTypes.JSONB, + defaultValue: [] + }), + queryInterface.addColumn('tokens', 'attributes', { + type: Sequelize.DataTypes.JSONB, + defaultValue: {} + }), + ]) + }, + + async down (queryInterface, Sequelize) { + return Promise.all([ + await queryInterface.removeColumn('tokens', 'properties'), + await queryInterface.removeColumn('tokens', 'attributes'), + ]); + } +}; diff --git a/utils/utils.js b/utils/utils.js index d2ee35d2..d4286715 100644 --- a/utils/utils.js +++ b/utils/utils.js @@ -187,7 +187,15 @@ function getTokenIdFromNestingAddress(address) { } function sanitizeUnicodeString(str) { - return str.replace(/\\u0000/g, ''); + // eslint-disable-next-line no-control-regex + return str.replace(/\\u0000|\x00/g, ''); +} + +function sanitizePropertiesValues(propertiesArr) { + return propertiesArr.map(({ key, value }) => ({ + key, + value: sanitizeUnicodeString(value), + })); } module.exports = { @@ -210,5 +218,6 @@ module.exports = { isNestingAddress, getCollectionIdFromNestingAddress, getTokenIdFromNestingAddress, - sanitizeUnicodeString + sanitizeUnicodeString, + sanitizePropertiesValues };