Skip to content

Optimize arcade-ts marketplace SQL #294

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ponderingdemocritus wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Optimize arcade-ts marketplace SQL #294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2b50779
Optimize arcade-ts SQL queries
ponderingdemocritus Mar 10, 2026
132e2e5
Fix arcade-ts lint failure
ponderingdemocritus Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 164 additions & 0 deletions packages/arcade-ts/src/marketplace/client.edge.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,168 @@ describe("createEdgeMarketplaceClient", () => {
expect.stringContaining("FROM tokens"),
);
});

it("uses keyset pagination instead of offset pagination", async () => {
mockedFetchToriisSql.mockResolvedValueOnce({
data: [
{
endpoint: "arcade-main",
data: [
{
contract_address: "0xabc",
token_id: "2",
metadata: JSON.stringify({ name: "Token 2" }),
},
],
},
],
errors: [],
} as any);

const client = await createEdgeMarketplaceClient({
chainId: constants.StarknetChainId.SN_MAIN,
});

const result = await client.listCollectionTokens({
address: "0xabc",
cursor: "1",
limit: 1,
fetchImages: false,
});

expect(result.error).toBeNull();
expect(result.page?.tokens[0]?.token_id).toBe("2");
expect(mockedFetchToriisSql).toHaveBeenCalledWith(
["arcade-main"],
expect.stringContaining("token_id > '1'"),
);
expect(mockedFetchToriisSql).not.toHaveBeenCalledWith(
["arcade-main"],
expect.stringContaining("OFFSET"),
);
});

it("continues fetching raw SQL pages until it fills a filtered page", async () => {
mockedFetchToriisSql
.mockResolvedValueOnce({
data: [
{
endpoint: "arcade-main",
data: [
{
contract_address: "0xabc",
token_id: "1",
metadata: JSON.stringify({
name: "Token 1",
attributes: [{ trait_type: "rarity", value: "common" }],
}),
},
],
},
],
errors: [],
} as any)
.mockResolvedValueOnce({
data: [
{
endpoint: "arcade-main",
data: [
{
contract_address: "0xabc",
token_id: "2",
metadata: JSON.stringify({
name: "Token 2",
attributes: [{ trait_type: "rarity", value: "legendary" }],
}),
},
],
},
],
errors: [],
} as any);

const client = await createEdgeMarketplaceClient({
chainId: constants.StarknetChainId.SN_MAIN,
});

const result = await client.listCollectionTokens({
address: "0xabc",
limit: 1,
attributeFilters: { rarity: ["legendary"] },
fetchImages: false,
});

expect(result.error).toBeNull();
expect(result.page?.tokens).toHaveLength(1);
expect(result.page?.tokens[0]?.token_id).toBe("2");
expect(result.page?.nextCursor).toBe("2");
expect(mockedFetchToriisSql).toHaveBeenCalledTimes(2);
expect(mockedFetchToriisSql.mock.calls[1]?.[1]).toContain("token_id > '1'");
});

it("verifies ownership with exact predicates and a SQL-side zero-balance filter", async () => {
mockedFetchToriisSql
.mockResolvedValueOnce({
data: [
{
endpoint: "arcade-main",
data: [
{
id: 7,
category: 2,
status: 1,
expiration: Math.floor(Date.now() / 1000) + 3600,
collection:
"0x0000000000000000000000000000000000000000000000000000000000000abc",
token_id: "1",
quantity: 1,
price: 1,
currency:
"0x0000000000000000000000000000000000000000000000000000000000000000",
owner:
"0x0000000000000000000000000000000000000000000000000000000000000def",
},
],
},
],
errors: [],
} as any)
.mockResolvedValueOnce({
data: [
{
endpoint: "arcade-main",
data: [
{
account_address:
"0x0000000000000000000000000000000000000000000000000000000000000def",
token_id: "1",
balance: "0x1",
},
],
},
],
errors: [],
} as any);

const client = await createEdgeMarketplaceClient({
chainId: constants.StarknetChainId.SN_MAIN,
});

const listings = await client.listCollectionListings({
collection: "0xabc",
limit: 1,
});

expect(listings).toHaveLength(1);
expect(mockedFetchToriisSql.mock.calls[0]?.[1]).toContain(
"collection = '0x0000000000000000000000000000000000000000000000000000000000000abc'",
);
expect(mockedFetchToriisSql.mock.calls[1]?.[1]).toContain(
"contract_address = '0x0000000000000000000000000000000000000000000000000000000000000abc'",
);
expect(mockedFetchToriisSql.mock.calls[1]?.[1]).toContain(
"balance != '0x0000000000000000000000000000000000000000000000000000000000000000'",
);
expect(mockedFetchToriisSql.mock.calls[1]?.[1]).not.toContain("lower(");
});
});
135 changes: 85 additions & 50 deletions packages/arcade-ts/src/marketplace/client.edge.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import {
} from "./utils";

const DEFAULT_LIMIT = 100;
const ZERO_BALANCE_HEX =
"0x0000000000000000000000000000000000000000000000000000000000000000";

const statusValueMap: Record<StatusType, number> = {
[StatusType.None]: 0,
Expand Down Expand Up @@ -78,6 +80,9 @@ const ensureProjectId = (

const escapeSqlValue = (value: string): string => value.replace(/'/g, "''");

const normalizeAddressForSql = (value: string): string =>
addAddressPadding(getChecksumAddress(value)).toLowerCase();

const extractRows = (data: any): any[] => {
if (!data) return [];
if (Array.isArray(data)) return data;
Expand Down Expand Up @@ -165,9 +170,9 @@ async function verifyListingsOwnership(
): Promise<OrderModel[]> {
if (!listings.length) return listings;

const collection = addAddressPadding(getChecksumAddress(collectionAddress));
const collection = normalizeAddressForSql(collectionAddress);
const owners = [
...new Set(listings.map((order) => getChecksumAddress(order.owner))),
...new Set(listings.map((order) => normalizeAddressForSql(order.owner))),
];
if (owners.length === 0) return [];

Expand All @@ -181,9 +186,11 @@ async function verifyListingsOwnership(

const sql = `SELECT account_address, token_id, balance
FROM token_balances
WHERE lower(contract_address) = lower('${escapeSqlValue(collection)}')
AND lower(account_address) IN (${ownerList})
AND token_id IN (${tokenIdList})`;
WHERE contract_address = '${escapeSqlValue(collection)}'
AND account_address IN (${ownerList})
AND token_id IN (${tokenIdList})
AND balance != '${ZERO_BALANCE_HEX}'
AND balance != '0'`;

const rows = await querySql(projectId, sql);
const ownership = new Set<string>();
Expand Down Expand Up @@ -218,31 +225,28 @@ export async function createEdgeMarketplaceClient(
): Promise<NormalizedCollection | null> => {
const { projectId: projectIdInput, address, fetchImages = true } = options;
const projectId = ensureProjectId(projectIdInput, defaultProject);
const collection = addAddressPadding(getChecksumAddress(address));
const collection = normalizeAddressForSql(address);

const rows = await querySql(
projectId,
`SELECT contract_address, contract_type, type, metadata, total_supply, token_id
`SELECT contract_address, contract_type, type, metadata, total_supply, token_id,
(SELECT metadata FROM tokens WHERE contract_address = '${escapeSqlValue(collection)}' LIMIT 1) AS fallback_metadata,
(SELECT token_id FROM tokens WHERE contract_address = '${escapeSqlValue(collection)}' LIMIT 1) AS fallback_token_id
FROM token_contracts
WHERE lower(contract_address) = lower('${escapeSqlValue(collection)}')
WHERE contract_address = '${escapeSqlValue(collection)}'
LIMIT 1`,
);

const contract = rows[0];
if (!contract) return null;

let tokenSample: any | undefined;
let metadataRaw = contract.metadata;
if (!metadataRaw) {
const tokenRows = await querySql(
projectId,
`SELECT token_id, metadata
FROM tokens
WHERE lower(contract_address) = lower('${escapeSqlValue(collection)}')
LIMIT 1`,
);
tokenSample = tokenRows[0];
if (tokenSample?.metadata) metadataRaw = tokenSample.metadata;
const metadataRaw = contract.metadata ?? contract.fallback_metadata;
if (contract.fallback_token_id || contract.fallback_metadata) {
tokenSample = {
token_id: contract.fallback_token_id,
metadata: contract.fallback_metadata,
};
}

const metadata = parseJsonSafe(metadataRaw, metadataRaw);
Expand Down Expand Up @@ -291,46 +295,81 @@ LIMIT 1`,
fetchImages = false,
} = options;
const projectId = ensureProjectId(project, defaultProject);
const collection = addAddressPadding(getChecksumAddress(address));
const offset = cursor ? Number.parseInt(cursor, 10) || 0 : 0;

const conditions = [
`lower(contract_address) = lower('${escapeSqlValue(collection)}')`,
const collection = normalizeAddressForSql(address);
const pageLimit = Math.max(1, Math.floor(limit));
const requiresPostFilter = Boolean(
attributeFilters && Object.keys(attributeFilters).length > 0,
);
const baseConditions = [
`contract_address = '${escapeSqlValue(collection)}'`,
];

if (tokenIds && tokenIds.length > 0) {
const values = [
...new Set(tokenIds.map((value) => escapeSqlValue(value))),
...new Set(
tokenIds
.map((value) => normalizeTokenIdForQuery(value) ?? value)
.map((value) => escapeSqlValue(value)),
),
];
conditions.push(
baseConditions.push(
`token_id IN (${values.map((v) => `'${v}'`).join(", ")})`,
);
}

const sql = `SELECT contract_address, token_id, metadata, name, symbol, decimals
try {
const tokens: NormalizedToken[] = [];
let nextCursor = cursor ? String(cursor) : null;
let hasMore = true;

while (tokens.length < pageLimit && hasMore) {
const conditions = [...baseConditions];
if (nextCursor) {
conditions.push(`token_id > '${escapeSqlValue(nextCursor)}'`);
}

const sql = `SELECT contract_address, token_id, metadata, name, symbol, decimals
FROM tokens
WHERE ${conditions.join(" AND ")}
ORDER BY token_id
LIMIT ${Math.max(1, Math.floor(limit))}
OFFSET ${Math.max(0, offset)}`;

try {
const rows = await querySql(projectId, sql);
const normalized = await normalizeTokens(rows as any[], projectId, {
fetchImages,
resolveTokenImage: resolveTokenImage ?? defaultResolveTokenImage,
});

const filtered = normalized.filter((token) =>
tokenMatchesAttributeFilters(token, attributeFilters),
) as NormalizedToken[];
LIMIT ${pageLimit}`;

const rows = await querySql(projectId, sql);
if (rows.length === 0) {
nextCursor = null;
break;
}

const normalized = await normalizeTokens(rows as any[], projectId, {
fetchImages,
resolveTokenImage: resolveTokenImage ?? defaultResolveTokenImage,
});

const filtered = requiresPostFilter
? normalized.filter((token) =>
tokenMatchesAttributeFilters(token, attributeFilters),
)
: normalized;

for (const token of filtered as NormalizedToken[]) {
tokens.push(token);
if (tokens.length === pageLimit) break;
}

const lastRow = rows[rows.length - 1];
nextCursor =
lastRow?.token_id != null ? String(lastRow.token_id) : nextCursor;
hasMore = rows.length === pageLimit;

if (!requiresPostFilter || !hasMore) {
break;
}
}

const nextCursor =
rows.length >= limit ? String(offset + rows.length) : null;
return {
page: {
tokens: filtered,
nextCursor,
tokens,
nextCursor: hasMore ? nextCursor : null,
},
error: null,
};
Expand All @@ -348,18 +387,14 @@ OFFSET ${Math.max(0, offset)}`;
const getCollectionOrders = async (
options: CollectionOrdersOptions,
): Promise<OrderModel[]> => {
const collection = addAddressPadding(
getChecksumAddress(options.collection),
);
const collection = normalizeAddressForSql(options.collection);
const tokenId = normalizeTokenIdForQuery(options.tokenId);
const status =
options.status != null ? statusValueMap[options.status] : undefined;
const category =
options.category != null ? categoryValueMap[options.category] : undefined;

const conditions = [
`lower(collection) = lower('${escapeSqlValue(collection)}')`,
];
const conditions = [`collection = '${escapeSqlValue(collection)}'`];
if (tokenId !== undefined) {
conditions.push(`token_id = '${escapeSqlValue(tokenId)}'`);
}
Expand Down
Loading