Skip to content

Commit

Permalink
perf: improve search index speed (#1193)
Browse files Browse the repository at this point in the history
  • Loading branch information
@caoxing9
caoxing9 authored Dec 27, 2024
1 parent 975f4d0 commit 12ec0e8
Show file tree
Hide file tree
Showing 7 changed files with 306 additions and 57 deletions.
9 changes: 6 additions & 3 deletions apps/nestjs-backend/src/db-provider/db.provider.interface.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { DriverClient, IFilter, ISortItem } from '@teable/core';
import type { Prisma } from '@teable/db-main-prisma';
import type { IAggregationField } from '@teable/openapi';
import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
import type { Knex } from 'knex';
import type { IFieldInstance } from '../features/field/model/factory';
import type { DateFieldDto } from '../features/field/model/field-dto/date-field.dto';
Expand Down Expand Up @@ -136,9 +136,12 @@ export interface IDbProvider {

searchIndexQuery(
originQueryBuilder: Knex.QueryBuilder,
dbTableName: string,
searchField: IFieldInstance[],
searchValue: string,
dbTableName: string
searchIndexRo: Partial<ISearchIndexByQueryRo>,
baseSortIndex?: string,
setFilterQuery?: (qb: Knex.QueryBuilder) => void,
setSortQuery?: (qb: Knex.QueryBuilder) => void
): Knex.QueryBuilder;

searchCountQuery(
Expand Down
23 changes: 14 additions & 9 deletions apps/nestjs-backend/src/db-provider/postgres.provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Logger } from '@nestjs/common';
import type { IFilter, ISortItem } from '@teable/core';
import { DriverClient } from '@teable/core';
import type { PrismaClient } from '@teable/db-main-prisma';
import type { IAggregationField } from '@teable/openapi';
import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
import type { Knex } from 'knex';
import type { IFieldInstance } from '../features/field/model/factory';
import type { SchemaType } from '../features/field/util';
Expand All @@ -23,7 +23,7 @@ import { FilterQueryPostgres } from './filter-query/postgres/filter-query.postgr
import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group-query.interface';
import { GroupQueryPostgres } from './group-query/group-query.postgres';
import { SearchQueryAbstract } from './search-query/abstract';
import { SearchQueryPostgres } from './search-query/search-query.postgres';
import { SearchQueryBuilder, SearchQueryPostgres } from './search-query/search-query.postgres';
import { SortQueryPostgres } from './sort-query/postgres/sort-query.postgres';
import type { ISortQueryInterface } from './sort-query/sort-query.interface';

Expand Down Expand Up @@ -331,17 +331,22 @@ export class PostgresProvider implements IDbProvider {

searchIndexQuery(
originQueryBuilder: Knex.QueryBuilder,
dbTableName: string,
searchField: IFieldInstance[],
searchValue: string,
dbTableName: string
searchIndexRo: ISearchIndexByQueryRo,
baseSortIndex?: string,
setFilterQuery?: (qb: Knex.QueryBuilder) => void,
setSortQuery?: (qb: Knex.QueryBuilder) => void
) {
return SearchQueryAbstract.buildSearchIndexQuery(
SearchQueryPostgres,
return new SearchQueryBuilder(
originQueryBuilder,
dbTableName,
searchField,
searchValue,
dbTableName
);
searchIndexRo,
baseSortIndex,
setFilterQuery,
setSortQuery
).getSearchIndexQuery();
}

shareFilterCollaboratorsQuery(
Expand Down
157 changes: 155 additions & 2 deletions apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { IDateFieldOptions } from '@teable/core';
import type { Knex } from 'knex';
import { CellValueType, type IDateFieldOptions } from '@teable/core';
import type { ISearchIndexByQueryRo } from '@teable/openapi';
import { type Knex } from 'knex';
import { get } from 'lodash';
import type { IFieldInstance } from '../../features/field/model/factory';
import { SearchQueryAbstract } from './abstract';
Expand Down Expand Up @@ -217,3 +218,155 @@ export class SearchQueryPostgres extends SearchQueryAbstract {
.toQuery();
}
}

export class SearchQueryBuilder {
constructor(
public queryBuilder: Knex.QueryBuilder,
public dbTableName: string,
public searchField: IFieldInstance[],
public searchIndexRo: ISearchIndexByQueryRo,
public baseSortIndex?: string,
public setFilterQuery?: (qb: Knex.QueryBuilder) => void,
public setSortQuery?: (qb: Knex.QueryBuilder) => void
) {
this.queryBuilder = queryBuilder;
this.dbTableName = dbTableName;
this.searchField = searchField;
this.baseSortIndex = baseSortIndex;
this.searchIndexRo = searchIndexRo;
this.setFilterQuery = setFilterQuery;
this.setSortQuery = setSortQuery;
}

getSearchQuery() {
const { queryBuilder, searchIndexRo, searchField } = this;
const { search } = searchIndexRo;
const searchValue = search?.[0];

if (!search || !searchField?.length || !searchValue) {
return queryBuilder;
}

return searchField.map((field) => {
const searchQueryBuilder = new SearchQueryPostgres(queryBuilder, field, searchValue);
if (field.isMultipleCellValue) {
switch (field.cellValueType) {
case CellValueType.DateTime:
return searchQueryBuilder.getMultipleDateSqlQuery();
case CellValueType.Number:
return searchQueryBuilder.getMultipleNumberSqlQuery();
case CellValueType.String:
if (field.isStructuredCellValue) {
return searchQueryBuilder.getMultipleJsonSqlQuery();
} else {
return searchQueryBuilder.getMultipleTextSqlQuery();
}
}
}

switch (field.cellValueType) {
case CellValueType.DateTime:
return searchQueryBuilder.getDateSqlQuery();
case CellValueType.Number:
return searchQueryBuilder.getNumberSqlQuery();
case CellValueType.String:
if (field.isStructuredCellValue) {
return searchQueryBuilder.getJsonSqlQuery();
} else {
return searchQueryBuilder.getTextSqlQuery();
}
}
});
}

getCaseWhenSqlBy() {
const { searchField, queryBuilder } = this;
const searchQuerySql = this.getSearchQuery() as string[];
return searchField.map(({ dbFieldName }, index) => {
const knexInstance = queryBuilder.client;
const searchSql = searchQuerySql[index];
return knexInstance.raw(
`
CASE WHEN ${searchSql} THEN ? END
`,
[dbFieldName]
);
});
}

getSearchIndexQuery() {
const {
queryBuilder,
dbTableName,
searchField,
searchIndexRo,
setFilterQuery,
setSortQuery,
baseSortIndex,
} = this;

const { search, groupBy, orderBy } = searchIndexRo;
const knexInstance = queryBuilder.client;

if (!search || !searchField.length) {
return queryBuilder;
}

const searchQuerySql = this.getSearchQuery() as string[];

const caseWhenQueryDbSql = this.getCaseWhenSqlBy() as string[];

queryBuilder.with('search_field_union_table', (qb) => {
qb.select('*').select(
knexInstance.raw(
`array_remove(
ARRAY [
${caseWhenQueryDbSql.join(',')}
],
NULL
) as matched_columns`
)
);

qb.from(dbTableName);

qb.where((subQb) => {
subQb.where((orWhere) => {
searchQuerySql.forEach((sql) => {
orWhere.orWhereRaw(sql);
});
});
if (this.searchIndexRo.filter && setFilterQuery) {
subQb.andWhere((andQb) => {
setFilterQuery?.(andQb);
});
}
});

if (orderBy?.length || groupBy?.length) {
setSortQuery?.(qb);
}

baseSortIndex && qb.orderBy(baseSortIndex, 'asc');
});

queryBuilder
.select('*', 'matched_column')
.select(
knexInstance.raw(
`CASE
${searchField.map((field) => knexInstance.raw(`WHEN matched_column = '${field.dbFieldName}' THEN ?`, [field.id])).join(' ')}
END AS "fieldId"`
)
)
.fromRaw(
`
"search_field_union_table",
LATERAL unnest(matched_columns) AS matched_column
`
)
.whereRaw(`array_length(matched_columns, 1) > 0`);

return queryBuilder;
}
}
97 changes: 96 additions & 1 deletion apps/nestjs-backend/src/db-provider/search-query/search-query.sqlite.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { IDateFieldOptions } from '@teable/core';
import { CellValueType, type IDateFieldOptions } from '@teable/core';
import type { ISearchIndexByQueryRo } from '@teable/openapi';
import type { Knex } from 'knex';
import { get } from 'lodash';
import type { IFieldInstance } from '../../features/field/model/factory';
Expand Down Expand Up @@ -211,3 +212,97 @@ export class SearchQuerySqlite extends SearchQueryAbstract {
.toQuery();
}
}

export class SearchQueryBuilder {
constructor(
public queryBuilder: Knex.QueryBuilder,
public dbTableName: string,
public searchField: IFieldInstance[],
public searchIndexRo: ISearchIndexByQueryRo,
public baseSortIndex?: string,
public setFilterQuery?: (qb: Knex.QueryBuilder) => void,
public setSortQuery?: (qb: Knex.QueryBuilder) => void
) {
this.queryBuilder = queryBuilder;
this.dbTableName = dbTableName;
this.searchField = searchField;
this.baseSortIndex = baseSortIndex;
this.searchIndexRo = searchIndexRo;
this.setFilterQuery = setFilterQuery;
this.setSortQuery = setSortQuery;
}

getSearchIndexQuery() {
const {
queryBuilder,
searchIndexRo,
dbTableName,
searchField,
baseSortIndex,
setFilterQuery,
setSortQuery,
} = this;
const { search, take, skip, filter, orderBy, groupBy } = searchIndexRo;
const knexInstance = queryBuilder.client;

if (!search || !searchField?.length) {
return queryBuilder;
}

queryBuilder.with('search_field_union_table', (qb) => {
for (let index = 0; index < searchField.length; index++) {
const currentWhereRaw = searchField[index];
const dbFieldName = searchField[index].dbFieldName;

// boolean field or new field which does not support search should be skipped
if (!currentWhereRaw || !dbFieldName) {
continue;
}

if (index === 0) {
qb.select('*', knexInstance.raw(`? as matched_column`, [dbFieldName]))
.whereRaw(`${currentWhereRaw}`)
.from(dbTableName);
} else {
qb.unionAll(function () {
this.select('*', knexInstance.raw(`? as matched_column`, [dbFieldName]))
.whereRaw(`${currentWhereRaw}`)
.from(dbTableName);
});
}
}
});

queryBuilder
.select('__id', '__auto_number', 'matched_column')
.select(
knexInstance.raw(
`CASE
${searchField.map((field) => `WHEN matched_column = '${field.dbFieldName}' THEN '${field.id}'`).join(' ')}
END AS "fieldId"`
)
)
.from('search_field_union_table');

if (orderBy?.length || groupBy?.length) {
setSortQuery?.(queryBuilder);
}

if (filter) {
setFilterQuery?.(queryBuilder);
}

baseSortIndex && queryBuilder.orderBy(baseSortIndex, 'asc');

const cases = searchField.map((field, index) => {
return knexInstance.raw(`CASE WHEN ?? = ? THEN ? END`, [
'matched_column',
field.dbFieldName,
index + 1,
]);
});
cases.length && queryBuilder.orderByRaw(cases.join(','));

return queryBuilder;
}
}
24 changes: 14 additions & 10 deletions apps/nestjs-backend/src/db-provider/sqlite.provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Logger } from '@nestjs/common';
import type { IFilter, ISortItem } from '@teable/core';
import { DriverClient } from '@teable/core';
import type { PrismaClient } from '@teable/db-main-prisma';
import type { IAggregationField } from '@teable/openapi';
import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
import type { Knex } from 'knex';
import type { IFieldInstance } from '../features/field/model/factory';
import type { SchemaType } from '../features/field/util';
Expand All @@ -24,7 +24,7 @@ import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group
import { GroupQuerySqlite } from './group-query/group-query.sqlite';
import { SearchQueryAbstract } from './search-query/abstract';
import { getOffset } from './search-query/get-offset';
import { SearchQuerySqlite } from './search-query/search-query.sqlite';
import { SearchQueryBuilder, SearchQuerySqlite } from './search-query/search-query.sqlite';
import type { ISortQueryInterface } from './sort-query/sort-query.interface';
import { SortQuerySqlite } from './sort-query/sqlite/sort-query.sqlite';

Expand Down Expand Up @@ -285,19 +285,23 @@ export class SqliteProvider implements IDbProvider {

searchIndexQuery(
originQueryBuilder: Knex.QueryBuilder,
dbTableName: string,
searchField: IFieldInstance[],
searchValue: string,
dbTableName: string
searchIndexRo: ISearchIndexByQueryRo,
baseSortIndex?: string,
setFilterQuery?: (qb: Knex.QueryBuilder) => void,
setSortQuery?: (qb: Knex.QueryBuilder) => void
) {
return SearchQueryAbstract.buildSearchIndexQuery(
SearchQuerySqlite,
return new SearchQueryBuilder(
originQueryBuilder,
dbTableName,
searchField,
searchValue,
dbTableName
);
searchIndexRo,
baseSortIndex,
setFilterQuery,
setSortQuery
).getSearchIndexQuery();
}

shareFilterCollaboratorsQuery(
originQueryBuilder: Knex.QueryBuilder,
dbFieldName: string,
Expand Down
Loading

0 comments on commit 12ec0e8

Please sign in to comment.