Skip to content

Commit

Permalink
feat: use footnote for perplexity search results (#9851)
Browse files Browse the repository at this point in the history
  • Loading branch information
akumatus committed Jan 22, 2025
1 parent f8a515e commit 862a9d0
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 42 deletions.
48 changes: 35 additions & 13 deletions packages/backend/server/src/__tests__/copilot.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1096,10 +1096,14 @@ test('CitationParser should replace citation placeholders with URLs', t => {
const citations = ['https://example1.com', 'https://example2.com'];

const parser = new CitationParser();
const result = parser.parse(content, citations);
const result = parser.parse(content, citations) + parser.end();

const expected = [
'This is [a] test sentence with [citations [^1]] and [^2] and [3].',
`[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
`[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
].join('\n\n');

const expected =
'This is [a] test sentence with [citations [[1](https://example1.com)]] and [[2](https://example2.com)] and [3].';
t.is(result, expected);
});

Expand Down Expand Up @@ -1130,10 +1134,18 @@ test('CitationParser should replace chunks of citation placeholders with URLs',
let result = contents.reduce((acc, current) => {
return acc + parser.parse(current, citations);
}, '');
result += parser.flush();

const expected =
'[[]]This is [a] test sentence with citations [[1](https://example1.com)] and [[2](https://example2.com)] and [[3](https://example3.com)] and [[4](https://example4.com)] and [[5](https://example5.com)] and [[6](https://example6.com)] and [7';
result += parser.end();

const expected = [
'[[]]This is [a] test sentence with citations [^1] and [^2] and [^3] and [^4] and [^5] and [^6] and [7',
`[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
`[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
`[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
`[^4]: {"type":"url","url":"${encodeURIComponent(citations[3])}"}`,
`[^5]: {"type":"url","url":"${encodeURIComponent(citations[4])}"}`,
`[^6]: {"type":"url","url":"${encodeURIComponent(citations[5])}"}`,
`[^7]: {"type":"url","url":"${encodeURIComponent(citations[6])}"}`,
].join('\n\n');
t.is(result, expected);
});

Expand All @@ -1147,9 +1159,14 @@ test('CitationParser should not replace citation already with URLs', t => {
];

const parser = new CitationParser();
const result = parser.parse(content, citations);

const expected = content;
const result = parser.parse(content, citations) + parser.end();

const expected = [
content,
`[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
`[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
`[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
].join('\n\n');
t.is(result, expected);
});

Expand All @@ -1169,8 +1186,13 @@ test('CitationParser should not replace chunks of citation already with URLs', t
let result = contents.reduce((acc, current) => {
return acc + parser.parse(current, citations);
}, '');
result += parser.flush();

const expected = contents.join('');
result += parser.end();

const expected = [
contents.join(''),
`[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
`[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
`[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
].join('\n\n');
t.is(result, expected);
});
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@ export class CitationParser {

private numberToken: string[] = [];

private citations: string[] = [];

public parse(content: string, citations: string[]) {
this.citations = citations;
let result = '';
const contentArray = content.split('');
for (const [index, char] of contentArray.entries()) {
Expand All @@ -85,7 +88,7 @@ export class CitationParser {
cIndex <= citations.length &&
contentArray[index + 1] !== this.PARENTHESES_OPEN
) {
const content = `[[${cIndex}](${citations[cIndex - 1]})]`;
const content = `[^${cIndex}]`;
result += content;
this.resetToken();
} else {
Expand Down Expand Up @@ -116,13 +119,26 @@ export class CitationParser {
return result;
}

public flush() {
const content = this.getFullContent();
public end() {
return this.flush() + this.getFootnotes();
}

private flush() {
const content = this.getTokenContent();
this.resetToken();
return content;
}

private getFullContent() {
private getFootnotes() {
const footnotes = this.citations.map((citation, index) => {
return `[^${index + 1}]: {"type":"url","url":"${encodeURIComponent(
citation
)}"}`;
});
return '\n\n' + footnotes.join('\n\n');
}

private getTokenContent() {
return this.startToken.concat(this.numberToken, this.endToken).join('');
}

Expand Down Expand Up @@ -208,7 +224,7 @@ export class PerplexityProvider implements CopilotTextToTextProvider {
const { content } = data.choices[0].message;
const { citations } = data;
let result = parser.parse(content, citations);
result += parser.flush();
result += parser.end();
return result;
}
} catch (e: any) {
Expand Down Expand Up @@ -277,7 +293,7 @@ export class PerplexityProvider implements CopilotTextToTextProvider {
}
},
flush(controller) {
controller.enqueue(parser.flush());
controller.enqueue(parser.end());
controller.enqueue(null);
},
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { css, html, nothing } from 'lit';
import { property, query, state } from 'lit/decorators.js';
import { repeat } from 'lit/directives/repeat.js';
import { styleMap } from 'lit/directives/style-map.js';
import { debounce, throttle } from 'lodash-es';
import { debounce } from 'lodash-es';

import {
EdgelessEditorActions,
Expand Down Expand Up @@ -132,9 +132,6 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
@query('.chat-panel-messages')
accessor messagesContainer: HTMLDivElement | null = null;

@query('.message:nth-last-child(2)')
accessor lastMessage: HTMLDivElement | null = null;

private _renderAIOnboarding() {
return this.isLoading ||
!this.host?.doc.get(FeatureFlagService).getFlag('enable_ai_onboarding')
Expand Down Expand Up @@ -192,16 +189,6 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
100
);

private readonly _scrollIntoView = () => {
if (!this.lastMessage) return;
this.lastMessage.scrollIntoView({ behavior: 'smooth' });
};

private readonly _throttledScrollIntoView = throttle(
this._scrollIntoView,
500
);

protected override render() {
const { items } = this.chatContextValue;
const { isLoading } = this;
Expand Down Expand Up @@ -300,12 +287,6 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
);
}

protected override updated() {
if (this.chatContextValue.status === 'transmitting') {
this._throttledScrollIntoView();
}
}

renderItem(item: ChatItem, isLast: boolean) {
const { status, error } = this.chatContextValue;
const { host } = this;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { Store } from '@blocksuite/affine/store';
import { css, html, type PropertyValues } from 'lit';
import { property, state } from 'lit/decorators.js';
import { createRef, type Ref, ref } from 'lit/directives/ref.js';
import { throttle } from 'lodash-es';

import { AIHelpIcon, SmallHintIcon } from '../_common/icons';
import { AIProvider } from '../provider';
Expand Down Expand Up @@ -191,6 +192,8 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
this._chatMessages.value?.scrollToEnd();
};

private readonly _throttledScrollToEnd = throttle(this._scrollToEnd, 1000);

private readonly _cleanupHistories = async () => {
const notification = this.host.std.getOptional(NotificationProvider);
if (!notification) return;
Expand Down Expand Up @@ -229,14 +232,20 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
}

if (
!this.isLoading &&
_changedProperties.has('chatContextValue') &&
(this.chatContextValue.status === 'loading' ||
this.chatContextValue.status === 'error' ||
this.chatContextValue.status === 'success')
) {
setTimeout(this._scrollToEnd, 500);
}

if (
_changedProperties.has('chatContextValue') &&
this.chatContextValue.status === 'transmitting'
) {
this._throttledScrollToEnd();
}
}

override connectedCallback() {
Expand Down
6 changes: 4 additions & 2 deletions tests/affine-cloud-copilot/e2e/copilot.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,7 @@ test.describe('chat panel', () => {
});
expect(history[1].name).toBe('AFFiNE AI');
expect(
await page.locator('chat-panel affine-link').count()
await page.locator('chat-panel affine-footnote-node').count()
).toBeGreaterThan(0);

await clearChat(page);
Expand All @@ -429,7 +429,9 @@ test.describe('chat panel', () => {
content: 'What is the weather in Shanghai today?',
});
expect(history[1].name).toBe('AFFiNE AI');
expect(await page.locator('chat-panel affine-link').count()).toBe(0);
expect(await page.locator('chat-panel affine-footnote-node').count()).toBe(
0
);
});

test('can trigger inline ai input and action panel by clicking Start with AI button', async ({
Expand Down

0 comments on commit 862a9d0

Please sign in to comment.