diff --git a/app/src/renderer/Resource/components/notebook/NotebookContents.svelte b/app/src/renderer/Resource/components/notebook/NotebookContents.svelte index 489a6d3bd..c59e66842 100644 --- a/app/src/renderer/Resource/components/notebook/NotebookContents.svelte +++ b/app/src/renderer/Resource/components/notebook/NotebookContents.svelte @@ -1,73 +1,131 @@ + + + + -{#snippet notesList(visibleItems, allItems)} - {#if allItems.length <= 0} - {#if (notebookId ? $ttyQuery : searchQuery).length > 0} -
-

Nothing found for "{notebookId ? $ttyQuery : searchQuery}"

-
+{#snippet loadingSnippet()} +
+ +
+{/snippet} + +{#snippet noResultsSnippet(categoryLabel: string)} +
+

No {categoryLabel} found for "{searchQuery}"

+
+{/snippet} + +{#snippet notesList({ resources, searchResults, pagination, loadMore })} + {#if searchQuery && searchResults?.length === 0} + {@render noResultsSnippet('notes')} + {:else if resources.length <= 0} + {#if searchQuery.length > 0} + {@render noResultsSnippet('notes')} {:else}
@@ -196,72 +303,62 @@ Jump start a new note by asking Surf's AI something in the input box above or create a blank note using the button.

- -
{/if} {:else} - {#each visibleItems as resource, i (typeof resource === 'string' ? resource : resource.id + i)} - - {#snippet children(resource: Resource)} - - {/snippet} - - {/each} - - {#if allItems.length > visibleItems.length} -
(showAll = !showAll)}> - +
+ {#each searchResults ?? resources as resource, i (typeof resource === 'string' ? resource : resource.id + i)} + + {#snippet children(resource: Resource)} + + {/snippet} + + {/each} +
+ + {#if pagination.hasMore && !searchQuery} +
+ {#if pagination.isLoadingMore} + {@render loadingSnippet()} + {/if}
{/if} {/if} {/snippet} -{#snippet sourcesList(visibleItems, allItems)} - {#if allItems.length <= 0} - {#if (notebookId ? $ttyQuery : searchQuery).length > 0} -
-

Nothing found for "{notebookId ? $ttyQuery : searchQuery}"

-
- {:else} -
-
-

Surf Media

- -

- Add media from across the web or your system to your notebook and to use it together - with Surf Notes to turn them into something great. -

- -

- Save web pages using the "Save" button while browsing, import local files or add - existing media from other notebooks by right-clicking them. -

- - - - - - -
+{#snippet sourcesList({ resources, searchResults, pagination, loadMore })} + {#if searchQuery && searchResults?.length === 0} + {@render noResultsSnippet('media')} + {:else if resources.length === 0} +
+
+

Surf Media

+ +

+ Add media from across the web or your system to your notebook and to use it together with + Surf Notes to turn them into something great. +

+ +

+ Save web pages using the "Save" button while browsing, import local files or add existing + media from other notebooks by right-clicking them. +

- {/if} +
{:else} -
- {#each visibleItems as resource, i (typeof resource === 'string' ? resource : resource.id + i)} +
+ {#each searchResults ?? resources as resource, i (typeof resource === 'string' ? resource : resource.id + i)} {#snippet children(resource: Resource)} {/each}
- {#if resourceRenderCnt < allItems.length} -
- Scroll to load more + + {#if pagination.hasMore && !searchQuery} +
+ {#if pagination.isLoadingMore} + {@render loadingSnippet()} + {/if}
{/if} {/if} @@ -305,395 +410,341 @@ /> {/if} -
- (showAllNotes = false)} - tabs={[ - ...conditionalArrayItem(notebookId === undefined, { - id: 'notebooks', - label: 'Notebooks', - icon: 'notebook' - }), - { - id: 'notes', - label: 'Notes', - icon: 'note' - }, - { - id: 'sources', - label: 'Media', - icon: 'link' - } - ]} - /> - - - - {#if activeTab === 'notes'} - - {:else if activeTab === 'sources'} - - {/if} -
- -{#if activeTab === 'notebooks'} - {#if !searchQuery || (searchQuery !== null && searchQuery.length > 0)} -
- {#if !searchQuery} -
{ - try { - const notebook = await notebookManager.createNotebook( - { - name: 'Untitled Notebook' - }, - true - ) - isNewNotebook = notebook - } catch (e) { - console.error('Failed to create notebook', e) - } - }} - > -
- - +
+
+

+ {notebookId ? 'Your Notebook' : 'Your Library'} +

+ +
+ +
+
+ {#each categories as category} +
+
+ + {#if !isCategoryCollapsed(category.id)} + + {/if}
-
- {/if} - - {#if !searchQuery || 'drafts'.includes(searchQuery.trim().toLowerCase())} -
{ - handleNotebookClick('drafts', event) - }} - > - {}} - /> -
- {/if} - - {#each notebooksList.slice(0, showAll ? Infinity : notebooksList.filter((e) => e.data.pinned).length) as notebook, i (notebook.id + i)} -
- handleNotebookClick(notebook.id, e)} - onpin={() => handlePinNotebook(notebook.id)} - onunpin={() => handleUnPinNotebook(notebook.id)} - {@attach contextMenu({ - canOpen: true, - items: [ - !notebook.data.pinned - ? { - type: 'action', - text: 'Add to Favorites', - icon: 'heart', - action: () => handlePinNotebook(notebook.id) + + {#if !isCategoryCollapsed(category.id)} +
+ {#if category.id === 'notebooks'} + {#if notebooksList.length <= 0 && searchQuery.length > 0} + {@render noResultsSnippet('notebooks')} + {/if} +
+ {#each notebooksList as notebook, i (notebook.id + i)} +
+ handleNotebookClick(notebook.id, e)} + canPin={notebook.id !== 'drafts'} + onpin={() => handlePinNotebook(notebook.id)} + onunpin={() => handleUnPinNotebook(notebook.id)} + {@attach contextMenu({ + canOpen: notebook.id !== 'drafts', + items: [ + !notebook.data?.pinned + ? { + type: 'action', + text: 'Add to Favorites', + icon: 'heart', + action: () => handlePinNotebook(notebook.id) + } + : { + type: 'action', + text: 'Remove from Favorites', + icon: 'heart.off', + action: () => handleUnPinNotebook(notebook.id) + }, + { + type: 'action', + text: 'Customize', + icon: 'edit', + action: () => (isCustomizingNotebook = notebook) + }, + { + type: 'action', + kind: 'danger', + text: 'Delete', + icon: 'trash', + action: () => handleDeleteNotebook(notebook) + } + ] + })} + /> +
+ {/each} +
+ {:else if category.id === 'notes'} +
    + + {#snippet children(loaderData)} + {@render notesList(loaderData)} + {/snippet} + {#snippet loading()} + {@render loadingSnippet()} + {/snippet} + +
+ {:else if category.id === 'sources'} + handleUnPinNotebook(notebook.id) - }, - /*{ - type: 'action', - text: 'Rename', - icon: 'edit', - action: () => (isRenamingNotebook = notebook.id) - },*/ - { - type: 'action', - text: 'Customize', - icon: 'edit', - action: () => (isCustomizingNotebook = notebook) - }, - - { - type: 'action', - kind: 'danger', - text: 'Delete', - icon: 'trash', - action: () => handleDeleteNotebook(notebook) - } - ] - })} - /> + }} + > + {#snippet children(loaderData)} + {@render sourcesList(loaderData)} + {/snippet} + {#snippet loading()} + {@render loadingSnippet()} + {/snippet} + + {/if} +
+ {/if}
{/each}
- - {#if notebooksList.length > notebooksList.slice(0, showAll ? Infinity : notebooksList.filter((e) => e.data.pinned).length).length} -
(showAll = !showAll)}> - -
- {/if} - {/if} -{:else if activeTab === 'notes'} -
    - - - {#if !notebookId} - - {#snippet children([resources, searchResult, searching])} - {@render notesList( - (searchResult ?? resources).slice(0, showAll ? Infinity : 6), - resources - )} - {/snippet} - - {#snippet loading()} -
    - -

    Loading…

    -
    - {/snippet} -     - {:else if notebookId === 'drafts'} - - {#snippet children([resources, searchResult, searching])} - {@render notesList( - (searchResult ?? resources).slice(0, showAll ? Infinity : 6), - resources - )} - {/snippet} - - {#snippet loading()} -
    - -

    Loading…

    -
    - {/snippet} -     - {:else} - - {#snippet children([notebook, searchResult, searching])} - {@render notesList( - filterNoteResources(notebook?.contents ?? [], searchResult).map((e) => e.entry_id), - filterNoteResources(notebook?.contents ?? [], searchResult).map((e) => e.entry_id) - )} - {/snippet} - - {#snippet loading()} -
    - -

    Loading…

    -
    - {/snippet} -     - {/if} -
-{:else if activeTab === 'sources'} - - - {#if !notebookId} - - {#snippet children([resources, searchResult, searching])} - {@render sourcesList( - (searchResult ?? resources).slice( - 0, - searchResult ? Infinity : showAll ? Infinity : resourceRenderCnt - ), - resources - )} - {/snippet} - - {#snippet loading()} -
- -

Loading…

-
- {/snippet} - - {:else if notebookId === 'drafts'} - - {#snippet children([resources, searchResult, searching])} - {@render sourcesList( - (searchResult ?? resources).slice( - 0, - searchResult ? Infinity : showAll ? Infinity : resourceRenderCnt - ), - resources - )} - {/snippet} - - {#snippet loading()} -
- -

Loading…

-
- {/snippet} - - {:else} - - {#snippet children([notebook, searchResult, searching])} - {@render sourcesList( - filterOtherResources(notebook?.contents ?? [], searchResult) - .slice(0, resourceRenderCnt) - .map((e) => e.entry_id), - filterOtherResources(notebook?.contents ?? [], searchResult).map((e) => e.entry_id) - )} - {/snippet} - - {#snippet loading()} -
- -

Loading…

-
- {/snippet} - - {/if} -{/if} +
+
diff --git a/app/src/renderer/Resource/components/notebook/NotebookSidebar.svelte b/app/src/renderer/Resource/components/notebook/NotebookSidebar.svelte index 174316552..dac9f678d 100644 --- a/app/src/renderer/Resource/components/notebook/NotebookSidebar.svelte +++ b/app/src/renderer/Resource/components/notebook/NotebookSidebar.svelte @@ -15,7 +15,7 @@ type OpenTarget, SpaceEntryOrigin } from '@deta/types' - import { NotebookLoader, SurfLoader, SourceCard } from '@deta/ui' + import { SurfLoader, SourceCard } from '@deta/ui' import { type Notebook } from '@deta/services/notebooks' import { type Resource, getResourceCtxItems } from '@deta/services/resources' import { @@ -252,7 +252,7 @@ {:else} - {#snippet children([notebook, searchResult, searching])}
@@ -661,7 +662,7 @@ {/if} {/snippet} - + {/if} diff --git a/app/src/renderer/Resource/layouts/NotebookLayout.svelte b/app/src/renderer/Resource/layouts/NotebookLayout.svelte index 9fcf232b1..569e41d79 100644 --- a/app/src/renderer/Resource/layouts/NotebookLayout.svelte +++ b/app/src/renderer/Resource/layouts/NotebookLayout.svelte @@ -12,44 +12,35 @@ transition: transform 123ms ease-out; } } + .notebook { position: relative; overflow-x: hidden; - - //.bg { - // content: ''; - // position: absolute; - // inset: -4px; - // background: - // linear-gradient(rgba(255, 255, 255, 0.65), rgba(255, 255, 255, 1)), - // url('https://i.imgur.com/7XbyivJ.png'); - // background-repeat: no-repeat; - // background-size: cover; - // background-position: 50% 30%; - // //background: #fff; - // z-index: -1; - // filter: blur(2px); - // pointer-events: none; - //} - height: 100vh; width: 100%; display: flex; flex-direction: column; - padding-inline: 1.5rem; - padding-block: 4.5rem; + // default for laptops (13-15") + padding-inline: 2rem; + padding-block: 2rem; + + // larger laptops and small external monitors (15-24") + @media screen and (min-width: 1440px) { + padding-inline: 3rem; + padding-block: 3rem; + } + + // large external monitors (27"+) + @media screen and (min-width: 1920px) { + padding-inline: 4rem; + padding-block: 4rem; + } - //&::before { - // content: ''; - // position: absolute; - // top: 0; - // left: 0; - // right: 0; - // height: 2rem; - // z-index: 0; - // pointer-events: none; - // background: linear-gradient(to bottom, var(--page-gradient-color), transparent); - //} + // ultra-wide or very large displays + @media screen and (min-width: 2560px) { + padding-inline: 5rem; + padding-block: 5rem; + } } diff --git a/packages/backend/src/api/message.rs b/packages/backend/src/api/message.rs index 5dabdad9d..a9d156a2b 100644 --- a/packages/backend/src/api/message.rs +++ b/packages/backend/src/api/message.rs @@ -114,8 +114,9 @@ pub enum ResourceMessage { RemoveResources(Vec), RemoveResourcesByTags(Vec), RecoverResource(String), - ListResourcesByTags(Vec), - ListResourcesByTagsNoSpace(Vec), + // Last Param is space filter where + // None = no space filter, Some("") = no space, Some(id) = specific space + ListResourcesByTags(Vec, PaginationParams, Option), ListAllResourcesAndSpaces(Vec), SearchResources(SearchResourcesParams), UpdateResource(Resource), diff --git a/packages/backend/src/api/mod.rs b/packages/backend/src/api/mod.rs index dc07b7f7b..50e0730ab 100644 --- a/packages/backend/src/api/mod.rs +++ b/packages/backend/src/api/mod.rs @@ -14,3 +14,23 @@ pub fn register_exported_functions(cx: &mut ModuleContext) -> NeonResult<()> { kv::register_exported_functions(cx)?; Ok(()) } + +pub fn parse_json_argument( + cx: &mut FunctionContext, + arg_index: usize, + arg_name: &str, +) -> Result { + let json_string = cx + .argument_opt(arg_index) + .and_then(|arg| arg.downcast::(cx).ok()) + .map(|js_string| js_string.value(cx)); + + match json_string + .map(|json_str| serde_json::from_str(&json_str)) + .transpose() + { + Ok(Some(value)) => Ok(value), + Ok(None) => Err(format!("{} must be provided", arg_name)), + Err(err) => Err(err.to_string()), + } +} diff --git a/packages/backend/src/api/store.rs b/packages/backend/src/api/store.rs index 32fe3976c..55b98d374 100644 --- a/packages/backend/src/api/store.rs +++ b/packages/backend/src/api/store.rs @@ -1,5 +1,5 @@ -use crate::store::models::SearchResourcesParams; -use crate::{api::message::*, store::models, worker::tunnel::WorkerTunnel}; +use super::{message::*, parse_json_argument}; +use crate::{store::models, worker::tunnel::WorkerTunnel}; use neon::prelude::*; use neon::types::JsDate; @@ -18,10 +18,6 @@ pub fn register_exported_functions(cx: &mut ModuleContext) -> NeonResult<()> { "js__store_list_resources_by_tags", js_list_resources_by_tags, )?; - cx.export_function( - "js__store_list_resources_by_tags_no_space", - js_list_resources_by_tags_no_space, - )?; cx.export_function( "js__store_list_all_resources_and_spaces", js_list_all_resources_and_spaces, @@ -469,47 +465,32 @@ fn js_recover_resource(mut cx: FunctionContext) -> JsResult { fn js_list_resources_by_tags(mut cx: FunctionContext) -> JsResult { let tunnel = cx.argument::>(0)?; - let resource_tags_json = cx - .argument_opt(1) - .and_then(|arg| arg.downcast::(&mut cx).ok()) - .map(|js_string| js_string.value(&mut cx)); - let resource_tags: Vec = match resource_tags_json - .map(|json_str| serde_json::from_str(&json_str)) - .transpose() - { - Ok(Some(tags)) => tags, - Ok(None) => return cx.throw_error("Resource tags must be provided"), - Err(err) => return cx.throw_error(err.to_string()), - }; - - let (deferred, promise) = cx.promise(); - tunnel.worker_send_js( - WorkerMessage::ResourceMessage(ResourceMessage::ListResourcesByTags(resource_tags)), - deferred, - ); + let resource_tags: Vec = + match parse_json_argument(&mut cx, 1, "Resource tags") { + Ok(tags) => tags, + Err(err) => return cx.throw_error(err), + }; - Ok(promise) -} - -fn js_list_resources_by_tags_no_space(mut cx: FunctionContext) -> JsResult { - let tunnel = cx.argument::>(0)?; + let pagination_params: models::PaginationParams = + match parse_json_argument(&mut cx, 2, "Pagination parameters") { + Ok(params) => params, + Err(err) => return cx.throw_error(err), + }; - let resource_tags_json = cx - .argument_opt(1) - .and_then(|arg| arg.downcast::(&mut cx).ok()) - .map(|js_string| js_string.value(&mut cx)); - let resource_tags: Vec = match resource_tags_json - .map(|json_str| serde_json::from_str(&json_str)) - .transpose() - { - Ok(Some(tags)) => tags, - Ok(None) => return cx.throw_error("Resource tags must be provided"), - Err(err) => return cx.throw_error(err.to_string()), - }; + // None = no space filter, Some("") = no space, Some(id) = specific space + let space_id = cx.argument_opt(3).and_then(|arg| { + arg.downcast::(&mut cx) + .ok() + .map(|js_string| js_string.value(&mut cx)) + }); let (deferred, promise) = cx.promise(); tunnel.worker_send_js( - WorkerMessage::ResourceMessage(ResourceMessage::ListResourcesByTagsNoSpace(resource_tags)), + WorkerMessage::ResourceMessage(ResourceMessage::ListResourcesByTags( + resource_tags, + pagination_params, + space_id, + )), deferred, ); @@ -572,18 +553,13 @@ fn js_search_resources(mut cx: FunctionContext) -> JsResult { .ok() .map(|js_number| js_number.value(&mut cx) as i64) }); - let include_annotations = cx.argument_opt(6).and_then(|arg| { - arg.downcast::(&mut cx) - .ok() - .map(|js_boolean| js_boolean.value(&mut cx)) - }); - let space_id = cx.argument_opt(7).and_then(|arg| { + let space_id = cx.argument_opt(6).and_then(|arg| { arg.downcast::(&mut cx) .ok() .map(|js_string| js_string.value(&mut cx)) }); - let keyword_limit = cx.argument_opt(8).and_then(|arg| { + let keyword_limit = cx.argument_opt(7).and_then(|arg| { arg.downcast::(&mut cx) .ok() .map(|js_number| js_number.value(&mut cx) as i64) @@ -591,16 +567,17 @@ fn js_search_resources(mut cx: FunctionContext) -> JsResult { let (deferred, promise) = cx.promise(); tunnel.worker_send_js( - WorkerMessage::ResourceMessage(ResourceMessage::SearchResources(SearchResourcesParams { - query, - resource_tag_filters, - semantic_search_enabled, - embeddings_distance_threshold, - embeddings_limit, - include_annotations, - space_id, - keyword_limit, - })), + WorkerMessage::ResourceMessage(ResourceMessage::SearchResources( + models::SearchResourcesParams { + query, + resource_tag_filters, + semantic_search_enabled, + embeddings_distance_threshold, + embeddings_limit, + space_id, + keyword_limit, + }, + )), deferred, ); diff --git a/packages/backend/src/store/models.rs b/packages/backend/src/store/models.rs index 92d72f1fc..695d834fb 100644 --- a/packages/backend/src/store/models.rs +++ b/packages/backend/src/store/models.rs @@ -8,6 +8,8 @@ use std::str::FromStr; use std::string::ToString; use strum_macros::EnumString; +use crate::{BackendError, BackendResult}; + pub fn default_horizon_tint() -> String { "hsl(275, 40%, 80%)".to_owned() } @@ -533,7 +535,9 @@ pub struct PostProcessingJob { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] #[serde(tag = "type")] +#[derive(Default)] pub enum ResourceProcessingState { + #[default] Pending, Started, Failed { message: String }, @@ -556,11 +560,6 @@ impl FromSql for ResourceProcessingState { } } -impl Default for ResourceProcessingState { - fn default() -> Self { - Self::Pending - } -} #[derive(Debug, Serialize, Deserialize, Clone)] pub struct LegacyResourceTextContent { @@ -951,7 +950,6 @@ pub struct SearchResourcesParams { pub semantic_search_enabled: Option, pub embeddings_distance_threshold: Option, pub embeddings_limit: Option, - pub include_annotations: Option, pub space_id: Option, pub keyword_limit: Option, } @@ -1000,3 +998,35 @@ pub struct App { pub icon: Option, pub meta: Option, } + +#[derive(Debug, Serialize, Deserialize)] +pub struct PaginationParams { + pub limit: usize, + pub cursor: Option, +} + +pub struct PaginationCursor {} + +// TODO: allowing different cursor formats? +impl PaginationCursor { + pub fn encode_date_id(datetime: &str, id: &str) -> String { + format!("{}|{}", datetime, id) + } + + pub fn decode_date_id(cursor: &str) -> BackendResult<(String, String)> { + let parts: Vec<&str> = cursor.split('|').collect(); + if parts.len() != 2 { + return Err(BackendError::GenericError( + "Invalid cursor format".to_string(), + )); + } + Ok((parts[0].to_string(), parts[1].to_string())) + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PaginatedResult { + pub items: Vec, + pub next_cursor: Option, + pub has_more: bool, +} diff --git a/packages/backend/src/store/resource_tags.rs b/packages/backend/src/store/resource_tags.rs index 2c671ebce..f7cba5ec9 100644 --- a/packages/backend/src/store/resource_tags.rs +++ b/packages/backend/src/store/resource_tags.rs @@ -4,6 +4,30 @@ use crate::{BackendError, BackendResult}; use rusqlite::OptionalExtension; use std::collections::HashMap; +fn process_paginated_results( + items: Vec<(String, String)>, // (updated_at, id) pairs + pagination: &PaginationParams, +) -> (Vec, Option, bool) { + let has_more = items.len() > pagination.limit; + let mut items = items; + + if has_more { + items.pop(); + } + + let next_cursor = if has_more { + items + .last() + .map(|(updated_at, id)| PaginationCursor::encode_date_id(updated_at, id)) + } else { + None + }; + + let resource_ids = items.into_iter().map(|(_, id)| id).collect(); + + (resource_ids, next_cursor, has_more) +} + pub fn list_resource_ids_by_tags_query( tag_filters: &Vec, param_start_index: usize, @@ -199,19 +223,127 @@ impl Database { pub fn list_resource_ids_by_tags( &self, tags: &Vec, + space_id: Option, ) -> BackendResult> { let mut result = Vec::new(); + let mut cursor: Option = None; + let page_size = 100; + + loop { + let paginated_result = self.list_resource_ids_by_tags_paginated( + tags, + PaginationParams { + limit: page_size, + cursor: cursor.clone(), + }, + space_id.clone(), + )?; + + result.extend(paginated_result.items); + + if !paginated_result.has_more { + break; + } + + cursor = paginated_result.next_cursor; + } + + Ok(result) + } + + fn list_resource_ids_by_tags_paginated_internal( + &self, + tags: &Vec, + pagination: PaginationParams, + // None = no space filter(all spaces), Some("") = does not belong to any space, Some(id) = specific space + space_filter: Option<&str>, + ) -> BackendResult> { if tags.is_empty() { - return Ok(result); + return Ok(PaginatedResult { + items: Vec::new(), + next_cursor: None, + has_more: false, + }); + } + + let (mut query, mut params) = list_resource_ids_by_tags_query(tags, 0); + + let param_offset = params.len(); + query = format!( + "SELECT r.updated_at, r.id + FROM ({}) rt + JOIN resources r ON rt.resource_id = r.id", + query + ); + + match space_filter { + Some("") => { + query = format!( + "{} AND rt.resource_id NOT IN (SELECT resource_id FROM space_entries WHERE manually_added = 1)", + query + ); + } + Some(space_id) => { + query = format!( + "{} AND rt.resource_id IN (SELECT resource_id FROM space_entries WHERE space_id = ?{})", + query, + param_offset + 1 + ); + params.push(space_id.to_string()); + } + None => {} } - let (query, params) = list_resource_ids_by_tags_query(tags, 0); + + let cursor_param_offset = params.len(); + if let Some(cursor_id) = &pagination.cursor { + let (updated_at, id) = PaginationCursor::decode_date_id(cursor_id)?; + query = format!( + "{} WHERE (r.updated_at < ?{} OR (r.updated_at = ?{} AND r.id < ?{}))", + query, + cursor_param_offset + 1, + cursor_param_offset + 2, + cursor_param_offset + 3 + ); + params.push(updated_at.clone()); + params.push(updated_at); + params.push(id); + } + + // NOTE: updated_at DESC is the primary ordering, but we need a secondary ordering on id DESC + query = format!( + "{} ORDER BY r.updated_at DESC, r.id DESC LIMIT ?{}", + query, + params.len() + 1 + ); + // fetch one extra to check has_more + params.push((pagination.limit + 1).to_string()); + let mut stmt = self.conn.prepare(&query)?; - let resource_ids = - stmt.query_map(rusqlite::params_from_iter(params.iter()), |row| row.get(0))?; - for resource_id in resource_ids { - result.push(resource_id?); + let mut rows = stmt.query(rusqlite::params_from_iter(params.iter()))?; + + let mut items = Vec::new(); + while let Some(row) = rows.next()? { + let updated_at: String = row.get(0)?; + let id: String = row.get(1)?; + items.push((updated_at, id)); } - Ok(result) + + let (resource_ids, next_cursor, has_more) = process_paginated_results(items, &pagination); + + Ok(PaginatedResult { + items: resource_ids, + next_cursor, + has_more, + }) + } + + pub fn list_resource_ids_by_tags_paginated( + &self, + tags: &Vec, + pagination: PaginationParams, + space_filter: Option, + ) -> BackendResult> { + self.list_resource_ids_by_tags_paginated_internal(tags, pagination, space_filter.as_deref()) } pub fn list_resource_ids_by_tags_space_id( @@ -240,31 +372,13 @@ impl Database { Ok(result) } - pub fn list_resource_ids_by_tags_no_space( + pub fn list_resource_ids_by_tags_space_id_paginated( &self, tags: &Vec, - ) -> BackendResult> { - if tags.is_empty() { - return Ok(Vec::new()); - } - let mut result = Vec::new(); - let (query, params) = list_resource_ids_by_tags_query(tags, 0); - - let final_query = format!( - "SELECT rt.resource_id FROM ({}) rt - JOIN resources r ON rt.resource_id = r.id - WHERE rt.resource_id NOT IN (SELECT resource_id FROM space_entries WHERE manually_added = 1) - ORDER BY r.created_at DESC", - query - ); - - let mut stmt = self.conn.prepare(&final_query)?; - let resource_ids = - stmt.query_map(rusqlite::params_from_iter(params.iter()), |row| row.get(0))?; - for resource_id in resource_ids { - result.push(resource_id?); - } - Ok(result) + space_id: &str, + pagination: PaginationParams, + ) -> BackendResult> { + self.list_resource_ids_by_tags_paginated_internal(tags, pagination, Some(space_id)) } } diff --git a/packages/backend/src/store/resources.rs b/packages/backend/src/store/resources.rs index ee44b2184..b92760479 100644 --- a/packages/backend/src/store/resources.rs +++ b/packages/backend/src/store/resources.rs @@ -152,6 +152,13 @@ impl Database { let mut stmt = self .conn .prepare("SELECT resource_id FROM space_entries WHERE space_id = ?1")?; + // TODO: document this behavior better + if space_id.is_empty() { + stmt = self. + conn. + prepare("SELECT id from resources WHERE id NOT IN (SELECT resource_id FROM space_entries WHERE manually_added = 1)")?; + } + let resource_ids = stmt.query_map(rusqlite::params![space_id], |row| row.get(0))?; for resource_id in resource_ids { result.push(resource_id?); diff --git a/packages/backend/src/store/search.rs b/packages/backend/src/store/search.rs index c95c62ee5..ddb560117 100644 --- a/packages/backend/src/store/search.rs +++ b/packages/backend/src/store/search.rs @@ -156,26 +156,6 @@ impl Database { Ok(results) } - // search for resources that match the given tags and only return the resource ids - pub fn list_resources_by_tags( - &self, - tags: Vec, - ) -> BackendResult { - let filtered_resource_ids = self.list_resource_ids_by_tags(&tags)?; - - if filtered_resource_ids.is_empty() { - return Ok(SearchResultSimple { - items: vec![], - total: 0, - }); - } - - Ok(SearchResultSimple { - total: filtered_resource_ids.len() as i64, - items: filtered_resource_ids, - }) - } - pub fn list_all_resources_and_spaces( &self, resource_tags: Vec, @@ -226,31 +206,10 @@ impl Database { Ok(items) } - // list all resources that are not in a space by list of tags - pub fn list_resources_by_tags_no_space( - &self, - tags: Vec, - ) -> BackendResult { - let filtered_resource_ids = self.list_resource_ids_by_tags_no_space(&tags)?; - - if filtered_resource_ids.is_empty() { - return Ok(SearchResultSimple { - items: vec![], - total: 0, - }); - } - - Ok(SearchResultSimple { - total: filtered_resource_ids.len() as i64, - items: filtered_resource_ids, - }) - } - pub fn search_resources( &self, keyword: &str, filtered_resource_ids: &Option>, - include_annotations: bool, keyword_limit: Option, ) -> BackendResult { // The Some value in filtered_resource_ids indicates that the search MUST have the filter ids @@ -283,19 +242,6 @@ impl Database { keyword_limit, )?); - if include_annotations { - let mut annotations = self.list_resource_annotations( - results - .iter() - .map(|item| item.resource.resource.id.as_str()) - .collect::>() - .as_ref(), - )?; - for item in results.iter_mut() { - item.resource.resource_annotations = - annotations.remove(item.resource.resource.id.as_str()) - } - } Ok(SearchResult { total: results.len() as i64, items: results, diff --git a/packages/backend/src/utils.rs b/packages/backend/src/utils.rs index a982d9183..43ef6905e 100644 --- a/packages/backend/src/utils.rs +++ b/packages/backend/src/utils.rs @@ -8,14 +8,14 @@ pub fn uuid_to_base62(uuid: &str) -> String { // Remove hyphens and convert to u128 let uuid = uuid.replace("-", ""); let mut num = u128::from_str_radix(&uuid, 16).unwrap_or(0); - + let mut result = Vec::new(); while num > 0 { let rem = (num % 62) as usize; result.push(BASE62_CHARS[rem]); num /= 62; } - + // Reverse and convert to string String::from_utf8(result.into_iter().rev().collect()).unwrap_or_default() } @@ -34,15 +34,11 @@ pub fn sanitize_filename(name: &str) -> String { /// Otherwise falls back to using just the resource ID pub fn get_resource_filename(resource_id: &str, metadata_name: Option<&str>) -> String { if let Some(name) = metadata_name { - let short_name = if name.len() > 150 { - &name[..150] - } else { - name - }; - + let short_name = if name.len() > 150 { &name[..150] } else { name }; + let sanitized_name = sanitize_filename(short_name); let short_id = uuid_to_base62(resource_id); - + format!("{}-{}", sanitized_name, short_id) } else { resource_id.to_string() @@ -50,7 +46,7 @@ pub fn get_resource_filename(resource_id: &str, metadata_name: Option<&str>) -> } /// Converts a resource type (MIME type) to a file extension. -/// +/// /// This function attempts to determine an appropriate file extension for a given resource type /// using the following logic: /// 1. For special space types (application/vnd.space.*), returns "jsong" @@ -79,10 +75,13 @@ pub fn get_resource_file_extension(resource_type: &str) -> String { "application/vnd.space.link", "application/vnd.space.article", "application/vnd.space.post", - "application/vnd.space.document.space-note" + "application/vnd.space.document.space-note", ]; - - if markdown_resource_types.iter().any(|&t| resource_type.starts_with(t)) { + + if markdown_resource_types + .iter() + .any(|&t| resource_type.starts_with(t)) + { return "md".to_string(); } @@ -114,23 +113,23 @@ mod tests { #[test] fn test_get_resource_filename() { let id = "550e8400-e29b-41d4-a716-446655440000"; - + // Test with metadata name let result = get_resource_filename(id, Some("My Test File")); assert!(result.starts_with("My Test File-")); assert!(result.contains('-')); - + // Test with problematic characters let result = get_resource_filename(id, Some("MyFile*.txt")); assert!(result.starts_with("My-Test-File-")); assert!(!result.contains('*')); assert!(!result.contains('<')); assert!(!result.contains('>')); - + // Test fallback to id let result = get_resource_filename(id, None); assert_eq!(result, id); - + // Test with leading periods let result = get_resource_filename(id, Some("...test")); assert!(result.starts_with("test-")); @@ -145,11 +144,20 @@ mod tests { #[test] fn test_get_resource_file_extension() { // Test space types - assert_eq!(get_resource_file_extension("application/vnd.space.article"), "md"); - assert_eq!(get_resource_file_extension("application/vnd.space.something"), "json"); + assert_eq!( + get_resource_file_extension("application/vnd.space.article"), + "md" + ); + assert_eq!( + get_resource_file_extension("application/vnd.space.something"), + "json" + ); // Test document space note - assert_eq!(get_resource_file_extension("application/vnd.space.document.space-note"), "md"); + assert_eq!( + get_resource_file_extension("application/vnd.space.document.space-note"), + "md" + ); // Test common MIME types assert_eq!(get_resource_file_extension("image/png"), "png"); @@ -161,4 +169,4 @@ mod tests { // Test fallback behavior assert_eq!(get_resource_file_extension("unknown/type"), "type"); } -} \ No newline at end of file +} diff --git a/packages/backend/src/worker/handlers/misc.rs b/packages/backend/src/worker/handlers/misc.rs index 5187bebc0..bcb40dc78 100644 --- a/packages/backend/src/worker/handlers/misc.rs +++ b/packages/backend/src/worker/handlers/misc.rs @@ -632,7 +632,6 @@ impl Worker { let db_results = self.db.search_resources( &query, &Some(ids.clone()), - false, Some(number_documents as i64), )?; diff --git a/packages/backend/src/worker/handlers/resource.rs b/packages/backend/src/worker/handlers/resource.rs index cd15596bd..2758a4833 100644 --- a/packages/backend/src/worker/handlers/resource.rs +++ b/packages/backend/src/worker/handlers/resource.rs @@ -8,11 +8,11 @@ use crate::{ db::Database, models::{ current_time, random_uuid, CompositeResource, EmbeddingResource, EmbeddingType, - InternalResourceTagNames, PostProcessingJob, Resource, ResourceMetadata, - ResourceOrSpace, ResourceProcessingState, ResourceTag, ResourceTagFilter, - ResourceTextContentMetadata, ResourceTextContentType, SearchEngine, - SearchResourcesParams, SearchResult, SearchResultItem, SearchResultSimple, - SearchResultSpaceItem, SpaceEntryExtended, SpaceEntryType, + InternalResourceTagNames, PaginatedResult, PaginationParams, PostProcessingJob, + Resource, ResourceMetadata, ResourceOrSpace, ResourceProcessingState, ResourceTag, + ResourceTagFilter, ResourceTextContentMetadata, ResourceTextContentType, SearchEngine, + SearchResourcesParams, SearchResult, SearchResultItem, SearchResultSpaceItem, + SpaceEntryExtended, SpaceEntryType, }, }, worker::{send_worker_response, Worker}, @@ -32,17 +32,17 @@ impl Worker { let resource_id = random_uuid(); let ct = current_time(); - let extension = crate::utils::get_resource_file_extension(&resource_type); + let extension = crate::utils::get_resource_file_extension(&resource_type); let name = metadata.as_ref().map(|m| m.name.as_ref()); let resource_name = crate::utils::get_resource_filename(&resource_id, name); let resource = Resource { id: resource_id.clone(), resource_path: Path::new(&self.resources_path) - .join(format!("{}.{}", resource_name, extension)) - .as_os_str() - .to_string_lossy() - .to_string(), + .join(format!("{}.{}", resource_name, extension)) + .as_os_str() + .to_string_lossy() + .to_string(), resource_type: resource_type.clone(), created_at: ct, updated_at: ct, @@ -214,7 +214,7 @@ impl Worker { #[instrument(level = "trace", skip(self))] pub fn remove_resources_by_tags(&mut self, tags: Vec) -> BackendResult<()> { - let ids = self.db.list_resource_ids_by_tags(&tags)?; + let ids = self.db.list_resource_ids_by_tags(&tags, None)?; self.remove_resources(ids) } @@ -232,8 +232,12 @@ impl Worker { pub fn list_resources_by_tags( &mut self, tags: Vec, - ) -> BackendResult { - self.db.list_resources_by_tags(tags) + pagination: PaginationParams, + // None = no space filter, Some("") = no space, Some(id) = specific space + space_filter: Option, + ) -> BackendResult> { + self.db + .list_resource_ids_by_tags_paginated(&tags, pagination, space_filter) } #[instrument(level = "trace", skip(self))] @@ -244,28 +248,15 @@ impl Worker { self.db.list_all_resources_and_spaces(tags) } - // Only return resource ids - pub fn list_resources_by_tags_no_space( - &mut self, - tags: Vec, - ) -> BackendResult { - self.db.list_resources_by_tags_no_space(tags) - } - fn get_filtered_ids_for_search( &mut self, resource_tag_filters: Option>, space_id: Option, ) -> BackendResult>> { if let Some(resource_tag_filters) = resource_tag_filters { - if let Some(space_id) = space_id { - return Ok(Some(self.db.list_resource_ids_by_tags_space_id( - &resource_tag_filters, - &space_id, - )?)); - } return Ok(Some( - self.db.list_resource_ids_by_tags(&resource_tag_filters)?, + self.db + .list_resource_ids_by_tags(&resource_tag_filters, space_id)?, )); } if let Some(space_id) = space_id { @@ -291,7 +282,6 @@ impl Worker { } } let keyword_limit = params.keyword_limit.unwrap_or(100); - let include_annotations = params.include_annotations.unwrap_or(false); let semantic_search_enabled = params.semantic_search_enabled.unwrap_or_default(); @@ -304,12 +294,9 @@ impl Worker { let filtered_resource_ids = self.get_filtered_ids_for_search(params.resource_tag_filters, params.space_id.clone())?; - let db_results = self.db.search_resources( - ¶ms.query, - &filtered_resource_ids, - include_annotations, - Some(keyword_limit), - )?; + let db_results = + self.db + .search_resources(¶ms.query, &filtered_resource_ids, Some(keyword_limit))?; for result in db_results.items { if result.resource.resource.resource_type.ends_with(".ignore") { @@ -753,18 +740,14 @@ pub fn handle_resource_message( let result = worker.remove_resources_by_tags(tags); send_worker_response(&mut worker.channel, oneshot, result); } - ResourceMessage::ListResourcesByTags(tags) => { - let result = worker.list_resources_by_tags(tags); + ResourceMessage::ListResourcesByTags(tags, pagination_params, space_filter) => { + let result = worker.list_resources_by_tags(tags, pagination_params, space_filter); send_worker_response(&mut worker.channel, oneshot, result); } ResourceMessage::ListAllResourcesAndSpaces(tags) => { let result = worker.list_all_resources_and_spaces(tags); send_worker_response(&mut worker.channel, oneshot, result); } - ResourceMessage::ListResourcesByTagsNoSpace(tags) => { - let result = worker.list_resources_by_tags_no_space(tags); - send_worker_response(&mut worker.channel, oneshot, result); - } ResourceMessage::SearchResources(search_params) => { let result = worker.search_resources(search_params); send_worker_response(&mut worker.channel, oneshot, result); diff --git a/packages/backend/src/worker/processor.rs b/packages/backend/src/worker/processor.rs index 664d98729..de31ed10d 100644 --- a/packages/backend/src/worker/processor.rs +++ b/packages/backend/src/worker/processor.rs @@ -268,21 +268,22 @@ fn is_markdown_file(file_name: &str) -> bool { fn parse_markdown_with_frontmatter(content: &str) -> BackendResult<(String, serde_yaml::Value)> { // Simple frontmatter parser - finds content between --- markers let parts: Vec<&str> = content.split("---").collect(); - + match parts.len() { // No frontmatter or invalid format 0 | 1 => Ok((content.to_string(), serde_yaml::Value::Null)), - + // Has frontmatter _ => { // Parse the YAML frontmatter (second part, index 1) let frontmatter_yaml = parts[1].trim(); - let frontmatter = serde_yaml::from_str(frontmatter_yaml) - .map_err(|e| BackendError::GenericError(format!("Failed to parse frontmatter: {}", e)))?; - + let frontmatter = serde_yaml::from_str(frontmatter_yaml).map_err(|e| { + BackendError::GenericError(format!("Failed to parse frontmatter: {}", e)) + })?; + // Get the content (everything after second ---) let content = parts[2..].join("---").trim().to_string(); - + Ok((content, frontmatter)) } } @@ -339,53 +340,68 @@ fn process_resource_data( } } - ResourceTextContentType::Post => { - process_file_data::(resource_data, resource_text_content_type, resource, |post_data| { + ResourceTextContentType::Post => process_file_data::( + resource_data, + resource_text_content_type, + resource, + |post_data| { let title = post_data.title.as_deref().unwrap_or_default(); let excerpt = post_data.excerpt.as_deref().unwrap_or_default(); let content = post_data.content_plain.as_deref().unwrap_or_default(); let author = post_data.author.as_deref().unwrap_or_default(); let site = post_data.site_name.as_deref().unwrap_or_default(); format!("{title} {excerpt} {content} {author} {site}") - }) - } + }, + ), - ResourceTextContentType::ChatMessage => { - process_file_data::(resource_data, resource_text_content_type, resource, |msg| { + ResourceTextContentType::ChatMessage => process_file_data::( + resource_data, + resource_text_content_type, + resource, + |msg| { let author = msg.author.as_deref().unwrap_or_default(); let content = msg.content_plain.as_deref().unwrap_or_default(); let platform = msg.platform_name.as_deref().unwrap_or_default(); format!("{author} {content} {platform}") - }) - } + }, + ), - ResourceTextContentType::Document => { - process_file_data::(resource_data, resource_text_content_type, resource, |doc| { + ResourceTextContentType::Document => process_file_data::( + resource_data, + resource_text_content_type, + resource, + |doc| { let author = doc.author.as_deref().unwrap_or_default(); let content = doc.content_plain.as_deref().unwrap_or_default(); let editor = doc.editor_name.as_deref().unwrap_or_default(); format!("{author} {content} {editor}") - }) - } + }, + ), - ResourceTextContentType::Article => { - process_file_data::(resource_data, resource_text_content_type, resource, |article| { + ResourceTextContentType::Article => process_file_data::( + resource_data, + resource_text_content_type, + resource, + |article| { let title = article.title.as_deref().unwrap_or_default(); let excerpt = article.excerpt.as_deref().unwrap_or_default(); let content = article.content_plain.as_deref().unwrap_or_default(); format!("{title} {excerpt} {content}") - }) - } + }, + ), - ResourceTextContentType::Link => { - process_file_data::(resource_data, resource_text_content_type, resource, |link| { + ResourceTextContentType::Link => process_file_data::( + resource_data, + resource_text_content_type, + resource, + |link| { let title = link.title.as_deref().unwrap_or_default(); let desc = link.description.as_deref().unwrap_or_default(); let url = link.url.as_deref().unwrap_or_default(); let content = link.content_plain.as_deref().unwrap_or_default(); format!("{title} {desc} {url}\n{content}") - }) - } + }, + ), ResourceTextContentType::ChatThread => process_file_data::( resource_data, @@ -506,11 +522,12 @@ where T: serde::de::DeserializeOwned, { // Check if this is a markdown file for supported resource types - if is_markdown_resource_type(&resource.resource.resource_type) && - is_markdown_file(&resource.resource.resource_path) { + if is_markdown_resource_type(&resource.resource.resource_type) + && is_markdown_file(&resource.resource.resource_path) + { // Parse markdown with frontmatter let (content, frontmatter) = parse_markdown_with_frontmatter(data)?; - + // Try to deserialize the frontmatter into our expected type match serde_yaml::from_value::(frontmatter) { Ok(parsed_data) => { diff --git a/packages/services/src/lib/resources/resources.svelte.ts b/packages/services/src/lib/resources/resources.svelte.ts index 10b0447d0..34d537b6d 100644 --- a/packages/services/src/lib/resources/resources.svelte.ts +++ b/packages/services/src/lib/resources/resources.svelte.ts @@ -27,6 +27,8 @@ import { type AiSFFSQueryResponse, type SFFSResourceMetadata, type SFFSResourceTag, + type SFFSPaginationParams, + type SFFSPaginatedResult, ResourceTypes, type SFFSResource, type ResourceDataLink, @@ -967,23 +969,63 @@ export class ResourceManager extends EventEmitterBase(resources) } - async listResourceIDsByTags(tags: SFFSResourceTag[], excludeWithinSpaces: boolean = false) { - const results = await this.sffs.listResourceIDsByTags(tags, excludeWithinSpaces) + async listResourceIDsByTags( + tags: SFFSResourceTag[], + paginationParams: SFFSPaginationParams, + // undefined = all spaces + // empty string = does not belong to any space + // non empty string = specific space + spaceId?: string + ) { + const results = await this.sffs.listResourceIDsByTags(tags, paginationParams, spaceId) return results } + // TODO: why leak SFFS types and still have this abstractions? async listResourcesByTags( + tags: SFFSResourceTag[], + paginationParams: SFFSPaginationParams, + opts: { + includeAnnotations?: boolean + // undefined = all spaces + // empty string = does not belong to any space + // non empty string = specific space + spaceId?: string + } = {} + ) { + const result = await this.sffs.listResourceIDsByTags(tags, paginationParams, opts?.spaceId) + // TODO: is this the right behavior? + if (!result) { + return [] + } + this.log.debug('found resource ids', result.items) + const resources = (await Promise.all( + result.items.map((id) => this.findOrGetResourceObject(id, opts)) + )) as Resource[] + return { + items: resources.filter((r) => r !== null), + next_cursor: result.next_cursor, + has_more: result.has_more + } as SFFSPaginatedResult + } + + async listAllResourcesByTags( tags: SFFSResourceTag[], opts: { includeAnnotations?: boolean; excludeWithinSpaces?: boolean } = {} ) { - const resourceIds = await this.sffs.listResourceIDsByTags( + const result = await this.sffs.listAllResourceIDsByTags( tags, opts?.excludeWithinSpaces ?? false ) - this.log.debug('found resource ids', resourceIds) - return (await Promise.all( - resourceIds.map((id) => this.findOrGetResourceObject(id, opts)) + // TODO: is this the right behavior? + if (!result) { + return [] + } + this.log.debug('found resource ids', result) + const resources = (await Promise.all( + result.map((id) => this.findOrGetResourceObject(id, opts)) )) as Resource[] + return resources.filter((r) => r !== null) } async listAllResourcesAndSpaces(tags: SFFSResourceTag[]) { @@ -1047,30 +1089,6 @@ export class ResourceManager extends EventEmitterBase - // ({ - // id: item.resource.id, - // engine: item.engine, - // cardIds: item.card_ids, - // resource: this.findOrCreateResourceObject(item.resource) - // }) as ResourceSearchResultItem - // ) - - // return results - // } - - async getResourceAnnotations() { - const resources = await this.listResourcesByTags([ - SearchResourceTags.ResourceType(ResourceTypes.ANNOTATION), - SearchResourceTags.Deleted(false) - ]) - - return resources as ResourceAnnotation[] - } - async getResourcesFromSourceURL(url: string, tags?: SFFSResourceTag[]) { const surfUrlMatch = url.match(/surf:\/\/resource\/([^\/]+)/) if (surfUrlMatch) { @@ -1083,7 +1101,7 @@ export class ResourceManager extends EventEmitterBase ({ @@ -453,16 +454,79 @@ export class SFFS { }) as SFFSRawResourceTag ) ) + } + + async listAllResourceIDsByTags( + tags: SFFSResourceTag[], + excludeWithinSpaces: boolean = false + ): Promise { + this.log.debug('listing all resources by tags', tags, excludeWithinSpaces) + + const tagsData = this.serializeTagsData(tags) + + const allResults: string[] = [] + let hasMore = true + let cursor: string | null = null + + while (hasMore) { + const paginationParams: SFFSPaginationParams = { + limit: 100, + ...(cursor && { cursor }) + } + const paginationData = JSON.stringify(paginationParams) - let raw: string - if (excludeWithinSpaces) { - raw = await this.backend.js__store_list_resources_by_tags_no_space(tagsData) - } else { - raw = await this.backend.js__store_list_resources_by_tags(tagsData) + let raw: string + if (excludeWithinSpaces) { + raw = await this.backend.js__store_list_resources_by_tags_no_space(tagsData, paginationData) + } else { + raw = await this.backend.js__store_list_resources_by_tags(tagsData, paginationData) + } + + const parsed = this.parseData>(raw) + if (!parsed) { + throw new Error( + 'failed to parse result of list resources by tags, unexpected data from backend' + ) + } + + allResults.push(...parsed.items) + hasMore = parsed.has_more + cursor = parsed.next_cursor } - const parsed = this.parseData<{ items: string[]; total: number }>(raw) - return parsed?.items ?? [] + this.log.debug(`fetched ${allResults.length} total resources by tags`) + return allResults + } + + async listResourceIDsByTags( + tags: SFFSResourceTag[], + paginationParams: SFFSPaginationParams, + // undefined = all spaces + // empty string = does not belong to any space + // non empty string = specific space + spaceId?: string + ) { + this.log.debug('listing resources by tags', tags, paginationParams, spaceId) + const tagsData = JSON.stringify( + tags.map( + (tag) => + ({ + id: '', + resource_id: '', + tag_name: tag.name, + tag_value: tag.value, + op: tag.op ?? 'eq' + }) as SFFSRawResourceTag + ) + ) + const paginationData = JSON.stringify(paginationParams) + const raw = await this.backend.js__store_list_resources_by_tags( + tagsData, + paginationData, + spaceId + ) + const parsed = this.parseData>(raw) + return parsed } async listAllResourcesAndSpaces(tags: SFFSResourceTag[]) { @@ -515,7 +579,6 @@ export class SFFS { parameters?.semanticEnabled, parameters?.semanticDistanceThreshold, parameters?.semanticLimit, - parameters?.includeAnnotations, parameters?.spaceId, parameters?.keywordLimit ) diff --git a/packages/services/src/lib/teletype/providers/ResourcesProvider.ts b/packages/services/src/lib/teletype/providers/ResourcesProvider.ts deleted file mode 100644 index bee046e56..000000000 --- a/packages/services/src/lib/teletype/providers/ResourcesProvider.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { type MentionItem } from '@deta/editor' -import type { ActionProvider, TeletypeAction } from '../types' -import { - generateUUID, - useLogScope, - prependProtocol, - SearchResourceTags, - truncate, - getFileKind -} from '@deta/utils' -import { Resource, ResourceJSON, useResourceManager } from '../../resources' -import { ResourceTagsBuiltInKeys, ResourceTypes } from '@deta/types' -import { useBrowser } from '../../browser' - -export class ResourcesProvider implements ActionProvider { - readonly name = 'resources-search' - readonly isLocal = false - readonly maxActions = 9 - - private readonly log = useLogScope('ResourcesProvider') - private readonly resourceManager = useResourceManager() - private readonly browser = useBrowser() - - canHandle(query: string): boolean { - return query.trim().length >= 2 - } - - async getActions(query: string, _mentions: MentionItem[]): Promise { - const actions: TeletypeAction[] = [] - const trimmedQuery = query.trim() - - if (trimmedQuery.length < 2) return actions - - try { - const resources = await this.searchResources(trimmedQuery) - let noteCount = 0 - let otherCount = 0 - - resources.forEach((resource, index) => { - if (resource.type === ResourceTypes.DOCUMENT_SPACE_NOTE) { - if (noteCount >= 3) return - noteCount++ - } else { - if (otherCount >= 6) return - otherCount++ - } - - actions.push( - this.createSearchAction(resource, 80 - index, ['search', 'suggestion', 'google']) - ) - }) - } catch (error) { - this.log.error('Failed to fetch search suggestions:', error) - } - - return actions - } - - private createSearchAction( - resource: Resource, - priority: number, - keywords: string[] - ): TeletypeAction { - const url = - resource.metadata?.sourceURI ?? - resource.tags?.find((tag) => tag.name === ResourceTagsBuiltInKeys.CANONICAL_URL)?.value - const data = (resource as ResourceJSON).parsedData - - return { - id: generateUUID(), - name: truncate( - data?.title || - resource.metadata?.name || - url || - `${resource.id} - ${resource.type}` || - 'Undefined', - 30 - ), - icon: url - ? `favicon;;${url}` - : resource.type === ResourceTypes.DOCUMENT_SPACE_NOTE - ? 'note' - : `file;;${getFileKind(resource.type)}`, - section: resource.type === ResourceTypes.DOCUMENT_SPACE_NOTE ? 'Your Notes' : 'Saved Sources', - priority, - keywords, - description: ``, - buttonText: 'Open', - handler: async () => { - this.log.debug('Handling resource open', resource) - await this.browser.openResourceInCurrentTab(resource) - } - } - } - - private async searchResources(query: string): Promise { - try { - const results = await this.resourceManager.searchResources( - query, - [...SearchResourceTags.NonHiddenDefaultTags()], - { - includeAnnotations: false, - semanticEnabled: this.resourceManager.config.settingsValue.use_semantic_search - // semanticLimit: 0, - // keywordLimit: 6 - } - ) - - const resources = results.resources.map((x) => x.resource) - - // await Promise.all( - // resources.map((resource) => { - // if (resource instanceof ResourceJSON) { - // return resource.getParsedData() - // } - // }) - // ) - - return resources - } catch (error) { - this.log.error('Error fetching Google suggestions:', error) - return [] - } - } -} diff --git a/packages/services/src/lib/teletype/teletypeServiceCore.ts b/packages/services/src/lib/teletype/teletypeServiceCore.ts index c085441f6..fe444516d 100644 --- a/packages/services/src/lib/teletype/teletypeServiceCore.ts +++ b/packages/services/src/lib/teletype/teletypeServiceCore.ts @@ -1,10 +1,8 @@ import { useLogScope } from '@deta/utils' import type { ActionProvider, TeletypeAction, TeletypeServiceOptions } from './types' import { SearchProvider } from './providers/SearchProvider' -import { AskProvider } from './providers/AskProvider' import { type TeletypeActionSerialized, useMessagePortPrimary } from '../messagePort' -import { MentionItemType, type MentionItem } from '@deta/editor' -import { ResourcesProvider } from './providers/ResourcesProvider' +import { type MentionItem } from '@deta/editor' import { useBrowser } from '../browser' import { HostnameProvider } from './providers/HostnameProvider' import type { Fn } from '@deta/types' @@ -72,7 +70,6 @@ export class TeletypeServiceCore { // Register external/async providers this.registerProvider(new HostnameProvider()) // Async Hostname suggestions this.registerProvider(new SearchProvider()) // Async Google suggestions - this.registerProvider(new ResourcesProvider()) // SFFS Resources search this.registerProvider(new NotebooksProvider()) // Notebooks search } diff --git a/packages/teletype/src/components/Teletype/TeletypeCore.svelte b/packages/teletype/src/components/Teletype/TeletypeCore.svelte index ba2124632..2d7027ff4 100644 --- a/packages/teletype/src/components/Teletype/TeletypeCore.svelte +++ b/packages/teletype/src/components/Teletype/TeletypeCore.svelte @@ -116,9 +116,7 @@ }) const placeholder = $derived( - hideNavigation - ? 'Search the notebook or ask a question...' - : 'Search the web, your notebooks, enter a URL or ask a question...' + hideNavigation ? 'Ask a question...' : 'Search the web, enter a URL or ask a question...' // $currentAction && $currentAction.placeholder ? $currentAction.placeholder : $placeholderText ) diff --git a/packages/types/src/resources.types.ts b/packages/types/src/resources.types.ts index 64b71f240..bc587984e 100644 --- a/packages/types/src/resources.types.ts +++ b/packages/types/src/resources.types.ts @@ -28,6 +28,17 @@ export interface SFFSResourceTag { op?: 'eq' | 'ne' | 'prefix' | 'suffix' | 'notexists' | 'neprefix' | 'nesuffix' } +export interface SFFSPaginationParams { + limit: number + cursor?: string +} + +export interface SFFSPaginatedResult { + items: T[] + next_cursor: string | null + has_more: boolean +} + export enum ResourceTagsBuiltInKeys { SAVED_WITH_ACTION = 'savedWithAction', TYPE = 'type', @@ -124,7 +135,6 @@ export interface SFFSResourceOrSpace { export type SFFSSearchResultEngine = 'keyword' | 'proximity' | 'semantic' | 'local' export interface SFFSSearchGeneralParameters { - includeAnnotations?: boolean spaceId?: string keywordLimit?: number // Limit for keyword-based search results } diff --git a/packages/ui/src/lib/components/Notebook/NotebookCover.svelte b/packages/ui/src/lib/components/Notebook/NotebookCover.svelte index bc4d2be85..3d8855e60 100644 --- a/packages/ui/src/lib/components/Notebook/NotebookCover.svelte +++ b/packages/ui/src/lib/components/Notebook/NotebookCover.svelte @@ -1,9 +1,9 @@
@@ -27,7 +31,7 @@ }} > {#if tab.icon} - + {/if} {tab.label}
@@ -61,7 +65,7 @@ &:hover:not(.disabled):not(.active) { background: light-dark(rgba(0, 0, 0, 0.075), rgba(255, 255, 255, 0.1)); - cursor: pointer; + cursor: pointer; } &.active { diff --git a/packages/ui/src/lib/components/Utils/SurfLoader.svelte b/packages/ui/src/lib/components/Utils/SurfLoader.svelte index 35efbf210..e49fd4b36 100644 --- a/packages/ui/src/lib/components/Utils/SurfLoader.svelte +++ b/packages/ui/src/lib/components/Utils/SurfLoader.svelte @@ -1,31 +1,54 @@ - -{#if isLoading} +{#if isLoading || searching} {@render loading?.()} {:else} - {@render children?.([resources, searchResults, searching])} + {@render children?.({ + resources, + searchResults, + searching, + pagination, + loadMore + })} {/if} -