refactor(rag): introduce doc_service and migrate Document off legacy doc managers

.gitignore

@@ -6,6 +6,7 @@ __pycache__
 test/
 dist/
 tmp/
+tmp*/
 build
 *.lock
 *.db

docs/en/API Reference/tools.md

@@ -200,35 +200,29 @@
     members: [find]
     exclude-members:
 
-::: lazyllm.tools.rag.DocManager
-    members: document, list_kb_groups, add_files, reparse_files
-	exclude-members:
+::: lazyllm.tools.rag.doc_service.DocServer
+    members: [upload, add, reparse, delete, transfer, patch_metadata, list_docs, get_doc, list_tasks, get_task, cancel_task, list_kbs, get_kb, list_chunks, list_algorithms, get_algorithm_info, create_kb, update_kb, batch_get_kbs, delete_kb, delete_kbs]
+    exclude-members:
 
-::: lazyllm.tools.rag.utils.SqliteDocListManager
-    members:
-      - table_inited
-      - get_status_cond_and_params
-      - validate_paths
-      - update_need_reparsing
-      - list_files
-      - get_docs
-      - set_docs_new_meta
-      - fetch_docs_changed_meta
-      - list_all_kb_group
-      - add_kb_group
-      - list_kb_group_files
-      - delete_unreferenced_doc
-      - get_docs_need_reparse
-      - get_existing_paths_by_pattern
-      - update_file_message
-      - update_file_status
-      - add_files_to_kb_group
-      - delete_files_from_kb_group
-      - get_file_status
-      - update_kb_group
-      - release
-      - get_status_cond_and_params
-	exclude-members:
+::: lazyllm.tools.rag.doc_service.base.AddFileItem
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.UploadRequest
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.AddRequest
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.TransferItem
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.TransferRequest
+    members:
+    exclude-members:
 
 ::: lazyllm.tools.rag.data_loaders.DirectoryReader
     members: load_data
@@ -425,9 +419,6 @@
 ::: lazyllm.tools.rag.rerank.ModuleReranker
     members: forward
     exclude-members:
-::: lazyllm.tools.rag.utils.DocListManager
-    members:
-    exclude-members:
 ::: lazyllm.tools.rag.global_metadata.GlobalMetadataDesc
     members:
     exclude-members:

docs/lazyllm-skill/assets/rag/document.md

@@ -91,8 +91,8 @@ def processYml(file):
 ...     print("Call the function processYml.")
 ...     return [DocNode(text=data)]
 ...
-doc1 = Document(dataset_path="your_files_path", create_ui=False)
-doc2 = Document(dataset_path="your_files_path", create_ui=False)
+doc1 = Document(dataset_path="your_files_path")
+doc2 = Document(dataset_path="your_files_path")
 doc1.add_reader("**/*.yml", YmlReader)
 print(doc1._impl._local_file_reader)
 {'**/*.yml': <class '__main__.YmlReader'>}
@@ -213,4 +213,3 @@ res = retriever(query=query)
 print(f"answer: {res}")
 ```
 
-

docs/zh/API Reference/tools.md

@@ -189,35 +189,29 @@
     members: [find]
     exclude-members:
 
-::: lazyllm.tools.rag.DocManager
-    members: document, list_kb_groups, add_files, reparse_files
-	exclude-members:
+::: lazyllm.tools.rag.doc_service.DocServer
+    members: [upload, add, reparse, delete, transfer, patch_metadata, list_docs, get_doc, list_tasks, get_task, cancel_task, list_kbs, get_kb, list_chunks, list_algorithms, get_algorithm_info, create_kb, update_kb, batch_get_kbs, delete_kb, delete_kbs]
+    exclude-members:
 
-::: lazyllm.tools.rag.utils.SqliteDocListManager
-    members:
-      - table_inited
-      - get_status_cond_and_params
-      - validate_paths
-      - update_need_reparsing
-      - list_files
-      - get_docs
-      - set_docs_new_meta
-      - fetch_docs_changed_meta
-      - list_all_kb_group
-      - add_kb_group
-      - list_kb_group_files
-      - delete_unreferenced_doc
-      - get_docs_need_reparse
-      - get_existing_paths_by_pattern
-      - update_file_message
-      - update_file_status
-      - add_files_to_kb_group
-      - delete_files_from_kb_group
-      - get_file_status
-      - update_kb_group
-      - release
-      - get_status_cond_and_params
-	exclude-members:
+::: lazyllm.tools.rag.doc_service.base.AddFileItem
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.UploadRequest
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.AddRequest
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.TransferItem
+    members:
+    exclude-members:
+
+::: lazyllm.tools.rag.doc_service.base.TransferRequest
+    members:
+    exclude-members:
 
 ::: lazyllm.tools.rag.data_loaders.DirectoryReader
     members: load_data
@@ -414,9 +408,6 @@
 ::: lazyllm.tools.rag.rerank.ModuleReranker
     members: forward
     exclude-members:
-::: lazyllm.tools.rag.utils.DocListManager
-    members:
-    exclude-members:
 ::: lazyllm.tools.rag.global_metadata.GlobalMetadataDesc
     members:
     exclude-members:

examples/rag/doc_service_mock_example.py

@@ -0,0 +1,85 @@
+'''Connect a Document to a deployed DocServer.
+
+Start DocServer first:
+    python examples/rag/doc_service_standalone.py --wait
+
+Run this example:
+    python examples/rag/doc_service_mock_example.py --doc-server-url http://127.0.0.1:8848
+'''
+
+from __future__ import annotations
+
+import argparse
+import os
+import tempfile
+import time
+
+from lazyllm import Document
+from lazyllm.tools.rag.doc_service import DocServer
+from lazyllm.tools.rag.doc_service.base import AddFileItem, AddRequest
+
+
+def _normalize_base_url(url: str) -> str:
+    url = url.rstrip('/')
+    if url.endswith('/_call') or url.endswith('/generate'):
+        return url.rsplit('/', 1)[0]
+    return url
+
+
+def _wait_task(server: DocServer, task_id: str, timeout: float = 30.0):
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        task = server.get_task(task_id)['data']
+        if task['status'] in {'SUCCESS', 'FAILED', 'CANCELED', 'DELETED'}:
+            return task
+        time.sleep(0.5)
+    raise TimeoutError(f'task {task_id} did not finish in time')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Connect a Document to an existing DocServer.')
+    parser.add_argument('--doc-server-url', type=str, required=True, help='Existing DocServer base URL.')
+    parser.add_argument('--algo-id', type=str, default='doc_service_demo_algo', help='Document algorithm ID.')
+    parser.add_argument('--kb-id', type=str, default='__default__', help='Knowledge base ID.')
+    args = parser.parse_args()
+
+    doc_server = DocServer(url=_normalize_base_url(args.doc_server_url))
+    with tempfile.TemporaryDirectory(prefix='lazyllm_doc_service_example_') as dataset_dir:
+        file_path = os.path.join(dataset_dir, 'demo.txt')
+        with open(file_path, 'w', encoding='utf-8') as file:
+            file.write('hello from a real doc_service example\n')
+
+        # Step 1: create a Document and bind it to the deployed DocServer.
+        document = Document(dataset_path=dataset_dir, manager=doc_server, name=args.algo_id)
+        document.start()
+
+        try:
+            base_url = _normalize_base_url(args.doc_server_url)
+            print(f'DocServer URL: {base_url}')
+            print(f'DocServer Docs: {base_url}/docs')
+
+            # Step 2: add a local file through the DocServer client.
+            response = doc_server.add(AddRequest(
+                kb_id=args.kb_id,
+                algo_id=args.algo_id,
+                items=[AddFileItem(file_path=file_path)],
+            ))
+            item = response['data']['items'][0]
+            print(f'Doc ID: {item["doc_id"]}')
+            print(f'Task ID: {item["task_id"]}')
+
+            # Step 3: wait for the asynchronous parse task to finish.
+            task = _wait_task(doc_server, item['task_id'])
+            print(f'Task Status: {task["status"]}')
+
+            # Step 4: list documents from the target knowledge base.
+            docs = doc_server.list_docs(
+                kb_id=args.kb_id, algo_id=args.algo_id, include_deleted_or_canceled=False
+            )['data']['items']
+            print(f'Doc Count In {args.kb_id}: {len(docs)}')
+        finally:
+            document.stop()
+
+
+if __name__ == '__main__':
+    main()