Translation

.gitpod.yml

@@ -0,0 +1,10 @@
+# This configuration file was automatically generated by Gitpod.
+# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml)
+# and commit this file to your remote git repository to share the goodness with others.
+
+# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
+
+tasks:
+  - init: make
+
+

backend/constants.py

@@ -1 +1,2 @@
 WEAVIATE_DOCS_INDEX_NAME = "LangChain_Combined_Docs_OpenAI_text_embedding_3_small"
+STOCK_API_URL="https://yfinance-fza5dthrg6dxd2c3.southeastasia-01.azurewebsites.net/"

backend/graph.py

@@ -32,38 +32,31 @@
 
 from backend.constants import WEAVIATE_DOCS_INDEX_NAME
 from backend.ingest import get_embeddings_model
+from backend.stock_utils import extract_and_fetch_stock_data, format_stock_info
 
-RESPONSE_TEMPLATE = """\
-You are an expert programmer and problem-solver, tasked with answering any question \
-about Langchain.
 
-Generate a comprehensive and informative answer of 80 words or less for the \
-given question based solely on the provided search results (URL and content). You must \
-only use information from the provided search results. Use an unbiased and \
-journalistic tone. Combine search results together into a coherent answer. Do not \
-repeat text. Cite search results using [${{number}}] notation. Only cite the most \
-relevant results that answer the question accurately. Place these citations at the end \
-of the sentence or paragraph that reference them - do not put them all at the end. If \
-different results refer to different entities within the same name, write separate \
-answers for each entity.
+RESPONSE_TEMPLATE = """
+You are an expert in stocks, finance, and cryptocurrencies, tasked with answering any question related to these domains. You can communicate fluently in both English and Chinese.
 
-You should use bullet points in your answer for readability. Put citations where they apply
-rather than putting them all at the end.
+Generate a comprehensive and informative answer of 500 words or less for the given question based solely on the provided search results (URL and content). You must only use information from the provided search results. Use an unbiased and journalistic tone. Combine search results together into a coherent answer. Do not repeat text. Cite search results using [${{number}}] notation. Only cite the most relevant results that answer the question accurately. Place these citations at the end of the sentence or paragraph that reference them - do not put them all at the end. If different results refer to different entities within the same name, write separate answers for each entity.
 
-If there is nothing in the context relevant to the question at hand, just say "Hmm, \
-I'm not sure." Don't try to make up an answer.
+You should use bullet points in your answer for readability. Put citations where they apply rather than putting them all at the end.
+
+If there is nothing in the context relevant to the question at hand, try your best to create an answer based on your own knowledge as a stock, finance, and crypto expert, but add a note at the end stating: "Note: This response is based on the AI's own knowledge as a stock, finance, and crypto expert, as no relevant information was found in the provided context."
 
-Anything between the following `context`  html blocks is retrieved from a knowledge \
-bank, not part of the conversation with the user. 
+If the input question is in Chinese, respond in Chinese. If the input question is in English, respond in English.
+
+Anything between the following context html blocks is retrieved from a knowledge bank, not part of the conversation with the user.
 
 <context>
-    {context} 
+
+{context}
+
 <context/>
 
-REMEMBER: If there is no relevant information within the context, just say "Hmm, I'm \
-not sure." Don't try to make up an answer. Anything between the preceding 'context' \
-html blocks is retrieved from a knowledge bank, not part of the conversation with the \
-user.\
+REMEMBER: If there is no relevant information within the context, create an answer based on your own knowledge as a stock, finance, and crypto expert and include the note about the source of the information. Anything between the preceding 'context' html blocks is retrieved from a knowledge bank, not part of the conversation with the user.
+
+如果输入的问题是中文,请用中文回答。如果输入的问题是英文,请用英文回答。
 """
 
 COHERE_RESPONSE_TEMPLATE = """\
@@ -137,6 +130,7 @@ class AgentState(TypedDict):
     # for convenience in evaluations
     answer: str
     feedback_urls: dict[str, list[str]]
+    stock_data: Optional[list[dict]]  # New field
 
 
 gpt_4o_mini = ChatOpenAI(model="gpt-4o-mini-2024-07-18", temperature=0, streaming=True)
@@ -204,6 +198,17 @@ class AgentState(TypedDict):
 )
 
 
+#for realtime stock info section
+def check_stock_symbols(state: AgentState) -> AgentState:
+    messages = convert_to_messages(state["messages"])
+    query = messages[-1].content
+    stock_data = extract_and_fetch_stock_data(query)
+    if stock_data:
+        state["stock_data"] = stock_data
+    return state
+
+
+
 @contextlib.contextmanager
 def get_retriever(k: Optional[int] = None) -> Iterator[BaseRetriever]:
     with weaviate.connect_to_weaviate_cloud(
@@ -305,7 +310,8 @@ def get_feedback_urls(config: RunnableConfig) -> dict[str, list[str]]:
     return key_to_token_urls
 
 
-def synthesize_response(
+
+def synthesize_response_old(
     state: AgentState,
     config: RunnableConfig,
     model: LanguageModelLike,
@@ -338,6 +344,44 @@ def synthesize_response(
         "feedback_urls": feedback_urls,
     }
 
+def synthesize_response(
+    state: AgentState,
+    config: RunnableConfig,
+    model: LanguageModelLike,
+    prompt_template: str,
+) -> AgentState:
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", prompt_template),
+            ("placeholder", "{chat_history}"),
+            ("human", "{question}"),
+        ]
+    )
+    response_synthesizer = prompt | model
+
+    # Include stock data in the context if available
+    context = format_docs(state["documents"])
+    if "stock_data" in state and state["stock_data"]:
+        stock_info = format_stock_info(state["stock_data"])
+        context = stock_info + "\n" + context
+
+    synthesized_response = response_synthesizer.invoke(
+        {
+            "question": state["query"],
+            "context": context,
+            "chat_history": get_chat_history(
+                convert_to_messages(state["messages"][:-1])
+            ),
+        }
+    )
+    feedback_urls = get_feedback_urls(config)
+    return {
+        "messages": [synthesized_response],
+        "answer": synthesized_response.content,
+        "feedback_urls": feedback_urls,
+    }
+
+
 
 def synthesize_response_default(
     state: AgentState, config: RunnableConfig
@@ -372,13 +416,17 @@ class InputSchema(TypedDict):
 workflow = StateGraph(AgentState, Configuration, input=InputSchema)
 
 # define nodes
+workflow.add_node("stock_symbol_check", check_stock_symbols)
 workflow.add_node("retriever", retrieve_documents)
 workflow.add_node("retriever_with_chat_history", retrieve_documents_with_chat_history)
 workflow.add_node("response_synthesizer", synthesize_response_default)
 workflow.add_node("response_synthesizer_cohere", synthesize_response_cohere)
 
-# set entry point to retrievers
-workflow.set_conditional_entry_point(route_to_retriever)
+# set entry point to stock symbol check
+workflow.set_entry_point("stock_symbol_check")
+
+# connect stock symbol check to retrievers
+workflow.add_conditional_edges("stock_symbol_check", route_to_retriever)
 
 # connect retrievers and response synthesizers
 workflow.add_conditional_edges("retriever", route_to_response_synthesizer)

backend/ingest.py

@@ -2,7 +2,9 @@
 import logging
 import os
 import re
+import requests
 from typing import Optional
+from lxml import etree
 
 import weaviate
 from bs4 import BeautifulSoup, SoupStrainer
@@ -122,7 +124,128 @@ def load_api_docs():
         ),
     ).load()
 
+# def extract_items_from_rss(feed_url: str):
+#     response = requests.get(feed_url)
+#     soup = BeautifulSoup(response.content, "xml")
+#     items = soup.find_all("item")
+#     news_items = []
 
+#     for item in items:
+#         title = item.find("title").get_text() if item.find("title") else "No Title"
+#         link = item.find("link").get_text() if item.find("link") else "No Link"
+#         identifier = item.find("dc:identifier").get_text() if item.find("dc:identifier") else "No Identifier"
+#         pub_date = item.find("pubDate").get_text() if item.find("pubDate") else "No Date"
+#         creator = item.find("dc:creator").get_text() if item.find("dc:creator") else "No Creator"
+#         thumbnail = item.find("media:thumbnail")['url'] if item.find("media:thumbnail") else "No Thumbnail"
+#         guid = item.find("guid").get_text() if item.find("guid") else "No GUID"
+#         description = item.find("description").get_text() if item.find("description") else "No Description"
+#         content_encoded = item.find("content:encoded").decode_contents() if item.find("content:encoded") else "No Content"
+
+#         news_item = {
+#             "title": title,
+#             "link": link,
+#             "identifier": identifier,
+#             "pub_date": pub_date,
+#             "creator": creator,
+#             "thumbnail": thumbnail,
+#             "guid": guid,
+#             "description": description,
+#             "content": content_encoded,
+#         }
+#         news_items.append(news_item)
+
+
+#     return news_items
+
+
+# def load_sample_news():
+#     feed_url = "https://cdn.feedcontrol.net/7512/12213-hIFHBiLc7Wh50.xml"
+#     items = extract_items_from_rss(feed_url)
+#     documents = []
+#     for item in items:
+#         doc = Document(
+#             page_content=item['content'],
+#             metadata={"title": item["title"], "source": item["link"], "identifier": item["identifier"], "pub_date": item["pub_date"], "creator": item[creator],
+#                      "thumbnail": item[thumbnail], "guid": item[guid], "description": item[description]}  
+#         )
+#         documents.append(doc)
+
+#     return documents
+
+
+# def load_sample_news():
+#     return SitemapLoader(
+#         "https://cdn.feedcontrol.net/7512/12213-hIFHBiLc7Wh50.xml",
+#         filter_urls=[],  
+#         parsing_function=simple_extractor,  
+#         default_parser="lxml",
+#         bs_kwargs={"parse_only": SoupStrainer(name=("article", "title", "html", "lang", "content"))},
+#         meta_function=metadata_extractor,  
+#     ).load()
+
+
+def generate_sitemap_xml(xml_string) -> str:
+    soup = BeautifulSoup(xml_string, "xml")
+    items = soup.find_all("item")
+
+    extracted_data = []
+    for item in items:
+        title = item.find("title").get_text() if item.find("title") else "No Title"
+        link = item.find("link").get_text() if item.find("link") else "No Link"
+        identifier = item.find("dc:identifier").get_text() if item.find("dc:identifier") else "No Identifier"
+        pub_date = item.find("pubDate").get_text() if item.find("pubDate") else "No Date"
+        creator = item.find("dc:creator").get_text() if item.find("dc:creator") else "No Creator"
+        thumbnail = item.find("media:thumbnail")['url'] if item.find("media:thumbnail") else "No Thumbnail"
+        guid = item.find("guid").get_text() if item.find("guid") else "No GUID"
+        description = item.find("description").get_text() if item.find("description") else "No Description"
+        content_encoded = item.find("content:encoded").decode_contents() if item.find("content:encoded") else "No Content"
+
+        extracted_data.append({
+            "title": title,
+            "link": link,
+            "identifier": identifier,
+            "pub_date": pub_date,
+            "creator": creator,
+            "thumbnail": thumbnail,
+            "guid": guid,
+            "description": description,
+            "content": content_encoded,
+        })
+
+
+    urlset = etree.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
+
+    for entry in extracted_data:
+        url = etree.SubElement(urlset, "url")
+
+        loc = etree.SubElement(url, "loc")
+        loc.text = entry['link']
+
+        lastmod = etree.SubElement(url, "lastmod")
+        lastmod.text = entry['pub_date']
+
+    return etree.tostring(urlset, pretty_print=True, xml_declaration=True, encoding='UTF-8').decode('utf-8')
+
+def load_sample_news():
+    response = requests.get("https://cdn.feedcontrol.net/7512/12213-hIFHBiLc7Wh50.xml")
+    xml_content = response.content
+    sitemap_xml = generate_sitemap_xml(xml_content)
+
+    with open('sample_news.xml', 'w', encoding='utf-8') as file:
+        file.write(sitemap_xml)
+    print("Sitemap XML has been saved to 'sitemap.xml'.")
+
+    return SitemapLoader(
+        "sample_news.xml",
+        is_local=True,
+        parsing_function=simple_extractor,  
+        default_parser="lxml",
+        bs_kwargs={"parse_only": SoupStrainer(name=("article", "title", "html", "lang", "content"))},
+          meta_function=lambda meta, soup: metadata_extractor(
+            meta, soup, title_suffix=" | sample_news"
+        ),
+    ).load()
+
 def ingest_docs():
     WEAVIATE_URL = os.environ["WEAVIATE_URL"]
     WEAVIATE_API_KEY = os.environ["WEAVIATE_API_KEY"]
@@ -157,12 +280,15 @@ def ingest_docs():
         logger.info(f"Loaded {len(docs_from_langsmith)} docs from LangSmith")
         docs_from_langgraph = load_langgraph_docs()
         logger.info(f"Loaded {len(docs_from_langgraph)} docs from LangGraph")
+        docs_from_sample_news = load_sample_news()
+        logger.info(f"Loaded {len(docs_from_sample_news)} docs from SampleNews")
 
         docs_transformed = text_splitter.split_documents(
             docs_from_documentation
             + docs_from_api
             + docs_from_langsmith
             + docs_from_langgraph
+            + docs_from_sample_news
         )
         docs_transformed = [
             doc for doc in docs_transformed if len(doc.page_content) > 10