From eceaee3c6642cf8f85f638faba5bc9f0ebb32948 Mon Sep 17 00:00:00 2001
From: chloedia <chloedaems0@gmail.com>
Date: Fri, 21 Jun 2024 10:40:59 +0200
Subject: [PATCH 1/2] add: celery + api difference assistant

---
 Pipfile                                       |    5 +-
 Pipfile.lock                                  | 1487 +++++++----------
 backend/celery_worker.py                      |  116 ++
 .../assistant/controller/assistant_routes.py  |   47 +-
 backend/modules/assistant/dto/inputs.py       |    3 +-
 backend/modules/assistant/ito/difference.py   |  178 --
 .../assistant/ito/difference/__init__.py      |    5 +
 .../ito/difference/difference_agent.py        |  133 ++
 .../assistant/ito/difference/query_engine.py  |  144 ++
 .../ito/difference/question_generator.py      |  118 ++
 .../assistant/ito/difference_assistant.py     |  287 ++++
 backend/modules/assistant/ito/ito.py          |   56 +-
 backend/modules/assistant/ito/summary.py      |   42 +-
 backend/requirements.txt                      |    3 +
 14 files changed, 1509 insertions(+), 1115 deletions(-)
 delete mode 100644 backend/modules/assistant/ito/difference.py
 create mode 100644 backend/modules/assistant/ito/difference/__init__.py
 create mode 100644 backend/modules/assistant/ito/difference/difference_agent.py
 create mode 100644 backend/modules/assistant/ito/difference/query_engine.py
 create mode 100644 backend/modules/assistant/ito/difference/question_generator.py
 create mode 100644 backend/modules/assistant/ito/difference_assistant.py

diff --git a/Pipfile b/Pipfile
index 8b2f5383ef55..18d2419949a6 100644
--- a/Pipfile
+++ b/Pipfile
@@ -49,9 +49,8 @@ langfuse = "*"
 pandasai = "*"
 colorlog = "*"
 psycopg2-binary = "*"
-psycopg2 = "*"
 celery = {extras = ["redis", "sqs"], version = "*"}
-unstructured = {extras = ["all-docs"], version = "*"}
+unstructured = "*"
 llama-parse = "*"
 llama-index = "*"
 lxml = {extras = ["html_clean"], version = "*"}
@@ -71,6 +70,8 @@ google-api-python-client = "*"
 google-auth-httplib2 = "*"
 google-auth-oauthlib = "*"
 msal = "*"
+llama-index-postprocessor-cohere-rerank = "*"
+megaparse = "*"
 
 [dev-packages]
 black = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 31c52a61fd74..409a519b1b64 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "ef45b462930863ab7c38532089d13fe9f0d48fab6397b114e2c61f593ac3e59f"
+            "sha256": "eee5ed26f89b9d5681cd28c6d69e97c62d8bb31357daa992918e846d15dcb213"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -122,12 +122,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.7.0"
         },
-        "antlr4-python3-runtime": {
-            "hashes": [
-                "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"
-            ],
-            "version": "==4.9.3"
-        },
         "anyio": {
             "hashes": [
                 "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94",
@@ -279,21 +273,21 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:38893db8269d25b72cc6fbab97633bfc863eefde5456847169d06149a16aa6e0",
-                "sha256:3c42bc309246a761413f6e152f307f009e80e7c9fd03dd9e6c0dc8ab8b3a8fc1"
+                "sha256:43a6e99f53a8d34b3b4dbe424dbcc6b894350dc41a85b0af7c7bc24a7ec2cead",
+                "sha256:a048ff980a81cd652724a73bc496c519b336fabe19cc8bfc6c53b2ff6eb22c7b"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==1.34.120"
+            "version": "==1.34.128"
         },
         "botocore": {
             "hashes": [
-                "sha256:5cc0fca43cb2aad54917a394a001ac9ba774d21ad6a08828002d54b601776f78",
-                "sha256:92bd739938078c7a0b110689a3eee21ecb3954d90653da013d9f98ef1165d6f7"
+                "sha256:8d8e03f7c8c080ecafda72036eb3b482d649f8417c90b5dca33b7c2c47adb0c9",
+                "sha256:db67fda136c372ab3fa432580c819c89ba18d28a6152a4d2a7ea40d44082892e"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==1.34.120"
+            "version": "==1.34.128"
         },
         "cachetools": {
             "hashes": [
@@ -516,13 +510,21 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.3.0"
         },
+        "cobble": {
+            "hashes": [
+                "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44",
+                "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==0.1.4"
+        },
         "cohere": {
             "hashes": [
-                "sha256:14acb2ccf272e958f79f9241ae972fd82e96b9b8ee9e6922a5687370761203ec",
-                "sha256:8b692dcb5e86b554e5884168a7d2454951ce102fbd983e9053ec933e06bf02fa"
+                "sha256:84ce7666ff8fbdf4f41fb5f6ca452ab2639a514bc88967a2854a9b1b820d6ea0",
+                "sha256:e1ed84b90eadd13c6a68ee28e378a0bb955f8945eadc6eb7ee126b3399cafd54"
             ],
             "markers": "python_version >= '3.8' and python_version < '4.0'",
-            "version": "==5.5.4"
+            "version": "==5.5.8"
         },
         "coloredlogs": {
             "hashes": [
@@ -647,20 +649,20 @@
         },
         "dataclasses-json": {
             "hashes": [
-                "sha256:0c09827d26fffda27f1be2fed7a7a01a29c5ddcd2eb6393ad5ebf9d77e9deae8",
-                "sha256:e54c5c87497741ad454070ba0ed411523d46beb5da102e221efb873801b0ba85"
+                "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a",
+                "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0"
             ],
             "markers": "python_version >= '3.7' and python_version < '4.0'",
-            "version": "==0.6.6"
+            "version": "==0.6.7"
         },
         "datasets": {
             "hashes": [
-                "sha256:e07ff15d75b1af75c87dd96323ba2a361128d495136652f37fd62f918d17bb4e",
-                "sha256:eccb82fb3bb5ee26ccc6d7a15b7f1f834e2cc4e59b7cff7733a003552bad51ef"
+                "sha256:3c4dbcd27e0f642b9d41d20ff2efa721a5e04b32b2ca4009e0fc9139e324553f",
+                "sha256:76ac02e3bdfff824492e20678f0b6b1b6d080515957fe834b00c2ba8d6b18e5e"
             ],
             "index": "pypi",
             "markers": "python_full_version >= '3.8.0'",
-            "version": "==2.19.2"
+            "version": "==2.20.0"
         },
         "debugpy": {
             "hashes": [
@@ -838,12 +840,12 @@
         },
         "duckduckgo-search": {
             "hashes": [
-                "sha256:10e5c4d09a4243fd9d85007dc4fe637456c3c3995fd2e1e9b49ffd6f75bb0afb",
-                "sha256:f0a18fe5f20323ba6bb11865ce32d4520bb90086a6ae62f5da510865f5a7dca8"
+                "sha256:42c83d58f4f1d717a580b89cc86861cbae59e46e75288243776c53349d006bf1",
+                "sha256:6139ab17579e96ca7c5ed9398365245a36ecca8e7432545e3115ef90a9304eb7"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==6.1.5"
+            "version": "==6.1.6"
         },
         "ecdsa": {
             "hashes": [
@@ -853,20 +855,13 @@
             "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.19.0"
         },
-        "effdet": {
-            "hashes": [
-                "sha256:10889a226228d515c948e3fcf811e64c0d78d7aa94823a300045653b9c284cb7",
-                "sha256:ac5589fd304a5650c201986b2ef5f8e10c111093a71b1c49fa6b8817710812b5"
-            ],
-            "version": "==0.4.1"
-        },
         "email-validator": {
             "hashes": [
-                "sha256:200a70680ba08904be6d1eef729205cc0d687634399a5924d842533efb824b84",
-                "sha256:97d882d174e2a65732fb43bfce81a3a834cbc1bde8bf419e30ef5ea976370a05"
+                "sha256:14c0f3d343c4beda37400421b39fa411bbe33a75df20825df73ad53e06a9f04c",
+                "sha256:d89f6324e13b1e39889eab7f9ca2f91dc9aebb6fa50a6d8bd4329ab50f251115"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.1.1"
+            "version": "==2.1.2"
         },
         "emoji": {
             "hashes": [
@@ -876,14 +871,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.12.1"
         },
-        "et-xmlfile": {
-            "hashes": [
-                "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c",
-                "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.1.0"
-        },
         "faker": {
             "hashes": [
                 "sha256:14ccb0aec342d33aa3889a864a56e5b3c2d56bce1b89f9189f4fbc128b9afc1e",
@@ -962,11 +949,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f",
-                "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"
+                "sha256:58a2549afdf9e02e10720eaa4d4470f56386d7a6f72edd7d0596337af8ed7ad8",
+                "sha256:71b3102950e91dfc1bb4209b64be4dc8854f40e5f534428d8684f953ac847fac"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.14.0"
+            "version": "==3.15.1"
         },
         "filetype": {
             "hashes": [
@@ -975,14 +962,20 @@
             ],
             "version": "==1.2.0"
         },
+        "fire": {
+            "hashes": [
+                "sha256:54ec5b996ecdd3c0309c800324a0703d6da512241bc73b553db959d98de0aa66"
+            ],
+            "version": "==0.6.0"
+        },
         "flake8": {
             "hashes": [
-                "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132",
-                "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"
+                "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a",
+                "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5"
             ],
             "index": "pypi",
             "markers": "python_full_version >= '3.8.1'",
-            "version": "==7.0.0"
+            "version": "==7.1.0"
         },
         "flake8-black": {
             "hashes": [
@@ -1163,11 +1156,11 @@
                 "http"
             ],
             "hashes": [
-                "sha256:918d18d41bf73f0e2b261824baeb1b124bcf771767e3a26425cd7dec3332f512",
-                "sha256:f39780e282d7d117ffb42bb96992f8a90795e4d0fb0f661a70ca39fe9c43ded9"
+                "sha256:1d021b0b0f933e3b3029ed808eb400c08ba101ca2de4b3483fbc9ca23fcee94a",
+                "sha256:e0fdbc446d67e182f49a70b82cf7889028a63588fde6b222521f10937b2b670c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.3.1"
+            "version": "==2024.5.0"
         },
         "gitdb": {
             "hashes": [
@@ -1199,20 +1192,20 @@
         },
         "google-api-python-client": {
             "hashes": [
-                "sha256:cde87700bd4d37f39f5e940292c1c6cd0910990b5b01f50b1332a8cea38e8595",
-                "sha256:d6340dc83b72d72333cee5d50f7dcfecbff66a8783164090e945f985ec4c374d"
+                "sha256:293092905b66a046d3187a99ac454e12b00cc2c70444f26eb2f1f9c1a82720b4",
+                "sha256:396fe676ea0dfed066654dcf9f8dea77a1342f9d9bb23bb88e45b7b81e773926"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==2.132.0"
+            "version": "==2.133.0"
         },
         "google-auth": {
             "hashes": [
-                "sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360",
-                "sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415"
+                "sha256:8df7da660f62757388b8a7f249df13549b3373f24388cb5d2f1dd91cc18180b5",
+                "sha256:ab630a1320f6720909ad76a7dbdb6841cdf5c66b328d690027e4867bdfb16688"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.29.0"
+            "version": "==2.30.0"
         },
         "google-auth-httplib2": {
             "hashes": [
@@ -1231,13 +1224,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.2.0"
         },
-        "google-cloud-vision": {
-            "hashes": [
-                "sha256:044330ad618c810333ff2296cd27ffd145f249638d1b35b270de6b460b00e8d2",
-                "sha256:a313088fcd9c016af0427c2447eea1ad01bc192ca4cc33babaace1be587bbf75"
-            ],
-            "version": "==3.7.2"
-        },
         "googleapis-common-protos": {
             "hashes": [
                 "sha256:0e1c2cdfcbc354b76e4a211a35ea35d6926a835cba1377073c4861db904a1877",
@@ -1318,64 +1304,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.0.3"
         },
-        "grpcio": {
-            "hashes": [
-                "sha256:03b43d0ccf99c557ec671c7dede64f023c7da9bb632ac65dbc57f166e4970040",
-                "sha256:0a12ddb1678ebc6a84ec6b0487feac020ee2b1659cbe69b80f06dbffdb249122",
-                "sha256:0a2813093ddb27418a4c99f9b1c223fab0b053157176a64cc9db0f4557b69bd9",
-                "sha256:0cc79c982ccb2feec8aad0e8fb0d168bcbca85bc77b080d0d3c5f2f15c24ea8f",
-                "sha256:1257b76748612aca0f89beec7fa0615727fd6f2a1ad580a9638816a4b2eb18fd",
-                "sha256:1262402af5a511c245c3ae918167eca57342c72320dffae5d9b51840c4b2f86d",
-                "sha256:19264fc964576ddb065368cae953f8d0514ecc6cb3da8903766d9fb9d4554c33",
-                "sha256:198908f9b22e2672a998870355e226a725aeab327ac4e6ff3a1399792ece4762",
-                "sha256:1de403fc1305fd96cfa75e83be3dee8538f2413a6b1685b8452301c7ba33c294",
-                "sha256:20405cb8b13fd779135df23fabadc53b86522d0f1cba8cca0e87968587f50650",
-                "sha256:2981c7365a9353f9b5c864595c510c983251b1ab403e05b1ccc70a3d9541a73b",
-                "sha256:2c3c1b90ab93fed424e454e93c0ed0b9d552bdf1b0929712b094f5ecfe7a23ad",
-                "sha256:39b9d0acaa8d835a6566c640f48b50054f422d03e77e49716d4c4e8e279665a1",
-                "sha256:3b64ae304c175671efdaa7ec9ae2cc36996b681eb63ca39c464958396697daff",
-                "sha256:4657d24c8063e6095f850b68f2d1ba3b39f2b287a38242dcabc166453e950c59",
-                "sha256:4d6dab6124225496010bd22690f2d9bd35c7cbb267b3f14e7a3eb05c911325d4",
-                "sha256:55260032b95c49bee69a423c2f5365baa9369d2f7d233e933564d8a47b893027",
-                "sha256:55697ecec192bc3f2f3cc13a295ab670f51de29884ca9ae6cd6247df55df2502",
-                "sha256:5841dd1f284bd1b3d8a6eca3a7f062b06f1eec09b184397e1d1d43447e89a7ae",
-                "sha256:58b1041e7c870bb30ee41d3090cbd6f0851f30ae4eb68228955d973d3efa2e61",
-                "sha256:5e42634a989c3aa6049f132266faf6b949ec2a6f7d302dbb5c15395b77d757eb",
-                "sha256:5e56462b05a6f860b72f0fa50dca06d5b26543a4e88d0396259a07dc30f4e5aa",
-                "sha256:5f8b75f64d5d324c565b263c67dbe4f0af595635bbdd93bb1a88189fc62ed2e5",
-                "sha256:62b4e6eb7bf901719fce0ca83e3ed474ae5022bb3827b0a501e056458c51c0a1",
-                "sha256:6503b64c8b2dfad299749cad1b595c650c91e5b2c8a1b775380fcf8d2cbba1e9",
-                "sha256:6c024ffc22d6dc59000faf8ad781696d81e8e38f4078cb0f2630b4a3cf231a90",
-                "sha256:73819689c169417a4f978e562d24f2def2be75739c4bed1992435d007819da1b",
-                "sha256:75dbbf415026d2862192fe1b28d71f209e2fd87079d98470db90bebe57b33179",
-                "sha256:8caee47e970b92b3dd948371230fcceb80d3f2277b3bf7fbd7c0564e7d39068e",
-                "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a",
-                "sha256:940e3ec884520155f68a3b712d045e077d61c520a195d1a5932c531f11883489",
-                "sha256:a011ac6c03cfe162ff2b727bcb530567826cec85eb8d4ad2bfb4bd023287a52d",
-                "sha256:a3a035c37ce7565b8f4f35ff683a4db34d24e53dc487e47438e434eb3f701b2a",
-                "sha256:a5e771d0252e871ce194d0fdcafd13971f1aae0ddacc5f25615030d5df55c3a2",
-                "sha256:ac15b6c2c80a4d1338b04d42a02d376a53395ddf0ec9ab157cbaf44191f3ffdd",
-                "sha256:b1a82e0b9b3022799c336e1fc0f6210adc019ae84efb7321d668129d28ee1efb",
-                "sha256:bac71b4b28bc9af61efcdc7630b166440bbfbaa80940c9a697271b5e1dabbc61",
-                "sha256:bbc5b1d78a7822b0a84c6f8917faa986c1a744e65d762ef6d8be9d75677af2ca",
-                "sha256:c1a786ac592b47573a5bb7e35665c08064a5d77ab88a076eec11f8ae86b3e3f6",
-                "sha256:c84ad903d0d94311a2b7eea608da163dace97c5fe9412ea311e72c3684925602",
-                "sha256:d4d29cc612e1332237877dfa7fe687157973aab1d63bd0f84cf06692f04c0367",
-                "sha256:e3d9f8d1221baa0ced7ec7322a981e28deb23749c76eeeb3d33e18b72935ab62",
-                "sha256:e7cd5c1325f6808b8ae31657d281aadb2a51ac11ab081ae335f4f7fc44c1721d",
-                "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd",
-                "sha256:ee73a2f5ca4ba44fa33b4d7d2c71e2c8a9e9f78d53f6507ad68e7d2ad5f64a22",
-                "sha256:f10193c69fc9d3d726e83bbf0f3d316f1847c3071c8c93d8090cf5f326b14309"
-            ],
-            "version": "==1.64.1"
-        },
-        "grpcio-status": {
-            "hashes": [
-                "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f",
-                "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a"
-            ],
-            "version": "==1.62.2"
-        },
         "h11": {
             "hashes": [
                 "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d",
@@ -1468,11 +1396,11 @@
         },
         "huggingface-hub": {
             "hashes": [
-                "sha256:1a1118a0b3dea3bab6c325d71be16f5ffe441d32f3ac7c348d6875911b694b5b",
-                "sha256:22222c41223f1b7c209ae5511d2d82907325a0e3cdbce5f66949d43c598ff3bc"
+                "sha256:35d99016433900e44ae7efe1c209164a5a81dbbcd53a52f99c281dcd7ce22431",
+                "sha256:3a0b957aa87150addf0cc7bd71b4d954b78e749850e1e7fb29ebbd2db64ca037"
             ],
             "markers": "python_full_version >= '3.8.0'",
-            "version": "==0.23.3"
+            "version": "==0.23.4"
         },
         "humanfriendly": {
             "hashes": [
@@ -1498,6 +1426,105 @@
             "markers": "python_version >= '3.5'",
             "version": "==3.7"
         },
+        "ijson": {
+            "hashes": [
+                "sha256:0015354011303175eae7e2ef5136414e91de2298e5a2e9580ed100b728c07e51",
+                "sha256:034642558afa57351a0ffe6de89e63907c4cf6849070cc10a3b2542dccda1afe",
+                "sha256:0420c24e50389bc251b43c8ed379ab3e3ba065ac8262d98beb6735ab14844460",
+                "sha256:04366e7e4a4078d410845e58a2987fd9c45e63df70773d7b6e87ceef771b51ee",
+                "sha256:0b003501ee0301dbf07d1597482009295e16d647bb177ce52076c2d5e64113e0",
+                "sha256:0ee57a28c6bf523d7cb0513096e4eb4dac16cd935695049de7608ec110c2b751",
+                "sha256:192e4b65495978b0bce0c78e859d14772e841724d3269fc1667dc6d2f53cc0ea",
+                "sha256:1efb521090dd6cefa7aafd120581947b29af1713c902ff54336b7c7130f04c47",
+                "sha256:25fd49031cdf5fd5f1fd21cb45259a64dad30b67e64f745cc8926af1c8c243d3",
+                "sha256:2636cb8c0f1023ef16173f4b9a233bcdb1df11c400c603d5f299fac143ca8d70",
+                "sha256:29ce02af5fbf9ba6abb70765e66930aedf73311c7d840478f1ccecac53fefbf3",
+                "sha256:2af323a8aec8a50fa9effa6d640691a30a9f8c4925bd5364a1ca97f1ac6b9b5c",
+                "sha256:30cfea40936afb33b57d24ceaf60d0a2e3d5c1f2335ba2623f21d560737cc730",
+                "sha256:33afc25057377a6a43c892de34d229a86f89ea6c4ca3dd3db0dcd17becae0dbb",
+                "sha256:36aa56d68ea8def26778eb21576ae13f27b4a47263a7a2581ab2ef58b8de4451",
+                "sha256:3917b2b3d0dbbe3296505da52b3cb0befbaf76119b2edaff30bd448af20b5400",
+                "sha256:3aba5c4f97f4e2ce854b5591a8b0711ca3b0c64d1b253b04ea7b004b0a197ef6",
+                "sha256:3c556f5553368dff690c11d0a1fb435d4ff1f84382d904ccc2dc53beb27ba62e",
+                "sha256:3dc1fb02c6ed0bae1b4bf96971258bf88aea72051b6e4cebae97cff7090c0607",
+                "sha256:3e8d8de44effe2dbd0d8f3eb9840344b2d5b4cc284a14eb8678aec31d1b6bea8",
+                "sha256:40ee3821ee90be0f0e95dcf9862d786a7439bd1113e370736bfdf197e9765bfb",
+                "sha256:44367090a5a876809eb24943f31e470ba372aaa0d7396b92b953dda953a95d14",
+                "sha256:45ff05de889f3dc3d37a59d02096948ce470699f2368b32113954818b21aa74a",
+                "sha256:4690e3af7b134298055993fcbea161598d23b6d3ede11b12dca6815d82d101d5",
+                "sha256:473f5d921fadc135d1ad698e2697025045cd8ed7e5e842258295012d8a3bc702",
+                "sha256:47c144117e5c0e2babb559bc8f3f76153863b8dd90b2d550c51dab5f4b84a87f",
+                "sha256:4ac6c3eeed25e3e2cb9b379b48196413e40ac4e2239d910bb33e4e7f6c137745",
+                "sha256:4b72178b1e565d06ab19319965022b36ef41bcea7ea153b32ec31194bec032a2",
+                "sha256:4e9ffe358d5fdd6b878a8a364e96e15ca7ca57b92a48f588378cef315a8b019e",
+                "sha256:501dce8eaa537e728aa35810656aa00460a2547dcb60937c8139f36ec344d7fc",
+                "sha256:5378d0baa59ae422905c5f182ea0fd74fe7e52a23e3821067a7d58c8306b2191",
+                "sha256:542c1e8fddf082159a5d759ee1412c73e944a9a2412077ed00b303ff796907dc",
+                "sha256:63afea5f2d50d931feb20dcc50954e23cef4127606cc0ecf7a27128ed9f9a9e6",
+                "sha256:658ba9cad0374d37b38c9893f4864f284cdcc7d32041f9808fba8c7bcaadf134",
+                "sha256:6b661a959226ad0d255e49b77dba1d13782f028589a42dc3172398dd3814c797",
+                "sha256:72e3488453754bdb45c878e31ce557ea87e1eb0f8b4fc610373da35e8074ce42",
+                "sha256:7914d0cf083471856e9bc2001102a20f08e82311dfc8cf1a91aa422f9414a0d6",
+                "sha256:7ab00721304af1ae1afa4313ecfa1bf16b07f55ef91e4a5b93aeaa3e2bd7917c",
+                "sha256:7d0b6b637d05dbdb29d0bfac2ed8425bb369e7af5271b0cc7cf8b801cb7360c2",
+                "sha256:7e2b3e9ca957153557d06c50a26abaf0d0d6c0ddf462271854c968277a6b5372",
+                "sha256:7f172e6ba1bee0d4c8f8ebd639577bfe429dee0f3f96775a067b8bae4492d8a0",
+                "sha256:7f7a5250599c366369fbf3bc4e176f5daa28eb6bc7d6130d02462ed335361675",
+                "sha256:844c0d1c04c40fd1b60f148dc829d3f69b2de789d0ba239c35136efe9a386529",
+                "sha256:8643c255a25824ddd0895c59f2319c019e13e949dc37162f876c41a283361527",
+                "sha256:8795e88adff5aa3c248c1edce932db003d37a623b5787669ccf205c422b91e4a",
+                "sha256:87c727691858fd3a1c085d9980d12395517fcbbf02c69fbb22dede8ee03422da",
+                "sha256:8851584fb931cffc0caa395f6980525fd5116eab8f73ece9d95e6f9c2c326c4c",
+                "sha256:891f95c036df1bc95309951940f8eea8537f102fa65715cdc5aae20b8523813b",
+                "sha256:8c85447569041939111b8c7dbf6f8fa7a0eb5b2c4aebb3c3bec0fb50d7025121",
+                "sha256:8e0ff16c224d9bfe4e9e6bd0395826096cda4a3ef51e6c301e1b61007ee2bd24",
+                "sha256:8f83f553f4cde6d3d4eaf58ec11c939c94a0ec545c5b287461cafb184f4b3a14",
+                "sha256:8f890d04ad33262d0c77ead53c85f13abfb82f2c8f078dfbf24b78f59534dfdd",
+                "sha256:8fdf3721a2aa7d96577970f5604bd81f426969c1822d467f07b3d844fa2fecc7",
+                "sha256:907f3a8674e489abdcb0206723e5560a5cb1fa42470dcc637942d7b10f28b695",
+                "sha256:92355f95a0e4da96d4c404aa3cff2ff033f9180a9515f813255e1526551298c1",
+                "sha256:97a9aea46e2a8371c4cf5386d881de833ed782901ac9f67ebcb63bb3b7d115af",
+                "sha256:988e959f2f3d59ebd9c2962ae71b97c0df58323910d0b368cc190ad07429d1bb",
+                "sha256:99f5c8ab048ee4233cc4f2b461b205cbe01194f6201018174ac269bf09995749",
+                "sha256:9cd5c03c63ae06d4f876b9844c5898d0044c7940ff7460db9f4cd984ac7862b5",
+                "sha256:a3b730ef664b2ef0e99dec01b6573b9b085c766400af363833e08ebc1e38eb2f",
+                "sha256:a716e05547a39b788deaf22725490855337fc36613288aa8ae1601dc8c525553",
+                "sha256:a7ec759c4a0fc820ad5dc6a58e9c391e7b16edcb618056baedbedbb9ea3b1524",
+                "sha256:aaa6bfc2180c31a45fac35d40e3312a3d09954638ce0b2e9424a88e24d262a13",
+                "sha256:ad04cf38164d983e85f9cba2804566c0160b47086dcca4cf059f7e26c5ace8ca",
+                "sha256:b2f73f0d0fce5300f23a1383d19b44d103bb113b57a69c36fd95b7c03099b181",
+                "sha256:b325f42e26659df1a0de66fdb5cde8dd48613da9c99c07d04e9fb9e254b7ee1c",
+                "sha256:b51bab2c4e545dde93cb6d6bb34bf63300b7cd06716f195dd92d9255df728331",
+                "sha256:b5c3e285e0735fd8c5a26d177eca8b52512cdd8687ca86ec77a0c66e9c510182",
+                "sha256:b73b493af9e947caed75d329676b1b801d673b17481962823a3e55fe529c8b8b",
+                "sha256:b9d85a02e77ee8ea6d9e3fd5d515bcc3d798d9c1ea54817e5feb97a9bc5d52fe",
+                "sha256:bdcfc88347fd981e53c33d832ce4d3e981a0d696b712fbcb45dcc1a43fe65c65",
+                "sha256:c594c0abe69d9d6099f4ece17763d53072f65ba60b372d8ba6de8695ce6ee39e",
+                "sha256:c8a9befb0c0369f0cf5c1b94178d0d78f66d9cebb9265b36be6e4f66236076b8",
+                "sha256:cd174b90db68c3bcca273e9391934a25d76929d727dc75224bf244446b28b03b",
+                "sha256:d5576415f3d76290b160aa093ff968f8bf6de7d681e16e463a0134106b506f49",
+                "sha256:d654d045adafdcc6c100e8e911508a2eedbd2a1b5f93f930ba13ea67d7704ee9",
+                "sha256:d92e339c69b585e7b1d857308ad3ca1636b899e4557897ccd91bb9e4a56c965b",
+                "sha256:da3b6987a0bc3e6d0f721b42c7a0198ef897ae50579547b0345f7f02486898f5",
+                "sha256:dd26b396bc3a1e85f4acebeadbf627fa6117b97f4c10b177d5779577c6607744",
+                "sha256:de7c1ddb80fa7a3ab045266dca169004b93f284756ad198306533b792774f10a",
+                "sha256:df3ab5e078cab19f7eaeef1d5f063103e1ebf8c26d059767b26a6a0ad8b250a3",
+                "sha256:e0155a8f079c688c2ccaea05de1ad69877995c547ba3d3612c1c336edc12a3a5",
+                "sha256:e10c14535abc7ddf3fd024aa36563cd8ab5d2bb6234a5d22c77c30e30fa4fb2b",
+                "sha256:e4396b55a364a03ff7e71a34828c3ed0c506814dd1f50e16ebed3fc447d5188e",
+                "sha256:e5589225c2da4bb732c9c370c5961c39a6db72cf69fb2a28868a5413ed7f39e6",
+                "sha256:e6576cdc36d5a09b0c1a3d81e13a45d41a6763188f9eaae2da2839e8a4240bce",
+                "sha256:e6850ae33529d1e43791b30575070670070d5fe007c37f5d06aebc1dd152ab3f",
+                "sha256:e9afd97339fc5a20f0542c971f90f3ca97e73d3050cdc488d540b63fae45329a",
+                "sha256:ead50635fb56577c07eff3e557dac39533e0fe603000684eea2af3ed1ad8f941",
+                "sha256:ed1336a2a6e5c427f419da0154e775834abcbc8ddd703004108121c6dd9eba9d",
+                "sha256:f0c819f83e4f7b7f7463b2dc10d626a8be0c85fbc7b3db0edc098c2b16ac968e",
+                "sha256:f64f01795119880023ba3ce43072283a393f0b90f52b66cc0ea1a89aa64a9ccb",
+                "sha256:f87a7e52f79059f9c58f6886c262061065eb6f7554a587be7ed3aa63e6b71b34",
+                "sha256:ff835906f84451e143f31c4ce8ad73d83ef4476b944c2a2da91aec8b649570e1"
+            ],
+            "version": "==3.3.0"
+        },
         "importlib-metadata": {
             "hashes": [
                 "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570",
@@ -1514,13 +1541,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.0.0"
         },
-        "iopath": {
-            "hashes": [
-                "sha256:3311c16a4d9137223e20f141655759933e1eda24f8bff166af834af3c645ef01"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.1.10"
-        },
         "jieba3k": {
             "hashes": [
                 "sha256:980a4f2636b778d312518066be90c7697d410dd5a472385f5afced71a2db1c10"
@@ -1649,11 +1669,11 @@
         },
         "jsonpointer": {
             "hashes": [
-                "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a",
-                "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"
+                "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942",
+                "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
-            "version": "==2.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.0"
         },
         "kiwisolver": {
             "hashes": [
@@ -1778,38 +1798,38 @@
         },
         "langchain": {
             "hashes": [
-                "sha256:58ca0c47bcdd156da66f50a0a4fcedc49bf6950827f4a6b06c8c4842d55805f3",
-                "sha256:9d61e50e9cdc2bea659bc5e6c03650ba048fda63a307490ae368e539f61a0d3a"
+                "sha256:9aded9a65348254e1c93dcdaacffe4d1b6a5e7f74ef80c160c88ff78ad299228",
+                "sha256:ffdbf4fcea46a10d461bcbda2402220fcfd72a0c70e9f4161ae0510067b9b3bd"
             ],
             "index": "pypi",
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.2.2"
+            "version": "==0.2.5"
         },
         "langchain-cohere": {
             "hashes": [
-                "sha256:d0be4e76079a74c4259fe4db2bab535d690efe0efac5e9e2fbf486476c0a85c8",
-                "sha256:f07bd53fadbebf744b8de1eebf977353f340f2010156821623a0c6247032ab9b"
+                "sha256:d3ef73d5050513ff3ca0f07c8f3f73b7773eec182312aae92138d3a0ad33e631",
+                "sha256:edbeca8d041186d2831b495d9a392a0a94d15b0e2c98863e0a0cd001fc888842"
             ],
             "index": "pypi",
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.1.5"
+            "version": "==0.1.8"
         },
         "langchain-community": {
             "hashes": [
-                "sha256:a3c35af215e47b700e7cb4e548fa8b45c6d46d52b5a5a65af2577c5a0104fc9f",
-                "sha256:aa895545be2f3f4aa2fea36f6da2e3b4ec50ce61ec986e8f146901a1e9138138"
+                "sha256:476787b8c8c213b67e7b0eceb53346e787f00fbae12d8e680985bd4f93b0bf64",
+                "sha256:bf37a334952e42c7676d083cf2d2c4cbfbb7de1949c4149fe19913e2b06c485f"
             ],
             "index": "pypi",
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.2.3"
+            "version": "==0.2.5"
         },
         "langchain-core": {
             "hashes": [
-                "sha256:5212f7ec78a525e88a178ed3aefe2fd7134b03fb92573dfbab9914f1d92d6ec5",
-                "sha256:82bdcc546eb0341cefcf1f4ecb3e49836fff003903afddda2d1312bb8491ef81"
+                "sha256:172c81c858dc1f3123cc72b7e44e10f44c92f8a761cae18c364081f6c208e9f6",
+                "sha256:2db866a4514672c4875b69d5590aa2ed50aa0d144874268bef68d74b5e7f33f9"
             ],
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.2.4"
+            "version": "==0.2.8"
         },
         "langchain-openai": {
             "hashes": [
@@ -1837,50 +1857,38 @@
         },
         "langfuse": {
             "hashes": [
-                "sha256:9c06590d62cab8d7f32a05a086a37e71452af6f1e2e966335192a74664f1d149",
-                "sha256:ae8564f78ee3cdc0bb7535e3b2763fd250b2d73a779ad86e7c329ff2e9562bb0"
+                "sha256:2cba244024bdf63aa3295bffada6b257497f9d18a3ca09026595470f46e09e5b",
+                "sha256:5117038d35d4629cfa37e8d89986132c926c52c05631cdfef4f9d54bc05591d8"
             ],
             "index": "pypi",
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==2.34.2"
+            "version": "==2.36.1"
         },
         "langgraph": {
             "hashes": [
-                "sha256:607c15a1292619ca1d0ce4bdcc65dade48a3e9cd604c966ecba47621d9842daf",
-                "sha256:e8e9904ac1515848b19b72d44013955fffc2deca1f81a486377f617c1f6ae725"
+                "sha256:25391195461e66783811f29dff34f071af8f3302382b80ac2fb524c12f0bc1f2",
+                "sha256:7eb628b25ed75d717c9521d98f147424df7dbd04cf0f12bfcf2b5c3122b04137"
             ],
             "index": "pypi",
             "markers": "python_version < '4.0' and python_full_version >= '3.9.0'",
-            "version": "==0.0.64"
+            "version": "==0.0.69"
         },
         "langsmith": {
             "hashes": [
-                "sha256:a9457731caf3001c76b909af8b0649a6bf4f265d5ebdeeb023cd253967ef1de8",
-                "sha256:c47e5f2d7a01939c7e1fb94328759e5d0158f89cbd411fcb185c4176a9a75e44"
+                "sha256:c7f2c23981917713b5515b773f37c84ff68a7adf803476e2ebb5adcb36a04202",
+                "sha256:d215718cfdcdf4a011126b7a3d4a37eee96d887e59ac1e628a57e24b2bfa3163"
             ],
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.1.74"
-        },
-        "layoutparser": {
-            "extras": [
-                "layoutmodels",
-                "tesseract"
-            ],
-            "hashes": [
-                "sha256:0dfb2194c36a5ad1075b8310f3cbc280c00306d1758cef127d20283f7ce085ea",
-                "sha256:269aedfab8a0caa50aef8d0fa62740fbee1f2964880daae3a0e6a0415363126a"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.3.4"
+            "version": "==0.1.79"
         },
         "litellm": {
             "hashes": [
-                "sha256:3edaa1189742afd7c7df2b122f77373d47154a8fb6df6187ff5875e188baa3e1",
-                "sha256:b3b8e4401f717c3a18595446bfdb80fc6bb74974aac4eae537fb7b3be37fbf9e"
+                "sha256:b32cd15c7cf9084aa0f92eb7103323741d63cc8a14e28b739054b0cb6682c431",
+                "sha256:b78571a39a3b7819ca9cb96278f4b15b97557e2cd50e9aeee9210a7e744b8667"
             ],
             "index": "pypi",
             "markers": "python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'",
-            "version": "==1.40.4"
+            "version": "==1.40.16"
         },
         "llama-cpp-python": {
             "hashes": [
@@ -1891,12 +1899,12 @@
         },
         "llama-index": {
             "hashes": [
-                "sha256:8f87ef3e7234688bc6e15f952a44d2034af7e8dd4ac71aeb441c25503a4dac7f",
-                "sha256:a79041a93570e10b21f370d04895fdc9a62e25159d7f2fc0d34feba927db501c"
+                "sha256:2adb116a8c67b1d8688277dff1559390a3e070ad684d5d0912bd475ab0df6bb6",
+                "sha256:a02957143158d09e6f8bb128a5362578c925161fcf3dffcb226d485f2f529209"
             ],
             "index": "pypi",
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.10.43"
+            "version": "==0.10.46"
         },
         "llama-index-agent-openai": {
             "hashes": [
@@ -1916,11 +1924,11 @@
         },
         "llama-index-core": {
             "hashes": [
-                "sha256:355a5f7808dfa09dc27b6bd564926fa697e0279344b829732a9f91b8000157ac",
-                "sha256:c7dce5158e112d3cbc8d331c303e830b2eabcae55c9636e8050dc85320522c1b"
+                "sha256:403acc4a6139a8228b686a3a5051bdc09699243c1689668a5755e2be7affda5c",
+                "sha256:6c38467fc6974d316a22afa4214e610636cbec16fcb7b065e3f011fef35bb75f"
             ],
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.10.43"
+            "version": "==0.10.46"
         },
         "llama-index-embeddings-openai": {
             "hashes": [
@@ -1962,6 +1970,15 @@
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
             "version": "==0.1.6"
         },
+        "llama-index-postprocessor-cohere-rerank": {
+            "hashes": [
+                "sha256:2d332d5127b66e6c0ef4c1f80acabd7961a1bb9ac1824f6d8c3ddcf3b3e3895f",
+                "sha256:f7d39e0754a2dcd67283ac629f183304a7edaf3552114dc44ea823bccc4e59e3"
+            ],
+            "index": "pypi",
+            "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
+            "version": "==0.1.7"
+        },
         "llama-index-program-openai": {
             "hashes": [
                 "sha256:4660b338503537c5edca1e0dab606af6ce372b4f1b597e2833c6b602447c5d8d",
@@ -1980,11 +1997,11 @@
         },
         "llama-index-readers-file": {
             "hashes": [
-                "sha256:32450d0a3edc6ef6af575f814beec39cd3a3351eaf0e3c97045bdd72a7a7b38d",
-                "sha256:fde8ecb588e703849e51dc0f075f56d1f5db3bc1479dd00c21b42e93b81b6267"
+                "sha256:238ddd98aa377d6a44322013eb848056037c80ad84571ea5bf451a640fff4d5c",
+                "sha256:bc659e432d441c445e110580340675aa60abae1d82add4f65e559dfe8add541b"
             ],
             "markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
-            "version": "==0.1.23"
+            "version": "==0.1.25"
         },
         "llama-index-readers-llama-parse": {
             "hashes": [
@@ -2169,12 +2186,13 @@
             ],
             "version": "==0.1.1"
         },
-        "markdown": {
+        "mammoth": {
             "hashes": [
-                "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f",
-                "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"
+                "sha256:7e8aa7db53f4aa7e9620b22bf8b716f1a16c84e969de1a0b1f920c756184e3d8",
+                "sha256:b2abf2340809b13a903c8b65a27846466290b869f0dd56e4a1e3072c4be1ea86"
             ],
-            "version": "==3.6"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.8.0"
         },
         "markdown-it-py": {
             "hashes": [
@@ -2184,6 +2202,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.0.0"
         },
+        "markdownify": {
+            "hashes": [
+                "sha256:1fb08c618b30e0ee7a31a39b998f44a18fb28ab254f55f4af06b6d35a2179e27",
+                "sha256:a3805abd8166dbb7b27783c5599d91f54f10d79894b2621404d85b333c7ce561"
+            ],
+            "index": "pypi",
+            "version": "==0.12.1"
+        },
         "markupsafe": {
             "hashes": [
                 "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf",
@@ -2309,6 +2335,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.1.2"
         },
+        "megaparse": {
+            "hashes": [
+                "sha256:583b54e039bad17375da3cae196041235737b4c35d584ee2c7b2b2f1b167d4ec",
+                "sha256:e423b13c4954fcb575d2a90b407cf791f84613ff06bbf4ac01d81297cd6407de"
+            ],
+            "index": "pypi",
+            "version": "==0.0.11"
+        },
         "monotonic": {
             "hashes": [
                 "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7",
@@ -2325,19 +2359,12 @@
         },
         "msal": {
             "hashes": [
-                "sha256:3064f80221a21cd535ad8c3fafbb3a3582cd9c7e9af0bb789ae14f726a0ca99b",
-                "sha256:80bbabe34567cb734efd2ec1869b2d98195c927455369d8077b3c542088c5c9d"
+                "sha256:563c2d70de77a2ca9786aab84cb4e133a38a6897e6676774edc23d610bfc9e7b",
+                "sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==1.28.0"
-        },
-        "msg-parser": {
-            "hashes": [
-                "sha256:0de858d4fcebb6c8f6f028da83a17a20fe01cdce67c490779cf43b3b0162aa66",
-                "sha256:d47a2f0b2a359cb189fad83cc991b63ea781ecc70d91410324273fbf93e95375"
-            ],
-            "version": "==1.2.0"
+            "version": "==1.28.1"
         },
         "multidict": {
             "hashes": [
@@ -2541,7 +2568,7 @@
                 "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
                 "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
             ],
-            "markers": "python_version >= '3.9'",
+            "markers": "python_version < '3.12'",
             "version": "==1.26.4"
         },
         "oauthlib": {
@@ -2552,53 +2579,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==3.2.2"
         },
-        "olefile": {
-            "hashes": [
-                "sha256:543c7da2a7adadf21214938bb79c83ea12b473a4b6ee4ad4bf854e7715e13d1f",
-                "sha256:599383381a0bf3dfbd932ca0ca6515acd174ed48870cbf7fee123d698c192c1c"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==0.47"
-        },
-        "omegaconf": {
-            "hashes": [
-                "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b",
-                "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.3.0"
-        },
-        "onnx": {
-            "hashes": [
-                "sha256:006ba5059c85ce43e89a1486cc0276d0f1a8ec9c6efd1a9334fd3fa0f6e33b64",
-                "sha256:0f3faf239b48418b3ea6fe73bd4d86807b903d0b2ebd20b8b8c84f83741b0f18",
-                "sha256:0fc189195a40b5862fb77d97410c89823197fe19c1088ce150444eec72f200c1",
-                "sha256:1521ea7cd3497ecaf57d3b5e72d637ca5ebca632122a0806a9df99bedbeecdf8",
-                "sha256:15abf94a7868eed6db15a8b5024ba570c891cae77ca4d0e7258dabdad76980df",
-                "sha256:18b22143836838591f6551b089196e69f60c47fabce52b4b72b4cb37522645aa",
-                "sha256:1c059fea6229c44d2d39c8f6e2f2f0d676d587c97f4c854c86f3e7bc97e0b31c",
-                "sha256:2bed6fe05905b073206cabbb4463c58050cf8d544192303c09927b229f93ac14",
-                "sha256:2fde4dd5bc278b3fc8148f460bce8807b2874c66f48529df9444cdbc9ecf456b",
-                "sha256:32e11d39bee04f927fab09f74c46cf76584094462311bab1aca9ccdae6ed3366",
-                "sha256:45cf20421aeac03872bea5fd6ebf92abe15c4d1461a2572eb839add5059e2a09",
-                "sha256:496ba17b16a74711081772e1b03f3207959972e351298e51abdc600051027a22",
-                "sha256:5798414332534a41404a7ff83677d49ced01d70160e1541484cce647f2295051",
-                "sha256:595b2830093f81361961295f7b0ebb6000423bcd04123d516d081c306002e387",
-                "sha256:6251910e554f811fdd070164b0bc76d76b067b95576cb9dad4d52ae64fe014b5",
-                "sha256:67f372db4fe8fe61e00b762af5b0833aa72b5baa37e7e2f47d8668964ebff411",
-                "sha256:8299193f0f2a3849bfc069641aa8e4f93696602da8d165632af8ee48ec7556b6",
-                "sha256:8884bf53b552873c0c9b072cb8625e7d4e8f3cc0529191632d24e3de58a3b93a",
-                "sha256:8c2b70d602acfb90056fbdc60ef26f4658f964591212a4e9dbbda922ff43061b",
-                "sha256:95aa20aa65a9035d7543e81713e8b0f611e213fc02171959ef4ee09311d1bf28",
-                "sha256:aa7518d6d27f357261a4014079dec364cad6fef827d0b3fe1d3ff59939a68394",
-                "sha256:b3d10405706807ec2ef493b2a78519fa0264cf190363e89478585aac1179b596",
-                "sha256:bb2d392e5b7060082c2fb38eb5c44f67eb34ff5f0681bd6f45beff9abc6f7094",
-                "sha256:c11e3b15eee46cd20767e505cc3ba97457ef5ac93c3e459cdfb77943ff8fe9a7",
-                "sha256:e69ad8c110d8c37d759cad019d498fdf3fd24e0bfaeb960e52fed0469a5d2974",
-                "sha256:f98e275b4f46a617a9c527e60c02531eae03cf67a04c26db8a1c20acee539533"
-            ],
-            "version": "==1.16.1"
-        },
         "onnxruntime": {
             "hashes": [
                 "sha256:0284c579c20ec8b1b472dd190290a040cc68b6caec790edb960f065d15cf164a",
@@ -2631,32 +2611,25 @@
         },
         "openai": {
             "hashes": [
-                "sha256:a15266827de20f407d4bf9837030b168074b5b29acd54f10bb38d5f53e95f083",
-                "sha256:a746cf070798a4048cfea00b0fc7cb9760ee7ead5a08c48115b914d1afbd1b53"
+                "sha256:018623c2f795424044675c6230fa3bfbf98d9e0aab45d8fd116f2efb2cfb6b7e",
+                "sha256:95c8e2da4acd6958e626186957d656597613587195abd0fb2527566a93e76770"
             ],
             "index": "pypi",
             "markers": "python_full_version >= '3.7.1'",
-            "version": "==1.31.1"
+            "version": "==1.34.0"
         },
-        "opencv-python": {
+        "opencv-python-headless": {
             "hashes": [
-                "sha256:2e3c2557b176f1e528417520a52c0600a92c1bb1c359f3df8e6411ab4293f065",
-                "sha256:5b49e530f7fd86f671514b39ffacdf5d14ceb073bc79d0de46bbb6b0cad78eaf",
-                "sha256:5f78652339957ec24b80a782becfb32f822d2008a865512121fad8c3ce233e9a",
-                "sha256:955c5ce8ac90c9e4636ad7f5c0d9c75b80abbe347182cfd09b0e3eec6e50472c",
-                "sha256:dbc021eaa310c4145c47cd648cb973db69bb5780d6e635386cd53d3ea76bd2d5",
-                "sha256:e6be19a0615aa8c4e0d34e0c7b133e26e386f4b7e9b557b69479104ab2c133ec",
-                "sha256:ff54adc9e4daaf438e669664af08bec4a268c7b7356079338b8e4fae03810f2c"
+                "sha256:377d08a7e48a1405b5e84afcbe4798464ce7ee17081c1c23619c8b398ff18295",
+                "sha256:46071015ff9ab40fccd8a163da0ee14ce9846349f06c6c8c0f2870856ffa45db",
+                "sha256:5ae454ebac0eb0a0b932e3406370aaf4212e6a3fdb5038cc86c7aea15a6851da",
+                "sha256:9092404b65458ed87ce932f613ffbb1106ed2c843577501e5768912360fc50ec",
+                "sha256:a4f4bcb07d8f8a7704d9c8564c224c8b064c63f430e95b61ac0bffaa374d330e",
+                "sha256:afcf28bd1209dd58810d33defb622b325d3cbe49dcd7a43a902982c33e5fad05",
+                "sha256:f2017c6101d7c2ef8d7bc3b414c37ff7f54d64413a1847d89970b6b7069b4e1a"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==4.10.0.82"
-        },
-        "openpyxl": {
-            "hashes": [
-                "sha256:25071b558db709de9e8782c3d3e058af3b23ffb2fc6f40c8f0c45a154eced2c3",
-                "sha256:8dd482e5350125b2388070bb2477927be2e8ebc27df61178709bc8c8751da2f9"
-            ],
-            "version": "==3.1.3"
+            "version": "==4.10.0.84"
         },
         "ordered-set": {
             "hashes": [
@@ -2668,55 +2641,55 @@
         },
         "orjson": {
             "hashes": [
-                "sha256:0943a96b3fa09bee1afdfccc2cb236c9c64715afa375b2af296c73d91c23eab2",
-                "sha256:0a62f9968bab8a676a164263e485f30a0b748255ee2f4ae49a0224be95f4532b",
-                "sha256:16bda83b5c61586f6f788333d3cf3ed19015e3b9019188c56983b5a299210eb5",
-                "sha256:1770e2a0eae728b050705206d84eda8b074b65ee835e7f85c919f5705b006c9b",
-                "sha256:17e0713fc159abc261eea0f4feda611d32eabc35708b74bef6ad44f6c78d5ea0",
-                "sha256:18566beb5acd76f3769c1d1a7ec06cdb81edc4d55d2765fb677e3eaa10fa99e0",
-                "sha256:1952c03439e4dce23482ac846e7961f9d4ec62086eb98ae76d97bd41d72644d7",
-                "sha256:1bd2218d5a3aa43060efe649ec564ebedec8ce6ae0a43654b81376216d5ebd42",
-                "sha256:1c23dfa91481de880890d17aa7b91d586a4746a4c2aa9a145bebdbaf233768d5",
-                "sha256:252124b198662eee80428f1af8c63f7ff077c88723fe206a25df8dc57a57b1fa",
-                "sha256:2b166507acae7ba2f7c315dcf185a9111ad5e992ac81f2d507aac39193c2c818",
-                "sha256:2e5e176c994ce4bd434d7aafb9ecc893c15f347d3d2bbd8e7ce0b63071c52e25",
-                "sha256:3582b34b70543a1ed6944aca75e219e1192661a63da4d039d088a09c67543b08",
-                "sha256:382e52aa4270a037d41f325e7d1dfa395b7de0c367800b6f337d8157367bf3a7",
-                "sha256:416b195f78ae461601893f482287cee1e3059ec49b4f99479aedf22a20b1098b",
-                "sha256:4ad1f26bea425041e0a1adad34630c4825a9e3adec49079b1fb6ac8d36f8b754",
-                "sha256:4c895383b1ec42b017dd2c75ae8a5b862fc489006afde06f14afbdd0309b2af0",
-                "sha256:5102f50c5fc46d94f2033fe00d392588564378260d64377aec702f21a7a22912",
-                "sha256:520de5e2ef0b4ae546bea25129d6c7c74edb43fc6cf5213f511a927f2b28148b",
-                "sha256:544a12eee96e3ab828dbfcb4d5a0023aa971b27143a1d35dc214c176fdfb29b3",
-                "sha256:73100d9abbbe730331f2242c1fc0bcb46a3ea3b4ae3348847e5a141265479700",
-                "sha256:831c6ef73f9aa53c5f40ae8f949ff7681b38eaddb6904aab89dca4d85099cb78",
-                "sha256:8bc7a4df90da5d535e18157220d7915780d07198b54f4de0110eca6b6c11e290",
-                "sha256:8d0b84403d287d4bfa9bf7d1dc298d5c1c5d9f444f3737929a66f2fe4fb8f134",
-                "sha256:8d40c7f7938c9c2b934b297412c067936d0b54e4b8ab916fd1a9eb8f54c02294",
-                "sha256:9059d15c30e675a58fdcd6f95465c1522b8426e092de9fff20edebfdc15e1cb0",
-                "sha256:93433b3c1f852660eb5abdc1f4dd0ced2be031ba30900433223b28ee0140cde5",
-                "sha256:978be58a68ade24f1af7758626806e13cff7748a677faf95fbb298359aa1e20d",
-                "sha256:99b880d7e34542db89f48d14ddecbd26f06838b12427d5a25d71baceb5ba119d",
-                "sha256:9a7bc9e8bc11bac40f905640acd41cbeaa87209e7e1f57ade386da658092dc16",
-                "sha256:9e253498bee561fe85d6325ba55ff2ff08fb5e7184cd6a4d7754133bd19c9195",
-                "sha256:9f3e87733823089a338ef9bbf363ef4de45e5c599a9bf50a7a9b82e86d0228da",
-                "sha256:9fb6c3f9f5490a3eb4ddd46fc1b6eadb0d6fc16fb3f07320149c3286a1409dd8",
-                "sha256:a39aa73e53bec8d410875683bfa3a8edf61e5a1c7bb4014f65f81d36467ea098",
-                "sha256:b69a58a37dab856491bf2d3bbf259775fdce262b727f96aafbda359cb1d114d8",
-                "sha256:b8d4d1a6868cde356f1402c8faeb50d62cee765a1f7ffcfd6de732ab0581e063",
-                "sha256:ba7f67aa7f983c4345eeda16054a4677289011a478ca947cd69c0a86ea45e534",
-                "sha256:be2719e5041e9fb76c8c2c06b9600fe8e8584e6980061ff88dcbc2691a16d20d",
-                "sha256:be2aab54313752c04f2cbaab4515291ef5af8c2256ce22abc007f89f42f49109",
-                "sha256:c0403ed9c706dcd2809f1600ed18f4aae50be263bd7112e54b50e2c2bc3ebd6d",
-                "sha256:c8334c0d87103bb9fbbe59b78129f1f40d1d1e8355bbed2ca71853af15fa4ed3",
-                "sha256:cb0175a5798bdc878956099f5c54b9837cb62cfbf5d0b86ba6d77e43861bcec2",
-                "sha256:ccaa0a401fc02e8828a5bedfd80f8cd389d24f65e5ca3954d72c6582495b4bcf",
-                "sha256:cf20465e74c6e17a104ecf01bf8cd3b7b252565b4ccee4548f18b012ff2f8069",
-                "sha256:d4a654ec1de8fdaae1d80d55cee65893cb06494e124681ab335218be6a0691e7",
-                "sha256:e852baafceff8da3c9defae29414cc8513a1586ad93e45f27b89a639c68e8176"
+                "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01",
+                "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa",
+                "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5",
+                "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04",
+                "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d",
+                "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e",
+                "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b",
+                "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b",
+                "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268",
+                "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211",
+                "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c",
+                "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c",
+                "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969",
+                "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a",
+                "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f",
+                "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932",
+                "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26",
+                "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6",
+                "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214",
+                "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2",
+                "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f",
+                "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96",
+                "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a",
+                "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d",
+                "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38",
+                "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807",
+                "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09",
+                "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6",
+                "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e",
+                "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5",
+                "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86",
+                "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63",
+                "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2",
+                "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4",
+                "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595",
+                "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228",
+                "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9",
+                "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7",
+                "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40",
+                "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3",
+                "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139",
+                "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1",
+                "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b",
+                "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47",
+                "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1",
+                "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.10.3"
+            "version": "==3.10.5"
         },
         "packaging": {
             "hashes": [
@@ -2761,12 +2734,20 @@
         },
         "pandasai": {
             "hashes": [
-                "sha256:38a2d065b2bbaa42c8fb88c6aa4e52ad6da6743bc770ccc457a95eb69ff37a82",
-                "sha256:b93517ab893f8bf6a47f3347d13d86652b9eee8a9bdb5a4513ce27f0c4f41cf4"
+                "sha256:9d37023916630eff90079ff2e55b9c3f38e420ecfc871d238de730193056fe00",
+                "sha256:f51438e60277297f5719458dbb09d6b473322b1b5aa84cdc045405021ec85fb2"
             ],
             "index": "pypi",
             "markers": "python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8' and python_version >= '3.9'",
-            "version": "==2.1.1"
+            "version": "==2.2.1"
+        },
+        "parameterized": {
+            "hashes": [
+                "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b",
+                "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.9.0"
         },
         "pathspec": {
             "hashes": [
@@ -2776,6 +2757,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.12.1"
         },
+        "pdf2docx": {
+            "hashes": [
+                "sha256:8c5f439ba8147d59c20d33dadb275ff75a7f19b3b6179a9afda29ae2962514dc",
+                "sha256:e723f639728193468602b69eade2fe62df5f5d339d2630228e325f4cc61c1918"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.5.8"
+        },
         "pdf2image": {
             "hashes": [
                 "sha256:eaa959bc116b420dd7ec415fcae49b98100dda3dd18cd2fdfa86d09f112f6d57",
@@ -2784,68 +2773,6 @@
             "index": "pypi",
             "version": "==1.17.0"
         },
-        "pdfminer.six": {
-            "hashes": [
-                "sha256:6004da3ad1a7a4d45930cb950393df89b068e73be365a6ff64a838d37bcb08c4",
-                "sha256:e8d3c3310e6fbc1fe414090123ab01351634b4ecb021232206c4c9a8ca3e3b8f"
-            ],
-            "version": "==20231228"
-        },
-        "pdfplumber": {
-            "hashes": [
-                "sha256:be41686fe9515da5f54fad254e2b6941a723b9afe99eb92008df30880b6d61b3",
-                "sha256:e0027b8fc0ab98329cd8d445f5324d779078a6b58406471234e3e9c265dd8a48"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.11.0"
-        },
-        "pikepdf": {
-            "hashes": [
-                "sha256:088491103d5e167c7906ead59c1b7310bb98a5a74228ca76b1e9c190b3086f03",
-                "sha256:110c1509e2dfcf3a6375f3012342973d5fc7ea8fe5c4db0a0940c69f1b86d6d3",
-                "sha256:18534b8b2d9e4d16fa97433e6f0b10dc6eaf8a743608e5313b2a0f6c0ceadd2f",
-                "sha256:19f5fa063a76cbde5f3e39f785131afdd2336aae7a3b5bb8ae0fbbe377a8c3e3",
-                "sha256:1e68aa6717d2faaa193aa41f2b00af9e205164a7e852e900019955776f3c62e8",
-                "sha256:23196e1437acd72dd90989ebd2c21deb39e7602203dd7faad2acf9e12242f02d",
-                "sha256:2b17d0a0f64cf886bc975458da0e8e5b81ea0c40f1909c5aefa6da550d7cc9db",
-                "sha256:2ddb0eade682ed0c3a2270db3741dca7a19e85d8d09d9227525d4343a21e084b",
-                "sha256:36cfa4e415539ee9d9bf389ab5f1a46b351334ad4bac47d978850f7d954a4f7c",
-                "sha256:3936ba5492bda7fea840f2d17d4f720ce3e7fb8dc30c8d7c72ae6712571642f9",
-                "sha256:40f37da54bc02f3714c17454e1aee28d02fdc28ea0b7903240eae9d53c360b2b",
-                "sha256:416c0d3d8a927a5213ed71c5529f98e4c9fef489588e1af434c955a4bb66f5f3",
-                "sha256:41a71dfd567eeddd9bd43bba99bf2fd48acd2057486e9ed07399f8ab1a19486e",
-                "sha256:447938e30fa22461da340a0a7adbd3c90c0a7f68051dd1b0b9b7c6f64fb17dfe",
-                "sha256:47145a7f564b123cc3d0f2a1b72bfe96defb42441534e3ec8697f3e7163398de",
-                "sha256:4af5c18a2ddbd5f16b0b35f4693edff68cbf69aecf19773b90f12804af4fc6e4",
-                "sha256:5e2fdfb216f1d03da1b46d683abcf73611a1972f1808e182d07f9362ad28c7b7",
-                "sha256:6e48a7d120a17961229fe10392158d5a93c60d98261d4b129a4381413b5e7eb1",
-                "sha256:7be94c2e5ab6621b95e0377ef2e707722419aa478bbf47cc1d83f98c8e6a9fd2",
-                "sha256:7c19d9d0d3703e35a0d4a75a81208360ae3eaec47b86bf92a0395f5c0a0fabad",
-                "sha256:81711d03207ae92e4db2300d12e16e19570383c7f62bb75a1d6194fc753655a9",
-                "sha256:8a3fb77c77d3d93ea14fcb57d5365845cf064009b5125605abd9344a35cd73ae",
-                "sha256:8f5c8b4ef2f96b3d319c0b2600c21d277cf4d3e4173240ea174b0472d7d39650",
-                "sha256:979e9a878b222fa1a5cfd37c9b2a49056f51f7ba0bf8e1c2f26f4520d8fd2abf",
-                "sha256:9ea9d134220cc72d7c60010b9d6ca807369900e477aeede39b69c8fbbae3b51b",
-                "sha256:a1618ace0625a91ed4841a67ff70b1f8b59bab6389fcbd4c1611352e0b3fc55e",
-                "sha256:a96f7fc82cde95511d28a9a4e4eb1277a3e600859a7405aabb9692f280e762eb",
-                "sha256:ac4310b751fb71e136a7bd613a2b4060823423fbe481d968e19c9a3425353346",
-                "sha256:b124471b1ab5635062fd0586c8c633a1cafb2ddb3168287ec3acce3e23395c9d",
-                "sha256:b2e18cab189d45457b4a4d2ee1f823be4ccdeff8fafd0150b2a0332863f1a9a6",
-                "sha256:b794d2ada55062d3a1c1c2cbbe8d6d054e8316547977a7b9a1027e66e14e1e99",
-                "sha256:b8d374a91d7a1e92f0b3b7e1e5e00cfb0f324562200b991b719e037f5cc6ed54",
-                "sha256:bcdc0d02a92a727740ef84d2f7512e05ea36657e132674764d303c60f3298a1f",
-                "sha256:c0851e88c21f13ddce16655e9f0ca9e8d684d3d362b370917d282f70b465aece",
-                "sha256:cae32c7294caab59f5b14ecc36f1340cb471f19f7199f33997a0bbc102a08b61",
-                "sha256:d2fd3686ac5dd6a3889459b0fdd86f3254077dc74ce225de20e4c282e4a2d19a",
-                "sha256:d49cd2bada9cb9f849bc3d5f09b89c3e29e9629d4f234218eb59d749c0b8a4f8",
-                "sha256:db53bacda92bf80686e962111b3eaf24322880e039ba7452663bea404df625a3",
-                "sha256:dc45f05a72697199539c985cb07ff216a40bf1eb95a5b42205d1c8808d63e332",
-                "sha256:e16b1b7b9727b22e51b46240a17ab3c7afef79cff9391d4fe16b3b3f53d434d4",
-                "sha256:e89f7a00742315198fce5e0c9597ecad5f475bc88129afc07677cdf72a00e3ba",
-                "sha256:f80d6d40fc6dc17eb4a7ed532a9cffc55d5ad2ee12dd50ccd01c678978ab74b0"
-            ],
-            "version": "==9.0.0"
-        },
         "pillow": {
             "hashes": [
                 "sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c",
@@ -2921,62 +2848,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==10.3.0"
         },
-        "pillow-heif": {
-            "hashes": [
-                "sha256:0075adeb324adb07ddbfbe8a5c79ed12e5d04e60e9a642ff9427e71b5b0adccd",
-                "sha256:078bc74fd767625e465b2c107228f9c398b9a128bdf81b3f18812d7c07be660f",
-                "sha256:1421d96aebdc9f5773213c8221ce547efb56e37a62da6698312edd4f281efb42",
-                "sha256:17963a73186961fe7792aef01c46e980635f3fcc1836393de39ec9c6776ca51e",
-                "sha256:1f4293ecbb81d255d8d887dce4708a58e87c86e53c6f1b1affc4c3105e1bcb8c",
-                "sha256:241cf6c510215c6df0ee948dfed06a20c099475250c5c6cac5e7a1ef9e0ec4c3",
-                "sha256:2673048f3cf1498327add70f16e1129be2a09cf4a31cbc02363f5760eb5ba955",
-                "sha256:28c980bf8d5239ee87986c9217a5954b07993d71d391949a9feafad0a9c5e9a7",
-                "sha256:2b7450303f08ec81d1a63a75052863bb687fc3be1fdd8a34d2c0fef627aacae5",
-                "sha256:331579ce4f5fa079595c529b06810886ff76f8ade3eb411a1c9c90853a708022",
-                "sha256:33e0b1549bcdfec363b3ba6fb55b3de882e1409b5b00f5a68a1a027f051e8ef2",
-                "sha256:3501f22985cbb427c76febf07a7e309cb828e485c0cf250a625733fc06fc1815",
-                "sha256:38fa2854ec7dbe6c875d64cc5b3cf5cc55f1c8a0248dc1c7c34e9d2505521b82",
-                "sha256:3a2681d4b62418813289987a9420059d724cd93542d0b05e0928fe4578517714",
-                "sha256:3f062c1be6f04804ffdf0bc452142eff38d7544c8655c04291d16e3b996e4dc4",
-                "sha256:40014105688478d6ca146fc04bff6c13f445d01bdea79417b34ee50c1e559190",
-                "sha256:4b6caa5b13b4dfc180507527254014530f6bedbeabc1de2238918bf5b2700c7e",
-                "sha256:4d95004bb77aa640f80617716aa21bc092ec06307f6f2ad423deeeda07b4d29c",
-                "sha256:502cebc90c11a6bffa2ea899088999c25fc99c8f322e047a266e541e3046b27c",
-                "sha256:5c7f7a94fc2d08ddcf55a6834c4c55b7dea9605656c565ce11c82e3f6e0454a8",
-                "sha256:7794c1a8304eeb841d72cb73aa64cc60c9e5dccb2c7612f8caf528505f78581f",
-                "sha256:792e5d88b7d016fe48ae2fd77a852ec8dcf9a7fad1f7f191d35bc173896fe378",
-                "sha256:7e424d6a34b9466d054706393e76b5abdd84fabdc0c72b19ca10435a76140de7",
-                "sha256:7ef47297d526147923f4ecc7ff681a5d5f4e6e3300017681f59968652a0d8afb",
-                "sha256:7fabd6534a38078a66ce8b7a5ae8ad37afd9863c930abd3031fb553f1ab4f01a",
-                "sha256:88ff22d2b162e7edd9cb9dd98de81455be04c40a99d1d3d3ebe1602b1a21c453",
-                "sha256:89ec30420ddc843c43916febbe31697552ed123396a1696715eea75169866c07",
-                "sha256:8e168d45b2ce63c1fe2334fd02927699b0097de72605f7571948010fd79e58f0",
-                "sha256:8f15dc73ced02a0ccfac93159d12deeaecfbe4335883a1a3309df0f01c26e6e6",
-                "sha256:9273af7224e0fb16c18637184a8ea9a8790105658daab04ad541982b8623e5c1",
-                "sha256:9923dfcc97ae9484d3514f2f6ec368e2ac97cd66f7b95359cc1b0ec0c1cd6157",
-                "sha256:9bf10a1686c2d51f4db8ebb78825f96f28d18d1878599e1c64e88cfbdb70a3d2",
-                "sha256:9fd829c257a763e3a2e8418a773c2808c90799ee3e6b405b5399cb4fdfbe336e",
-                "sha256:a146be0c8e7bef204eeaa14799b2fca8a4a52ad972850975e23ef10cee4e7de7",
-                "sha256:a27abb523a07b17c118c09f1a00f92cde2295f8e997600024d4b57df3c5ba818",
-                "sha256:a8856cf5f0d53f83d814ae5c8d34433e5e5ad9f3e328480257cd6e9fbdb4a458",
-                "sha256:b50160331754b603524e6ed33c386f478fd66fb345fa6433a507a01c8de642c6",
-                "sha256:bbd9cc527bbd53c3e7588e16aad170e11cfd180b7e9bd84f18fb020ddec11408",
-                "sha256:be41b7fadd4a9355d24936f6fad83bb8130fe55ba228ec298ad316392bb6f38b",
-                "sha256:beb6576cbe5a9404a8f2ad9ec68f6b0c406e5e9f5d5573722dc3244898dc9866",
-                "sha256:c2ad68e3e4be40adfc5290bf6daa1569dd7d18501e17779d217ce5cd8c1e338d",
-                "sha256:c7db96ac172e2654676986e8c35fa32bffdd5b429a8c86b9d628c0333c570d82",
-                "sha256:d4595ec975db845d84ab90cbf0678f15b0068b8b83c01d1db7ea524e31bab4b4",
-                "sha256:d9e465d92cf01093e3e4c33776af97368add23ac1c8d0007f34b8d3e3390d6ad",
-                "sha256:e0492e4fd6d3334b9eed3651058216ef62f04afa099cfc6b05815c1bf0da2c38",
-                "sha256:e5edd98192f74e4c7cffdd62953b2987e2b1e0d6a55d5c940306bed71f40206a",
-                "sha256:f613dfd05fd62a8b7b57649bfa5db1501be41e18b5e15dd4a2fc12d3e3ddfdaa",
-                "sha256:f63a1d8f95811569df5df9b6b11674038929c2f696221f2a393aee5ac1e535b4",
-                "sha256:fb3efbe8efd26203589794988b11ea9bf3dea2d3bcf218e658f779d526dfcf80",
-                "sha256:fba5c46f84031f1186bdea2a0c95f82958f8c29321200e73d7ac5e79ee460c83",
-                "sha256:fea4410ce02e295079db5b2617579ba016671d334ac1888a1d4b34aedb56b866"
-            ],
-            "version": "==0.16.0"
-        },
         "platformdirs": {
             "hashes": [
                 "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee",
@@ -3007,14 +2878,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.5.0"
         },
-        "portalocker": {
-            "hashes": [
-                "sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33",
-                "sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2.8.2"
-        },
         "postgrest": {
             "hashes": [
                 "sha256:7b3802a514dc1e0fc8b5bbdeb2c99af35a0bd910e4ddb17855ca4e3422350c84",
@@ -3041,11 +2904,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:45abe60a8300f3c618b23c16c4bb98c6fc80af8ce8b17c7ae92db48db3ee63c1",
-                "sha256:869c50d682152336e23c4db7f74667639b5047494202ffe7670817053fd57795"
+                "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10",
+                "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.0.46"
+            "version": "==3.0.47"
         },
         "proto-plus": {
             "hashes": [
@@ -3094,26 +2957,6 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==5.9.8"
         },
-        "psycopg2": {
-            "hashes": [
-                "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981",
-                "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516",
-                "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3",
-                "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa",
-                "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a",
-                "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693",
-                "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372",
-                "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e",
-                "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59",
-                "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156",
-                "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024",
-                "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913",
-                "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"
-            ],
-            "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==2.9.9"
-        },
         "psycopg2-binary": {
             "hashes": [
                 "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9",
@@ -3267,39 +3110,13 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.4.0"
         },
-        "pycocotools": {
-            "hashes": [
-                "sha256:0ac4f30bac1503c780072053e6922971392fa3628b2e6967192bfca1f14736e2",
-                "sha256:121017ca87e2eec4e9081636d1a79519b50f473959defc5671c2d1ce0eec482e",
-                "sha256:1dbc429018149dc34e206ea32ee6297ff30b55a8615a3f7f4c6e3842f9df73db",
-                "sha256:254506c0eecabb3abbde17640f82103c0c04d53148ae920657664cab9cd649fc",
-                "sha256:363a6be808125306ace1a163c0b9ba479ee08eceec1fbd3889a88bd8245f73dc",
-                "sha256:3e6f7bfa1c5fb206a614bf2382c923d56092219a12dfd0fec3b5f83c13e29e00",
-                "sha256:4a8ec6f439638120e11f49120e1ddb6c66e0b1f293d7884207d02703a73d25a1",
-                "sha256:4d517ec315e53ef8df9f6b0899ebc4c79bd61fd715383861949bb1c3fca2c6d5",
-                "sha256:623b941bbafecbfee574aedbf3cb257f7a879f4fdb79394e6d3fb9c76e7ad6cf",
-                "sha256:625388f52e543f6f798f75f1ec125fe519580f22e72ccbd75eee0355ce336e18",
-                "sha256:6469089b9b36a1f645dc9ee830f29d261e99b4b3be73cb260688fd8b6d02760c",
-                "sha256:6541340f26bae32e044eedc5d8ccdac5bd0cb64eb2b0a342dac859b696edd0aa",
-                "sha256:66864bec8b30d47faa946bb55c8e8d6b7acb9fba0c17ff6aaa37abd78cda962a",
-                "sha256:73dc251ae4a06b7c10747ca7e2d29faabb4f13e5fc43760945966845581e79ae",
-                "sha256:8def3c46349e919999d6d5a1d6b7e587e6891524cc28f8b4a11e463bf0914621",
-                "sha256:9eb5d46900375adaba88eedb5cbc29d8cbcf43e82505d67378df1c3b720a8c5f",
-                "sha256:a6683a002fcb4500edbcec94bdf48be69f578a9aa5c638db38614df1f45cc935",
-                "sha256:b29086b6ce7b73e4ddaf3045006f5c059f344a2720605cd4474814017ff2af53",
-                "sha256:da8b7815196eebf0adabf67fcc459126cbc6498bbc6ab1fd144c371465d86879",
-                "sha256:f1a675728e459d72be6e3bb3546672bb37c7daffdc2e5335aa7b834aece2b560"
-            ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.0.7"
-        },
         "pycodestyle": {
             "hashes": [
-                "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f",
-                "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"
+                "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c",
+                "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.11.1"
+            "version": "==2.12.0"
         },
         "pycparser": {
             "hashes": [
@@ -3352,11 +3169,11 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:c46c76a40bb1296728d7a8b99aa73dd70a48c3510111ff290034f860c99c419e",
-                "sha256:ea91b002777bf643bb20dd717c028ec43216b24a6001a280f83877fd2655d0b4"
+                "sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52",
+                "sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.7.3"
+            "version": "==2.7.4"
         },
         "pydantic-core": {
             "hashes": [
@@ -3445,12 +3262,12 @@
         },
         "pydantic-settings": {
             "hashes": [
-                "sha256:acb2c213140dfff9669f4fe9f8180d43914f51626db28ab2db7308a576cce51a",
-                "sha256:e34bbd649803a6bb3e2f0f58fb0edff1f0c7f556849fda106cc21bcce12c30ab"
+                "sha256:87fda838b64b5039b970cd47c3e8a1ee460ce136278ff672980af21516f6e6ce",
+                "sha256:e4ed62ad851670975ec11285141db888fd24947f9440bd4380d7d8788d4965de"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==2.3.1"
+            "version": "==2.3.3"
         },
         "pyee": {
             "hashes": [
@@ -3554,6 +3371,56 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.8.0"
         },
+        "pymupdf": {
+            "hashes": [
+                "sha256:04170c32b44ca2fb11ef18ff2ff66bf594d7f77c8047e12257c95e21feabf8bf",
+                "sha256:0741a8bae81fe72ae12984c969a1f2bd47aa97e757d3a457ee46386c70b260c3",
+                "sha256:131cf58e9932c84b0d367621cd580301a0d5d67590ce97a4d484b5c6c12bc8a5",
+                "sha256:13acfd2f616f0628fe0cf1461438f112a455d143924bb0fcfb305701ff9698e2",
+                "sha256:2be7be2e8e3db25f8590a2cbf9790cf316ae8719beb5a29c37f6ebcf1f1cd781",
+                "sha256:318befef651f1b98c6de4980d725970c93eddd3f183fa0bf5e47decfbed918a9",
+                "sha256:4634f3e89aec71686b56c6db0e5932314d5c220ba452bf82b00650ff3bbed662",
+                "sha256:499dc8216b98862aae3b39d2da06f8e943a4daebf83bb283ca782ec910b886ce",
+                "sha256:4f6a998f5e61479bb0d21335cb845917f0c1c78344262b9e24c98f8f0ed742bd",
+                "sha256:65f999519b87747f2220194eb9251c3d18dc7191da56acd9b38641c8f093caaf",
+                "sha256:6b951ea8cb68077618e3bd009a558c04aced4989cfd6d38733d3b4f84cfd0cec",
+                "sha256:6f8514e345702c1978c8888a5515f0b38325d688fd377e7fc1d0744b92cca934",
+                "sha256:7d7431abdf4f96da6f1159dab4cb9aa9d4b6e20ec91d87994c3c88ee802aeda7",
+                "sha256:8b2b1b08ce5b36168c625ed2a3621a6fd56e77f433a68ab8cd0445b616a59ec6",
+                "sha256:8d6e5e3a81979ec2cc1a1647395fbc224a57dbe91a6e65d3ab3e4644e1137b90",
+                "sha256:8e3f23755b5e131e529b0dd6c26a3751eb25441212f25f981b235b8ace22b1b0",
+                "sha256:968d30ab20be828aab56bb04621bbb9f962c59ab5f1788236473c372fe797093",
+                "sha256:9f75be74de4852e7addd8b8a1baf49eb2fc9698fc522ce5a98fc046f2a1ae19c",
+                "sha256:9fbf1c9f1cdc3d6663bdccf314c9c52d021a77bb1c933f1cf99289ab452aea9b",
+                "sha256:a0dbaaed46db52f36ca74c0950b1d10b48a955cfa4526e8edbb7e5fb72a79cb9",
+                "sha256:a6b3abfa5da334014b7221bd3ec7978f433064acbf502c9fb7b08973a3e59e5d",
+                "sha256:b0d660a5e157791e0ffecffc2b98c8a038adcad57d84c068a25e4f21c4044c13",
+                "sha256:b2b089cd75b28479d999aa0d02e25c2148a3048459777457dbb9f583a6d9926e",
+                "sha256:b8db22130355e3ad106556dc5ca320d25bfb1d1d7917775a665740fd801e7ea7",
+                "sha256:c2ee61ba4b6302d67c88e7e0c5b3fbea2c4c4c05f567182c465a1b9af6067e51",
+                "sha256:c817a45c583cdc796e4ebe4d01663b20e1e9f5a16f3959aa0291d1896e81ebb7",
+                "sha256:ddecb4098b843bf50f86e50fe418dcc382fe536225cbf07d98249f5db45c154b",
+                "sha256:dfe44284e1c753376d111f3edf8c11f7daca96c72fe65f97d41ed71704a640e1",
+                "sha256:e1025960650ed601f2707ba5623c08d73a4cef15ebbbf65cbbbb4e0eadd71a03",
+                "sha256:eb145b5372b6faf6a4ec86dee51c12a55bd30c0c9df04fc1146d61710726cd08",
+                "sha256:f40354702fafbde31dfc3fe45372d3dfaa3be775d8055e9daf3cc932f8b064e7"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.24.5"
+        },
+        "pymupdfb": {
+            "hashes": [
+                "sha256:0d606a10cb828cefc9f864bf67bc9d46e8007af55e643f022b59d378af4151a8",
+                "sha256:3e7aab000d707c40e3254cd60152897b90952ed9a3567584d70974292f4912ce",
+                "sha256:7cc5da3031d160e0f01dbb88567ddca70adc82f062a3a5b4e2dd2a57646f442c",
+                "sha256:ad51d21086a16199684a3eebcb47d9c8460fc27e7bebae77f5fe64e8c34ebf34",
+                "sha256:d2ccca660042896d4af479f979ec10674c5a0b3cd2d9ecb0011f08dc82380cce",
+                "sha256:e88289bd4b4afe5966a028774b302f37d4b51dad5c5e6720dd04524910db6c6e",
+                "sha256:f39588fd2b7a63e2456df42cd8925c316202e0eb77d115d9c01ba032b2c9086f"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.24.3"
+        },
         "pypandoc": {
             "hashes": [
                 "sha256:31652073c7960c2b03570bd1e94f602ca9bc3e70099df5ead4cea98ff5151c1e",
@@ -3580,25 +3447,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==4.2.0"
         },
-        "pypdfium2": {
-            "hashes": [
-                "sha256:0dfa61421b5eb68e1188b0b2231e7ba35735aef2d867d86e48ee6cab6975195e",
-                "sha256:119b2969a6d6b1e8d55e99caaf05290294f2d0fe49c12a3f17102d01c441bd29",
-                "sha256:3d0dd3ecaffd0b6dbda3da663220e705cb563918249bda26058c6036752ba3a2",
-                "sha256:48b5b7e5566665bc1015b9d69c1ebabe21f6aee468b509531c3c8318eeee2e16",
-                "sha256:4e55689f4b06e2d2406203e771f78789bd4f190731b5d57383d05cf611d829de",
-                "sha256:4e6e50f5ce7f65a40a33d7c9edc39f23140c57e37144c2d6d9e9262a2a854854",
-                "sha256:5eda3641a2da7a7a0b2f4dbd71d706401a656fea521b6b6faa0675b15d31a163",
-                "sha256:90dbb2ac07be53219f56be09961eb95cf2473f834d01a42d901d13ccfad64b4c",
-                "sha256:b33ceded0b6ff5b2b93bc1fe0ad4b71aa6b7e7bd5875f1ca0cdfb6ba6ac01aab",
-                "sha256:cc3bf29b0db8c76cdfaac1ec1cde8edf211a7de7390fbf8934ad2aa9b4d6dfad",
-                "sha256:ee2410f15d576d976c2ab2558c93d392a25fb9f6635e8dd0a8a3a5241b275e0e",
-                "sha256:f1f78d2189e0ddf9ac2b7a9b9bd4f0c66f54d1389ff6c17e9fd9dc034d06eb3f",
-                "sha256:f33bd79e7a09d5f7acca3b0b69ff6c8a488869a7fab48fdf400fec6e20b9c8be"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==4.30.0"
-        },
         "pyreqwest-impersonate": {
             "hashes": [
                 "sha256:05213f5f014ecc6732d859a0f51b3dff0424748cc6e2d0d9a42aa1f7108b4eaa",
@@ -3655,12 +3503,12 @@
         },
         "pyright": {
             "hashes": [
-                "sha256:10e4d60be411f6d960cd39b0b58bf2ff76f2c83b9aeb102ffa9d9fda2e1303cb",
-                "sha256:c09e73ccc894976bcd6d6a5784aa84d724dbd9ceb7b873b39d475ca61c2de071"
+                "sha256:89de6502ae02f1552d0c4df4b46867887a419849f379db617695ef9308cf01eb",
+                "sha256:b1e5522ceb246ee6bc293a43d6d0162719d6467c1f1e9b81cee741aa11cdacbd"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==1.1.366"
+            "version": "==1.1.367"
         },
         "pysbd": {
             "hashes": [
@@ -3734,6 +3582,7 @@
                 "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe",
                 "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd"
             ],
+            "markers": "python_version >= '3.7'",
             "version": "==1.1.2"
         },
         "python-dotenv": {
@@ -3957,20 +3806,20 @@
         },
         "realtime": {
             "hashes": [
-                "sha256:4abbb3218b6ce8bd8d9d3b1112661d325e36ceab67a0e918673d0fd8fca04fb1",
-                "sha256:93342fbcb8812ed8d81733f2782c1199376f0471e78014675420c7d31f2f327d"
+                "sha256:2be0d8a6305513d423604ee319216108fc20105cb7438922d5c8958c48f40a47",
+                "sha256:c66918a106d8ef348d1821f2dbf6683d8833825580d95b2fdea9995406b42838"
             ],
             "markers": "python_version >= '3.8' and python_version < '4.0'",
-            "version": "==1.0.5"
+            "version": "==1.0.6"
         },
         "redis": {
             "hashes": [
-                "sha256:30b47d4ebb6b7a0b9b40c1275a19b87bb6f46b3bed82a89012cf56dea4024ada",
-                "sha256:3417688621acf6ee368dec4a04dd95881be24efd34c79f00d31f62bb528800ae"
+                "sha256:38473cd7c6389ad3e44a91f4c3eaf6bcb8a9f746007f29bf4fb20824ff0b2197",
+                "sha256:c0d6d990850c627bbf7be01c5c4cbaadf67b48593e913bb71c9819c30df37eee"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==5.0.5"
+            "version": "==5.0.6"
         },
         "regex": {
             "hashes": [
@@ -4090,12 +3939,12 @@
         },
         "resend": {
             "hashes": [
-                "sha256:5bae8db4dfd6b246377bd1f934e64b4ad8b83f7af3ace4c7d5d76eec56550e2b",
-                "sha256:b8c3da84698d7b1f5a869cb63a06e022e3910a4980640183cc54da46c77e0c53"
+                "sha256:7f2a221983fab74a09f669c0c14a75daf547ffa4b4930141626d9cca55bca767",
+                "sha256:92dc8e035c2ce8cf8210c1c322e86b0a4f509e0c82a80932d3323cd2f3a43d2d"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.0"
+            "version": "==2.1.0"
         },
         "retry": {
             "hashes": [
@@ -4128,112 +3977,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.10.1"
         },
-        "safetensors": {
-            "hashes": [
-                "sha256:018b691383026a2436a22b648873ed11444a364324e7088b99cd2503dd828400",
-                "sha256:01e4b22e3284cd866edeabe4f4d896229495da457229408d2e1e4810c5187121",
-                "sha256:01feb3089e5932d7e662eda77c3ecc389f97c0883c4a12b5cfdc32b589a811c3",
-                "sha256:02318f01e332cc23ffb4f6716e05a492c5f18b1d13e343c49265149396284a44",
-                "sha256:02ef3a24face643456020536591fbd3c717c5abaa2737ec428ccbbc86dffa7a4",
-                "sha256:03a4447c784917c9bf01d8f2ac5080bc15c41692202cd5f406afba16629e84d6",
-                "sha256:084fc436e317f83f7071fc6a62ca1c513b2103db325cd09952914b50f51cf78f",
-                "sha256:0bf4f9d6323d9f86eef5567eabd88f070691cf031d4c0df27a40d3b4aaee755b",
-                "sha256:0d52c958dc210265157573f81d34adf54e255bc2b59ded6218500c9b15a750eb",
-                "sha256:0d5ffc6a80f715c30af253e0e288ad1cd97a3d0086c9c87995e5093ebc075e50",
-                "sha256:0d9cd8e1560dfc514b6d7859247dc6a86ad2f83151a62c577428d5102d872721",
-                "sha256:0dd37306546b58d3043eb044c8103a02792cc024b51d1dd16bd3dd1f334cb3ed",
-                "sha256:1139eb436fd201c133d03c81209d39ac57e129f5e74e34bb9ab60f8d9b726270",
-                "sha256:19bbdf95de2cf64f25cd614c5236c8b06eb2cfa47cbf64311f4b5d80224623a3",
-                "sha256:1ab6527a20586d94291c96e00a668fa03f86189b8a9defa2cdd34a1a01acc7d5",
-                "sha256:1b89381517891a7bb7d1405d828b2bf5d75528299f8231e9346b8eba092227f9",
-                "sha256:1f598b713cc1a4eb31d3b3203557ac308acf21c8f41104cdd74bf640c6e538e3",
-                "sha256:22d21760dc6ebae42e9c058d75aa9907d9f35e38f896e3c69ba0e7b213033856",
-                "sha256:22f3b5d65e440cec0de8edaa672efa888030802e11c09b3d6203bff60ebff05a",
-                "sha256:2a0deb16a1d3ea90c244ceb42d2c6c276059616be21a19ac7101aa97da448faf",
-                "sha256:2a1f4430cc0c9d6afa01214a4b3919d0a029637df8e09675ceef1ca3f0dfa0df",
-                "sha256:2d603846a8585b9432a0fd415db1d4c57c0f860eb4aea21f92559ff9902bae4d",
-                "sha256:2f85fc50c4e07a21e95c24e07460fe6f7e2859d0ce88092838352b798ce711c2",
-                "sha256:309b10dbcab63269ecbf0e2ca10ce59223bb756ca5d431ce9c9eeabd446569da",
-                "sha256:3615a96dd2dcc30eb66d82bc76cda2565f4f7bfa89fcb0e31ba3cea8a1a9ecbb",
-                "sha256:38e2a8666178224a51cca61d3cb4c88704f696eac8f72a49a598a93bbd8a4af9",
-                "sha256:393e6e391467d1b2b829c77e47d726f3b9b93630e6a045b1d1fca67dc78bf632",
-                "sha256:3f9cdca09052f585e62328c1c2923c70f46814715c795be65f0b93f57ec98a02",
-                "sha256:41a727a7f5e6ad9f1db6951adee21bbdadc632363d79dc434876369a17de6ad6",
-                "sha256:420a98f593ff9930f5822560d14c395ccbc57342ddff3b463bc0b3d6b1951550",
-                "sha256:446e9fe52c051aeab12aac63d1017e0f68a02a92a027b901c4f8e931b24e5397",
-                "sha256:455d538aa1aae4a8b279344a08136d3f16334247907b18a5c3c7fa88ef0d3c46",
-                "sha256:4f9bac020faba7f5dc481e881b14b6425265feabb5bfc552551d21189c0eddc3",
-                "sha256:53c4879b9c6bd7cd25d114ee0ef95420e2812e676314300624594940a8d6a91f",
-                "sha256:5757e4688f20df083e233b47de43845d1adb7e17b6cf7da5f8444416fc53828d",
-                "sha256:585c9ae13a205807b63bef8a37994f30c917ff800ab8a1ca9c9b5d73024f97ee",
-                "sha256:5d07cbca5b99babb692d76d8151bec46f461f8ad8daafbfd96b2fca40cadae65",
-                "sha256:5fc6775529fb9f0ce2266edd3e5d3f10aab068e49f765e11f6f2a63b5367021d",
-                "sha256:622afd28968ef3e9786562d352659a37de4481a4070f4ebac883f98c5836563e",
-                "sha256:6f9568f380f513a60139971169c4a358b8731509cc19112369902eddb33faa4d",
-                "sha256:70a5319ef409e7f88686a46607cbc3c428271069d8b770076feaf913664a07ac",
-                "sha256:74707624b81f1b7f2b93f5619d4a9f00934d5948005a03f2c1845ffbfff42212",
-                "sha256:7c4fa560ebd4522adddb71dcd25d09bf211b5634003f015a4b815b7647d62ebe",
-                "sha256:7de32d0d34b6623bb56ca278f90db081f85fb9c5d327e3c18fd23ac64f465768",
-                "sha256:840b7ac0eff5633e1d053cc9db12fdf56b566e9403b4950b2dc85393d9b88d67",
-                "sha256:840caf38d86aa7014fe37ade5d0d84e23dcfbc798b8078015831996ecbc206a3",
-                "sha256:8651c7299cbd8b4161a36cd6a322fa07d39cd23535b144d02f1c1972d0c62f3c",
-                "sha256:868ad1b6fc41209ab6bd12f63923e8baeb1a086814cb2e81a65ed3d497e0cf8f",
-                "sha256:88887f69f7a00cf02b954cdc3034ffb383b2303bc0ab481d4716e2da51ddc10e",
-                "sha256:89f9f17b0dacb913ed87d57afbc8aad85ea42c1085bd5de2f20d83d13e9fc4b2",
-                "sha256:8c496c5401c1b9c46d41a7688e8ff5b0310a3b9bae31ce0f0ae870e1ea2b8caf",
-                "sha256:8cf18888606dad030455d18f6c381720e57fc6a4170ee1966adb7ebc98d4d6a3",
-                "sha256:8d22c1a10dff3f64d0d68abb8298a3fd88ccff79f408a3e15b3e7f637ef5c980",
-                "sha256:90964917f5b0fa0fa07e9a051fbef100250c04d150b7026ccbf87a34a54012e0",
-                "sha256:9bfb92f82574d9e58401d79c70c716985dc049b635fef6eecbb024c79b2c46ad",
-                "sha256:9c6ad011c1b4e3acff058d6b090f1da8e55a332fbf84695cf3100c649cc452d1",
-                "sha256:a11c374eb63a9c16c5ed146457241182f310902bd2a9c18255781bb832b6748b",
-                "sha256:a7cef55929dcbef24af3eb40bedec35d82c3c2fa46338bb13ecf3c5720af8a61",
-                "sha256:a844cdb5d7cbc22f5f16c7e2a0271170750763c4db08381b7f696dbd2c78a361",
-                "sha256:ae7613a119a71a497d012ccc83775c308b9c1dab454806291427f84397d852fd",
-                "sha256:b1648568667f820b8c48317c7006221dc40aced1869908c187f493838a1362bc",
-                "sha256:b1e31be7945f66be23f4ec1682bb47faa3df34cb89fc68527de6554d3c4258a4",
-                "sha256:b277482120df46e27a58082df06a15aebda4481e30a1c21eefd0921ae7e03f65",
-                "sha256:b7ffba80aa49bd09195145a7fd233a7781173b422eeb995096f2b30591639517",
-                "sha256:b852e47eb08475c2c1bd8131207b405793bfc20d6f45aff893d3baaad449ed14",
-                "sha256:bb4f8c5d0358a31e9a08daeebb68f5e161cdd4018855426d3f0c23bb51087055",
-                "sha256:bbae3b4b9d997971431c346edbfe6e41e98424a097860ee872721e176040a893",
-                "sha256:befdf0167ad626f22f6aac6163477fcefa342224a22f11fdd05abb3995c1783c",
-                "sha256:c0acbe31340ab150423347e5b9cc595867d814244ac14218932a5cf1dd38eb39",
-                "sha256:c41e1893d1206aa7054029681778d9a58b3529d4c807002c156d58426c225173",
-                "sha256:c59d51f182c729f47e841510b70b967b0752039f79f1de23bcdd86462a9b09ee",
-                "sha256:cd6fff9e56df398abc5866b19a32124815b656613c1c5ec0f9350906fd798aac",
-                "sha256:cdd0a3b5da66e7f377474599814dbf5cbf135ff059cc73694de129b58a5e8a2c",
-                "sha256:cf476bca34e1340ee3294ef13e2c625833f83d096cfdf69a5342475602004f95",
-                "sha256:d0dd4a1db09db2dba0f94d15addc7e7cd3a7b0d393aa4c7518c39ae7374623c3",
-                "sha256:d1456f814655b224d4bf6e7915c51ce74e389b413be791203092b7ff78c936dd",
-                "sha256:d14d30c25897b2bf19b6fb5ff7e26cc40006ad53fd4a88244fdf26517d852dd7",
-                "sha256:d244bcafeb1bc06d47cfee71727e775bca88a8efda77a13e7306aae3813fa7e4",
-                "sha256:d8815b5e1dac85fc534a97fd339e12404db557878c090f90442247e87c8aeaea",
-                "sha256:d88b33980222085dd6001ae2cad87c6068e0991d4f5ccf44975d216db3b57376",
-                "sha256:d8c5093206ef4b198600ae484230402af6713dab1bd5b8e231905d754022bec7",
-                "sha256:d9c289f140a9ae4853fc2236a2ffc9a9f2d5eae0cb673167e0f1b8c18c0961ac",
-                "sha256:dcf5705cab159ce0130cd56057f5f3425023c407e170bca60b4868048bae64fd",
-                "sha256:e011cc162503c19f4b1fd63dfcddf73739c7a243a17dac09b78e57a00983ab35",
-                "sha256:e066e8861eef6387b7c772344d1fe1f9a72800e04ee9a54239d460c400c72aab",
-                "sha256:e0b2104df1579d6ba9052c0ae0e3137c9698b2d85b0645507e6fd1813b70931a",
-                "sha256:e375d975159ac534c7161269de24ddcd490df2157b55c1a6eeace6cbb56903f0",
-                "sha256:e4119532cd10dba04b423e0f86aecb96cfa5a602238c0aa012f70c3a40c44b50",
-                "sha256:e7dbbde64b6c534548696808a0e01276d28ea5773bc9a2dfb97a88cd3dffe3df",
-                "sha256:e9afd5358719f1b2cf425fad638fc3c887997d6782da317096877e5b15b2ce93",
-                "sha256:ec4b52ce9a396260eb9731eb6aea41a7320de22ed73a1042c2230af0212758ce",
-                "sha256:edb5698a7bc282089f64c96c477846950358a46ede85a1c040e0230344fdde10",
-                "sha256:ee463219d9ec6c2be1d331ab13a8e0cd50d2f32240a81d498266d77d07b7e71e",
-                "sha256:efcc860be094b8d19ac61b452ec635c7acb9afa77beb218b1d7784c6d41fe8ad",
-                "sha256:f5e6883af9a68c0028f70a4c19d5a6ab6238a379be36ad300a22318316c00cb0",
-                "sha256:f9650713b2cfa9537a2baf7dd9fee458b24a0aaaa6cafcea8bdd5fb2b8efdc34",
-                "sha256:faefeb3b81bdfb4e5a55b9bbdf3d8d8753f65506e1d67d03f5c851a6c87150e9",
-                "sha256:fb9c65bd82f9ef3ce4970dc19ee86be5f6f93d032159acf35e663c6bea02b237",
-                "sha256:fe746d03ed8d193674a26105e4f0fe6c726f5bb602ffc695b409eaf02f04763d",
-                "sha256:fef5d70683643618244a4f5221053567ca3e77c2531e42ad48ae05fae909f542"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.4.3"
-        },
         "scipy": {
             "hashes": [
                 "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d",
@@ -4270,11 +4013,11 @@
                 "fastapi"
             ],
             "hashes": [
-                "sha256:05453f921c561b51159f712c2cd267b595e5c195b38b337d03baeb42719dd3c7",
-                "sha256:75d2e1fd8a887fdf63df612f5d37991c248a080b886d80714560bdded2bb9051"
+                "sha256:1f87acdce4a43a523ae5aa21a3fc37522d73ebd9ec04b1dbf01aa3d173852def",
+                "sha256:fbc40a78a8a9c6675133031116144f0d0940376fa6e4e1acd5624c90b0aaf58b"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.5.0"
+            "version": "==2.5.1"
         },
         "setuptools": {
             "hashes": [
@@ -4388,6 +4131,83 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.0.30"
         },
+        "sqlglot": {
+            "extras": [
+                "rs"
+            ],
+            "hashes": [
+                "sha256:545ee36ab9f94bb227715d6bff63872509280c584046854d4d977daff169c6e2",
+                "sha256:734dedf43c1a453eb0ab02dd9e245290855bb44ee3a569acb661fd378484955f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==25.2.0"
+        },
+        "sqlglotrs": {
+            "hashes": [
+                "sha256:139d8230197715069d2c821466c87903c0b80a2927ec74631b99a16a12d5c2c5",
+                "sha256:14277a7ff7ac8294f163724dc3add921c4d636aecddb862c2961a33bfd3e9fd8",
+                "sha256:19b3b28025ddb94cffa33c5f6888f658a3c4bb650f5aff01ef839a680d534a09",
+                "sha256:22c1c97a7deb92eba77a2926432d9b94142e47babc5c178678a9adf5d9fda18a",
+                "sha256:3138708198781a37297050d652f9319acac1613039bd5f05a9a34b20474b6327",
+                "sha256:31a338f6193d816f98d3c2f84f79aeb854d07502fa199661f93b50340772eca9",
+                "sha256:33c01004c53bc51bc5a046476245f7cfd1dde04ae21f5babf35cf5273f672b05",
+                "sha256:3bdd9e95ee186861d49f8222c03f279c4a65651c2115da32647ad669955ddff3",
+                "sha256:40e472511fe4864125283d8c9383a8a1476a283026efc4c451953f37b4606fe6",
+                "sha256:4193658fa42a0d52965f5fb18f5b14a61e8af601d1f01c89949519d3a6244a60",
+                "sha256:437a37ef9e026ae87bc5627a1e6115686998863cad4b673bce9c22ac45b26ee1",
+                "sha256:49112ebbb6ca54e732c4852026e7ea94e37203f685369c6c6e001fe2b0749668",
+                "sha256:4ad8145e04eb1e495921459a1ff1377338d70e11f8ab99e45488a4beac89b457",
+                "sha256:4e158c851adf35c96679279995efc74f7750a9726b191d55faea9c9a8d67d4c0",
+                "sha256:50c0592a22cbb989b968e8e931085c2aa75f6c3a3ee22473490fb806352b9f94",
+                "sha256:52d0d27fc9c4037923713156a6e150852d9d60229c69a979a196cebf530810ce",
+                "sha256:53bf99cd1ba2d2cd661bf19f15a9d22ab6c44a0a00ba8b8344adb80ead3dcf76",
+                "sha256:546175a73de8f69f0452f23bb5c9a1ba8cadb84a37dce4bff6db23918d658d86",
+                "sha256:55ff0a89e81930185e041898ecbb7efba1434719eea4ac858fe7536e5db9a198",
+                "sha256:5979ba0d97d1a4b37a71da8624ec5fe5ca57931a99f132b26f4c71e370c46b49",
+                "sha256:5c8e80d3a795514a815e1f9d42d07e663aa8a2492406b9046b95a78eeada9c6c",
+                "sha256:5da691c33d476fc7ba5bfb3249281e66f95cf2291ee45419df83c3d4494acd06",
+                "sha256:624a5cf4a94ebaabd357cfb278ba7b9af64ae9e478ee7159143ba75c9ed93eb0",
+                "sha256:6326846542455ffe3902a12fd50f46ebf07c5ff2ae1fdfce82a2d379660afb0a",
+                "sha256:65b1bcfaea8ac29217b5d7c7a537dfd99123a9ff7b7dd6da261664854f543a74",
+                "sha256:68a43bbbaa91b126e7bfca245e6587c5c49ac834f0d2c9b9ced2ac5b262fa937",
+                "sha256:691289985cdc964be47c00caae232c2d4592f8f9d43d106edbf519490418d286",
+                "sha256:6a79c4a0f61043a3a1811ec25a0066e35d904bfcd10585cf5fbfca88717d3214",
+                "sha256:7a7569ccb2336ca123ebe7c9da5eb25901c6a0ab18ae5c378a80fa0a01ca2501",
+                "sha256:7cadb7937123e423f5ca44e26d780b17988602bf06e9d5740b82358032a07879",
+                "sha256:7d28fc71002cecefd7fe69b0635f8d73be49718f0cb833a150051db3dbca03be",
+                "sha256:7f20c18f974becc919e4e7efbea2e18e3c3baf4efe3343b4f52484738c16662c",
+                "sha256:7f78c43a30359192b1fed5ec489a5dbe0744e1a7d4dd51f93be8330b1db4c010",
+                "sha256:82eb150625aaa095f1ae0febe788f6e8f62b1d83626d306ee35ccb6c74c5239e",
+                "sha256:8a861bfae792ee90629a589f6a93606fdf8befeec5f4a15d1f410abb3bf34017",
+                "sha256:8cd36137860b73c43a8210ccf9a65a2a340d6a2da20b835c83a0294df24da421",
+                "sha256:8ff463164aa6a7b3f535e807dd18f2fc0e120cea2cb58eee770bafda86106243",
+                "sha256:9074469c2848ada49b9b77e73b204bc0877751fdd2509d89c58d818f69e87377",
+                "sha256:92cd529718d26151f8a07d3256039f5219fb5a2072b6a3d1a9e8d663bd869f6a",
+                "sha256:92cf50a514e036b6b16d0a5ce9ba1eace7703d5723eec50dfb8ecb033b577988",
+                "sha256:97cb819e85926842cade238236e83c26ab2cd7514f3014d03f5f6d259e90c255",
+                "sha256:9ad0de561b8f76bd5648da4dcb1e537992c85d9f25deff3d687f322129d67929",
+                "sha256:a419dc4557df37267a0863aab9452ef4c7b12d7609aff4d324a1aa5517d4477d",
+                "sha256:a7e125f1de363c7714f816d8ca62e6565cd4b1cb3dbffba0dc373232e504b873",
+                "sha256:a9788dc4927c1dbf3d9cc9b4bb2cf78ae020ebf91cefd4ce4904f477f6915c3c",
+                "sha256:ad4f6ebb7fa9197de8ee0f5e37df77e539c4b0206a62e9016d0b8b73430f2631",
+                "sha256:ae38566b864ef48fe9a6220a6ae0e25108faf8ade1f003123ece8002a2152e4a",
+                "sha256:b34bc5e54d0cf40cfc769faad4ec2488900a5352ef82dae7eccfcd6646d4d0df",
+                "sha256:b76bfaa4f8d64e3b0e53580c4534bcef39d05688a4fddf4e47df9b70b6759605",
+                "sha256:ba12c7c07984e350b61c3351c90e2da3633f380a35428fcac47513dd7e51f3fd",
+                "sha256:bb2880fb143b89c057ed5c90b6c26e39d4f0326cd42f99fd9a84d3402584b820",
+                "sha256:bb5f5cde7a89a16a98bfdbefe4dfb8a694a9726fd264bab5a68bee06182a0065",
+                "sha256:bc1029b3c944fc1318edd7c7e9375806fb2b007a177c17dbfaefa7fe68e35956",
+                "sha256:bff066c033c91632aed9e53bcde2a5ef1cd48f86e175353df40ed6a83ed0c996",
+                "sha256:c7bab712a94f47001432e3370d3d780eb603ff182055de525353047d89b891e4",
+                "sha256:cc0d6d630d4aedc09b034e8a889e2c1583dbc30e431f6100b21a26a13d193798",
+                "sha256:d01bce5e470a22f2a3574230ef4f2137c76848d086c1fd9fad450d3b8cdbd86e",
+                "sha256:df922de02ff3bf2acbaa917b72c3a15dc34eef6bd14246a8b64db884841dc54b",
+                "sha256:e0165f39b0278a87a36e40745d056e4df1034b22e3f9b6cead7fb3a338736e62",
+                "sha256:e111fb7a22efbc5862bb16f50e3bd9685d29e4848ba0999dba1f8f09d29e7a89",
+                "sha256:f40914dab4bf836cf00a8ae5006d17fc3ec2290b38cf23c8d9a8eed09b978409"
+            ],
+            "version": "==0.2.6"
+        },
         "starlette": {
             "hashes": [
                 "sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee",
@@ -4420,12 +4240,12 @@
         },
         "supabase": {
             "hashes": [
-                "sha256:133dc832dfdd617f2f90ac5b52664df96ac8a9302ac6656ee769dc3f545812f0",
-                "sha256:13e5ed9e9377a1a69e70ad18ed7b82997cf13ffcd28173952f7503e4d5067771"
+                "sha256:74a1f24f04fede1967ef084b50dea688228f7b10eb2f9d73350fe2251a865188",
+                "sha256:c50e0eba5b03de3abd5ac0f887957ca43558ba44c4d17bb44e73ec454b41734c"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8' and python_version < '4.0'",
-            "version": "==2.5.0"
+            "version": "==2.5.1"
         },
         "supafunc": {
             "hashes": [
@@ -4468,6 +4288,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==8.3.0"
         },
+        "termcolor": {
+            "hashes": [
+                "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63",
+                "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.4.0"
+        },
         "tiktoken": {
             "hashes": [
                 "sha256:03c6c40ff1db0f48a7b4d2dafeae73a5607aacb472fa11f125e7baf9dce73704",
@@ -4511,14 +4339,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.7.0"
         },
-        "timm": {
-            "hashes": [
-                "sha256:83920a7efe2cfd503b2a1257dc8808d6ff7dcd18a4b79f451c283e7d71497329",
-                "sha256:d1ec86f7765aa79fbc7491508fa6e285d38a38f10bf4fe44ba2e9c70f91f0f5b"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.0.3"
-        },
         "tinysegmenter": {
             "hashes": [
                 "sha256:ed1f6d2e806a4758a73be589754384cbadadc7e1a414c81a166fc9adf2d40c6d"
@@ -4535,188 +4355,126 @@
         },
         "tokenizers": {
             "hashes": [
-                "sha256:0143e7d9dcd811855c1ce1ab9bf5d96d29bf5e528fd6c7824d0465741e8c10fd",
-                "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda",
-                "sha256:02458bee6f5f3139f1ebbb6d042b283af712c0981f5bc50edf771d6b762d5e4f",
-                "sha256:054c1cc9c6d68f7ffa4e810b3d5131e0ba511b6e4be34157aa08ee54c2f8d9ee",
-                "sha256:05a77cbfebe28a61ab5c3891f9939cc24798b63fa236d84e5f29f3a85a200c00",
-                "sha256:064ff87bb6acdbd693666de9a4b692add41308a2c0ec0770d6385737117215f2",
-                "sha256:06cd0487b1cbfabefb2cc52fbd6b1f8d4c37799bd6c6e1641281adaa6b2504a7",
-                "sha256:0774bccc6608eca23eb9d620196687c8b2360624619623cf4ba9dc9bd53e8b51",
-                "sha256:0cf6b7f1d4dc59af960e6ffdc4faffe6460bbfa8dce27a58bf75755ffdb2526d",
-                "sha256:0ef06b9707baeb98b316577acb04f4852239d856b93e9ec3a299622f6084e4be",
-                "sha256:0ff110ecc57b7aa4a594396525a3451ad70988e517237fe91c540997c4e50e29",
-                "sha256:107089f135b4ae7817affe6264f8c7a5c5b4fd9a90f9439ed495f54fcea56fb4",
-                "sha256:112a1dd436d2cc06e6ffdc0b06d55ac019a35a63afd26475205cb4b1bf0bfbff",
-                "sha256:13ca3611de8d9ddfbc4dc39ef54ab1d2d4aaa114ac8727dfdc6a6ec4be017378",
-                "sha256:158be8ea8554e5ed69acc1ce3fbb23a06060bd4bbb09029431ad6b9a466a7121",
-                "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc",
-                "sha256:1ddba9a2b0c8c81633eca0bb2e1aa5b3a15362b1277f1ae64176d0f6eba78ab1",
-                "sha256:20ea60479de6fc7b8ae756b4b097572372d7e4032e2521c1bbf3d90c90a99ff0",
-                "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c",
-                "sha256:237d1bf3361cf2e6463e6c140628e6406766e8b27274f5fcc62c747ae3c6f094",
-                "sha256:2735ecbbf37e52db4ea970e539fd2d450d213517b77745114f92867f3fc246eb",
-                "sha256:2ef09bbc16519f6c25d0c7fc0c6a33a6f62923e263c9d7cca4e58b8c61572afb",
-                "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f",
-                "sha256:361abdc068e8afe9c5b818769a48624687fb6aaed49636ee39bec4e95e1a215b",
-                "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e",
-                "sha256:3835738be1de66624fff2f4f6f6684775da4e9c00bde053be7564cbf3545cc66",
-                "sha256:38bfb0204ff3246ca4d5e726e8cc8403bfc931090151e6eede54d0e0cf162ef0",
-                "sha256:38d7ab43c6825abfc0b661d95f39c7f8af2449364f01d331f3b51c94dcff7221",
-                "sha256:3b919afe4df7eb6ac7cafd2bd14fb507d3f408db7a68c43117f579c984a73843",
-                "sha256:3ef5dd1d39797044642dbe53eb2bc56435308432e9c7907728da74c69ee2adca",
-                "sha256:3f5e64b0389a2be47091d8cc53c87859783b837ea1a06edd9d8e04004df55a5c",
-                "sha256:40b6a4c78da863ff26dbd5ad9a8ecc33d8a8d97b535172601cf00aee9d7ce9ce",
-                "sha256:41e39b41e5531d6b2122a77532dbea60e171ef87a3820b5a3888daa847df4153",
-                "sha256:44f2a832cd0825295f7179eaf173381dc45230f9227ec4b44378322d900447c9",
-                "sha256:454c203164e07a860dbeb3b1f4a733be52b0edbb4dd2e5bd75023ffa8b49403a",
-                "sha256:4620cca5c2817177ee8706f860364cc3a8845bc1e291aaf661fb899e5d1c45b0",
-                "sha256:473c83c5e2359bb81b0b6fde870b41b2764fcdd36d997485e07e72cc3a62264a",
-                "sha256:48e2b9335be2bc0171df9281385c2ed06a15f5cf121c44094338306ab7b33f2c",
-                "sha256:494fdbe5932d3416de2a85fc2470b797e6f3226c12845cadf054dd906afd0442",
-                "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b",
-                "sha256:4c4b89038a684f40a6b15d6b09f49650ac64d951ad0f2a3ea9169687bbf2a8ba",
-                "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d",
-                "sha256:4eeb12daf02a59e29f578a865f55d87cd103ce62bd8a3a5874f8fdeaa82e336b",
-                "sha256:4fe1f74a902bee74a3b25aff180fbfbf4f8b444ab37c4d496af7afd13a784ed2",
-                "sha256:508711a108684111ec8af89d3a9e9e08755247eda27d0ba5e3c50e9da1600f6d",
-                "sha256:5179c271aa5de9c71712e31cb5a79e436ecd0d7532a408fa42a8dbfa4bc23fd9",
-                "sha256:524e60da0135e106b254bd71f0659be9f89d83f006ea9093ce4d1fab498c6d0d",
-                "sha256:52f6130c9cbf70544287575a985bf44ae1bda2da7e8c24e97716080593638012",
-                "sha256:5645938a42d78c4885086767c70923abad047163d809c16da75d6b290cb30bbe",
-                "sha256:5ab2a4d21dcf76af60e05af8063138849eb1d6553a0d059f6534357bce8ba364",
-                "sha256:620beacc3373277700d0e27718aa8b25f7b383eb8001fba94ee00aeea1459d89",
-                "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78",
-                "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104",
-                "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3",
-                "sha256:6a9b648a58281c4672212fab04e60648fde574877d0139cd4b4f93fe28ca8944",
-                "sha256:6d76f00f5c32da36c61f41c58346a4fa7f0a61be02f4301fd30ad59834977cc3",
-                "sha256:6fc7083ab404019fc9acafe78662c192673c1e696bd598d16dc005bd663a5cf9",
-                "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18",
-                "sha256:7c0d8b52664ab2d4a8d6686eb5effc68b78608a9008f086a122a7b2996befbab",
-                "sha256:7c7d18b733be6bbca8a55084027f7be428c947ddf871c500ee603e375013ffba",
-                "sha256:7ca22bd897537a0080521445d91a58886c8c04084a6a19e6c78c586e0cfa92a5",
-                "sha256:7ef789f83eb0f9baeb4d09a86cd639c0a5518528f9992f38b28e819df397eb06",
-                "sha256:82f8652a74cc107052328b87ea8b34291c0f55b96d8fb261b3880216a9f9e48e",
-                "sha256:865c60ae6eaebdde7da66191ee9b7db52e542ed8ee9d2c653b6d190a9351b980",
-                "sha256:89cd1cb93e4b12ff39bb2d626ad77e35209de9309a71e4d3d4672667b4b256e7",
-                "sha256:8b9ec69247a23747669ec4b0ca10f8e3dfb3545d550258129bd62291aabe8605",
-                "sha256:918fbb0eab96fe08e72a8c2b5461e9cce95585d82a58688e7f01c2bd546c79d0",
-                "sha256:93268e788825f52de4c7bdcb6ebc1fcd4a5442c02e730faa9b6b08f23ead0e24",
-                "sha256:936bf3842db5b2048eaa53dade907b1160f318e7c90c74bfab86f1e47720bdd6",
-                "sha256:968fa1fb3c27398b28a4eca1cbd1e19355c4d3a6007f7398d48826bbe3a0f728",
-                "sha256:9ba9f6895af58487ca4f54e8a664a322f16c26bbb442effd01087eba391a719e",
-                "sha256:9c861d35e8286a53e06e9e28d030b5a05bcbf5ac9d7229e561e53c352a85b1fc",
-                "sha256:9e0480c452217edd35eca56fafe2029fb4d368b7c0475f8dfa3c5c9c400a7456",
-                "sha256:a308a607ca9de2c64c1b9ba79ec9a403969715a1b8ba5f998a676826f1a7039d",
-                "sha256:a33ab881c8fe70474980577e033d0bc9a27b7ab8272896e500708b212995d834",
-                "sha256:a47acfac7e511f6bbfcf2d3fb8c26979c780a91e06fb5b9a43831b2c0153d024",
-                "sha256:a907d76dcfda37023ba203ab4ceeb21bc5683436ebefbd895a0841fd52f6f6f2",
-                "sha256:a9b9b070fdad06e347563b88c278995735292ded1132f8657084989a4c84a6d5",
-                "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3",
-                "sha256:b8fcfa81bcb9447df582c5bc96a031e6df4da2a774b8080d4f02c0c16b42be0b",
-                "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b",
-                "sha256:c2d60f5246f4da9373f75ff18d64c69cbf60c3bca597290cea01059c336d2470",
-                "sha256:c73e2e74bbb07910da0d37c326869f34113137b23eadad3fc00856e6b3d9930c",
-                "sha256:c9a09cd26cca2e1c349f91aa665309ddb48d71636370749414fbf67bc83c5343",
-                "sha256:c9a2ebdd2ad4ec7a68e7615086e633857c85e2f18025bd05d2a4399e6c5f7169",
-                "sha256:cc90102ed17271cf0a1262babe5939e0134b3890345d11a19c3145184b706055",
-                "sha256:ccd73a82751c523b3fc31ff8194702e4af4db21dc20e55b30ecc2079c5d43cb7",
-                "sha256:ccec77aa7150e38eec6878a493bf8c263ff1fa8a62404e16c6203c64c1f16a26",
-                "sha256:cf27fd43472e07b57cf420eee1e814549203d56de00b5af8659cb99885472f1f",
-                "sha256:cf7fd9a5141634fa3aa8d6b7be362e6ae1b4cda60da81388fa533e0b552c98fd",
-                "sha256:cfed5c64e5be23d7ee0f0e98081a25c2a46b0b77ce99a4f0605b1ec43dd481fa",
-                "sha256:d0222c5b7c9b26c0b4822a82f6a7011de0a9d3060e1da176f66274b70f846b98",
-                "sha256:d05a1b06f986d41aed5f2de464c003004b2df8aaf66f2b7628254bcbfb72a438",
-                "sha256:d44ba80988ff9424e33e0a49445072ac7029d8c0e1601ad25a0ca5f41ed0c1d6",
-                "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693",
-                "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7",
-                "sha256:daa348f02d15160cb35439098ac96e3a53bacf35885072611cd9e5be7d333daa",
-                "sha256:db35825f6d54215f6b6009a7ff3eedee0848c99a6271c870d2826fbbedf31a38",
-                "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388",
-                "sha256:dce74266919b892f82b1b86025a613956ea0ea62a4843d4c4237be2c5498ed3a",
-                "sha256:de19c4dc503c612847edf833c82e9f73cd79926a384af9d801dcf93f110cea4e",
-                "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6",
-                "sha256:e6e9c6e019dd5484be5beafc775ae6c925f4c69a3487040ed09b45e13df2cb91",
-                "sha256:ea09acd2fe3324174063d61ad620dec3bcf042b495515f27f638270a7d466e8b",
-                "sha256:ea621a7eef4b70e1f7a4e84dd989ae3f0eeb50fc8690254eacc08acb623e82f1",
-                "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028",
-                "sha256:f33dfbdec3784093a9aebb3680d1f91336c56d86cc70ddf88708251da1fe9064",
-                "sha256:f3f40604f5042ff210ba82743dda2b6aa3e55aa12df4e9f2378ee01a17e2855e",
-                "sha256:f86593c18d2e6248e72fb91c77d413a815153b8ea4e31f7cd443bdf28e467670",
-                "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"
+                "sha256:01d62812454c188306755c94755465505836fd616f75067abcae529c35edeb57",
+                "sha256:02e81bf089ebf0e7f4df34fa0207519f07e66d8491d963618252f2e0729e0b46",
+                "sha256:04ce49e82d100594715ac1b2ce87d1a36e61891a91de774755f743babcd0dd52",
+                "sha256:07f9295349bbbcedae8cefdbcfa7f686aa420be8aca5d4f7d1ae6016c128c0c5",
+                "sha256:08a44864e42fa6d7d76d7be4bec62c9982f6f6248b4aa42f7302aa01e0abfd26",
+                "sha256:0b5ca92bfa717759c052e345770792d02d1f43b06f9e790ca0a1db62838816f3",
+                "sha256:0b9394bd204842a2a1fd37fe29935353742be4a3460b6ccbaefa93f58a8df43d",
+                "sha256:0bcce02bf1ad9882345b34d5bd25ed4949a480cf0e656bbd468f4d8986f7a3f1",
+                "sha256:0e64bfde9a723274e9a71630c3e9494ed7b4c0f76a1faacf7fe294cd26f7ae7c",
+                "sha256:10a707cc6c4b6b183ec5dbfc5c34f3064e18cf62b4a938cb41699e33a99e03c1",
+                "sha256:16baac68651701364b0289979ecec728546133e8e8fe38f66fe48ad07996b88b",
+                "sha256:1de5bc8652252d9357a666e609cb1453d4f8e160eb1fb2830ee369dd658e8975",
+                "sha256:1f0360cbea28ea99944ac089c00de7b2e3e1c58f479fb8613b6d8d511ce98267",
+                "sha256:2e8a3dd055e515df7054378dc9d6fa8c8c34e1f32777fb9a01fea81496b3f9d3",
+                "sha256:3174c76efd9d08f836bfccaca7cfec3f4d1c0a4cf3acbc7236ad577cc423c840",
+                "sha256:35583cd46d16f07c054efd18b5d46af4a2f070a2dd0a47914e66f3ff5efb2b1e",
+                "sha256:39c1ec76ea1027438fafe16ecb0fb84795e62e9d643444c1090179e63808c69d",
+                "sha256:3b11853f17b54c2fe47742c56d8a33bf49ce31caf531e87ac0d7d13d327c9334",
+                "sha256:427c4f0f3df9109314d4f75b8d1f65d9477033e67ffaec4bca53293d3aca286d",
+                "sha256:43350270bfc16b06ad3f6f07eab21f089adb835544417afda0f83256a8bf8b75",
+                "sha256:453e4422efdfc9c6b6bf2eae00d5e323f263fff62b29a8c9cd526c5003f3f642",
+                "sha256:4692ab92f91b87769d950ca14dbb61f8a9ef36a62f94bad6c82cc84a51f76f6a",
+                "sha256:4ad23d37d68cf00d54af184586d79b84075ada495e7c5c0f601f051b162112dc",
+                "sha256:4f3fefdc0446b1a1e6d81cd4c07088ac015665d2e812f6dbba4a06267d1a2c95",
+                "sha256:56ae39d4036b753994476a1b935584071093b55c7a72e3b8288e68c313ca26e7",
+                "sha256:5c88d1481f1882c2e53e6bb06491e474e420d9ac7bdff172610c4f9ad3898059",
+                "sha256:61b7fe8886f2e104d4caf9218b157b106207e0f2a4905c9c7ac98890688aabeb",
+                "sha256:621d670e1b1c281a1c9698ed89451395d318802ff88d1fc1accff0867a06f153",
+                "sha256:6258c2ef6f06259f70a682491c78561d492e885adeaf9f64f5389f78aa49a051",
+                "sha256:6309271f57b397aa0aff0cbbe632ca9d70430839ca3178bf0f06f825924eca22",
+                "sha256:638e43936cc8b2cbb9f9d8dde0fe5e7e30766a3318d2342999ae27f68fdc9bd6",
+                "sha256:63c38f45d8f2a2ec0f3a20073cccb335b9f99f73b3c69483cd52ebc75369d8a1",
+                "sha256:670b802d4d82bbbb832ddb0d41df7015b3e549714c0e77f9bed3e74d42400fbe",
+                "sha256:6852c5b2a853b8b0ddc5993cd4f33bfffdca4fcc5d52f89dd4b8eada99379285",
+                "sha256:6b2da5c32ed869bebd990c9420df49813709e953674c0722ff471a116d97b22d",
+                "sha256:6c330c0eb815d212893c67a032e9dc1b38a803eccb32f3e8172c19cc69fbb439",
+                "sha256:6f8a20266e695ec9d7a946a019c1d5ca4eddb6613d4f466888eee04f16eedb85",
+                "sha256:706a37cc5332f85f26efbe2bdc9ef8a9b372b77e4645331a405073e4b3a8c1c6",
+                "sha256:71e3ec71f0e78780851fef28c2a9babe20270404c921b756d7c532d280349214",
+                "sha256:72791f9bb1ca78e3ae525d4782e85272c63faaef9940d92142aa3eb79f3407a3",
+                "sha256:76951121890fea8330d3a0df9a954b3f2a37e3ec20e5b0530e9a0044ca2e11fe",
+                "sha256:78e769eb3b2c79687d9cb0f89ef77223e8e279b75c0a968e637ca7043a84463f",
+                "sha256:7c9d5b6c0e7a1e979bec10ff960fae925e947aab95619a6fdb4c1d8ff3708ce3",
+                "sha256:7fb297edec6c6841ab2e4e8f357209519188e4a59b557ea4fafcf4691d1b4c98",
+                "sha256:7ff898780a155ea053f5d934925f3902be2ed1f4d916461e1a93019cc7250837",
+                "sha256:82c8b8063de6c0468f08e82c4e198763e7b97aabfe573fd4cf7b33930ca4df77",
+                "sha256:85aa3ab4b03d5e99fdd31660872249df5e855334b6c333e0bc13032ff4469c4a",
+                "sha256:89183e55fb86e61d848ff83753f64cded119f5d6e1f553d14ffee3700d0a4a49",
+                "sha256:8a6298bde623725ca31c9035a04bf2ef63208d266acd2bed8c2cb7d2b7d53ce6",
+                "sha256:8b01afb7193d47439f091cd8f070a1ced347ad0f9144952a30a41836902fe09e",
+                "sha256:952078130b3d101e05ecfc7fc3640282d74ed26bcf691400f872563fca15ac97",
+                "sha256:952b80dac1a6492170f8c2429bd11fcaa14377e097d12a1dbe0ef2fb2241e16c",
+                "sha256:9620b78e0b2d52ef07b0d428323fb34e8ea1219c5eac98c2596311f20f1f9266",
+                "sha256:9ed240c56b4403e22b9584ee37d87b8bfa14865134e3e1c3fb4b2c42fafd3256",
+                "sha256:a179856d1caee06577220ebcfa332af046d576fb73454b8f4d4b0ba8324423ea",
+                "sha256:a2b718f316b596f36e1dae097a7d5b91fc5b85e90bf08b01ff139bd8953b25af",
+                "sha256:ac11016d0a04aa6487b1513a3a36e7bee7eec0e5d30057c9c0408067345c48d2",
+                "sha256:ad57d59341710b94a7d9dbea13f5c1e7d76fd8d9bcd944a7a6ab0b0da6e0cc66",
+                "sha256:b07c538ba956843833fee1190cf769c60dc62e1cf934ed50d77d5502194d63b1",
+                "sha256:b279ab506ec4445166ac476fb4d3cc383accde1ea152998509a94d82547c8e2a",
+                "sha256:b2edbc75744235eea94d595a8b70fe279dd42f3296f76d5a86dde1d46e35f574",
+                "sha256:b342d2ce8fc8d00f376af068e3274e2e8649562e3bc6ae4a67784ded6b99428d",
+                "sha256:b4399b59d1af5645bcee2072a463318114c39b8547437a7c2d6a186a1b5a0e2d",
+                "sha256:b4c89aa46c269e4e70c4d4f9d6bc644fcc39bb409cb2a81227923404dd6f5227",
+                "sha256:b70bfbe3a82d3e3fb2a5e9b22a39f8d1740c96c68b6ace0086b39074f08ab89a",
+                "sha256:b82931fa619dbad979c0ee8e54dd5278acc418209cc897e42fac041f5366d626",
+                "sha256:bac0b0eb952412b0b196ca7a40e7dce4ed6f6926489313414010f2e6b9ec2adf",
+                "sha256:bb9dfe7dae85bc6119d705a76dc068c062b8b575abe3595e3c6276480e67e3f1",
+                "sha256:bcd266ae85c3d39df2f7e7d0e07f6c41a55e9a3123bb11f854412952deacd828",
+                "sha256:bea6f9947e9419c2fda21ae6c32871e3d398cba549b93f4a65a2d369662d9403",
+                "sha256:c27b99889bd58b7e301468c0838c5ed75e60c66df0d4db80c08f43462f82e0d3",
+                "sha256:c2a0d47a89b48d7daa241e004e71fb5a50533718897a4cd6235cb846d511a478",
+                "sha256:c5c2ff13d157afe413bf7e25789879dd463e5a4abfb529a2d8f8473d8042e28f",
+                "sha256:c85cf76561fbd01e0d9ea2d1cbe711a65400092bc52b5242b16cfd22e51f0c58",
+                "sha256:ca407133536f19bdec44b3da117ef0d12e43f6d4b56ac4c765f37eca501c7bda",
+                "sha256:cbf001afbbed111a79ca47d75941e9e5361297a87d186cbfc11ed45e30b5daba",
+                "sha256:ce05fde79d2bc2e46ac08aacbc142bead21614d937aac950be88dc79f9db9022",
+                "sha256:d16ff18907f4909dca9b076b9c2d899114dd6abceeb074eca0c93e2353f943aa",
+                "sha256:d26194ef6c13302f446d39972aaa36a1dda6450bc8949f5eb4c27f51191375bd",
+                "sha256:d8c5d59d7b59885eab559d5bc082b2985555a54cda04dda4c65528d90ad252ad",
+                "sha256:d924204a3dbe50b75630bd16f821ebda6a5f729928df30f582fb5aade90c818a",
+                "sha256:dadc509cc8a9fe460bd274c0e16ac4184d0958117cf026e0ea8b32b438171594",
+                "sha256:dd26e3afe8a7b61422df3176e06664503d3f5973b94f45d5c45987e1cb711876",
+                "sha256:ddf672ed719b4ed82b51499100f5417d7d9f6fb05a65e232249268f35de5ed14",
+                "sha256:dfedf31824ca4915b511b03441784ff640378191918264268e6923da48104acc",
+                "sha256:e28cab1582e0eec38b1f38c1c1fb2e56bce5dc180acb1724574fc5f47da2a4fe",
+                "sha256:e742d76ad84acbdb1a8e4694f915fe59ff6edc381c97d6dfdd054954e3478ad4",
+                "sha256:e83a31c9cf181a0a3ef0abad2b5f6b43399faf5da7e696196ddd110d332519ee",
+                "sha256:e8d1ed93beda54bbd6131a2cb363a576eac746d5c26ba5b7556bc6f964425594",
+                "sha256:e8ff5b90eabdcdaa19af697885f70fe0b714ce16709cf43d4952f1f85299e73a",
+                "sha256:ec11802450a2487cdf0e634b750a04cbdc1c4d066b97d94ce7dd2cb51ebb325b",
+                "sha256:ecb2651956eea2aa0a2d099434134b1b68f1c31f9a5084d6d53f08ed43d45ff2",
+                "sha256:ed69af290c2b65169f0ba9034d1dc39a5db9459b32f1dd8b5f3f32a3fcf06eab",
+                "sha256:eddd5783a4a6309ce23432353cdb36220e25cbb779bfa9122320666508b44b88",
+                "sha256:ee59e6680ed0fdbe6b724cf38bd70400a0c1dd623b07ac729087270caeac88e3",
+                "sha256:f03727225feaf340ceeb7e00604825addef622d551cbd46b7b775ac834c1e1c4",
+                "sha256:f3bbb7a0c5fcb692950b041ae11067ac54826204318922da754f908d95619fbc",
+                "sha256:f8a9c828277133af13f3859d1b6bf1c3cb6e9e1637df0e45312e6b7c2e622b1f",
+                "sha256:f97660f6c43efd3e0bfd3f2e3e5615bf215680bad6ee3d469df6454b8c6e8256",
+                "sha256:f9939ca7e58c2758c01b40324a59c034ce0cebad18e0d4563a9b1beab3018243"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.15.2"
-        },
-        "torch": {
-            "hashes": [
-                "sha256:07e9ba746832b8d069cacb45f312cadd8ad02b81ea527ec9766c0e7404bb3feb",
-                "sha256:224259821fe3e4c6f7edf1528e4fe4ac779c77addaa74215eb0b63a5c474d66c",
-                "sha256:2bb5af780c55be68fe100feb0528d2edebace1d55cb2e351de735809ba7391eb",
-                "sha256:32b05fe0d1ada7f69c9f86c14ff69b0ef1957a5a54199bacba63d22d8fab720b",
-                "sha256:3c333dc2ebc189561514eda06e81df22bf8fb64e2384746b2cb9f04f96d1d4c8",
-                "sha256:3d7a7f7ef21a7520510553dc3938b0c57c116a7daee20736a9e25cbc0e832bdc",
-                "sha256:462d1c07dbf6bb5d9d2f3316fee73a24f3d12cd8dacf681ad46ef6418f7f6626",
-                "sha256:4777f6cefa0c2b5fa87223c213e7b6f417cf254a45e5829be4ccd1b2a4ee1011",
-                "sha256:490cc3d917d1fe0bd027057dfe9941dc1d6d8e3cae76140f5dd9a7e5bc7130ab",
-                "sha256:5802530783bd465fe66c2df99123c9a54be06da118fbd785a25ab0a88123758a",
-                "sha256:605a25b23944be5ab7c3467e843580e1d888b8066e5aaf17ff7bf9cc30001cc3",
-                "sha256:7c09a94362778428484bcf995f6004b04952106aee0ef45ff0b4bab484f5498d",
-                "sha256:a486c0b1976a118805fc7c9641d02df7afbb0c21e6b555d3bb985c9f9601b61a",
-                "sha256:a7dd4ed388ad1f3d502bf09453d5fe596c7b121de7e0cfaca1e2017782e9bbac",
-                "sha256:aaa872abde9a3d4f91580f6396d54888620f4a0b92e3976a6034759df4b961ad",
-                "sha256:b2ec81b61bb094ea4a9dee1cd3f7b76a44555375719ad29f05c0ca8ef596ad39",
-                "sha256:bee0bd33dc58aa8fc8a7527876e9b9a0e812ad08122054a5bff2ce5abf005b10",
-                "sha256:e5fdccbf6f1334b2203a61a0e03821d5845f1421defe311dabeae2fc8fbeac2d",
-                "sha256:f2357eb0965583a0954d6f9ad005bba0091f956aef879822274b1bcdb11bd308",
-                "sha256:ff60bf7ce3de1d43ad3f6969983f321a31f0a45df3690921720bcad6a8596cc4"
-            ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==2.3.1"
-        },
-        "torchvision": {
-            "hashes": [
-                "sha256:02085a2ffc7461f5c0edb07d6f3455ee1806561f37736b903da820067eea58c7",
-                "sha256:0b3bda0aa5b416eeb547143b8eeaf17720bdba9cf516dc991aacb81811aa96a5",
-                "sha256:13d24d904f65e62d66a1e0c41faec630bc193867b8a4a01166769e8a8e8df8e9",
-                "sha256:2be6f0bf7c455c89a51a1dbb6f668d36c6edc479f49ac912d745d10df5715657",
-                "sha256:3e694e54b0548dad99c12af6bf0c8e4f3350137d391dcd19af22a1c5f89322b3",
-                "sha256:52f7436140045dc2239cdc502aa76b2bd8bd676d64244ff154d304aa69852046",
-                "sha256:54bfcd352abb396d5c9c237d200167c178bd136051b138e1e8ef46ce367c2773",
-                "sha256:573ff523c739405edb085f65cb592f482d28a30e29b0be4c4ba08040b3ae785f",
-                "sha256:5c8366a1aeee49e9ea9e64b30d199debdf06b1bd7610a76165eb5d7869c3bde5",
-                "sha256:64a2662dbf30db9055d8b201d6e56f312a504e5ccd9d144c57c41622d3c524cb",
-                "sha256:80b5d794dd0fdba787adc22f1a367a5ead452327686473cb260dd94364bc56a6",
-                "sha256:9077cf590cdb3a5e8fdf5cdb71797f8c67713f974cf0228ecb17fcd670ab42f9",
-                "sha256:9726c316a2501df8503e5a5dc46a631afd4c515a958972e5b7f7b9c87d2125c0",
-                "sha256:975b8594c0f5288875408acbb74946eea786c5b008d129c0d045d0ead23742bc",
-                "sha256:b1c3864fa9378c88bce8ad0ef3599f4f25397897ce612e1c245c74b97092f35e",
-                "sha256:ceb993a882f1ae7ae373ed39c28d7e3e802205b0e59a7ed84ef4028f0bba8d7f",
-                "sha256:da83c8bbd34d8bee48bfa1d1b40e0844bc3cba10ed825a5a8cbe3ce7b62264cd",
-                "sha256:ed6340b69a63a625e512a66127210d412551d9c5f2ad2978130c6a45bf56cd4a",
-                "sha256:ef7bbbc60b38e831a75e547c66ca1784f2ac27100f9e4ddbe9614cef6cbcd942",
-                "sha256:f118d887bfde3a948a41d56587525401e5cac1b7db2eaca203324d6ed2b1caca"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.18.1"
+            "version": "==0.19.1"
         },
         "tornado": {
             "hashes": [
-                "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0",
-                "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63",
-                "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263",
-                "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052",
-                "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f",
-                "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee",
-                "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78",
-                "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579",
-                "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212",
-                "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e",
-                "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"
+                "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8",
+                "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f",
+                "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4",
+                "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3",
+                "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14",
+                "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842",
+                "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9",
+                "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698",
+                "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7",
+                "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d",
+                "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.4"
+            "version": "==6.4.1"
         },
         "tqdm": {
             "hashes": [
@@ -4726,14 +4484,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==4.66.4"
         },
-        "transformers": {
-            "hashes": [
-                "sha256:2586e5ff4150f122716fc40f5530e92871befc051848fbe82600969c535b762d",
-                "sha256:7838034a12cca3168247f9d2d1dba6724c9de3ae0f73a108258c6b8fc5912601"
-            ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==4.39.3"
-        },
         "typer": {
             "hashes": [
                 "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914",
@@ -4752,11 +4502,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a",
-                "sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1"
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.12.1"
+            "version": "==4.12.2"
         },
         "typing-inspect": {
             "hashes": [
@@ -4871,33 +4621,20 @@
                 "all-docs"
             ],
             "hashes": [
-                "sha256:d79104a574fd4de07548ee12ef63c34fab0a454a514d6252a261c7e8f15b9b1f",
-                "sha256:f4f3b5f18fcc174d3d606cbc3a9affa418494a8ea96424a112d0080fc9e93560"
+                "sha256:5dddf44908faa0f5250c02f41768fd4ce3792496d3e00258fd33f68c24b902c1",
+                "sha256:ab7d83016e46d3c221f6dae8b1040492ee69f7a97cb53c0f2109b54f0b764d19"
             ],
+            "index": "pypi",
             "markers": "python_version < '3.13' and python_full_version >= '3.9.0'",
-            "version": "==0.14.4"
+            "version": "==0.14.6"
         },
         "unstructured-client": {
             "hashes": [
-                "sha256:7aa03ee385450e32340a73ebdef09b8645771f08884e3ff838a518ab675878d5",
-                "sha256:b7745b5213e67a6af50d78f884e3eeae3741f4a8d82e390f3408c3b55245a264"
+                "sha256:9161ca25365764712b66cbb50e7cb22bc59a89a613de2bb2e2b171ef7b17e5b0",
+                "sha256:95df14697b1f79cc2a4580c7f143bd463d3dd5fab605220d2e4888321342decb"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.23.0"
-        },
-        "unstructured-inference": {
-            "hashes": [
-                "sha256:828013a0775b6bde401bd22914427c3f84f80014dedc3b9b8c08e8066642b20e",
-                "sha256:d2638c0f41cbe03685565e421332240cedba65073410342c76862e727ad74ee3"
-            ],
-            "version": "==0.7.33"
-        },
-        "unstructured.pytesseract": {
-            "hashes": [
-                "sha256:6ed42530fc697bb08d1ae4884cc517ee808620c1c1414efe8d5d90334da068d3",
-                "sha256:751a21d67b1f109036bf4daf796d3e04631697a355efd650f3373412b249de2e"
-            ],
-            "version": "==0.3.12"
+            "version": "==0.23.7"
         },
         "uritemplate": {
             "hashes": [
@@ -4909,11 +4646,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
-                "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+                "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+                "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
         },
         "uvicorn": {
             "extras": [
@@ -5706,11 +5443,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:45abe60a8300f3c618b23c16c4bb98c6fc80af8ce8b17c7ae92db48db3ee63c1",
-                "sha256:869c50d682152336e23c4db7f74667639b5047494202ffe7670817053fd57795"
+                "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10",
+                "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.0.46"
+            "version": "==3.0.47"
         },
         "psutil": {
             "hashes": [
@@ -5875,20 +5612,20 @@
         },
         "tornado": {
             "hashes": [
-                "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0",
-                "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63",
-                "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263",
-                "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052",
-                "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f",
-                "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee",
-                "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78",
-                "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579",
-                "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212",
-                "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e",
-                "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"
+                "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8",
+                "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f",
+                "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4",
+                "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3",
+                "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14",
+                "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842",
+                "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9",
+                "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698",
+                "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7",
+                "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d",
+                "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.4"
+            "version": "==6.4.1"
         },
         "traitlets": {
             "hashes": [
@@ -5900,11 +5637,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a",
-                "sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1"
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.12.1"
+            "version": "==4.12.2"
         },
         "wcwidth": {
             "hashes": [
diff --git a/backend/celery_worker.py b/backend/celery_worker.py
index 3b910ae82890..c200c24e1fee 100644
--- a/backend/celery_worker.py
+++ b/backend/celery_worker.py
@@ -1,14 +1,21 @@
+import asyncio
 import os
 from datetime import datetime
+from io import BytesIO
 from tempfile import NamedTemporaryFile
+from typing import List
 from uuid import UUID
 
 from celery.schedules import crontab
 from celery_config import celery
+from fastapi import UploadFile
 from logger import get_logger
 from middlewares.auth.auth_bearer import AuthBearer
 from models.files import File
 from models.settings import get_supabase_client, get_supabase_db
+from modules.assistant.dto.inputs import InputAssistant
+from modules.assistant.ito.difference_assistant import DifferenceAssistant
+from modules.assistant.ito.summary import SummaryAssistant
 from modules.brain.integrations.Notion.Notion_connector import NotionConnector
 from modules.brain.service.brain_service import BrainService
 from modules.brain.service.brain_vector_service import BrainVectorService
@@ -16,6 +23,7 @@
 from modules.notification.entity.notification import NotificationsStatusEnum
 from modules.notification.service.notification_service import NotificationService
 from modules.onboarding.service.onboarding_service import OnboardingService
+from modules.user.entity.user_identity import UserIdentity
 from packages.files.crawl.crawler import CrawlWebsite, slugify
 from packages.files.parsers.github import process_github
 from packages.files.processors import filter_file
@@ -302,6 +310,110 @@ def check_if_is_premium_user():
     return True
 
 
+@celery.task(name="process_assistant_task")
+def process_assistant_task(
+    input_in: str,
+    files_name: List[str],
+    current_user: dict,
+    notification_id=None,
+) -> None:
+
+    logger.debug(f"Input: {input}")
+    logger.debug(type(input))
+    _input = InputAssistant.model_validate(input_in)
+    # _input = InputAssistant(**json.loads(input))  # type: ignore
+    # _input = InputAssistant(json.dumps(_input))
+    _current_user = UserIdentity(**current_user)  # type: ignore
+    try:
+        files = []
+        supabase_client = get_supabase_client()
+
+        for file_name in files_name:
+            tmp_name = file_name.replace("/", "_")
+            base_file_name = os.path.basename(file_name)
+            _, file_extension = os.path.splitext(base_file_name)
+
+            with NamedTemporaryFile(suffix="_" + tmp_name, delete=False) as tmp_file:
+                res = supabase_client.storage.from_("quivr").download(file_name)
+                tmp_file.write(res)
+                tmp_file.flush()
+
+                file_instance = File(
+                    file_name=base_file_name,
+                    tmp_file_path=tmp_file.name,
+                    bytes_content=res,
+                    file_size=len(res),
+                    file_extension=file_extension,
+                )
+                upload_file = UploadFile(
+                    filename=file_instance.file_name,
+                    size=file_instance.file_size,
+                    file=BytesIO(file_instance.bytes_content),
+                    headers='{"content-type": "application/pdf"}',  # type : ignore
+                )
+                files.append(upload_file)
+
+    except Exception as e:
+        logger.exception(e)
+        if notification_id:
+            notification_service.update_notification_by_id(
+                notification_id,
+                NotificationUpdatableProperties(
+                    status=NotificationsStatusEnum.ERROR,
+                    description=f"An error occurred while processing the file: {e}",
+                ),
+            )
+        return
+    loop = asyncio.get_event_loop()
+
+    asyncio.set_event_loop(asyncio.new_event_loop())
+
+    if _input.name.lower() == "summary":
+        summary_assistant = SummaryAssistant(
+            input=_input, files=files, current_user=_current_user
+        )
+        try:
+            summary_assistant.check_input()
+            loop.run_until_complete(summary_assistant.process_assistant())
+        except ValueError as e:
+            logger.error(f"ValueError in SummaryAssistant: {e}")
+            if notification_id:
+                notification_service.update_notification_by_id(
+                    notification_id,
+                    NotificationUpdatableProperties(
+                        status=NotificationsStatusEnum.ERROR,
+                        description=f"Error in summary processing: {e}",
+                    ),
+                )
+    elif _input.name.lower() == "difference":
+        difference_assistant = DifferenceAssistant(
+            input=_input, files=files, current_user=_current_user
+        )
+        try:
+            difference_assistant.check_input()
+            loop.run_until_complete(difference_assistant.process_assistant())
+        except ValueError as e:
+            logger.error(f"ValueError in DifferenceAssistant: {e}")
+            if notification_id:
+                notification_service.update_notification_by_id(
+                    notification_id,
+                    NotificationUpdatableProperties(
+                        status=NotificationsStatusEnum.ERROR,
+                        description=f"Error in difference processing: {e}",
+                    ),
+                )
+    else:
+        logger.error("Invalid assistant name provided.")
+        if notification_id:
+            notification_service.update_notification_by_id(
+                notification_id,
+                NotificationUpdatableProperties(
+                    status=NotificationsStatusEnum.ERROR,
+                    description="Invalid assistant name provided.",
+                ),
+            )
+
+
 celery.conf.beat_schedule = {
     "remove_onboarding_more_than_x_days_task": {
         "task": f"{__name__}.remove_onboarding_more_than_x_days_task",
@@ -319,4 +431,8 @@ def check_if_is_premium_user():
         "task": "check_if_is_premium_user",
         "schedule": crontab(minute="*/1", hour="*"),
     },
+    "process_assistant": {
+        "task": "process_assistant_task",
+        "schedule": crontab(minute="*/1", hour="*"),
+    },
 }
diff --git a/backend/modules/assistant/controller/assistant_routes.py b/backend/modules/assistant/controller/assistant_routes.py
index 4cdd6f6c8117..217733c18a9b 100644
--- a/backend/modules/assistant/controller/assistant_routes.py
+++ b/backend/modules/assistant/controller/assistant_routes.py
@@ -1,19 +1,23 @@
 from typing import List
 
-from fastapi import APIRouter, Depends, HTTPException, UploadFile
+from celery_worker import process_assistant_task
+from fastapi import APIRouter, Depends, UploadFile
 from logger import get_logger
 from middlewares.auth import AuthBearer, get_current_user
 from modules.assistant.dto.inputs import InputAssistant
 from modules.assistant.dto.outputs import AssistantOutput
-from modules.assistant.ito.difference import DifferenceAssistant
-from modules.assistant.ito.summary import SummaryAssistant, summary_inputs
+from modules.assistant.ito.difference_assistant import difference_inputs
+from modules.assistant.ito.summary import summary_inputs
 from modules.assistant.service.assistant import Assistant
+from modules.notification.service.notification_service import NotificationService
+from modules.upload.service.upload_file import upload_file_storage
 from modules.user.entity.user_identity import UserIdentity
 
 assistant_router = APIRouter()
 logger = get_logger(__name__)
 
 assistant_service = Assistant()
+notification_service = NotificationService()
 
 
 @assistant_router.get(
@@ -27,10 +31,10 @@ async def list_assistants(
     """
 
     summary = summary_inputs()
-    # difference = difference_inputs()
+    difference = difference_inputs()
     # crawler = crawler_inputs()
     # audio_transcript = audio_transcript_inputs()
-    return [summary]
+    return [summary, difference]
 
 
 @assistant_router.post(
@@ -40,25 +44,18 @@ async def list_assistants(
 )
 async def process_assistant(
     input: InputAssistant,
-    files: List[UploadFile] = None,
+    files: List[UploadFile] = None,  # type: ignore
     current_user: UserIdentity = Depends(get_current_user),
 ):
-    if input.name.lower() == "summary":
-        summary_assistant = SummaryAssistant(
-            input=input, files=files, current_user=current_user
-        )
-        try:
-            summary_assistant.check_input()
-            return await summary_assistant.process_assistant()
-        except ValueError as e:
-            raise HTTPException(status_code=400, detail=str(e))
-    elif input.name.lower() == "difference":
-        difference_assistant = DifferenceAssistant(
-            input=input, files=files, current_user=current_user
-        )
-        try:
-            difference_assistant.check_input()
-            return await difference_assistant.process_assistant()
-        except ValueError as e:
-            raise HTTPException(status_code=400, detail=str(e))
-    return {"message": "Assistant not found"}
+    files_names = []
+    for file in files:
+        file_content = await file.read()
+        upload_file_storage(file_content, str(file.filename), upsert="true")
+        files_names.append(file.filename)
+
+    process_assistant_task.delay(
+        input_in=input.model_dump_json(),
+        files_name=files_names,
+        current_user=current_user.model_dump(),
+    )
+    return {"message": "Assistant is working in the back"}
diff --git a/backend/modules/assistant/dto/inputs.py b/backend/modules/assistant/dto/inputs.py
index 631f3e4fee74..f75c86c5528c 100644
--- a/backend/modules/assistant/dto/inputs.py
+++ b/backend/modules/assistant/dto/inputs.py
@@ -79,4 +79,5 @@ class InputAssistant(BaseModel):
     @model_validator(mode="before")
     @classmethod
     def to_py_dict(cls, data):
-        return json.loads(data)
+        if isinstance(data, str):
+            return json.loads(data)
diff --git a/backend/modules/assistant/ito/difference.py b/backend/modules/assistant/ito/difference.py
deleted file mode 100644
index 16fddc98cc71..000000000000
--- a/backend/modules/assistant/ito/difference.py
+++ /dev/null
@@ -1,178 +0,0 @@
-import asyncio
-import os
-import tempfile
-from typing import List
-
-import nest_asyncio
-import uvloop
-from fastapi import UploadFile
-from langchain.prompts import HumanMessagePromptTemplate, SystemMessagePromptTemplate
-from langchain_community.chat_models import ChatLiteLLM
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate, PromptTemplate
-from llama_parse import LlamaParse
-from logger import get_logger
-from modules.assistant.dto.inputs import InputAssistant
-from modules.assistant.dto.outputs import (
-    AssistantOutput,
-    InputFile,
-    Inputs,
-    OutputBrain,
-    OutputEmail,
-    Outputs,
-)
-from modules.assistant.ito.ito import ITO
-from modules.user.entity.user_identity import UserIdentity
-
-if not isinstance(asyncio.get_event_loop(), uvloop.Loop):
-    nest_asyncio.apply()
-
-
-logger = get_logger(__name__)
-
-
-class DifferenceAssistant(ITO):
-
-    def __init__(
-        self,
-        input: InputAssistant,
-        files: List[UploadFile] = None,
-        current_user: UserIdentity = None,
-        **kwargs,
-    ):
-        super().__init__(
-            input=input,
-            files=files,
-            current_user=current_user,
-            **kwargs,
-        )
-
-    def check_input(self):
-        if not self.files:
-            raise ValueError("No file was uploaded")
-        if len(self.files) != 2:
-            raise ValueError("Only two files can be uploaded")
-        if not self.input.inputs.files:
-            raise ValueError("No files key were given in the input")
-        if len(self.input.inputs.files) != 2:
-            raise ValueError("Only two files can be uploaded")
-        if not self.input.inputs.files[0].key == "doc_1":
-            raise ValueError("The key of the first file should be doc_1")
-        if not self.input.inputs.files[1].key == "doc_2":
-            raise ValueError("The key of the second file should be doc_2")
-        if not self.input.inputs.files[0].value:
-            raise ValueError("No file was uploaded")
-        if not (
-            self.input.outputs.brain.activated or self.input.outputs.email.activated
-        ):
-            raise ValueError("No output was selected")
-        return True
-
-    async def process_assistant(self):
-
-        document_1 = self.files[0]
-        document_2 = self.files[1]
-
-        # Get the file extensions
-        document_1_ext = os.path.splitext(document_1.filename)[1]
-        document_2_ext = os.path.splitext(document_2.filename)[1]
-
-        # Create temporary files with the same extension as the original files
-        document_1_tmp = tempfile.NamedTemporaryFile(
-            suffix=document_1_ext, delete=False
-        )
-        document_2_tmp = tempfile.NamedTemporaryFile(
-            suffix=document_2_ext, delete=False
-        )
-
-        document_1_tmp.write(document_1.file.read())
-        document_2_tmp.write(document_2.file.read())
-
-        parser = LlamaParse(
-            result_type="markdown"  # "markdown" and "text" are available
-        )
-
-        document_1_llama_parsed = parser.load_data(document_1_tmp.name)
-        document_2_llama_parsed = parser.load_data(document_2_tmp.name)
-
-        document_1_tmp.close()
-        document_2_tmp.close()
-
-        document_1_to_langchain = document_1_llama_parsed[0].to_langchain_format()
-        document_2_to_langchain = document_2_llama_parsed[0].to_langchain_format()
-
-        llm = ChatLiteLLM(model="gpt-4o")
-
-        human_prompt = """Given the following two documents, find the difference between them:
-
-        Document 1:
-        {document_1}
-        Document 2:
-        {document_2}
-        Difference:
-        """
-        CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(human_prompt)
-
-        system_message_template = """
-        You are an expert in finding the difference between two documents. You look deeply into what makes the two documents different and provide a detailed analysis if needed of the differences between the two documents.
-        If no differences are found, simply say that there are no differences.
-        """
-
-        ANSWER_PROMPT = ChatPromptTemplate.from_messages(
-            [
-                SystemMessagePromptTemplate.from_template(system_message_template),
-                HumanMessagePromptTemplate.from_template(human_prompt),
-            ]
-        )
-
-        final_inputs = {
-            "document_1": document_1_to_langchain.page_content,
-            "document_2": document_2_to_langchain.page_content,
-        }
-
-        output_parser = StrOutputParser()
-
-        chain = ANSWER_PROMPT | llm | output_parser
-        result = chain.invoke(final_inputs)
-
-        return result
-
-
-def difference_inputs():
-    output = AssistantOutput(
-        name="difference",
-        description="Finds the difference between two sets of documents",
-        tags=["new"],
-        input_description="Two documents to compare",
-        output_description="The difference between the two documents",
-        icon_url="https://quivr-cms.s3.eu-west-3.amazonaws.com/report_94bea8b918.png",
-        inputs=Inputs(
-            files=[
-                InputFile(
-                    key="doc_1",
-                    allowed_extensions=["pdf"],
-                    required=True,
-                    description="The first document to compare",
-                ),
-                InputFile(
-                    key="doc_2",
-                    allowed_extensions=["pdf"],
-                    required=True,
-                    description="The second document to compare",
-                ),
-            ]
-        ),
-        outputs=Outputs(
-            brain=OutputBrain(
-                required=True,
-                description="The brain to which upload the document",
-                type="uuid",
-            ),
-            email=OutputEmail(
-                required=True,
-                description="Send the document by email",
-                type="str",
-            ),
-        ),
-    )
-    return output
diff --git a/backend/modules/assistant/ito/difference/__init__.py b/backend/modules/assistant/ito/difference/__init__.py
new file mode 100644
index 000000000000..46c2975fa5db
--- /dev/null
+++ b/backend/modules/assistant/ito/difference/__init__.py
@@ -0,0 +1,5 @@
+from .difference_agent import ContextType, DifferenceAgent
+from .query_engine import DiffQueryEngine
+from .question_generator import QuestionGenerator
+
+__all__ = ["DifferenceAgent", "DiffQueryEngine", "QuestionGenerator", "ContextType"]
diff --git a/backend/modules/assistant/ito/difference/difference_agent.py b/backend/modules/assistant/ito/difference/difference_agent.py
new file mode 100644
index 000000000000..12f413595970
--- /dev/null
+++ b/backend/modules/assistant/ito/difference/difference_agent.py
@@ -0,0 +1,133 @@
+import asyncio
+
+import pandas as pd
+from pydantic import BaseModel, Field
+
+from .query_engine import DecisionEnum, DiffQueryEngine
+from .question_generator import QuestionGenerator
+
+
+class ResponseType(BaseModel):
+    """Represents an ingredient and its associated decision."""
+
+    name: str | None = Field(description="Name of the ingredient")
+    detailed_answer: str | None = Field(description="Detailed answer with explanations")
+    decision: DecisionEnum | None = Field(
+        description="Decision made: authorized / to avoid / forbidden"
+    )
+
+
+class ContextType(BaseModel):
+    """Represents the context given and the specific category if needed."""
+
+    category: str = Field(description="Category of the context")
+    context: str = Field(description="Context to add to the query")
+
+
+"""
+
+"""
+
+
+class DifferenceAgent:
+    def __init__(
+        self,
+        diff_query_engine: DiffQueryEngine,
+        document_context: str | None = None,
+        question_generator: QuestionGenerator | None = None,
+    ):
+        self.document_context = document_context
+        self.question_generator = (
+            question_generator
+            if question_generator
+            else QuestionGenerator(document_context=document_context)
+        )
+        self.diff_query_engine = diff_query_engine
+
+        self.questions = None
+        self.generated_df = pd.DataFrame()
+
+    def generate_questions(
+        self, target_content: str, language_verification: bool = False
+    ) -> list[str] | None:
+
+        self.questions = self.question_generator.generate_questions(
+            target_content=target_content, language_verification=language_verification
+        )
+
+    def run(
+        self,
+        target_content: str | None = None,
+        language_verification: bool = False,
+        additional_context: ContextType | None = None,
+        verbose: bool = False,
+        n_retry: int = 3,
+    ):
+        if self.questions is None and target_content:
+            self.questions = self.generate_questions(
+                target_content=target_content,
+                language_verification=language_verification,
+            )
+        elif self.questions is None:
+            raise Exception(
+                f"Please provide a source path and tab name to generate questions"
+            )
+
+        print("Querying generated questions to the reference document...")
+        analysis = []
+
+        async def query_all(questions):
+            return await asyncio.gather(*[query_one(question) for question in questions])  # type: ignore
+
+        async def query_one(question):
+            response = ResponseType(name=None, detailed_answer=None, decision=None)
+
+            for i in range(n_retry):
+                try:
+                    response: ResponseType = await self.diff_query_engine.query_engine.aquery(question[:-1])  # type: ignore
+                    nodes_to_update = [response.source_nodes[int(i)] for i in response.response.used_chunks]  # type: ignore
+                    self.diff_query_engine.update_query_engine(nodes_to_update)
+
+                except Exception as e:
+                    print(e)
+                    if verbose:
+                        print(f"Error with question: {question}")
+                        print("Retry ...")
+                        if i == 2:
+                            print(
+                                f"{n_retry} repeted errors with the same question, deleting the question."
+                            )
+                            break
+                        continue
+                break
+            return {
+                "decision": (
+                    response.decision.value if response.decision else response.decision
+                ),
+                "name": response.name,
+                "detailed_answer": response.detailed_answer,
+            }
+
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        try:
+            analysis = loop.run_until_complete(query_all(self.questions))
+            self.generated_df = pd.DataFrame(analysis)
+            return self.generated_df
+        finally:
+            if loop.is_running():
+                loop.close()
+
+    def get_detail(self, category, iteration, name):
+        return self.generated_df.loc[
+            self.generated_df["name"] == name,
+            [f"{category}.{iteration}.detail", f"{category}.{iteration}"],
+        ]
+
+    """
+    Later : DVC (git) / clearML 
+    """
diff --git a/backend/modules/assistant/ito/difference/query_engine.py b/backend/modules/assistant/ito/difference/query_engine.py
new file mode 100644
index 000000000000..f9a2d342b600
--- /dev/null
+++ b/backend/modules/assistant/ito/difference/query_engine.py
@@ -0,0 +1,144 @@
+import asyncio
+import math
+import os
+import pickle
+from enum import Enum
+from typing import List, Optional
+
+import nest_asyncio
+import uvloop
+from langchain_core.pydantic_v1 import BaseModel, Field
+from llama_index.core import Document, QueryBundle, VectorStoreIndex
+from llama_index.core.node_parser import MarkdownNodeParser
+from llama_index.core.postprocessor.types import BaseNodePostprocessor
+from llama_index.core.schema import BaseNode, MetadataMode, NodeWithScore
+from llama_index.llms.openai import OpenAI
+from llama_index.postprocessor.cohere_rerank import CohereRerank
+
+
+class DecisionEnum(str, Enum):
+    authorized = "Authorized"
+    to_avoid = "To Avoid"
+    forbidden = "Forbidden"
+
+
+class Ingredient(BaseModel):
+    """Represents an ingredient and its associated decision."""
+
+    name: str = Field(description="Name of the ingredient")
+    detailed_answer: str = Field(description="Detailed answer with explanations")
+    decision: DecisionEnum = Field(
+        description="Decision made: authorized / to avoid / forbidden"
+    )
+    used_chunks: list[int] = Field(
+        description="List of chunks id used to generate the answer, None if the response was not found in the chunks. -  focus on those with high information density that directly answer the query. Never select a table of content or any text similar."
+    )
+    # more_info: str = Field(description="More information on how the chunks (with ID) were used to generate the answer, start with look into chunk id to ..., then look into chunk id ..., keep it short")
+
+
+class AddChunkId(BaseNodePostprocessor):
+    def _postprocess_nodes(  # type: ignore
+        self, nodes: List[NodeWithScore], query_bundle: Optional[QueryBundle]
+    ) -> List[NodeWithScore]:
+        for i, n in enumerate(nodes):
+            content = n.node.get_content()
+            n.node.set_content(f"Chunk #{i} : \n {content}")
+        return nodes
+
+
+class GeneticRerank(BaseNodePostprocessor):
+    # top_n: int = Field(description="Top N nodes to return.")
+    def _postprocess_nodes(  # type: ignore
+        self, nodes: List[NodeWithScore], query_bundle: Optional[QueryBundle]
+    ) -> List[NodeWithScore]:  # type: ignore
+        retrieved_nodes = {}
+        for n in nodes:
+            if n.score is None:
+                n.score = 0.0
+            n.score += math.tanh(0.05 * float(n.node.metadata["info_density"]))
+            retrieved_nodes[n.node.id_] = n.score
+
+        # sort nodes by score
+        sorted_ids = sorted(
+            retrieved_nodes, key=lambda n: retrieved_nodes[n], reverse=True
+        )
+
+        # return a list of topk nodes retrieve by id
+        sorted_nodes = [node for node in nodes if node.node.id_ in sorted_ids[:5]]
+        # return a list of topk nodes
+        return sorted_nodes
+        # return nodes
+
+
+class DiffQueryEngine:
+    def __init__(self, source_md: List[str], top_k: int = 15):
+        self.cleaned_nodes: List[BaseNode] | None = None
+        self.query_engine = None
+        self.top_k = top_k
+        self.get_query_engine(source_md)
+
+    def get_query_engine(self, source_md: List[str]) -> None:
+        """Get a query engine for markdown files."""
+        if not isinstance(asyncio.get_event_loop(), uvloop.Loop):
+            nest_asyncio.apply()
+
+        md_document = [Document(text=sub_source_md) for sub_source_md in source_md]
+        node_parser = MarkdownNodeParser()
+
+        if not os.path.exists("./charte.pkl"):
+            raw_nodes = node_parser.get_nodes_from_documents(
+                md_document, show_progress=False
+            )
+            self.cleaned_nodes = [
+                node
+                for node in raw_nodes
+                if node.get_content(metadata_mode=MetadataMode.EMBED) != ""
+            ]
+            for node in self.cleaned_nodes:
+                node.metadata["info_density"] = 0
+
+            pickle.dump(self.cleaned_nodes, open("./charte.pkl", "wb"))
+        else:
+            self.cleaned_nodes = pickle.load(open("./charte.pkl", "rb"))
+
+        if not self.cleaned_nodes:
+            raise ValueError("No nodes found in the document.")
+
+        # FIXME: Delete this line once nodes saved :
+        for node in self.cleaned_nodes:
+            node.metadata["info_density"] = 0
+
+        llm = OpenAI(model="gpt-4o", temperature=0.0)  # 0.1
+
+        vector_index = VectorStoreIndex(self.cleaned_nodes)
+        # vector_retriever = vector_index.as_retriever(similarity_top_k=top_k, outpuy_cls = Ingredient)
+
+        # query_engine = RetrieverQueryEngine.from_args(vector_retriever)
+        self.query_engine = vector_index.as_query_engine(
+            output_cls=Ingredient,
+            similarity_top_k=self.top_k,
+            llm=llm,
+            node_postprocessors=[CohereRerank(top_n=10), GeneticRerank(), AddChunkId()],
+            response_mode="tree_summarize",
+        )
+
+    def update_query_engine(self, modified_nodes: List[BaseNode]) -> None:
+        """Update the query engine with modified nodes."""
+
+        if not self.cleaned_nodes:
+            raise ValueError("No nodes found in the document.")
+        modified_nodes_ids = [node.id_ for node in modified_nodes]
+
+        for node in self.cleaned_nodes:
+            if node.id_ in modified_nodes_ids:
+                print("Node found")
+                node.metadata["info_density"] += 1
+
+        vector_index = VectorStoreIndex(self.cleaned_nodes)
+        self.query_engine = vector_index.as_query_engine(
+            output_cls=Ingredient,
+            similarity_top_k=self.top_k,
+            llm=OpenAI(model="gpt-4o", temperature=0.0),
+            node_postprocessors=[CohereRerank(top_n=10), GeneticRerank(), AddChunkId()],
+            response_mode="tree_summarize",
+        )
diff --git a/backend/modules/assistant/ito/difference/question_generator.py b/backend/modules/assistant/ito/difference/question_generator.py
new file mode 100644
index 000000000000..0c53d1833e9b
--- /dev/null
+++ b/backend/modules/assistant/ito/difference/question_generator.py
@@ -0,0 +1,118 @@
+from typing import List
+
+import pandas as pd
+from bs4 import BeautifulSoup
+from langchain.output_parsers import PydanticOutputParser
+from langchain_core.prompts import PromptTemplate
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel, Field
+
+GENERIC_PROMPT = """You will be generating questions to verify the compliance of items based on a provided document. Here is the document:
+<document>
+{DOCUMENT}
+</document>
+
+<document_context>
+{DOCUMENT_CONTEXT}
+</document_context>
+
+Please follow these steps to generate the questions:
+
+Carefully review the document and items table to understand the context and requirements. Review the input language to know in which language you should generate the questions.
+Extract the individual raw items from the items table. For composed items, break them down into their sub-items. Focus on the raw items only, not the composed items themselves.
+For each raw item, formulate a specific question to verify its compliance with the requirements outlined in the document. The question should be in the format: "Is [item information] compliant with the requirements?"
+When generating each question, include all relevant information about the item so that the question can be answered without needing to refer back to the items table. This is important because the team verifying the items will not have access to the table.
+Before providing the final questions, write out your thought process and reasoning inside <scratchpad> tags. Explain how you extracted the raw items and formulated each question.
+Finally, avoiding repetitive questions, output the generated questions inside <questions> tags, with each question on a separate line.
+
+Remember, the goal is to create specific, informative questions for each raw item to verify compliance with the requirements outlined in the document. Make sure to provide all necessary context within the questions themselves. Write the most question possible.
+"""
+# answer in the language of the question only
+
+DOCUMENT_CONTEXT = """
+The document is a list of specificity for a pastry product, every ingredient listed must be verified to be accepted by the Coup de Pates company.
+Extract the individual raw ingredients from the ingredients table. For composed ingredients, break them down into their sub-ingredients. Focus on the raw ingredients only, not the composed ingredients themselves.
+When generating each question, include all relevant information about the ingredient so that the question can be answered without needing to refer back to the ingredients table. This is important because the team verifying the ingredients will not have access to the table.
+Look for the most detail in each ingredient, such as the labels (RSPCO, élevés en cage), do not add the country it is from, do not add the name of the parent ingredient (ex: BASE VERTE)
+Note that a recipe that is not precised to be gluten free is by default not gluten free, also try to note which king of product it is (a pastry ? a salted dish ?)
+Do not ask for an ingredient that is not raw, only raw ingredients are to be verified such as sugar, additifs, oil, colorants, etc.
+
+The questions must be specific to a unique ingredient at each time.
+The number of questions should match the number of unique ingredients in the document.
+The question will be asked directly to the "Charte Qualité" team, they won't have access to the provided document.
+
+Exemple : 
+Document states : The product is composed of a green base containing sugar, flour, and eggs from outdors raised chicken.
+questions : ["Is Sugar compliant with the requirements?", "Is Flour compliant with the requirements?", "Is Eggs from outdors raised chicken compliant with the requirements?]
+"""
+
+
+class QuestionType(BaseModel):
+    """Represent a list of questions translated in a specific language."""
+
+    language: str = Field(description="Language of the generated questions.")
+    questions: List[str] = Field(
+        description="List of generated questions in the right language."
+    )
+
+
+class QuestionGenerator:
+    def __init__(self, document_context=None):
+        self.generic_prompt = GENERIC_PROMPT
+        self.document_context = (
+            document_context if document_context else DOCUMENT_CONTEXT
+        )
+
+    def table_to_text(self, df):
+        text_rows = []
+        for _, row in df.iterrows():
+            row_text = " | ".join(str(value) for value in row.values if pd.notna(value))
+            if row_text:
+                text_rows.append("|" + row_text + "|")
+        return "\n".join(text_rows)
+
+    def generate_questions(
+        self, target_content: str, language_verification: bool = False
+    ) -> list[str]:
+        target_text = target_content
+
+        print("Generating Questions ...")
+
+        prompt = PromptTemplate(
+            template=self.generic_prompt,
+            input_variables=["DOCUMENT", "DOCUMENT_CONTEXT"],
+        )
+
+        llm = ChatOpenAI(model="gpt-4o", temperature=0.0)
+        llm_chain = prompt | llm
+
+        questions = llm_chain.invoke(
+            {"DOCUMENT": target_text, "DOCUMENT_CONTEXT": self.document_context}
+        )
+
+        soup = BeautifulSoup(str(questions.content), "html.parser")
+        questions_content = soup.find("questions").text  # type: ignore
+
+        if language_verification:
+            print("Verifying language and translating questions ...")
+            model = ChatOpenAI(model="gpt-4o", temperature=0.0)
+
+            # Set up a parser + inject instructions into the prompt template.
+            parser = PydanticOutputParser(pydantic_object=QuestionType)
+
+            prompt = PromptTemplate(
+                template="Translate the following questions in french.\n{format_instructions}\n{questions}\n",
+                input_variables=["questions"],
+                partial_variables={
+                    "format_instructions": parser.get_format_instructions()
+                },
+            )
+
+            # And a query intended to prompt a language model to populate the data structure.
+            prompt_and_model = prompt | model
+            questions = prompt_and_model.invoke({"questions": questions_content})
+            translated_question: QuestionType = parser.invoke(questions)
+
+            return translated_question.questions
+
+        return str(questions_content).split("\n")
diff --git a/backend/modules/assistant/ito/difference_assistant.py b/backend/modules/assistant/ito/difference_assistant.py
new file mode 100644
index 000000000000..ab34b8f72db9
--- /dev/null
+++ b/backend/modules/assistant/ito/difference_assistant.py
@@ -0,0 +1,287 @@
+import asyncio
+import os
+import tempfile
+import traceback
+from typing import List
+
+import nest_asyncio
+import pandas as pd
+import uvloop
+from fastapi import UploadFile
+from logger import get_logger
+from megaparse.Converter import MegaParse
+from modules.assistant.dto.inputs import InputAssistant
+from modules.assistant.dto.outputs import (
+    AssistantOutput,
+    InputFile,
+    Inputs,
+    OutputBrain,
+    OutputEmail,
+    Outputs,
+)
+from modules.assistant.ito.difference.difference_agent import ContextType
+from modules.assistant.ito.ito import ITO
+from modules.upload.service import upload_file
+from modules.user.entity.user_identity import UserIdentity
+
+from .difference import DifferenceAgent, DiffQueryEngine
+
+if not isinstance(asyncio.get_event_loop(), uvloop.Loop):
+    nest_asyncio.apply()
+
+
+logger = get_logger(__name__)
+
+
+class DifferenceAssistant(ITO):
+
+    def __init__(
+        self,
+        input: InputAssistant,
+        files: List[UploadFile] = None,  # type: ignore
+        current_user: UserIdentity = None,  # type: ignore
+        **kwargs,
+    ):
+        super().__init__(
+            input=input,
+            files=files,
+            current_user=current_user,
+            **kwargs,
+        )
+
+    def check_input(self):
+        if not self.files:
+            raise ValueError("No file was uploaded")
+        if len(self.files) != 2:
+            raise ValueError("Only two files can be uploaded")
+        if not self.input.inputs.files:
+            raise ValueError("No files key were given in the input")
+        if len(self.input.inputs.files) != 2:
+            raise ValueError("Only two files can be uploaded")
+        if not self.input.inputs.files[0].key == "ref_doc":
+            raise ValueError("The key of the first file should be ref_doc")
+        if not self.input.inputs.files[1].key == "target_doc":
+            raise ValueError("The key of the second file should be target_doc")
+        if not self.input.inputs.files[0].value:
+            raise ValueError("No file was uploaded")
+        if not (
+            self.input.outputs.brain.activated or self.input.outputs.email.activated  # type: ignore
+        ):
+            raise ValueError("No output was selected")
+        return True
+
+    async def process_assistant(self):
+        print("\nRunning Difference Assistant\n")
+        tab_name = ""
+        category = ""
+
+        for elements in self.input.inputs.texts:  # type: ignore
+            if elements.key == "tab_name":
+                tab_name = elements.value
+            elif elements.key == "category":
+                category = elements.value
+        print("tab name: ", tab_name, "category : ", category)
+
+        # breakpoint()
+
+        ## Process the documents
+        ##----------------------
+        document_1 = self.files[0]
+        document_2 = self.files[1]
+
+        # Get the file extensions
+        document_1_ext = os.path.splitext(document_1.filename)[1]  # type: ignore
+        document_2_ext = os.path.splitext(document_2.filename)[1]  # type: ignore
+
+        # Create temporary files with the same extension as the original files
+        document_1_tmp = tempfile.NamedTemporaryFile(
+            suffix=document_1_ext, delete=False
+        )
+        document_2_tmp = tempfile.NamedTemporaryFile(
+            suffix=document_2_ext, delete=False
+        )
+
+        document_1_tmp.write(document_1.file.read())
+        document_2_tmp.write(document_2.file.read())
+
+        print("\nRoad to MegaParse, currently parsing the documents\n")
+        print(document_1_tmp.name)
+        print(document_2_tmp.name)
+
+        for key in self.input.inputs.files:  # type: ignore
+            if key.key == "ref_doc":
+                ref_doc_name = key.value
+            elif key.key == "target_doc":
+                target_doc_name = key.value
+
+        print(target_doc_name, "\n")
+        print(ref_doc_name, "\n")
+
+        # breakpoint()
+        try:
+            megaparse = MegaParse(
+                file_path=document_1_tmp.name,
+                llama_parse_api_key=os.getenv("LLAMA_CLOUD_API_KEY"),
+            )
+            ref_doc_md = megaparse.convert(gpt4o_cleaner=True)
+        except Exception as e:
+            print(e)
+            print(traceback.format_exc())
+
+            raise ValueError("Error parsing the reference document")
+
+        additional_context = ContextType(
+            category=category,
+            context=f"pour les produits de la catégorie {category} ?",
+        )
+
+        # FIXME: Add a check if the pkl name file already exists before launching megaparse
+        megaparse = MegaParse(file_path=document_2_tmp.name)
+        target_doc_md = megaparse.convert_tab(tab_name=tab_name)  # type: ignore
+        ##----------------------
+        ref_doc_md = [ref_doc_md]
+        target_doc_md = [target_doc_md]
+
+        print("\nDocuments parsed successfully\n")
+
+        ## Create diff_query_engine
+        ##----------------------
+        diff_query_engine = DiffQueryEngine(ref_doc_md, 15)  # type: ignore
+        ##----------------------
+
+        DOCUMENT_CONTEXT = None  # type: ignore
+
+        print("\nCreating the Difference Agent\n")
+        ## Create the Difference Agent
+        ##----------------------
+        agent = DifferenceAgent(diff_query_engine)
+        print("\nDifference Agent created successfully, generating questions ...\n")
+        questions = agent.generate_questions(
+            target_content=target_doc_md[0], language_verification=True
+        )
+
+        print("\nQuestions generated successfully, asking questions to Quivr\n")
+
+        diff_df = pd.DataFrame()
+
+        diff_df = agent.run(additional_context=additional_context)
+
+        diff_df = diff_df.dropna()
+        print("\nNice, the process has succeeded, giving back the json ;)\n")
+
+        content = generate_authorisation_report(diff_df)
+
+        original_filename = "Classic Ingredient Comparison"
+        file_description = "Difference Report"
+        processed_file = self.create_and_upload_processed_file(
+            content,
+            original_filename=original_filename,
+            file_description=file_description,
+        )  # FIXME change name
+        file_to_upload = processed_file["file_to_upload"]
+        new_filename = processed_file["new_filename"]
+
+        # Email the file if required
+        if self.input.outputs.email.activated:
+            await self.send_output_by_email(
+                file_to_upload,
+                new_filename,
+                "Summary",
+                f"{file_description} of {original_filename}",
+                brain_id=(
+                    self.input.outputs.brain.value
+                    if self.input.outputs.brain.activated
+                    and self.input.outputs.brain.value
+                    else None
+                ),
+            )
+
+        # Reset to start of file before upload
+        file_to_upload.file.seek(0)
+
+        # Upload the file if required
+        if self.input.outputs.brain.activated:
+            await upload_file(
+                uploadFile=file_to_upload,
+                brain_id=self.input.outputs.brain.value,
+                current_user=self.current_user,
+                chat_id=None,
+            )
+
+        return {"message": f"{file_description} generated successfully"}
+
+
+def generate_authorisation_report(df: pd.DataFrame):
+    # Ensure the dataframe has the required columns
+    if "name" not in df.columns or "decision" not in df.columns:
+        raise ValueError("DataFrame must contain 'name' and 'decision' columns")
+
+    # Extract forbidden and to avoid ingredients
+    forbidden_ingredients = df[df["decision"] == "Forbidden"][
+        ["name", "detailed_answer"]
+    ].values.tolist()
+    to_avoid_ingredients = df[df["decision"] == "To Avoid"][
+        ["name", "detailed_answer"]
+    ].values.tolist()
+
+    # Create the markdown content
+    markdown_content = "# Coup de Pates\n\n"
+    markdown_content += "## Rapport d'analyse\n\n"
+
+    if forbidden_ingredients:
+        markdown_content += "### Ingrédients utilisés **Interdits**\n\n"
+        for ingredient, detail in forbidden_ingredients:
+            markdown_content += f"**{ingredient}**: *{detail}*\n\n"
+    else:
+        markdown_content += "### Ingrédients utilisés **Interdits**\n- Aucun\n\n"
+
+    markdown_content += "\n"
+
+    if to_avoid_ingredients:
+        markdown_content += "### Ingrédients utilisés **à éviter**\n"
+        for ingredient, detail in to_avoid_ingredients:
+            markdown_content += f"**{ingredient}** : *{detail}*\n\n"
+    else:
+        markdown_content += "### To Avoid Ingredients\n- None\n\n"
+
+    return markdown_content
+
+
+def difference_inputs():
+    output = AssistantOutput(
+        name="difference",
+        description="Finds the difference between two sets of documents",
+        tags=["new"],
+        input_description="Two documents to compare",
+        output_description="The difference between the two documents",
+        icon_url="https://quivr-cms.s3.eu-west-3.amazonaws.com/report_94bea8b918.png",
+        inputs=Inputs(
+            files=[
+                InputFile(
+                    key="ref_doc",
+                    allowed_extensions=["pdf"],
+                    required=True,
+                    description="The reference document",
+                ),
+                InputFile(
+                    key="target_doc",
+                    allowed_extensions=["pdf", "xlsx"],
+                    required=True,
+                    description="The target document to compare to the reference document.",
+                ),
+            ],
+        ),
+        outputs=Outputs(
+            brain=OutputBrain(
+                required=True,
+                description="The brain to which upload the document",
+                type="uuid",
+            ),
+            email=OutputEmail(
+                required=True,
+                description="Send the document by email",
+                type="str",
+            ),
+        ),
+    )
+    return output
diff --git a/backend/modules/assistant/ito/ito.py b/backend/modules/assistant/ito/ito.py
index dc008178113a..38c0df8f2f47 100644
--- a/backend/modules/assistant/ito/ito.py
+++ b/backend/modules/assistant/ito/ito.py
@@ -1,7 +1,6 @@
 import os
-import random
 import re
-import string
+import uuid
 from abc import abstractmethod
 from io import BytesIO
 from tempfile import NamedTemporaryFile
@@ -13,7 +12,6 @@
 from modules.assistant.ito.utils.pdf_generator import PDFGenerator, PDFModel
 from modules.chat.controller.chat.utils import update_user_usage
 from modules.contact_support.controller.settings import ContactsSettings
-from modules.upload.controller.upload_routes import upload_file
 from modules.user.entity.user_identity import UserIdentity
 from modules.user.service.user_usage import UserUsage
 from packages.emails.send_email import send_email
@@ -62,11 +60,12 @@ def increase_usage_user(self):
     def calculate_pricing(self):
         return 20
 
-    def generate_pdf(self, filename: str, title: str, content: str):
+    def generate_pdf(self, file_io: BytesIO, title: str, content: str):
         pdf_model = PDFModel(title=title, content=content)
         pdf = PDFGenerator(pdf_model)
         pdf.print_pdf()
-        pdf.output(filename, "F")
+        pdf_content = pdf.output(dest="S")
+        file_io.write(pdf_content)
 
     @abstractmethod
     async def process_assistant(self):
@@ -111,6 +110,7 @@ async def send_output_by_email(
             </div>
             """
             params = {
+                "from": mail_from,
                 "sender": mail_from,
                 "to": [mail_to],
                 "subject": "Quivr Ingestion Processed",
@@ -128,42 +128,40 @@ async def uploadfile_to_file(self, uploadFile: UploadFile):
         tmp_file.flush()  # Make sure all data is written to disk
         return tmp_file
 
-    async def create_and_upload_processed_file(
+    def create_and_upload_processed_file(
         self, processed_content: str, original_filename: str, file_description: str
     ) -> dict:
         """Handles creation and uploading of the processed file."""
-        # remove any special characters from the filename that aren't http safe
 
-        new_filename = (
-            original_filename.split(".")[0]
-            + "_"
-            + file_description.lower().replace(" ", "_")
-            + "_"
-            + str(random.randint(1000, 9999))
-            + ".pdf"
+        # Generate a new filename
+        base_filename = original_filename.rsplit(".", 1)[0]
+        safe_description = re.sub(
+            r"[^A-Za-z0-9_]", "", file_description.lower().replace(" ", "_")
         )
-        new_filename = unidecode(new_filename)
-        new_filename = re.sub(
-            "[^{}0-9a-zA-Z]".format(re.escape(string.punctuation)), "", new_filename
+        new_filename = (
+            f"{unidecode(base_filename)}_{safe_description}_{uuid.uuid4().hex}.pdf"
         )
 
+        # Generate PDF in-memory
+        content_io = BytesIO()
         self.generate_pdf(
-            new_filename,
+            content_io,
             f"{file_description} of {original_filename}",
             processed_content,
         )
-
-        content_io = BytesIO()
-        with open(new_filename, "rb") as f:
-            content_io.write(f.read())
         content_io.seek(0)
 
+        # Prepare file for upload
         file_to_upload = UploadFile(
             filename=new_filename,
             file=content_io,
             headers={"content-type": "application/pdf"},
         )
 
+        return {"file_to_upload": file_to_upload, "new_filename": new_filename}
+
+
+"""        # Email the file if required
         if self.input.outputs.email.activated:
             await self.send_output_by_email(
                 file_to_upload,
@@ -172,24 +170,20 @@ async def create_and_upload_processed_file(
                 f"{file_description} of {original_filename}",
                 brain_id=(
                     self.input.outputs.brain.value
-                    if (
-                        self.input.outputs.brain.activated
-                        and self.input.outputs.brain.value
-                    )
+                    if self.input.outputs.brain.activated
+                    and self.input.outputs.brain.value
                     else None
                 ),
             )
 
         # Reset to start of file before upload
         file_to_upload.file.seek(0)
+
+        # Upload the file if required
         if self.input.outputs.brain.activated:
             await upload_file(
                 uploadFile=file_to_upload,
                 brain_id=self.input.outputs.brain.value,
                 current_user=self.current_user,
                 chat_id=None,
-            )
-
-        os.remove(new_filename)
-
-        return {"message": f"{file_description} generated successfully"}
+            )"""
diff --git a/backend/modules/assistant/ito/summary.py b/backend/modules/assistant/ito/summary.py
index 4c713aa2f95c..e83358eff12f 100644
--- a/backend/modules/assistant/ito/summary.py
+++ b/backend/modules/assistant/ito/summary.py
@@ -23,6 +23,7 @@
     Outputs,
 )
 from modules.assistant.ito.ito import ITO
+from modules.upload.service import upload_file
 from modules.user.entity.user_identity import UserIdentity
 
 logger = get_logger(__name__)
@@ -157,9 +158,44 @@ async def process_assistant(self):
 
         content = map_reduce_chain.run(split_docs)
 
-        return await self.create_and_upload_processed_file(
-            content, self.files[0].filename, "Summary"
-        )
+        original_filename = "Classic Ingredient Comparison"
+        file_description = "Difference Report"
+        processed_file = self.create_and_upload_processed_file(
+            content,
+            original_filename=original_filename,
+            file_description=file_description,
+        )  # FIXME change name
+        file_to_upload = processed_file["file_to_upload"]
+        new_filename = processed_file["new_filename"]
+
+        # Email the file if required
+        if self.input.outputs.email.activated:
+            await self.send_output_by_email(
+                file_to_upload,
+                new_filename,
+                "Summary",
+                f"{file_description} of {original_filename}",
+                brain_id=(
+                    self.input.outputs.brain.value
+                    if self.input.outputs.brain.activated
+                    and self.input.outputs.brain.value
+                    else None
+                ),
+            )
+
+        # Reset to start of file before upload
+        file_to_upload.file.seek(0)
+
+        # Upload the file if required
+        if self.input.outputs.brain.activated:
+            await upload_file(
+                uploadFile=file_to_upload,
+                brain_id=self.input.outputs.brain.value,
+                current_user=self.current_user,
+                chat_id=None,
+            )
+
+        return {"message": f"{file_description} generated successfully"}
 
 
 def summary_inputs():
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 74d6379085f7..4029d4a3f439 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -307,3 +307,6 @@ xlsxwriter==3.2.0; python_version >= '3.6'
 xxhash==3.4.1; python_version >= '3.7'
 yarl==1.9.4; python_version >= '3.7'
 zipp==3.19.2; python_version >= '3.8'
+megaparse; python_version >= '3.8'
+llama-index-postprocessor-cohere-rerank; python_version >= '3.8'
+unstructured; python_version >= '3.8'
\ No newline at end of file

From eee933253747e08b5b3f5884dac47f83df3b4846 Mon Sep 17 00:00:00 2001
From: chloedia <chloedaems0@gmail.com>
Date: Fri, 21 Jun 2024 17:21:01 +0200
Subject: [PATCH 2/2] fix: PR comments & summary

---
 backend/celery_worker.py                      | 43 ++++------
 backend/modules/assistant/dto/inputs.py       |  3 +-
 .../ito/difference/difference_agent.py        | 81 ++++++++-----------
 .../assistant/ito/difference/query_engine.py  |  4 -
 .../assistant/ito/difference_assistant.py     | 48 ++++++++---
 backend/modules/assistant/ito/summary.py      | 25 +++++-
 backend/packages/files/parsers/common.py      |  4 -
 7 files changed, 109 insertions(+), 99 deletions(-)

diff --git a/backend/celery_worker.py b/backend/celery_worker.py
index c200c24e1fee..eb81571904f0 100644
--- a/backend/celery_worker.py
+++ b/backend/celery_worker.py
@@ -321,8 +321,6 @@ def process_assistant_task(
     logger.debug(f"Input: {input}")
     logger.debug(type(input))
     _input = InputAssistant.model_validate(input_in)
-    # _input = InputAssistant(**json.loads(input))  # type: ignore
-    # _input = InputAssistant(json.dumps(_input))
     _current_user = UserIdentity(**current_user)  # type: ignore
     try:
         files = []
@@ -333,25 +331,14 @@ def process_assistant_task(
             base_file_name = os.path.basename(file_name)
             _, file_extension = os.path.splitext(base_file_name)
 
-            with NamedTemporaryFile(suffix="_" + tmp_name, delete=False) as tmp_file:
-                res = supabase_client.storage.from_("quivr").download(file_name)
-                tmp_file.write(res)
-                tmp_file.flush()
-
-                file_instance = File(
-                    file_name=base_file_name,
-                    tmp_file_path=tmp_file.name,
-                    bytes_content=res,
-                    file_size=len(res),
-                    file_extension=file_extension,
-                )
-                upload_file = UploadFile(
-                    filename=file_instance.file_name,
-                    size=file_instance.file_size,
-                    file=BytesIO(file_instance.bytes_content),
-                    headers='{"content-type": "application/pdf"}',  # type : ignore
-                )
-                files.append(upload_file)
+            res = supabase_client.storage.from_("quivr").download(file_name)
+            upload_file = UploadFile(
+                filename=base_file_name,
+                size=len(res),
+                file=BytesIO(res),
+                headers='{"content-type": "application/pdf"}',  # type : ignore
+            )
+            files.append(upload_file)
 
     except Exception as e:
         logger.exception(e)
@@ -363,12 +350,14 @@ def process_assistant_task(
                     description=f"An error occurred while processing the file: {e}",
                 ),
             )
-        return
+        raise e
     loop = asyncio.get_event_loop()
 
-    asyncio.set_event_loop(asyncio.new_event_loop())
+    # asyncio.set_event_loop(asyncio.new_event_loop())
 
     if _input.name.lower() == "summary":
+        logger.debug(f"N Files given : {len(files)}")
+        logger.debug(f"Input given : {input_in}")
         summary_assistant = SummaryAssistant(
             input=_input, files=files, current_user=_current_user
         )
@@ -431,8 +420,8 @@ def process_assistant_task(
         "task": "check_if_is_premium_user",
         "schedule": crontab(minute="*/1", hour="*"),
     },
-    "process_assistant": {
-        "task": "process_assistant_task",
-        "schedule": crontab(minute="*/1", hour="*"),
-    },
+    # "process_assistant": {
+    #     "task": "process_assistant_task",
+    #     "schedule": crontab(minute="*/1", hour="*"),
+    # },
 }
diff --git a/backend/modules/assistant/dto/inputs.py b/backend/modules/assistant/dto/inputs.py
index f75c86c5528c..631f3e4fee74 100644
--- a/backend/modules/assistant/dto/inputs.py
+++ b/backend/modules/assistant/dto/inputs.py
@@ -79,5 +79,4 @@ class InputAssistant(BaseModel):
     @model_validator(mode="before")
     @classmethod
     def to_py_dict(cls, data):
-        if isinstance(data, str):
-            return json.loads(data)
+        return json.loads(data)
diff --git a/backend/modules/assistant/ito/difference/difference_agent.py b/backend/modules/assistant/ito/difference/difference_agent.py
index 12f413595970..3fde7a7301f3 100644
--- a/backend/modules/assistant/ito/difference/difference_agent.py
+++ b/backend/modules/assistant/ito/difference/difference_agent.py
@@ -1,4 +1,5 @@
 import asyncio
+import time
 
 import pandas as pd
 from pydantic import BaseModel, Field
@@ -55,7 +56,34 @@ def generate_questions(
             target_content=target_content, language_verification=language_verification
         )
 
-    def run(
+    async def query_one(self, question: str, n_retry: int = 3, verbose: bool = False):
+        response = ResponseType(name=None, detailed_answer=None, decision=None)
+        retry_count = 0
+        while retry_count < n_retry and response.name is None:
+            try:
+                response: ResponseType = await self.diff_query_engine.query_engine.aquery(question[:-1])  # type: ignore
+                nodes_to_update = [response.source_nodes[int(i)] for i in response.response.used_chunks]  # type: ignore
+                self.diff_query_engine.update_query_engine(nodes_to_update)
+
+            except Exception as e:
+                # Exponential backoff
+                time.sleep(2**retry_count)
+                retry_count += 1
+                if verbose:
+                    print(f"Error with question: {question}")
+                    print("Retry ...")
+        if retry_count == n_retry:
+            print("n_retry reached, skipping question ...")
+
+        return {
+            "decision": (
+                response.decision.value if response.decision else response.decision
+            ),
+            "name": response.name,
+            "detailed_answer": response.detailed_answer,
+        }
+
+    async def run(
         self,
         target_content: str | None = None,
         language_verification: bool = False,
@@ -76,51 +104,12 @@ def run(
         print("Querying generated questions to the reference document...")
         analysis = []
 
-        async def query_all(questions):
-            return await asyncio.gather(*[query_one(question) for question in questions])  # type: ignore
-
-        async def query_one(question):
-            response = ResponseType(name=None, detailed_answer=None, decision=None)
-
-            for i in range(n_retry):
-                try:
-                    response: ResponseType = await self.diff_query_engine.query_engine.aquery(question[:-1])  # type: ignore
-                    nodes_to_update = [response.source_nodes[int(i)] for i in response.response.used_chunks]  # type: ignore
-                    self.diff_query_engine.update_query_engine(nodes_to_update)
-
-                except Exception as e:
-                    print(e)
-                    if verbose:
-                        print(f"Error with question: {question}")
-                        print("Retry ...")
-                        if i == 2:
-                            print(
-                                f"{n_retry} repeted errors with the same question, deleting the question."
-                            )
-                            break
-                        continue
-                break
-            return {
-                "decision": (
-                    response.decision.value if response.decision else response.decision
-                ),
-                "name": response.name,
-                "detailed_answer": response.detailed_answer,
-            }
-
-        try:
-            loop = asyncio.get_event_loop()
-        except RuntimeError:
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-
-        try:
-            analysis = loop.run_until_complete(query_all(self.questions))
-            self.generated_df = pd.DataFrame(analysis)
-            return self.generated_df
-        finally:
-            if loop.is_running():
-                loop.close()
+        async def query_all(questions: list[str] | None, n_retry: int = 3):
+            return await asyncio.gather(*[self.query_one(question, n_retry=n_retry) for question in questions])  # type: ignore
+
+        analysis = await query_all(self.questions, n_retry=n_retry)
+        self.generated_df = pd.DataFrame(analysis)
+        return self.generated_df
 
     def get_detail(self, category, iteration, name):
         return self.generated_df.loc[
diff --git a/backend/modules/assistant/ito/difference/query_engine.py b/backend/modules/assistant/ito/difference/query_engine.py
index f9a2d342b600..148f4dfc327d 100644
--- a/backend/modules/assistant/ito/difference/query_engine.py
+++ b/backend/modules/assistant/ito/difference/query_engine.py
@@ -5,7 +5,6 @@
 from enum import Enum
 from typing import List, Optional
 
-import nest_asyncio
 import uvloop
 from langchain_core.pydantic_v1 import BaseModel, Field
 from llama_index.core import Document, QueryBundle, VectorStoreIndex
@@ -33,7 +32,6 @@ class Ingredient(BaseModel):
     used_chunks: list[int] = Field(
         description="List of chunks id used to generate the answer, None if the response was not found in the chunks. -  focus on those with high information density that directly answer the query. Never select a table of content or any text similar."
     )
-    # more_info: str = Field(description="More information on how the chunks (with ID) were used to generate the answer, start with look into chunk id to ..., then look into chunk id ..., keep it short")
 
 
 class AddChunkId(BaseNodePostprocessor):
@@ -79,8 +77,6 @@ def __init__(self, source_md: List[str], top_k: int = 15):
 
     def get_query_engine(self, source_md: List[str]) -> None:
         """Get a query engine for markdown files."""
-        if not isinstance(asyncio.get_event_loop(), uvloop.Loop):
-            nest_asyncio.apply()
 
         md_document = [Document(text=sub_source_md) for sub_source_md in source_md]
         node_parser = MarkdownNodeParser()
diff --git a/backend/modules/assistant/ito/difference_assistant.py b/backend/modules/assistant/ito/difference_assistant.py
index ab34b8f72db9..b22d389e34f5 100644
--- a/backend/modules/assistant/ito/difference_assistant.py
+++ b/backend/modules/assistant/ito/difference_assistant.py
@@ -2,11 +2,11 @@
 import os
 import tempfile
 import traceback
+from concurrent.futures import ThreadPoolExecutor
 from typing import List
 
-import nest_asyncio
+import httpx
 import pandas as pd
-import uvloop
 from fastapi import UploadFile
 from logger import get_logger
 from megaparse.Converter import MegaParse
@@ -21,16 +21,31 @@
 )
 from modules.assistant.ito.difference.difference_agent import ContextType
 from modules.assistant.ito.ito import ITO
-from modules.upload.service import upload_file
+
+# from modules.upload.controller.upload_routes import upload_file #FIXME circular import
 from modules.user.entity.user_identity import UserIdentity
 
 from .difference import DifferenceAgent, DiffQueryEngine
 
-if not isinstance(asyncio.get_event_loop(), uvloop.Loop):
-    nest_asyncio.apply()
+logger = get_logger(__name__)
 
 
-logger = get_logger(__name__)
+# FIXME: PATCHER -> find another solution
+async def upload_file_to_api(upload_file, brain_id, current_user):
+    url = "http://localhost:5050/upload"
+    headers = {
+        "Authorization": f"Bearer {current_user.token}",
+        "Content-Type": "application/json",
+    }
+    data = {
+        "uploadFile": upload_file,
+        "brain_id": brain_id,
+        "chat_id": None,
+    }
+    async with httpx.AsyncClient() as client:
+        response = await client.post(url, headers=headers, json=data)
+        response.raise_for_status()  # Raise an error for 4xx/5xx responses
+        return response.json()
 
 
 class DifferenceAssistant(ITO):
@@ -117,13 +132,23 @@ async def process_assistant(self):
         print(target_doc_name, "\n")
         print(ref_doc_name, "\n")
 
+        def run_megaparse():
+            megaparse = MegaParse(
+                file_path=document_1_tmp.name,
+                llama_parse_api_key=os.getenv("LLAMA_CLOUD_API_KEY"),
+            )
+            return megaparse.convert(gpt4o_cleaner=True)
+
         # breakpoint()
         try:
             megaparse = MegaParse(
                 file_path=document_1_tmp.name,
                 llama_parse_api_key=os.getenv("LLAMA_CLOUD_API_KEY"),
             )
-            ref_doc_md = megaparse.convert(gpt4o_cleaner=True)
+            loop = asyncio.get_event_loop()
+            executor = ThreadPoolExecutor(max_workers=1)
+            ref_doc_md = await loop.run_in_executor(executor, run_megaparse)  # type: ignore
+            # ref_doc_md = megaparse.convert(gpt4o_cleaner=True)
         except Exception as e:
             print(e)
             print(traceback.format_exc())
@@ -164,7 +189,7 @@ async def process_assistant(self):
 
         diff_df = pd.DataFrame()
 
-        diff_df = agent.run(additional_context=additional_context)
+        diff_df = await agent.run(additional_context=additional_context)
 
         diff_df = diff_df.dropna()
         print("\nNice, the process has succeeded, giving back the json ;)\n")
@@ -186,7 +211,7 @@ async def process_assistant(self):
             await self.send_output_by_email(
                 file_to_upload,
                 new_filename,
-                "Summary",
+                "Difference",
                 f"{file_description} of {original_filename}",
                 brain_id=(
                     self.input.outputs.brain.value
@@ -201,11 +226,10 @@ async def process_assistant(self):
 
         # Upload the file if required
         if self.input.outputs.brain.activated:
-            await upload_file(
-                uploadFile=file_to_upload,
+            response = await upload_file_to_api(
+                upload_file=file_to_upload,
                 brain_id=self.input.outputs.brain.value,
                 current_user=self.current_user,
-                chat_id=None,
             )
 
         return {"message": f"{file_description} generated successfully"}
diff --git a/backend/modules/assistant/ito/summary.py b/backend/modules/assistant/ito/summary.py
index e83358eff12f..0e67a9f5b6b0 100644
--- a/backend/modules/assistant/ito/summary.py
+++ b/backend/modules/assistant/ito/summary.py
@@ -1,6 +1,7 @@
 import tempfile
 from typing import List
 
+import httpx
 from fastapi import UploadFile
 from langchain.chains import (
     MapReduceDocumentsChain,
@@ -23,12 +24,29 @@
     Outputs,
 )
 from modules.assistant.ito.ito import ITO
-from modules.upload.service import upload_file
 from modules.user.entity.user_identity import UserIdentity
 
 logger = get_logger(__name__)
 
 
+# FIXME: PATCHER -> find another solution
+async def upload_file_to_api(upload_file, brain_id, current_user):
+    url = "http://localhost:5050/upload"
+    headers = {
+        "Authorization": f"Bearer {current_user.token}",
+        "Content-Type": "application/json",
+    }
+    data = {
+        "uploadFile": upload_file,
+        "brain_id": brain_id,
+        "chat_id": None,
+    }
+    async with httpx.AsyncClient() as client:
+        response = await client.post(url, headers=headers, json=data)
+        response.raise_for_status()  # Raise an error for 4xx/5xx responses
+        return response.json()
+
+
 class SummaryAssistant(ITO):
 
     def __init__(
@@ -188,11 +206,10 @@ async def process_assistant(self):
 
         # Upload the file if required
         if self.input.outputs.brain.activated:
-            await upload_file(
-                uploadFile=file_to_upload,
+            response = await upload_file_to_api(
+                upload_file=file_to_upload,
                 brain_id=self.input.outputs.brain.value,
                 current_user=self.current_user,
-                chat_id=None,
             )
 
         return {"message": f"{file_description} generated successfully"}
diff --git a/backend/packages/files/parsers/common.py b/backend/packages/files/parsers/common.py
index 6b56aeb30a65..2a642bb96ddb 100644
--- a/backend/packages/files/parsers/common.py
+++ b/backend/packages/files/parsers/common.py
@@ -3,7 +3,6 @@
 import tempfile
 import time
 
-import nest_asyncio
 import tiktoken
 import uvloop
 from langchain.schema import Document
@@ -15,9 +14,6 @@
 from modules.upload.service.upload_file import DocumentSerializable
 from packages.embeddings.vectors import Neurons
 
-if not isinstance(asyncio.get_event_loop(), uvloop.Loop):
-    nest_asyncio.apply()
-
 logger = get_logger(__name__)