diff --git a/backend-server/Dockerfile b/backend-server/Dockerfile index 0fc71a89..ad201759 100644 --- a/backend-server/Dockerfile +++ b/backend-server/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y \ unzip \ git -RUN docker-php-ext-install pdo pdo_mysql +RUN docker-php-ext-install pdo pdo_mysql zip # Set the working directory WORKDIR /var/www/html diff --git a/backend-server/app/Http/Controllers/ChatbotController.php b/backend-server/app/Http/Controllers/ChatbotController.php index 9111a525..5e05f0ef 100644 --- a/backend-server/app/Http/Controllers/ChatbotController.php +++ b/backend-server/app/Http/Controllers/ChatbotController.php @@ -4,13 +4,16 @@ use App\Http\Events\ChatbotWasCreated; use App\Http\Events\CodebaseDataSourceWasAdded; +use App\Http\Events\JsonDataSourceWasAdded; use App\Http\Events\PdfDataSourceWasAdded; use App\Http\Requests\CreateChatbotRequest; use App\Http\Requests\CreateChatbotViaCodebaseRequest; +use App\Http\Requests\CreateChatbotViaJsonFlowRequest; use App\Http\Requests\CreateChatbotViaPdfFlowRequest; use App\Http\Requests\SendChatMessageRequest; use App\Http\Requests\UpdateCharacterSettingsRequest; use App\Http\Responses\ChatbotResponse; +use App\Http\Services\HandleJsonDataSource; use App\Http\Services\HandlePdfDataSource; use App\Models\Chatbot; use App\Models\ChatHistory; @@ -102,6 +105,39 @@ public function createViaPdfFlow(CreateChatbotViaPdfFlowRequest $request): Redir return redirect()->route('onboarding.config', ['id' => $chatbot->getId()->toString()]); } + /** + * Create a chatbot via Json flow. + * + * @param \App\Http\Requests\CreateChatbotViaJsonFlowRequest $request + * @return \Illuminate\Http\RedirectResponse + */ + public function createViaJsonFlow(CreateChatbotViaJsonFlowRequest $request): RedirectResponse + { + // Create a new Chatbot instance + $chatbot = new Chatbot(); + + // Set the properties of the chatbot + $chatbot->setId(Uuid::uuid4()); + $chatbot->setName($request->getName()); + $chatbot->setToken(Str::random(20)); + $chatbot->setPromptMessage($request->getPromptMessage()); + + // Save the chatbot to the database + $chatbot->save(); + + // Get the JSON files from the request + $files = $request->file('jsonfiles'); + + // Handle the JSON data source + $dataSource = (new HandleJsonDataSource($chatbot, $files))->handle(); // todo this should be moved to an event listener similar to the one in the previous method + + // Trigger the JsonDataSourceWasAdded event + event(new JsonDataSourceWasAdded($chatbot->getId(), $dataSource->getId())); + + // Redirect to the onboarding configuration page + return redirect()->route('onboarding.config', ['id' => $chatbot->getId()->toString()]); + } + /** * Update character settings for a chatbot. * diff --git a/backend-server/app/Http/Controllers/ChatbotSettingController.php b/backend-server/app/Http/Controllers/ChatbotSettingController.php index e3c1654b..a04ac396 100644 --- a/backend-server/app/Http/Controllers/ChatbotSettingController.php +++ b/backend-server/app/Http/Controllers/ChatbotSettingController.php @@ -99,12 +99,14 @@ public function dataSettings(Request $request, $id) $websiteDataSources = $bot->getWebsiteDataSources()->get(); $pdfDataSources = $bot->getPdfFilesDataSources()->get(); $codebaseDataSources = $bot->getCodebaseDataSources()->get(); + $jsonDataSources = $bot->getJsonFilesDataSources()->get(); return view('settings-data', [ 'bot' => $bot, 'websiteDataSources' => $websiteDataSources, 'pdfDataSources' => $pdfDataSources, 'codebaseDataSources' => $codebaseDataSources, + 'jsonDataSources' => $jsonDataSources, ]); } diff --git a/backend-server/app/Http/Controllers/JsonDataSourceController.php b/backend-server/app/Http/Controllers/JsonDataSourceController.php new file mode 100644 index 00000000..393d9070 --- /dev/null +++ b/backend-server/app/Http/Controllers/JsonDataSourceController.php @@ -0,0 +1,31 @@ +firstOrFail(); + $files = $request->file('jsonfiles'); + $dataSource = (new HandleJsonDataSource($bot, $files))->handle(); + event(new JsonDataSourceWasAdded($bot->getId(), $dataSource->getId())); + + return redirect()->route('chatbot.settings-data', ['id' => $bot->getId()])->with('success', 'Your files have been uploaded successfully, we are training the model now, it should take around 5 minutes to reflect.'); + } + + public function show($id) + { + /** @var Chatbot $bot */ + $bot = Chatbot::where('id', $id)->firstOrFail(); + $jsonDataSources = $bot->getJsonFilesDataSources()->get(); + return view('onboarding.other-data-sources-json', ['bot' => $bot, 'jsonDataSources' => $jsonDataSources]); + } + +} diff --git a/backend-server/app/Http/Controllers/OnboardingController.php b/backend-server/app/Http/Controllers/OnboardingController.php index ae2a005a..81d36bbe 100644 --- a/backend-server/app/Http/Controllers/OnboardingController.php +++ b/backend-server/app/Http/Controllers/OnboardingController.php @@ -26,6 +26,11 @@ public function dataSourcesPdf() return view('onboarding.step-2-pdf'); } + public function dataSourcesJson() + { + return view('onboarding.step-2-json'); + } + public function config() { return view('onboarding.step-3'); diff --git a/backend-server/app/Http/Events/JsonDataSourceWasAdded.php b/backend-server/app/Http/Events/JsonDataSourceWasAdded.php new file mode 100644 index 00000000..eb48717b --- /dev/null +++ b/backend-server/app/Http/Events/JsonDataSourceWasAdded.php @@ -0,0 +1,29 @@ +chatbotId; + } + + public function getJsonDataSourceId(): UuidInterface + { + return $this->jsonDataSourceId; + } +} diff --git a/backend-server/app/Http/Listeners/IngestJsonDataSource.php b/backend-server/app/Http/Listeners/IngestJsonDataSource.php new file mode 100644 index 00000000..02dce602 --- /dev/null +++ b/backend-server/app/Http/Listeners/IngestJsonDataSource.php @@ -0,0 +1,48 @@ +getChatbotId(); + $jsonDataSourceId = $event->getJsonDataSourceId(); + + /** @var JsonDataSource $jsonDataSource */ + $jsonDataSource = JsonDataSource::where('id', $jsonDataSourceId)->firstOrFail(); + $files = $jsonDataSource->getFiles(); + + $requestBody = [ + 'type' => 'json', + 'shared_folder' => $jsonDataSource->getFolderName(), + 'namespace' => $botId, + ]; + + // Call to ingest service endpoint + $client = new Client(); + $response = $client->request('POST', "http://llm-server:3000/api/ingest", [ + 'json' => $requestBody, + ]); + + if ($response->getStatusCode() !== 200) { + throw new Exception('Ingest service returned an error: ' . $response->getBody()->getContents()); + } + } +} diff --git a/backend-server/app/Http/Requests/CreateChatbotViaJsonFlowRequest.php b/backend-server/app/Http/Requests/CreateChatbotViaJsonFlowRequest.php new file mode 100644 index 00000000..484b5b0f --- /dev/null +++ b/backend-server/app/Http/Requests/CreateChatbotViaJsonFlowRequest.php @@ -0,0 +1,37 @@ + 'required', + ]; + } + + public function getName(): string + { + return $this->get('name', 'My first chatbot'); + } + + public function getWebsite(): string + { + return $this->get('website'); + } + + public function getPromptMessage(): string + { + return $this->get('prompt_message', ""); + } + + public function getFiles() + { + return $this->get('jsonfiles'); + } +} diff --git a/backend-server/app/Http/Requests/UploadJsonFilesRequest.php b/backend-server/app/Http/Requests/UploadJsonFilesRequest.php new file mode 100644 index 00000000..b9038669 --- /dev/null +++ b/backend-server/app/Http/Requests/UploadJsonFilesRequest.php @@ -0,0 +1,18 @@ + 'required', + ]; + } +} diff --git a/backend-server/app/Http/Services/HandleJsonDataSource.php b/backend-server/app/Http/Services/HandleJsonDataSource.php new file mode 100644 index 00000000..f90561ca --- /dev/null +++ b/backend-server/app/Http/Services/HandleJsonDataSource.php @@ -0,0 +1,73 @@ +setChatbotId($this->bot->getId()); + $dataSource->setId(Uuid::uuid4()); + + $files = $this->files; + $filesUrls = []; + $folderName = Str::random(20); + foreach ($files as $file) { + $extension = $file->getClientOriginalExtension(); + if($extension == "json") + { + $fileName = Str::random(20) . '.' . $extension; + // random folder name + try { + $file->storeAs($folderName, $fileName, ['disk' => 'shared_volume']); + $filesUrls[] = $fileName; + } catch (\Exception $e) { + // Handle exception + } + } + else if ($extension == "zip") + { + $zip = new \ZipArchive(); + + $fileName = Str::random(20) . '.' . $extension; + $file->storeAs('', $fileName, ['disk' => 'shared_volume']); + + // Open the ZIP archive + $pathToFile = '/app/shared_data/' . $fileName; + $result = $zip->open($pathToFile); + if ($result === true) { + // Extract all files to the specified directory + $zip->extractTo('/app/shared_data/' . $folderName . "/"); + + // Get the list of file paths + for ($i = 0; $i < $zip->numFiles; $i++) { + $filesUrls[] = $zip->getNameIndex($i); + } + + // Close the ZIP archive + $zip->close(); + } else { + echo "Failed to open the ZIP archive with error code: $result $pathToFile"; + } + } + } + + $dataSource->setFiles($filesUrls); + $dataSource->setFolderName($folderName); + + $dataSource->save(); + return $dataSource; + } +} diff --git a/backend-server/app/Models/Chatbot.php b/backend-server/app/Models/Chatbot.php index f468014b..f0d17e02 100644 --- a/backend-server/app/Models/Chatbot.php +++ b/backend-server/app/Models/Chatbot.php @@ -130,6 +130,11 @@ public function getCodebaseDataSources() return $this->hasMany(CodebaseDataSource::class); } + public function getJsonFilesDataSources() + { + return $this->hasMany(JsonDataSource::class); + } + public function getCreatedAt(): \DateTimeInterface { return $this->created_at; diff --git a/backend-server/app/Models/JsonDataSource.php b/backend-server/app/Models/JsonDataSource.php new file mode 100644 index 00000000..a1f1d3ca --- /dev/null +++ b/backend-server/app/Models/JsonDataSource.php @@ -0,0 +1,66 @@ + 'string', + 'chatbot_id' => 'string', + 'files' => 'array', + ]; + + public function setChatbotId(UuidInterface $chatbotId): void + { + $this->chatbot_id = $chatbotId; + } + + public function getId(): UuidInterface + { + return Uuid::fromString($this->id); + } + + public function getChatbotId(): UuidInterface + { + return Uuid::fromString($this->chatbot_id); + } + + public function setId(UuidInterface $id): void + { + $this->id = $id; + } + + public function setFiles($files): void + { + $this->files = $files; + } + + public function setFolderName($folderName): void + { + $this->folder_name = $folderName; + } + + public function getFolderName(): string + { + return $this->folder_name; + } + + public function getFiles() + { + return $this->files; + } + + public function getCreatedAt(): \DateTimeInterface + { + return $this->created_at; + } +} diff --git a/backend-server/app/Providers/EventServiceProvider.php b/backend-server/app/Providers/EventServiceProvider.php index 378c5ac4..48a035a6 100644 --- a/backend-server/app/Providers/EventServiceProvider.php +++ b/backend-server/app/Providers/EventServiceProvider.php @@ -3,15 +3,17 @@ namespace App\Providers; use App\Http\Events\ChatbotWasCreated; +use App\Http\Events\JsonDataSourceWasAdded; use App\Http\Events\PdfDataSourceWasAdded; use App\Http\Events\WebsiteDataSourceCrawlingWasCompleted; use App\Http\Events\WebsiteDataSourceWasAdded; use App\Http\Events\CodebaseDataSourceWasAdded; use App\Http\Listeners\CreateWebsiteDataSourceIfNeeded; +use App\Http\Listeners\IngestCodebaseDataSource; +use App\Http\Listeners\IngestJsonDataSource; use App\Http\Listeners\IngestPdfDataSource; use App\Http\Listeners\IngestWebsiteDataSource; use App\Http\Listeners\StartRecursiveCrawler; -use App\Http\Listeners\IngestCodebaseDataSource; use Illuminate\Auth\Events\Registered; use Illuminate\Auth\Listeners\SendEmailVerificationNotification; use Illuminate\Foundation\Support\Providers\EventServiceProvider as ServiceProvider; @@ -37,6 +39,9 @@ class EventServiceProvider extends ServiceProvider PdfDataSourceWasAdded::class => [ IngestPdfDataSource::class ], + JsonDataSourceWasAdded::class => [ + IngestJsonDataSource::class + ], WebsiteDataSourceCrawlingWasCompleted::class => [ IngestWebsiteDataSource::class ], diff --git a/backend-server/composer.json b/backend-server/composer.json index 1e2b6dd6..8fe15fa8 100644 --- a/backend-server/composer.json +++ b/backend-server/composer.json @@ -2,7 +2,10 @@ "name": "laravel/laravel", "type": "project", "description": "The Laravel Framework.", - "keywords": ["framework", "laravel"], + "keywords": [ + "framework", + "laravel" + ], "license": "MIT", "require": { "php": "^8.1", diff --git a/backend-server/database/migrations/2023_06_30_222525_create_json_data_sources_table.php b/backend-server/database/migrations/2023_06_30_222525_create_json_data_sources_table.php new file mode 100644 index 00000000..1b025202 --- /dev/null +++ b/backend-server/database/migrations/2023_06_30_222525_create_json_data_sources_table.php @@ -0,0 +1,30 @@ +uuid('id')->primary(); + $table->uuid('chatbot_id'); + $table->json('files'); + $table->string('folder_name')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('json_data_sources'); + } +}; diff --git a/backend-server/resources/views/onboarding/other-data-sources-json.blade.php b/backend-server/resources/views/onboarding/other-data-sources-json.blade.php new file mode 100644 index 00000000..c4c990e7 --- /dev/null +++ b/backend-server/resources/views/onboarding/other-data-sources-json.blade.php @@ -0,0 +1,189 @@ +@extends('layout.app', ['title' => __('Dashboard')]) +@section('content') + + +
+ + + + + +
+
+ +

Upload Json or Zip file as a source ✨

+ + @if ($errors->has('jsonfiles')) +
+
+
+
+ + + +
+
+
+ +
+
+
+ @endif +
+ @csrf +
+ +
+
+
+ ⬆️ +
+

Click to upload or drag & drop + +

+ + you can upload up to +
+ + + +
+ +
+ +
+ +
+
+
Make sure that your files are scannable (text not images) 🫶 +
+
+ You can upload multiple files at once and we will process them in the background. +
+
+
+
+
+ <- + Back + +
+
+ +
+
+
+ +@endsection + +@section('scripts') + + +@endsection diff --git a/backend-server/resources/views/onboarding/step-1.blade.php b/backend-server/resources/views/onboarding/step-1.blade.php index a6026f49..7f7e62c5 100644 --- a/backend-server/resources/views/onboarding/step-1.blade.php +++ b/backend-server/resources/views/onboarding/step-1.blade.php @@ -106,6 +106,19 @@ class="w-full h-full text-left py-3 px-4 rounded bg-white border border-slate-20 +
  • + +
  • +
  • + + + + @endif +
    + @csrf +
    + +
    +
    +
    + ⬆️ +
    +

    Click to upload or drag & drop + +

    + + you can upload up to 5 json files, and we will process the first ±100k words +
    + + + +
    + +
    + +
    + +
    +
    +
    Make sure that your files are + scannable (text not images) 🫶 +
    +
    + You can upload multiple files at once and we will process them in the background. +
    +
    +
    +
    +
    + <- + Back + +
    +
    + + + + + +@endsection + +@section('scripts') + + +@endsection diff --git a/backend-server/resources/views/settings-data.blade.php b/backend-server/resources/views/settings-data.blade.php index 1e12f1ed..3d377c14 100644 --- a/backend-server/resources/views/settings-data.blade.php +++ b/backend-server/resources/views/settings-data.blade.php @@ -436,6 +436,193 @@ class="text-right text-slate-700 font-medium"> @endforeach +
    + @php + /** @var \App\Models\JsonDataSource[] $jsonDataSources */ + @endphp + @foreach($jsonDataSources as $source) +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    + User 01 +
    +
    + Bulk Upload of Json files + +
    +
    +
    +
    + {{count($source->getFiles())}} files scanned +
    +
    + @if($source->getCreatedAt()->diffInMinutes(now()) <= 3) +
    + In progress (eta: {{$source->getCreatedAt()->addMinutes(3)->diffForHumans()}}) +
    + @else +
    + Completed +
    + @endif +
    +
    + + + +
    Re-sync +
    + + + + soon +
    +
    +
    +
    + + +
    + +
    +
    +
    +
    + +
    + @endforeach +
    @php @@ -594,6 +781,17 @@ class="w-full h-full text-left py-3 px-4 rounded bg-white border border-slate-20
  • +
  • + +