Add support for importing JSON files

backend-server/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y \
     unzip \
     git
 
-RUN docker-php-ext-install pdo pdo_mysql
+RUN docker-php-ext-install pdo pdo_mysql zip
 
 # Set the working directory
 WORKDIR /var/www/html

backend-server/app/Http/Controllers/ChatbotController.php

@@ -4,13 +4,16 @@
 
 use App\Http\Events\ChatbotWasCreated;
 use App\Http\Events\CodebaseDataSourceWasAdded;
+use App\Http\Events\JsonDataSourceWasAdded;
 use App\Http\Events\PdfDataSourceWasAdded;
 use App\Http\Requests\CreateChatbotRequest;
 use App\Http\Requests\CreateChatbotViaCodebaseRequest;
+use App\Http\Requests\CreateChatbotViaJsonFlowRequest;
 use App\Http\Requests\CreateChatbotViaPdfFlowRequest;
 use App\Http\Requests\SendChatMessageRequest;
 use App\Http\Requests\UpdateCharacterSettingsRequest;
 use App\Http\Responses\ChatbotResponse;
+use App\Http\Services\HandleJsonDataSource;
 use App\Http\Services\HandlePdfDataSource;
 use App\Models\Chatbot;
 use App\Models\ChatHistory;
@@ -102,6 +105,39 @@ public function createViaPdfFlow(CreateChatbotViaPdfFlowRequest $request): Redir
         return redirect()->route('onboarding.config', ['id' => $chatbot->getId()->toString()]);
     }
 
+    /**
+     * Create a chatbot via Json flow.
+     *
+     * @param  \App\Http\Requests\CreateChatbotViaJsonFlowRequest  $request
+     * @return \Illuminate\Http\RedirectResponse
+     */
+    public function createViaJsonFlow(CreateChatbotViaJsonFlowRequest $request): RedirectResponse
+    {
+        // Create a new Chatbot instance
+        $chatbot = new Chatbot();
+
+        // Set the properties of the chatbot
+        $chatbot->setId(Uuid::uuid4());
+        $chatbot->setName($request->getName());
+        $chatbot->setToken(Str::random(20));
+        $chatbot->setPromptMessage($request->getPromptMessage());
+
+        // Save the chatbot to the database
+        $chatbot->save();
+
+        // Get the JSON files from the request
+        $files = $request->file('jsonfiles');
+
+        // Handle the JSON data source
+        $dataSource = (new HandleJsonDataSource($chatbot, $files))->handle(); // todo this should be moved to an event listener similar to the one in the previous method
+
+        // Trigger the JsonDataSourceWasAdded event
+        event(new JsonDataSourceWasAdded($chatbot->getId(), $dataSource->getId()));
+
+        // Redirect to the onboarding configuration page
+        return redirect()->route('onboarding.config', ['id' => $chatbot->getId()->toString()]);
+    }
+
     /**
      * Update character settings for a chatbot.
      *

backend-server/app/Http/Controllers/ChatbotSettingController.php

@@ -99,12 +99,14 @@ public function dataSettings(Request $request, $id)
         $websiteDataSources = $bot->getWebsiteDataSources()->get();
         $pdfDataSources = $bot->getPdfFilesDataSources()->get();
         $codebaseDataSources = $bot->getCodebaseDataSources()->get();
+        $jsonDataSources = $bot->getJsonFilesDataSources()->get();
 
         return view('settings-data', [
             'bot' => $bot,
             'websiteDataSources' => $websiteDataSources,
             'pdfDataSources' => $pdfDataSources,
             'codebaseDataSources' => $codebaseDataSources,
+            'jsonDataSources' => $jsonDataSources,
         ]);
     }
 

backend-server/app/Http/Controllers/JsonDataSourceController.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Http\Events\JsonDataSourceWasAdded;
+use App\Http\Requests\UploadJsonFilesRequest;
+use App\Http\Services\HandleJsonDataSource;
+use App\Models\Chatbot;
+
+class JsonDataSourceController extends Controller
+{
+    public function create(UploadJsonFilesRequest $request, $id)
+    {
+        /** @var Chatbot $bot */
+        $bot = Chatbot::where('id', $id)->firstOrFail();
+        $files = $request->file('jsonfiles');
+        $dataSource = (new HandleJsonDataSource($bot, $files))->handle();
+        event(new JsonDataSourceWasAdded($bot->getId(), $dataSource->getId()));
+
+        return redirect()->route('chatbot.settings-data', ['id' => $bot->getId()])->with('success', 'Your files have been uploaded successfully, we are training the model now, it should take around 5 minutes to reflect.');
+    }
+
+    public function show($id)
+    {
+        /** @var Chatbot $bot */
+        $bot = Chatbot::where('id', $id)->firstOrFail();
+        $jsonDataSources = $bot->getJsonFilesDataSources()->get();
+        return view('onboarding.other-data-sources-json', ['bot' => $bot, 'jsonDataSources' => $jsonDataSources]);
+    }
+
+}

backend-server/app/Http/Controllers/OnboardingController.php

@@ -26,6 +26,11 @@ public function dataSourcesPdf()
         return view('onboarding.step-2-pdf');
     }
 
+    public function dataSourcesJson()
+    {
+        return view('onboarding.step-2-json');
+    }
+
     public function config()
     {
         return view('onboarding.step-3');

backend-server/app/Http/Events/JsonDataSourceWasAdded.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace App\Http\Events;
+
+use Illuminate\Broadcasting\InteractsWithSockets;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+use Ramsey\Uuid\UuidInterface;
+
+class JsonDataSourceWasAdded
+{
+    use Dispatchable, InteractsWithSockets, SerializesModels;
+
+    public function __construct(
+        private UuidInterface $chatbotId,
+        private UuidInterface $jsonDataSourceId,
+    ) {
+    }
+
+    public function getChatbotId(): UuidInterface
+    {
+        return $this->chatbotId;
+    }
+
+    public function getJsonDataSourceId(): UuidInterface
+    {
+        return $this->jsonDataSourceId;
+    }
+}

backend-server/app/Http/Listeners/IngestJsonDataSource.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace App\Http\Listeners;
+
+use App\Http\Events\JsonDataSourceWasAdded;
+use App\Models\JsonDataSource;
+use Exception;
+use GuzzleHttp\Client;
+use GuzzleHttp\Exception\GuzzleException;
+use Illuminate\Contracts\Queue\ShouldQueue;
+
+class IngestJsonDataSource implements ShouldQueue
+{
+
+    /**
+     * @throws GuzzleException
+     * @throws Exception
+     */
+    public function handle($event)
+    {
+        if (!$event instanceof JsonDataSourceWasAdded) {
+            return;
+        }
+
+        $botId = $event->getChatbotId();
+        $jsonDataSourceId = $event->getJsonDataSourceId();
+
+        /** @var JsonDataSource $jsonDataSource */
+        $jsonDataSource = JsonDataSource::where('id', $jsonDataSourceId)->firstOrFail();
+        $files = $jsonDataSource->getFiles();
+
+        $requestBody = [
+            'type' => 'json',
+            'shared_folder' => $jsonDataSource->getFolderName(),
+            'namespace' => $botId,
+        ];
+
+        // Call to ingest service endpoint
+        $client = new Client();
+        $response = $client->request('POST', "http://llm-server:3000/api/ingest", [
+            'json' => $requestBody,
+        ]);
+
+        if ($response->getStatusCode() !== 200) {
+            throw new Exception('Ingest service returned an error: ' . $response->getBody()->getContents());
+        }
+    }
+}

backend-server/app/Http/Requests/CreateChatbotViaJsonFlowRequest.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace App\Http\Requests;
+
+
+
+use Illuminate\Foundation\Http\FormRequest;
+
+class CreateChatbotViaJsonFlowRequest extends FormRequest
+{
+    public function rules(): array
+    {
+        return [
+            'jsonfiles' => 'required',
+        ];
+    }
+
+    public function getName(): string
+    {
+        return $this->get('name', 'My first chatbot');
+    }
+
+    public function getWebsite(): string
+    {
+        return $this->get('website');
+    }
+
+    public function getPromptMessage(): string
+    {
+        return $this->get('prompt_message', "");
+    }
+
+    public function getFiles()
+    {
+        return $this->get('jsonfiles');
+    }
+}

backend-server/app/Http/Requests/UploadJsonFilesRequest.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Http\Requests;
+
+
+use Illuminate\Foundation\Http\FormRequest;
+use Ramsey\Uuid\Uuid;
+use Ramsey\Uuid\UuidInterface;
+
+class UploadJsonFilesRequest extends FormRequest
+{
+    public function rules(): array
+    {
+        return [
+          'jsonfiles' => 'required',
+        ];
+    }
+}

backend-server/app/Http/Services/HandleJsonDataSource.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace App\Http\Services;
+
+use App\Http\Requests\UploadJsonFilesRequest;
+use App\Models\Chatbot;
+use App\Models\JsonDataSource;
+use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Str;
+use Ramsey\Uuid\Uuid;
+
+class HandleJsonDataSource
+{
+    public function __construct(private readonly Chatbot $bot, private $files)
+    {
+    }
+
+    public function handle(): JsonDataSource
+    {
+        $dataSource = new JsonDataSource();
+        $dataSource->setChatbotId($this->bot->getId());
+        $dataSource->setId(Uuid::uuid4());
+
+        $files = $this->files;
+        $filesUrls = [];
+        $folderName = Str::random(20);
+        foreach ($files as $file) {
+            $extension = $file->getClientOriginalExtension();
+            if($extension == "json")
+            {
+                $fileName = Str::random(20) . '.' . $extension;
+                // random folder name
+                try {
+                    $file->storeAs($folderName, $fileName, ['disk' => 'shared_volume']);
+                    $filesUrls[] = $fileName;
+                } catch (\Exception $e) {
+                    // Handle exception
+                }
+            }
+            else if ($extension == "zip")
+            {
+                $zip = new \ZipArchive();
+
+                $fileName = Str::random(20) . '.' . $extension;
+                $file->storeAs('', $fileName, ['disk' => 'shared_volume']);
+
+                // Open the ZIP archive
+                $pathToFile = '/app/shared_data/' . $fileName;
+                $result = $zip->open($pathToFile);
+                if ($result === true) {
+                    // Extract all files to the specified directory
+                    $zip->extractTo('/app/shared_data/' . $folderName . "/");
+
+                    // Get the list of file paths
+                    for ($i = 0; $i < $zip->numFiles; $i++) {
+                        $filesUrls[] = $zip->getNameIndex($i);
+                    }
+
+                    // Close the ZIP archive
+                    $zip->close();
+                } else {
+                    echo "Failed to open the ZIP archive with error code: $result $pathToFile";
+                }
+            }
+        }
+
+        $dataSource->setFiles($filesUrls);
+        $dataSource->setFolderName($folderName);
+
+        $dataSource->save();
+        return $dataSource;
+    }
+}

backend-server/app/Models/Chatbot.php

@@ -130,6 +130,11 @@ public function getCodebaseDataSources()
         return $this->hasMany(CodebaseDataSource::class);
     }
 
+    public function getJsonFilesDataSources()
+    {
+        return $this->hasMany(JsonDataSource::class);
+    }
+
     public function getCreatedAt(): \DateTimeInterface
     {
         return $this->created_at;