diff --git a/dj_backend_server/Makefile b/dj_backend_server/Makefile index 42392f9e..93c3dcfd 100644 --- a/dj_backend_server/Makefile +++ b/dj_backend_server/Makefile @@ -1,5 +1,17 @@ # Makefile to Run docker-compose for Django App +# first run "make preinstall" to create .env file and venv environment +# you will need to edit .env file, it will be created from example.env +# after .env is done, run "make install" to create run the nginx and docker-compose. +# doublecheck if the nginx.conf is correct, if not, edit it and run "make install" again +# if you want to run the app in development mode, run "make dev-start" +# to stop the app, run "make dev-stop" +# to run migrations, run "make force_migrate" +# to see the logs, run "make logs" +# to enter the python container, run "make exec" +# to restart the python and celery containers, run "make restart" +# to stop the app, run "make down" (in dev mode, run "make dev-stop") + # Check if Docker and Docker Compose are installed DOCKER := $(shell command -v docker 2> /dev/null) DOCKER_COMPOSE := $(shell command -v docker-compose 2> /dev/null) @@ -13,45 +25,50 @@ ifndef DOCKER_COMPOSE $(error $(shell tput setaf 1)"Docker Compose is not installed. Please install Docker Compose before proceeding."$(shell tput sgr0)) endif -.env.docker: - @echo $(shell tput setaf 1)"Error: The .env.docker file is missing. Please create it before proceeding. Refer example.env.docker or readme file in dj_backend_server/readme.md"$(shell tput sgr0) - exit 1 +.env: + @if [ ! -f .env ]; then \ + echo "\033[1;31mError: The .env file is missing.\033[0m"; \ + if [ -f example.env ]; then \ + echo "An example.env file has been found."; \ + read -p "Do you want to copy example.env to .env? [y/N] " yn; \ + case $$yn in \ + [Yy]*) cp example.env .env; \ + echo "The .env file has been created from example.env."; \ + echo "Please edit the .env file and set the values for OPENAI_API_KEY and APP_URL.";; \ + *) echo "Exiting. Please create a .env file manually."; exit 1;; \ + esac; \ + else \ + echo "No example.env file found. Please create a .env file manually."; \ + exit 1; \ + fi; \ + fi venv: -ifndef venv - ifeq ($(OS), Darwin) - @echo $(shell tput setaf 2)"Creating a virtual environment..."$(shell tput sgr0) - python3 -m venv venv - else - @echo $(shell tput setaf 2)"Creating a virtual environment..."$(shell tput sgr0) - python3 -m venv venv - endif -endif - -activate-venv: - @echo $$(tput setaf 3) "Activating virtual environment..." $$(tput sgr0) - . venv/bin/activate - - -install-requirements: activate-venv - @echo $$(tput setaf 2) "Installing Python dependencies..." $$(tput sgr0) - pip install -r requirements.txt + @if [ ! -d "venv" ]; then \ + echo "\033[1;32mCreating a virtual environment...\033[0m"; \ + python3 -m venv venv; \ + else \ + echo "Virtual environment already exists."; \ + fi -install: .env install-requirements - $(DOCKER_COMPOSE) up -d +install-requirements: venv + @echo "Installing requirements..." + @venv/bin/pip install -r requirements.txt nginx: - if [ ! -f .env.docker ]; then \ - echo $$(tput setaf 1)"Error: .env.docker file not found. Please create it before proceeding."$$(tput sgr0); \ + @if [ ! -f .env ]; then \ + echo $$(tput setaf 1)"Error: .env file not found. Please create it before proceeding."$$(tput sgr0); \ exit 1; \ - fi; \ - export APP_URL=$$(grep APP_URL .env.docker | cut -d '=' -f2- | sed -e "s#http[s]\?://##" -e "s#'##g"); \ + fi + @export APP_URL=$$(grep APP_URL .env | cut -d '=' -f2- | sed -e "s#http[s]\?://##" -e "s#'##g"); \ if [ -z "$$APP_URL" ]; then \ - echo $$(tput setaf 1)"Error: APP_URL is not set in .env.docker. Please set it before proceeding."$$(tput sgr0); \ + echo $$(tput setaf 1)"Error: APP_URL is not set in .env. Please set it before proceeding."$$(tput sgr0); \ exit 1; \ - fi; \ - envsubst '$${APP_URL}' < nginx/nginx.template.conf > nginx/nginx.conf; \ - echo $$(tput setaf 2) "Installing NGINX conf file with APP_URL $$APP_URL"$$(tput sgr0) + fi + @if [ ! -f nginx/nginx.conf ]; then \ + envsubst '$${APP_URL}' < nginx/nginx.template.conf > nginx/nginx.conf; \ + echo $$(tput setaf 2) "Installing NGINX conf file with APP_URL $$APP_URL"$$(tput sgr0); \ + fi down: $(DOCKER_COMPOSE) down --remove-orphans @@ -72,6 +89,11 @@ else COMPOSE_FILE := docker-compose.linux.yaml endif +install: nginx + $(DOCKER_COMPOSE) up -d + +pre-install: .env venv install-requirements + # celery -A dj_backend_server worker --loglevel=info & dev-start: $(DOCKER_COMPOSE) -f docker-compose.linux.yaml up -d @@ -89,9 +111,6 @@ force_migrate: @echo $(shell tput setaf 2)"Running migrations inside the Docker container..." $(DOCKER) exec -it web python manage.py makemigrations web $(DOCKER) exec -it web python manage.py migrate - -all: venv install-requirements install down dev-start dev-stop - @echo $$(tput setaf 3)"All steps executed successfully."$$(tput sgr0) logs: $(DOCKER_COMPOSE) logs -f @@ -104,4 +123,4 @@ restart: $(DOCKER) restart oc_celery make logs -.PHONY: venv install-requirements install down dev-start dev-stop nginx \ No newline at end of file +.PHONY: install .env venv activate-venv install-requirements down dev-start dev-stop nginx diff --git a/dj_backend_server/docker-compose.linux.yaml b/dj_backend_server/docker-compose.linux.yaml index 388bee19..73c002e7 100644 --- a/dj_backend_server/docker-compose.linux.yaml +++ b/dj_backend_server/docker-compose.linux.yaml @@ -17,7 +17,7 @@ services: volumes: - database:/var/lib/mysql healthcheck: - test: "exit 0" + test: "exit 0" networks: - openchat_network @@ -74,7 +74,7 @@ services: condition: service_healthy env_file: - .env - entrypoint: ["/entrypoint.sh"] + entrypoint: [ "/entrypoint.sh" ] command: > sh -c "python manage.py migrate auth && python manage.py migrate && @@ -90,25 +90,25 @@ services: ports: - "8080:8080" environment: - ADMINER_DEFAULT_SERVER: db # Change this to match your database server name + ADMINER_DEFAULT_SERVER: db # Change this to match your database server name depends_on: - mysql networks: - openchat_network - + celery_worker: build: context: . dockerfile: Dockerfile container_name: oc_celery volumes: - - ./website_data_sources:/app/website_data_sources + - ./website_data_sources:/app/website_data_sources - ./:/app/ # - ./llama-2-7b-chat.ggmlv3.q4_K_M.bin:/app/llama-2-7b-chat.ggmlv3.q4_K_M.bin:ro depends_on: - web env_file: - - .env.docker + - .env command: celery -A dj_backend_server worker --loglevel=info networks: - openchat_network @@ -124,14 +124,15 @@ services: networks: - openchat_network healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: [ "CMD", "redis-cli", "ping" ] interval: 30s timeout: 10s - retries: 5 + retries: 5 networks: openchat_network: + volumes: qdrant_storage: database: diff --git a/dj_backend_server/docker-compose.yaml b/dj_backend_server/docker-compose.yaml index c348fdde..bf2d58ab 100644 --- a/dj_backend_server/docker-compose.yaml +++ b/dj_backend_server/docker-compose.yaml @@ -32,8 +32,7 @@ services: build: context: . dockerfile: Dockerfile - container_name: - web + container_name: web ports: - "8001:8000" volumes: @@ -42,7 +41,7 @@ services: depends_on: - mysql env_file: - - .env.docker + - .env command: > sh -c "python manage.py migrate auth && python manage.py migrate && @@ -57,12 +56,12 @@ services: ports: - "8080:8080" environment: - ADMINER_DEFAULT_SERVER: db # Change this to match your database server name + ADMINER_DEFAULT_SERVER: db # Change this to match your database server name depends_on: - mysql networks: - openchat_network - + celery_worker: build: context: . @@ -74,7 +73,7 @@ services: depends_on: - web env_file: - - .env.docker + - .env command: celery -A dj_backend_server worker --loglevel=info networks: - openchat_network @@ -88,7 +87,7 @@ services: - redis_data:/data networks: - openchat_network - + nginx: image: nginx container_name: oc_nginx @@ -109,10 +108,11 @@ services: - web - redis - celery_worker - + networks: openchat_network: + volumes: qdrant_storage: database: diff --git a/dj_backend_server/example.env.docker b/dj_backend_server/example.env.docker deleted file mode 100644 index 09d4a75f..00000000 --- a/dj_backend_server/example.env.docker +++ /dev/null @@ -1,72 +0,0 @@ -# "azure" | "openai" | llama2 -OPENAI_API_TYPE=openai -OPENAI_API_MODEL=gpt-3.5-turbo -OPENAI_API_TEMPERATURE=1 - -# If using azure -# AZURE_OPENAI_API_BASE= -# AZURE_OPENAI_API_KEY= -# AZURE_OPENAI_API_VERSION=2023-03-15-preview -# AZURE_OPENAI_EMBEDDING_MODEL_NAME= -# AZURE_OPENAI_DEPLOYMENT_NAME= -# AZURE_OPENAI_COMPLETION_MODEL=gpt-35-turbo - - -# For openai -OPENAI_API_KEY= - - - -# "azure" | "openai" | llama2 -EMBEDDING_PROVIDER=openai - -# Vector Store, PINECONE|QDRANT -STORE=QDRANT - - -# if using pinecone -# PINECONE_API_KEY= -# PINECONE_ENV= -# VECTOR_STORE_INDEX_NAME= - - -# if using qdrant -QDRANT_URL=http://qdrant:6333 - - -# optional, defaults to 15 -MAX_PAGES_CRAWL=15 - -# --- these will change if you decide to start testing the software -CELERY_BROKER_URL=redis://redis:6379/0 -CELERY_RESULT_BACKEND=redis://redis:6379/0 -DATABASE_NAME=openchat -DATABASE_USER=dbuser -DATABASE_PASSWORD=dbpass -DATABASE_HOST=mysql -DATABASE_PORT=3306 - -# retrieval_qa | conversation_retrieval, retrieval_qa works better with azure openai - -# Add Allowed Hosts here, no quote, just IP or domain, separated by a comma -ALLOWED_HOSTS=localhost,0.0.0.0 -APP_URL=http://0.0.0.0:8000 - -# use 'external' if you want to use below services. -PDF_LIBRARY = 'internal' - -#PDF API - OCRWebService.com (REST API). https://www.ocrwebservice.com/api/restguide -#Extract text from scanned images and PDF documents and convert into editable formats. -#Please create new account with ocrwebservice.com via http://www.ocrwebservice.com/account/signup and get license code -OCR_LICCODE = 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXX' -OCR_USERNAME = 'username' -OCR_LANGUAGE = 'english' -# Advantage to clean up the OCR text which can be messy and full with garbage, but will generate a cost with LLM if is paid. Use carefully. -# Use 1 to enable, 0 to disable. -OCR_LLM = 0 - -# Replace in Chat JS and Search JS english language strings with these (use your own language) -LNG_BOT='Bot is Thinking...' -LNG_ERROR='Error sending the message.' -LNG_WRITE='Ask a question...' -LNG_ASK='Write a reply...' \ No newline at end of file diff --git a/docs/django_release.md b/docs/django_release.md index b95bb3ab..ebe0b626 100644 --- a/docs/django_release.md +++ b/docs/django_release.md @@ -27,11 +27,42 @@ To ensure the application works seamlessly with your environment, you'll need to 1. switch to python directory, `cd dj_backend_server` To initiate the app, execute the following command: +First run "make preinstall" to create .env file and venv environment + +```bash +make preinstall +``` + +You will need to edit .env file, it will be created from example.env +After .env is done, run "make install" to create run the nginx and docker-compose. +Doublecheck if the nginx.conf is correct, if not, edit it and run "make install" again + ```bash make install ``` -To stop the server, use: +If you want to run the app in development mode, run "make dev-start" and to stop the app, run "make dev-stop" + +To see the logs, run "make logs" + +```bash +make logs +``` + +To enter the python container, run "make exec" + +```bash +make exec +``` + +To restart the python and celery containers, run "make restart" + +```bash +make restart +``` + +To stop the server run "make down" (in dev mode, run "make dev-stop"), use: + ```bash make down ```