Skip to content

Commit ad0e30b

Browse files
mudlermudler
authored
refactor: move backends into the backends directory (#1279)
* refactor: move backends into the backends directory Signed-off-by: Ettore Di Giacinto <[email protected]> * refactor: move main close to implementation for every backend Signed-off-by: Ettore Di Giacinto <[email protected]> --------- Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 5546118 commit ad0e30b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+156
-190
lines changed

‎.github/workflows/test.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ jobs:
7878
sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
7979
8080
sudo rm -rfv /usr/bin/conda || true
81-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/huggingface
81+
PATH=$PATH:/opt/conda/bin make -C backend/python/huggingface
8282
8383
# Pre-build piper before we start tests in order to have shared libraries in place
8484
make go-piper && \

‎Dockerfile

+9-19
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ ARG TARGETARCH
1212
ARG TARGETVARIANT
1313

1414
ENV BUILD_TYPE=${BUILD_TYPE}
15-
ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/run.sh,autogptq:/build/extra/grpc/autogptq/run.sh,bark:/build/extra/grpc/bark/run.sh,diffusers:/build/extra/grpc/diffusers/run.sh,exllama:/build/extra/grpc/exllama/run.sh,vall-e-x:/build/extra/grpc/vall-e-x/run.sh,vllm:/build/extra/grpc/vllm/run.sh"
15+
ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/backend/python/huggingface/run.sh,autogptq:/build/backend/python/autogptq/run.sh,bark:/build/backend/python/bark/run.sh,diffusers:/build/backend/python/diffusers/run.sh,exllama:/build/backend/python/exllama/run.sh,vall-e-x:/build/backend/python/vall-e-x/run.sh,vllm:/build/backend/python/vllm/run.sh"
1616
ENV GALLERIES='[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/index.yaml"}, {"url": "github:go-skynet/model-gallery/huggingface.yaml","name":"huggingface"}]'
1717
ARG GO_TAGS="stablediffusion tts"
1818

@@ -64,20 +64,10 @@ RUN curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmo
6464
apt-get update && \
6565
apt-get install -y conda
6666

67-
COPY extra/requirements.txt /build/extra/requirements.txt
6867
ENV PATH="/root/.cargo/bin:${PATH}"
6968
RUN pip install --upgrade pip
7069
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
71-
#RUN if [ "${TARGETARCH}" = "amd64" ]; then \
72-
# pip install git+https://github.com/suno-ai/bark.git diffusers invisible_watermark transformers accelerate safetensors;\
73-
# fi
74-
#RUN if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "amd64" ]; then \
75-
# pip install torch vllm && pip install auto-gptq https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu${CUDA_MAJOR_VERSION}${CUDA_MINOR_VERSION}-cp39-cp39-linux_x86_64.whl;\
76-
# fi
77-
#RUN pip install -r /build/extra/requirements.txt && rm -rf /build/extra/requirements.txt
7870

79-
# Vall-e-X
80-
RUN git clone https://github.com/Plachtaa/VALL-E-X.git /usr/lib/vall-e-x && cd /usr/lib/vall-e-x && pip install -r requirements.txt
8171

8272
# \
8373
# ; fi
@@ -162,25 +152,25 @@ COPY --from=builder /build/backend-assets/grpc/stablediffusion ./backend-assets/
162152

163153
## Duplicated from Makefile to avoid having a big layer that's hard to push
164154
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
165-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/autogptq \
155+
PATH=$PATH:/opt/conda/bin make -C backend/python/autogptq \
166156
; fi
167157
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
168-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/bark \
158+
PATH=$PATH:/opt/conda/bin make -C backend/python/bark \
169159
; fi
170160
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
171-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/diffusers \
161+
PATH=$PATH:/opt/conda/bin make -C backend/python/diffusers \
172162
; fi
173163
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
174-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/vllm \
164+
PATH=$PATH:/opt/conda/bin make -C backend/python/vllm \
175165
; fi
176166
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
177-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/huggingface \
167+
PATH=$PATH:/opt/conda/bin make -C backend/python/huggingface \
178168
; fi
179169
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
180-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/vall-e-x \
170+
PATH=$PATH:/opt/conda/bin make -C backend/python/vall-e-x \
181171
; fi
182172
RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
183-
PATH=$PATH:/opt/conda/bin make -C extra/grpc/exllama \
173+
PATH=$PATH:/opt/conda/bin make -C backend/python/exllama \
184174
; fi
185175

186176
# Copy VALLE-X as it's not a real "lib"
@@ -190,7 +180,7 @@ RUN if [ -d /usr/lib/vall-e-x ]; then \
190180

191181
# we also copy exllama libs over to resolve exllama import error
192182
RUN if [ -d /usr/local/lib/python3.9/dist-packages/exllama ]; then \
193-
cp -rfv /usr/local/lib/python3.9/dist-packages/exllama extra/grpc/exllama/;\
183+
cp -rfv /usr/local/lib/python3.9/dist-packages/exllama backend/python/exllama/;\
194184
fi
195185

196186
# Define the health check command

‎Makefile

+33-33
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ test: prepare test-models/testmodel grpcs
306306
@echo 'Running tests'
307307
export GO_TAGS="tts stablediffusion"
308308
$(MAKE) prepare-test
309-
HUGGINGFACE_GRPC=$(abspath ./)/extra/grpc/huggingface/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \
309+
HUGGINGFACE_GRPC=$(abspath ./)/backend/python/huggingface/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \
310310
$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="!gpt4all && !llama && !llama-gguf" --flake-attempts 5 --fail-fast -v -r ./api ./pkg
311311
$(MAKE) test-gpt4all
312312
$(MAKE) test-llama
@@ -374,27 +374,27 @@ protogen: protogen-go protogen-python
374374

375375
protogen-go:
376376
protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
377-
pkg/grpc/proto/backend.proto
377+
backend/backend.proto
378378

379379
protogen-python:
380-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/huggingface/ --grpc_python_out=extra/grpc/huggingface/ pkg/grpc/proto/backend.proto
381-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/autogptq/ --grpc_python_out=extra/grpc/autogptq/ pkg/grpc/proto/backend.proto
382-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/exllama/ --grpc_python_out=extra/grpc/exllama/ pkg/grpc/proto/backend.proto
383-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/bark/ --grpc_python_out=extra/grpc/bark/ pkg/grpc/proto/backend.proto
384-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/diffusers/ --grpc_python_out=extra/grpc/diffusers/ pkg/grpc/proto/backend.proto
385-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/vall-e-x/ --grpc_python_out=extra/grpc/vall-e-x/ pkg/grpc/proto/backend.proto
386-
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/vllm/ --grpc_python_out=extra/grpc/vllm/ pkg/grpc/proto/backend.proto
380+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/huggingface/ --grpc_python_out=backend/python/huggingface/ backend/backend.proto
381+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/autogptq/ --grpc_python_out=backend/python/autogptq/ backend/backend.proto
382+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/exllama/ --grpc_python_out=backend/python/exllama/ backend/backend.proto
383+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/bark/ --grpc_python_out=backend/python/bark/ backend/backend.proto
384+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/diffusers/ --grpc_python_out=backend/python/diffusers/ backend/backend.proto
385+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/vall-e-x/ --grpc_python_out=backend/python/vall-e-x/ backend/backend.proto
386+
python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/vllm/ --grpc_python_out=backend/python/vllm/ backend/backend.proto
387387

388388
## GRPC
389389
# Note: it is duplicated in the Dockerfile
390390
prepare-extra-conda-environments:
391-
$(MAKE) -C extra/grpc/autogptq
392-
$(MAKE) -C extra/grpc/bark
393-
$(MAKE) -C extra/grpc/diffusers
394-
$(MAKE) -C extra/grpc/vllm
395-
$(MAKE) -C extra/grpc/huggingface
396-
$(MAKE) -C extra/grpc/vall-e-x
397-
$(MAKE) -C extra/grpc/exllama
391+
$(MAKE) -C backend/python/autogptq
392+
$(MAKE) -C backend/python/bark
393+
$(MAKE) -C backend/python/diffusers
394+
$(MAKE) -C backend/python/vllm
395+
$(MAKE) -C backend/python/huggingface
396+
$(MAKE) -C backend/python/vall-e-x
397+
$(MAKE) -C backend/python/exllama
398398

399399

400400
backend-assets/grpc:
@@ -403,7 +403,7 @@ backend-assets/grpc:
403403
backend-assets/grpc/llama: backend-assets/grpc go-llama/libbinding.a
404404
$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama
405405
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama LIBRARY_PATH=$(shell pwd)/go-llama \
406-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama ./cmd/grpc/llama/
406+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama ./backend/go/llm/llama/
407407
# TODO: every binary should have its own folder instead, so can have different metal implementations
408408
ifeq ($(BUILD_TYPE),metal)
409409
cp go-llama/build/bin/ggml-metal.metal backend-assets/grpc/
@@ -443,68 +443,68 @@ endif
443443
backend-assets/grpc/llama-stable: backend-assets/grpc go-llama-stable/libbinding.a
444444
$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama-stable
445445
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama-stable LIBRARY_PATH=$(shell pwd)/go-llama \
446-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-stable ./cmd/grpc/llama-stable/
446+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-stable ./backend/go/llm/llama-stable/
447447

448448
backend-assets/grpc/gpt4all: backend-assets/grpc backend-assets/gpt4all gpt4all/gpt4all-bindings/golang/libgpt4all.a
449449
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/gpt4all/gpt4all-bindings/golang/ LIBRARY_PATH=$(shell pwd)/gpt4all/gpt4all-bindings/golang/ \
450-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt4all ./cmd/grpc/gpt4all/
450+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt4all ./backend/go/llm/gpt4all/
451451

452452
backend-assets/grpc/dolly: backend-assets/grpc go-ggml-transformers/libtransformers.a
453453
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
454-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/dolly ./cmd/grpc/dolly/
454+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/dolly ./backend/go/llm/dolly/
455455

456456
backend-assets/grpc/gpt2: backend-assets/grpc go-ggml-transformers/libtransformers.a
457457
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
458-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt2 ./cmd/grpc/gpt2/
458+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt2 ./backend/go/llm/gpt2/
459459

460460
backend-assets/grpc/gptj: backend-assets/grpc go-ggml-transformers/libtransformers.a
461461
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
462-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptj ./cmd/grpc/gptj/
462+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptj ./backend/go/llm/gptj/
463463

464464
backend-assets/grpc/gptneox: backend-assets/grpc go-ggml-transformers/libtransformers.a
465465
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
466-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptneox ./cmd/grpc/gptneox/
466+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptneox ./backend/go/llm/gptneox/
467467

468468
backend-assets/grpc/mpt: backend-assets/grpc go-ggml-transformers/libtransformers.a
469469
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
470-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/mpt ./cmd/grpc/mpt/
470+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/mpt ./backend/go/llm/mpt/
471471

472472
backend-assets/grpc/replit: backend-assets/grpc go-ggml-transformers/libtransformers.a
473473
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
474-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/replit ./cmd/grpc/replit/
474+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/replit ./backend/go/llm/replit/
475475

476476
backend-assets/grpc/falcon-ggml: backend-assets/grpc go-ggml-transformers/libtransformers.a
477477
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
478-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/falcon-ggml ./cmd/grpc/falcon-ggml/
478+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/falcon-ggml ./backend/go/llm/falcon-ggml/
479479

480480
backend-assets/grpc/starcoder: backend-assets/grpc go-ggml-transformers/libtransformers.a
481481
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
482-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/starcoder ./cmd/grpc/starcoder/
482+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/starcoder ./backend/go/llm/starcoder/
483483

484484
backend-assets/grpc/rwkv: backend-assets/grpc go-rwkv/librwkv.a
485485
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-rwkv LIBRARY_PATH=$(shell pwd)/go-rwkv \
486-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./cmd/grpc/rwkv/
486+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./backend/go/llm/rwkv
487487

488488
backend-assets/grpc/bert-embeddings: backend-assets/grpc go-bert/libgobert.a
489489
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-bert LIBRARY_PATH=$(shell pwd)/go-bert \
490-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bert-embeddings ./cmd/grpc/bert-embeddings/
490+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bert-embeddings ./backend/go/llm/bert/
491491

492492
backend-assets/grpc/langchain-huggingface: backend-assets/grpc
493-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./cmd/grpc/langchain-huggingface/
493+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./backend/go/llm/langchain/
494494

495495
backend-assets/grpc/stablediffusion: backend-assets/grpc
496496
if [ ! -f backend-assets/grpc/stablediffusion ]; then \
497497
$(MAKE) go-stable-diffusion/libstablediffusion.a; \
498498
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-stable-diffusion/ LIBRARY_PATH=$(shell pwd)/go-stable-diffusion/ \
499-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./cmd/grpc/stablediffusion/; \
499+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./backend/go/image/; \
500500
fi
501501

502502
backend-assets/grpc/piper: backend-assets/grpc backend-assets/espeak-ng-data go-piper/libpiper_binding.a
503503
CGO_CXXFLAGS="$(PIPER_CGO_CXXFLAGS)" CGO_LDFLAGS="$(PIPER_CGO_LDFLAGS)" LIBRARY_PATH=$(shell pwd)/go-piper \
504-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./cmd/grpc/piper/
504+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./backend/go/tts/
505505

506506
backend-assets/grpc/whisper: backend-assets/grpc whisper.cpp/libwhisper.a
507507
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/whisper.cpp LIBRARY_PATH=$(shell pwd)/whisper.cpp \
508-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./cmd/grpc/whisper/
508+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./backend/go/transcribe/
509509

510510
grpcs: prepare $(GRPC_BACKENDS)

‎pkg/grpc/proto/backend.proto ‎backend/backend.proto

File renamed without changes.

‎backend/cpp/llama/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ include_directories(${Protobuf_INCLUDE_DIRS})
3636
message(STATUS "Using protobuf version ${Protobuf_VERSION} | Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS} | CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}")
3737

3838
# Proto file
39-
get_filename_component(hw_proto "../../../../../../pkg/grpc/proto/backend.proto" ABSOLUTE)
39+
get_filename_component(hw_proto "../../../../../../backend/backend.proto" ABSOLUTE)
4040
get_filename_component(hw_proto_path "${hw_proto}" PATH)
4141

4242
# Generated sources

‎cmd/grpc/bert-embeddings/main.go ‎backend/go/image/main.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ package main
55
import (
66
"flag"
77

8-
bert "github.com/go-skynet/LocalAI/pkg/backend/llm/bert"
98
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
109
)
1110

@@ -16,7 +15,7 @@ var (
1615
func main() {
1716
flag.Parse()
1817

19-
if err := grpc.StartServer(*addr, &bert.Embeddings{}); err != nil {
18+
if err := grpc.StartServer(*addr, &StableDiffusion{}); err != nil {
2019
panic(err)
2120
}
2221
}

‎pkg/backend/image/stablediffusion.go ‎backend/go/image/stablediffusion.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package image
1+
package main
22

33
// This is a wrapper to statisfy the GRPC service interface
44
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)

‎pkg/backend/llm/bert/bert.go ‎backend/go/llm/bert/bert.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package bert
1+
package main
22

33
// This is a wrapper to statisfy the GRPC service interface
44
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)

‎cmd/grpc/rwkv/main.go ‎backend/go/llm/bert/main.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ package main
55
import (
66
"flag"
77

8-
rwkv "github.com/go-skynet/LocalAI/pkg/backend/llm/rwkv"
9-
108
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
119
)
1210

@@ -17,7 +15,7 @@ var (
1715
func main() {
1816
flag.Parse()
1917

20-
if err := grpc.StartServer(*addr, &rwkv.LLM{}); err != nil {
18+
if err := grpc.StartServer(*addr, &Embeddings{}); err != nil {
2119
panic(err)
2220
}
2321
}

‎cmd/grpc/dolly/main.go ‎backend/go/llm/dolly/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ package main
55
import (
66
"flag"
77

8-
transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
8+
transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
99

1010
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
1111
)

‎cmd/grpc/falcon-ggml/main.go ‎backend/go/llm/falcon-ggml/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ package main
55
import (
66
"flag"
77

8-
transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
8+
transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
99

1010
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
1111
)

‎cmd/grpc/gpt2/main.go ‎backend/go/llm/gpt2/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ package main
55
import (
66
"flag"
77

8-
transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
8+
transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
99

1010
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
1111
)

‎pkg/backend/llm/gpt4all/gpt4all.go ‎backend/go/llm/gpt4all/gpt4all.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package gpt4all
1+
package main
22

33
// This is a wrapper to statisfy the GRPC service interface
44
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)

‎cmd/grpc/gpt4all/main.go ‎backend/go/llm/gpt4all/main.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ package main
55
import (
66
"flag"
77

8-
gpt4all "github.com/go-skynet/LocalAI/pkg/backend/llm/gpt4all"
9-
108
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
119
)
1210

@@ -17,7 +15,7 @@ var (
1715
func main() {
1816
flag.Parse()
1917

20-
if err := grpc.StartServer(*addr, &gpt4all.LLM{}); err != nil {
18+
if err := grpc.StartServer(*addr, &LLM{}); err != nil {
2119
panic(err)
2220
}
2321
}

‎cmd/grpc/gptj/main.go ‎backend/go/llm/gptj/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ package main
55
import (
66
"flag"
77

8-
transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
8+
transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
99

1010
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
1111
)

‎cmd/grpc/gptneox/main.go ‎backend/go/llm/gptneox/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ package main
55
import (
66
"flag"
77

8-
transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
8+
transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
99

1010
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
1111
)

‎pkg/backend/llm/langchain/langchain.go ‎backend/go/llm/langchain/langchain.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package langchain
1+
package main
22

33
// This is a wrapper to statisfy the GRPC service interface
44
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)

‎backend/go/llm/langchain/main.go

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package main
2+
3+
// Note: this is started internally by LocalAI and a server is allocated for each model
4+
5+
import (
6+
"flag"
7+
8+
grpc "github.com/go-skynet/LocalAI/pkg/grpc"
9+
)
10+
11+
var (
12+
addr = flag.String("addr", "localhost:50051", "the address to connect to")
13+
)
14+
15+
func main() {
16+
flag.Parse()
17+
18+
if err := grpc.StartServer(*addr, &LLM{}); err != nil {
19+
panic(err)
20+
}
21+
}

0 commit comments

Comments
 (0)