-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathMakefile
243 lines (192 loc) · 9.03 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
PROJECT := dateilager
VERSION := $(shell git describe --tags --abbrev=0)
BUILD_FLAGS := -ldflags="-s -w -X github.com/gadget-inc/dateilager/pkg/version.Version=$(VERSION)"
DB_HOST ?= 127.0.0.1
DB_USER ?= postgres
DB_PASS ?= password
DB_URI := postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl
GRPC_PORT ?= 5051
GRPC_SERVER ?= localhost:$(GRPC_PORT)
DEV_TOKEN_ADMIN ?= v2.public.eyJzdWIiOiJhZG1pbiIsImlhdCI6IjIwMjEtMTAtMTVUMTE6MjA6MDAuMDM0WiJ9WtEey8KfQQRy21xoHq1C5KQatEevk8RxS47k4bRfMwVCPHumZmVuk6ADcfDHTmSnMtEGfFXdxnYOhRP6Clb_Dw
DEV_TOKEN_PROJECT_1 ?= v2.public.eyJzdWIiOiIxIiwiaWF0IjoiMjAyMS0xMC0xNVQxMToyMDowMC4wMzVaIn2MQ14RfIGpoEycCuvRu9J3CZp6PppUXf5l5w8uKKydN3C31z6f6GgOEPNcnwODqBnX7Pjarpz4i2uzWEqLgQYD
PKG_GO_FILES := $(shell find pkg/ -type f -name '*.go')
INTERNAL_GO_FILES := $(shell find internal/ -type f -name '*.go')
PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto')
PROTO_TS_FILES := $(shell find js/src/pb -type f -name '*.ts')
MIGRATE_DIR := ./migrations
SERVICE := $(PROJECT).server
.PHONY: install migrate migrate-create clean build release
.PHONY: test test-one test-fuzz test-js lint-js typecheck-js
.PHONY: reset-db setup-local server server-profile js-install
.PHONY: client-update client-large-update client-get client-rebuild client-pack
.PHONY: client-gc-contents client-gc-project client-gc-random-projects
.PHONY: webui health upload-container-image gen-docs
.PHONY: load-test-new load-test-get load-test-update
install:
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]
go install github.com/grpc-ecosystem/[email protected]
go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/[email protected]
go install github.com/bojand/ghz/cmd/[email protected]
go install github.com/gadget-inc/fsdiff/cmd/[email protected]
go install github.com/stamblerre/gocode@latest
go install golang.org/x/tools/cmd/goimports@latest
cd js && npm ci
migrate:
migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up
migrate-create:
ifndef name
$(error name variable must be set)
else
mkdir -p $(MIGRATE_DIR)
migrate create -ext sql -dir $(MIGRATE_DIR) -seq $(name)
endif
clean:
rm -f bin/*
rm -f release/*
rm -f internal/pb/*.pb.go
rm -rf js/dist
rm -rf js/node_modules
rm -rf js/src/pb
rm -rf input
internal/pb/%.pb.go: internal/pb/%.proto
protoc --experimental_allow_proto3_optional --go_out=. --go_opt=paths=source_relative $^
internal/pb/%_grpc.pb.go: internal/pb/%.proto
protoc --experimental_allow_proto3_optional --go-grpc_out=. --go-grpc_opt=paths=source_relative $^
bin/%: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 go build $(BUILD_FLAGS) -o $@ $<
js-install:
cd js && npm install
js/src/pb: $(PROTO_FILES)
cd js && mkdir -p ./src/pb && npx protoc --experimental_allow_proto3_optional --ts_out ./src/pb --ts_opt long_type_bigint,ts_nocheck,eslint_disable,add_pb_suffix --proto_path ../internal/pb/ ../$^
js/dist: js-install $(PROTO_TS_FILES)
cd js && npm run build
development/server.key:
mkcert -cert-file development/server.crt -key-file development/server.key localhost
development/server.crt: development/server.key
build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client bin/webui js/src/pb js/dist development/server.crt
release/%_linux_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $<
release/%_macos_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $<
release/%_macos_arm64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build $(BUILD_FLAGS) -o $@ $<
release/migrations.tar.gz: migrations/*
tar -zcf $@ migrations
release/assets.tar.gz: assets/*
tar -zcf $@ assets
release: build
release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64
release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64
release: release/webui_linux_amd64 release/webui_macos_amd64 release/webui_macos_arm64
release: release/assets.tar.gz release/migrations.tar.gz
test: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
test: migrate
cd test && go test
test-one: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
test-one: migrate
ifndef name
$(error name variable must be set)
else
cd test && go test -run $(name)
endif
test-fuzz: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
test-fuzz: export DL_SKIP_SSL_VERIFICATION=1
test-fuzz: reset-db
go run cmd/fuzz-test/main.go --server $(GRPC_SERVER) --iterations 50 --projects 5
test-js: js-install
cd js && npm run test
lint-js: js-install
cd js && npm run lint
typecheck-js: js-install
cd js && npm run typecheck
reset-db: migrate
psql $(DB_URI) -c "truncate dl.objects; truncate dl.contents; truncate dl.projects; truncate dl.cache_versions;"
setup-local: reset-db
psql $(DB_URI) -c "insert into dl.projects (id, latest_version, pack_patterns) values (1, 0, '{\"node_modules/.*/\"}');"
server: export DL_ENV=dev
server: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go
go run cmd/server/main.go --dburi $(DB_URI) --port $(GRPC_PORT)
server-profile: export DL_ENV=dev
server-profile: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go
go run cmd/server/main.go --dburi $(DB_URI) --port $(GRPC_PORT) --profile cpu.prof --log-level info
client-update: export DL_TOKEN=$(DEV_TOKEN_PROJECT_1)
client-update: export DL_SKIP_SSL_VERIFICATION=1
client-update:
development/scripts/simple_input.sh 1
go run cmd/client/main.go update --server $(GRPC_SERVER) --project 1 --dir input/simple
development/scripts/simple_input.sh 2
go run cmd/client/main.go update --server $(GRPC_SERVER) --project 1 --dir input/simple
development/scripts/simple_input.sh 3
go run cmd/client/main.go update --server $(GRPC_SERVER) --project 1 --dir input/simple
client-large-update: export DL_TOKEN=$(DEV_TOKEN_PROJECT_1)
client-large-update: export DL_SKIP_SSL_VERIFICATION=1
client-large-update:
development/scripts/complex_input.sh 1
go run cmd/client/main.go update --server $(GRPC_SERVER) --project 1 --dir input/complex
development/scripts/complex_input.sh 2
go run cmd/client/main.go update --server $(GRPC_SERVER) --project 1 --dir input/complex
development/scripts/complex_input.sh 3
go run cmd/client/main.go update --server $(GRPC_SERVER) --project 1 --dir input/complex
client-get: export DL_TOKEN=$(DEV_TOKEN_PROJECT_1)
client-get: export DL_SKIP_SSL_VERIFICATION=1
client-get:
ifndef to_version
go run cmd/client/main.go get --server $(GRPC_SERVER) --project 1 --prefix "$(prefix)"
else
go run cmd/client/main.go get --server $(GRPC_SERVER) --project 1 --to $(to_version) --prefix "$(prefix)"
endif
client-rebuild: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
client-rebuild: export DL_SKIP_SSL_VERIFICATION=1
client-rebuild:
ifndef to_version
go run cmd/client/main.go rebuild --server $(GRPC_SERVER) --project 1 --prefix "$(prefix)" --dir $(dir)
else
go run cmd/client/main.go rebuild --server $(GRPC_SERVER) --project 1 --to $(to_version) --prefix "$(prefix)" --dir $(dir)
endif
client-gc-contents: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
client-gc-contents: export DL_SKIP_SSL_VERIFICATION=1
client-gc-contents:
go run cmd/client/main.go gc --server $(GRPC_SERVER) --mode contents --sample 25
client-gc-project: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
client-gc-project: export DL_SKIP_SSL_VERIFICATION=1
client-gc-project:
go run cmd/client/main.go gc --server $(GRPC_SERVER) --mode project --project 1 --keep 1
client-gc-random-projects: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
client-gc-random-projects: export DL_SKIP_SSL_VERIFICATION=1
client-gc-random-projects:
go run cmd/client/main.go gc --server $(GRPC_SERVER) --mode random-projects --sample 25 --keep 1
webui: export DL_ENV=dev
webui: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
webui: export DL_SKIP_SSL_VERIFICATION=1
webui:
go run cmd/webui/main.go --server $(GRPC_SERVER)
health:
grpc-health-probe -addr $(GRPC_SERVER)
grpc-health-probe -addr $(GRPC_SERVER) -service $(SERVICE)
upload-container-image: release
ifndef version
$(error version variable must be set)
else
docker build -t gcr.io/gadget-core-production/dateilager:$(version) -t gcr.io/gadget-core-production/dateilager:latest .
docker push gcr.io/gadget-core-production/dateilager:$(version)
docker push gcr.io/gadget-core-production/dateilager:latest
endif
gen-docs:
go run cmd/gen-docs/main.go
define load-test
ghz --cert=development/server.crt --key=development/server.key \
--proto internal/pb/fs.proto --call "pb.Fs.$(1)" \
--total $(3) --concurrency $(4) --rps $(if $5,$5,0) \
--data-file "development/scripts/load-tests/$(2)" \
--metadata '{"authorization": "Bearer $(DEV_TOKEN_ADMIN)"}' \
localhost:$(GRPC_PORT)
endef
load-test-new: reset-db
$(call load-test,NewProject,new.json,100,1)
load-test-update:
$(call load-test,Update,update.json,10000,20)
load-test-get:
$(call load-test,Get,get.json,100000,40,5000)
load-test-get-compress:
$(call load-test,GetCompress,get-compress.json,100000,40,5000)