-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
273 lines (227 loc) · 8.43 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
SHELL := /bin/bash
include .env
PROJECT_ID=$(PROJECT)-$(ENV)
GOOGLE_CLOUD_PROJECT=$(PROJECT_ID)
.EXPORT_ALL_VARIABLES:
.PHONY: all test check quality run docker
# -- bucket definitions
DEPLOY_BUCKET := $(PROJECT_ID)-gcs-deploy
check: poetry-lock clean-code clean-coverage quality test poetry-build docker-build
test: prepare-test poetry-test
quality: prepare-quality poetry-quality
run: prepare-run poetry-run
docker: docker-build docker-run
prepare-quality:
@poetry install --only nox,fmt,lint,type_check,docs
poetry-quality:
@poetry run nox -s fmt_check
@poetry run nox -s lint
@poetry run nox -s type_check
@poetry run nox -s docs
poetry-lock:
@poetry env use python3.9
@poetry install
@poetry lock --check
clean-coverage:
@rm -f .coverage*
@rm -f coverage.xml
prepare-test:
@poetry install --only nox
poetry-test:
@poetry run nox -s test-3.9
@rm -f .coverage*
@rm -f coverage.xml
@poetry run nox -s test-3.10
clean-code:
@poetry run isort .
@poetry run black .
poetry-build:
@poetry build
prepare-run:
@poetry install --only main
poetry-run:
@poetry run uvicorn dbt_serverless.main:app --host 0.0.0.0 --port 8080 --reload
poetry-pulumi:
@poetry run python -m iac-pulumi.main
docker-build:
@docker build --tag dbt-serverless .
docker-run:
@docker run \
--rm \
--interactive \
--tty \
-p 8080:8080 \
-v "$(HOME)/.config/gcloud:/gcp/config:ro" \
-v /gcp/config/logs \
--env CLOUDSDK_CONFIG=/gcp/config \
--env GOOGLE_APPLICATION_CREDENTIALS=/gcp/config/application_default_credentials.json \
--env GOOGLE_CLOUD_PROJECT=$(PROJECT_ID) \
--env DBT_PROJECT=$(DBT_PROJECT) \
--env DBT_DATASET=$(DBT_DATASET) \
--env DBT_PROFILES_DIR=$(DBT_PROJECT) \
dbt-serverless
@docker rmi -f $$(docker images -f "dangling=true" -q)
@docker volume prune -f
# ---------------------------------------------------------------------------------------- #
# This target will perform the complete setup of the current repository.
# ---------------------------------------------------------------------------------------- #
all: create-project create-bucket create-artifactregistry build deploy-app
build: check build-app
deploy: deploy-app iac-clean iac-deploy
gcloud:
gcloud projects add-iam-policy-binding $(PROJECT_ID) \
--member=user:$(ACCOUNT) \
--role=roles/iam.serviceAccountTokenCreator
gcloud-auth:
@gcloud auth print-access-token --project $(PROJECT)
# -- This target triggers the creation of the necessary project
.PHONY: create-project
create-project:
@echo "[$@] :: creating project..."
@echo "$(PROJECT_ID)"
@gcloud projects create $(PROJECT_ID) --name=$(PROJECT_ID) --organization=$(ORG_ID) --folder=$(FOLDER_ID)
@echo "[$@] :: linking billing account to project..."
@gcloud beta billing projects link $(PROJECT_ID) --billing-account=$(BILLING_ID)
@echo "[$@] :: project creation is over."
create-artifactregistry:
@echo "[$@] :: enabling apis..."
@gcloud services enable artifactregistry.googleapis.com --project $(PROJECT_ID)
@echo "[$@] :: creating repository..."
@gcloud artifacts repositories create $(REPOSITORY_ID) \
--project $(PROJECT_ID) \
--location $(REGION) \
--repository-format docker \
--description "Docker repository"
@echo "[$@] :: apis enabled"
# -- This target triggers the creation of the necessary buckets
.PHONY: create-bucket
create-bucket:
@echo "[$@] :: creating bucket..."
@gsutil ls -p $(PROJECT_ID) gs://$(DEPLOY_BUCKET) 2>/dev/null || \
gsutil mb -l EU -p $(PROJECT_ID) gs://$(DEPLOY_BUCKET);
@gsutil versioning set on gs://$(DEPLOY_BUCKET);
@echo "[$@] :: bucket creation is over."
# -- This target triggers the deletion of the gcloud project
.PHONY: delete-project
delete-project:
@echo "[$@] :: deleting project..."
@gcloud beta billing projects unlink $(PROJECT_ID)
@gcloud projects delete $(PROJECT_ID)
@echo "[$@] :: deletion is over."
.PHONY: clean
clean: iac-clean
# ---------------------------------------------------------------------------------------- #
# -- < Building > --
# ---------------------------------------------------------------------------------------- #
build-app:
@echo "[$@] :: building the Docker image"
@set -euo pipefail; \
docker build \
--tag $(REGION)-docker.pkg.dev/$(PROJECT_ID)/$(REPOSITORY_ID)/dbt-serverless:latest \
.
@echo "[$@] :: docker build is over."
deploy-app:
@echo "[$@] :: Pushing docker image"
@docker push $(REGION)-docker.pkg.dev/$(PROJECT_ID)/$(REPOSITORY_ID)/dbt-serverless:latest;
@echo "[$@] :: docker push is over."
# ---------------------------------------------------------------------------------------- #
# -- < IaC > --
# ---------------------------------------------------------------------------------------- #
# -- terraform variables declaration
IAC_DIR = iac/
DBT_DIR = $(DBT_PROJECT)/
TF_DIR = $(IAC_DIR).terraform/
TF_INIT = $(TF_DIR)terraform.tfstate
TF_VARS = $(IAC_DIR)terraform.tfvars
TF_PLAN = $(IAC_DIR)tfplan
TF_STATE = $(wildcard $(IAC_DIR)*.tfstate $(TF_DIR)*.tfstate)
TF_FILES = $(wildcard $(IAC_DIR)*.tf)
# -- this target will clean the local terraform infrastructure
.PHONY: iac-clean
iac-clean:
@echo "[$@] :: cleaning the infrastructure intermediary files"
@rm -fr $(TF_PLAN) $(TF_VARS);
@if [ ! -f $(IAC_DIR).iac-env ] || [ $$(cat $(IAC_DIR).iac-env || echo -n) != $(PROJECT_ID) ]; then \
echo "[$@] :: env has changed, removing also $(TF_DIR) and $(IAC_DIR).terraform.lock.hcl"; \
rm -rf $(TF_DIR) $(IAC_DIR).terraform.lock.hcl; \
fi;
@echo "[$@] :: infrastructure cleaning DONE"
# -- this target will initialize the terraform initialization
.PHONY: iac-init
iac-init: $(TF_INIT) # provided for convenience
$(TF_INIT):
@set -euo pipefail; \
if [ ! -d $(TF_DIR) ]; then \
function remove_me() { if (( $$? != 0 )); then rm -fr $(TF_DIR); fi; }; \
trap remove_me EXIT; \
echo "[iac-init] :: initializing terraform"; \
echo "$(PROJECT_ID)" > $(IAC_DIR).iac-env; \
cd $(IAC_DIR) && terraform init \
-backend-config=bucket=$(DEPLOY_BUCKET) \
-backend-config=prefix=terraform-state/$(ENV) \
-input=false; \
else \
echo "[iac-init] :: terraform already initialized"; \
fi;
# -- internal definition for easing changes
define HERE_TF_VARS
project = "$(PROJECT_ID)"
zone = "$(ZONE)"
region = "$(REGION)"
env = "$(ENV)"
repository_id = "$(REPOSITORY_ID)"
github_owner = "$(GITHUB_OWNER)"
github_repo = "$(GITHUB_REPO)"
github_token = "$(GITHUB_TOKEN)"
pypi_token = "$(PYPI_TOKEN)"
codecov_token = "$(CODECOV_TOKEN)"
endef
export HERE_TF_VARS
# -- this target will create the terraform.tfvars file
.PHONY: iac-prepare
iac-prepare: $(TF_VARS) # provided for convenience
$(TF_VARS): $(TF_INIT)
@echo "[iac-prepare] :: generation of $(TF_VARS) file";
@echo "$$HERE_TF_VARS" > $(TF_VARS);
@echo "[iac-prepare] :: generation of $(TF_VARS) file DONE.";
# -- this target will create the tfplan file whenever the variables file and any *.tf
# file have changed
.PHONY: iac-plan iac-plan-clean
iac-plan-clean:
@rm -f $(TF_PLAN)
iac-plan: iac-plan-clean $(TF_PLAN) # provided for convenience
$(TF_PLAN): $(TF_VARS) $(TF_FILES)
@echo "[iac-plan] :: planning the iac in $(PROJECT_ID)";
@set -euo pipefail; \
cd $(IAC_DIR) && terraform plan \
-var-file $(shell basename $(TF_VARS)) \
-out=$(shell basename $(TF_PLAN));
@echo "[iac-plan] :: planning the iac for $(PROJECT_ID) DONE.";
# -- this target will only trigger the iac of the current parent
.PHONY: iac-validate
iac-validate:
@echo "[$@] :: validating the infrastructure for $(PROJECT_ID)"
@set -euo pipefail; \
cd $(IAC_DIR) && terraform validate;
@echo "[$@] :: infrastructure validated on $(PROJECT_ID)"
# -- this target will only trigger the iac of the current parent
.PHONY: iac-sec
iac-sec:
@echo "[$@] :: checking the infrastructure security for $(PROJECT_ID)"
@tfsec .
@echo "[$@] :: security checked on $(PROJECT_ID)"
# -- this target will only trigger the iac of the current parent
.PHONY: iac-version
iac-version:
@cd $(IAC_DIR) && terraform -version
# -- this target will only trigger the iac of the current parent
.PHONY: iac-deploy
iac-deploy: iac-clean $(TF_PLAN)
@echo "[$@] :: applying the infrastructure for $(PROJECT_ID)"
@set -euo pipefail; \
cd $(IAC_DIR) && terraform apply -auto-approve -input=false $(shell basename $(TF_PLAN));
@echo "[$@] :: infrastructure applied on $(PROJECT_ID)"
# -- this target re-initializes the git working tree removing untracked and ignored files
.PHONY: reinit
reinit:
@rm -rf $(IAC_DIR).terraform* $(IAC_DIR)terraform.tfstate* $(IAC_DIR)tfplan