-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
441 lines (404 loc) · 12.6 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
stages:
- check
- build
- test
- deploy
variables:
# We don't need Husky to install the Git hooks for CI.
CARGO_HUSKY_DONT_INSTALL_HOOKS: "true"
# fs-mistrust doesn't like umask 0
FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true"
default:
before_script:
# gitlab fetch strategy doesn't reset permissions
- (while [ "$PWD" != / ]; do chmod go-w . && cd ..; done)
check-editorconfig:
stage: check
image: mstruebing/editorconfig-checker
script:
- ec
shellcheck:
stage: check
image: koalaman/shellcheck-alpine
script:
- apk add git bash
- ./maint/shellcheck_all
maint-checks:
stage: check
image: debian:bookworm-slim
script:
- apt-get update && apt-get install -y python3-toml python-is-python3
- ./maint/check_toposort
- ./maint/add_warning --check
- ./maint/shebang
# non-blocking for now, see
# https://gitlab.torproject.org/tpo/core/arti/-/issues/581
# https://gitlab.torproject.org/tpo/core/arti/-/issues/601
doc-features:
stage: check
image: debian:bookworm-slim
allow_failure: true
script:
- apt-get update && apt-get install -y python3-toml python-is-python3
- ./maint/check_doc_features
# This should always be in the last testing stage, so that if it fails all the other steps still run
# But it should run before any deployument.
blocking-todos:
stage: test
needs: []
image: debian:bookworm-slim
script:
- apt update && apt install -y git
- ./maint/check_todos
rust-checks:
# This is too slow (and the cacheing of the "cargo build" too flaky) to be a "check"
stage: build
image: rust:bookworm
script:
- rustup show
- rustup component add rustfmt
- ./maint/via-cargo-install-in-ci cargo-sort cargo-license
- cargo fmt -- --check
- ./maint/check_licenses
- ./maint/cargo_sort
- ./maint/check_tree
- ./maint/check_all_lockfiles
cache:
paths:
- cache
cargo-audit:
# This can start to fail even when our code doesn't change.
# Usually the new advisory is not a huge concern.
# Run it last, separately, so if we think we may want to merge anyway,
# all the other tests will have been run.
stage: test
image: rust:bookworm
script:
- rustup show
- ./maint/via-cargo-install-in-ci cargo-audit
- ./maint/cargo_audit
cache:
paths:
- cache
rust-latest:
stage: build
image: rust:bookworm
script:
- rustup show
- cargo check --locked --verbose --target x86_64-unknown-linux-gnu
- cargo test --verbose --target x86_64-unknown-linux-gnu
- rustup component add clippy
- rustup show
- cargo clippy --all-features --all-targets -- --cfg ci_arti_stable -D warnings
- cargo build --verbose --release -p arti-bench --target x86_64-unknown-linux-gnu
- cargo build --locked --verbose --target x86_64-unknown-linux-gnu -p arti
- ./maint/preserve target/x86_64-unknown-linux-gnu/debug/arti target/x86_64-unknown-linux-gnu/release/arti-bench
after_script:
- cargo clean
artifacts:
paths:
- artifacts
expire_in: 1 hours
rust-latest-arti-extra-features:
stage: build
image: rust:bookworm
script:
- rustup show
# Build the arti binary for use in chutney and shadow integration tests.
#
# Note: we enable the `experimental-api` feature instead of `experimental`,
# because we don't want to build with `rpc` enabled. The `rpc` feature causes
# the RPC listener to try to bind to a Unix domain socket, and pathname Unix
# domain sockets are not currently supported by shadow.
#
# Consider enabling the rpc feature when shadow starts supporting pathname
# addresses, or when we add a config setting for disabling rpc.
#
# Note: `-p arti` is *not* already implied by `--bin arti`. If we omit it,
# we'll get the union of all features needed by anything in the workspace,
# including examples.
- cargo build --verbose
--target x86_64-unknown-linux-gnu
-p arti -p tor-circmgr
--bin arti
--features full,experimental-api,arti-client/keymgr,tor-circmgr/ntor_v3,onion-service-service,vanguards
- ./maint/preserve target/x86_64-unknown-linux-gnu/debug/arti
# Save the full-featured binary under a different name to prevent it from being
# overwritten by the other jobs that preserve the arti binary.
- mv artifacts/target/x86_64-unknown-linux-gnu/debug/arti artifacts/target/x86_64-unknown-linux-gnu/debug/arti-extra
after_script:
- cargo clean
artifacts:
paths:
- artifacts
expire_in: 1 hours
rust-latest-async-std-rustls:
stage: build
image: rust:bookworm
script:
- rustup show
- rustup component add clippy
- cd crates/arti-client && cargo clippy --no-default-features --features=async-std,rustls
after_script:
- cargo clean
rust-nightly:
stage: test
image: rustlang/rust:nightly
# In case there is a bug in rust:nightly, you can instead pin an older
# version of the Docker image until that bug is fixed. To find the
# SHA256 ID of the last working version of nightly, look at the logs
# from the last successful CI run. Here is an example of how to do so:
#
# image: rustlang/rust@sha256:415b7c22ab4a8a3ec3efc9cc8d7b018964f0c6757fff27bbd110e0ed92566321
allow_failure: true
script:
- rustup show
- cargo build --verbose --target x86_64-unknown-linux-gnu --all-features
- cargo test --verbose --target x86_64-unknown-linux-gnu --all-features
- rustup component add clippy
# We check these extra warnings on CI only, since we don't want to forbid them while developing.
# The --cfg allows us to write #![cfg_attr(ci_arti_nightly, allow(clippy::needless_borrow))]
# and similar, for when nightly clippy has false positives. (There is no builtin
# cfg flag for nightly, and a cargo "feature" would be bad with --all-features.)
- (echo; cat clippy-nightly.toml) >>clippy.toml
- cargo clippy --all-features --tests -- --cfg ci_arti_nightly -D clippy::dbg_macro
- RUSTDOCFLAGS="-Dwarnings --cfg docsrs" cargo doc --all-features --document-private-items --no-deps
after_script:
- cargo clean
coverage:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
stage: test
image: rust:bookworm
script:
- apt-get update && apt-get install -y python3-pip python3-setuptools python3-bs4 python3-lxml
- rustup component add llvm-tools
- ./maint/via-cargo-install-in-ci grcov
# Generate report
- ./maint/with_coverage -f cobertura -o coverage.xml cargo test --verbose --all-features
after_script:
- cargo clean
cache:
paths:
- cache
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
tags:
- tpa
minimal-versions:
stage: test
image: rust:1.70
needs: ["rust-checks"]
script:
- rustup install nightly
- ./maint/downgrade_dependencies
- cargo test --verbose --target x86_64-unknown-linux-gnu --all-features
after_script:
- cargo clean
build-repro:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null
stage: build
# If you upgrade this image, also change the one in docker_reproducible_build
image: rust:1.76.0-alpine3.18
script:
- apk add bash
- ./maint/reproducible_build linux windows macos
# no after_script:, we don't build in the project dir
artifacts:
paths:
- arti-linux
- arti-windows.exe
- arti-macos
expire_in: 1 day
cache:
paths:
- osxcross/target
tags:
- tpa
- amd64
integration-chutney:
stage: test
rules:
# Job never runs. See arti#810.
- when: never
image: debian:12-slim
script:
- ./maint/preserve -u
- apt update
- apt install -y tor git python3 curl dnsutils
# arti runtime dependencies
- apt install -y libsqlite3-0 libssl3
- ./tests/chutney/setup proxy
- ./tests/chutney/test
- ./tests/chutney/stop-arti
- RUST_LOG=debug target/x86_64-unknown-linux-gnu/release/arti-bench -c ./chutney/net/nodes/arti.toml --socks5 127.0.0.1:9008 -o benchmark_results.json
- ./tests/chutney/teardown
artifacts:
paths:
- benchmark_results.json
integration-shadow:
variables:
JOB_SHADOW_REPO: "https://github.com/shadow/shadow.git"
JOB_SHADOW_BRANCH: "main"
JOB_SHADOW_COMMIT: "v3.1.0"
JOB_TGEN_REPO: "https://github.com/shadow/tgen.git"
JOB_TGEN_BRANCH: "main"
JOB_TGEN_COMMIT: "v1.1.2"
stage: test
cache:
- key: $CI_JOB_NAME-shadow-$JOB_SHADOW_COMMIT
paths:
- opt/shadow
- key: $CI_JOB_NAME-tgen-$JOB_TGEN_COMMIT
paths:
- opt/tgen
image: debian:12-slim
script:
- ./maint/preserve -u
- ./maint/ci_log_span start "shadow_install_deps[collapsed=true]" "Installing shadow and tgen"
# We're going to install binaries to $HOME/.local/bin
- 'export PATH=$HOME/.local/bin:$PATH'
- apt-get update
- apt-get install -y git tor stow
- mkdir -p ~/src
- mkdir -p ~/.local
# arti runtime dependencies
- apt install -y libsqlite3-0 libssl3
# Build shadow
- |
if [ -f opt/shadow/bin/shadow ]
then
echo "Using shadow binary from cache"
else
echo "Building shadow"
git clone --shallow-since=2021-08-01 -b $JOB_SHADOW_BRANCH $JOB_SHADOW_REPO ~/src/shadow
pushd ~/src/shadow
git checkout $JOB_SHADOW_COMMIT
export CC=gcc CXX=g++ CONTAINER=debian:12-slim BUILDTYPE=release RUSTPROFILE=minimal
ci/container_scripts/install_deps.sh
ci/container_scripts/install_extra_deps.sh
export PATH="$HOME/.cargo/bin:${PATH}"
./setup build --jobs $(nproc) --prefix $CI_PROJECT_DIR/opt/shadow
./setup install
popd
fi
- apt-get install -y libglib2.0-0
- stow -d opt -t $HOME/.local shadow
# Build tgen
- |
if [ -f opt/tgen/bin/tgen ]
then
echo "Using tgen binary from cache"
else
echo "Building tgen"
git clone --shallow-since=2022-01-01 -b $JOB_TGEN_BRANCH $JOB_TGEN_REPO ~/src/tgen
pushd ~/src/tgen
git checkout $JOB_TGEN_COMMIT
apt-get install -y cmake gcc libglib2.0-0 libglib2.0-dev libigraph-dev make
mkdir build
cd build
cmake .. -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/opt/tgen
make --jobs $(nproc)
make install
popd
fi
- apt-get install -y libigraph3 libglib2.0-0
- stow -d opt -t $HOME/.local tgen
# Ensure newly installed executables can be found
- hash -r
- DEBIAN_FRONTEND=noninteractive apt-get install -y tshark
- ./maint/ci_log_span end "shadow_install_deps"
# Run tests
- pushd tests/shadow
- ./run.sh
artifacts:
paths:
- tests/shadow
when: always
expire_in: 1 week
tags:
- amd64
- tpa
rust-latest-test-all-features:
stage: test
image: rust:bookworm
script:
- rustup show
- cargo test --target x86_64-unknown-linux-gnu --locked --workspace --all-features
after_script:
- cargo clean
every-crate:
stage: test
image: rust:bookworm
needs: ["rust-checks", "rust-latest-async-std-rustls"]
script:
- apt-get update && apt-get install -y python3-toml python-is-python3
- ./maint/every-crate
matrix-test:
stage: test
image: rust:bookworm
needs: ["rust-checks", "rust-latest-async-std-rustls"]
script:
- apt-get update && apt-get install -y python3-toml python-is-python3
- ./maint/matrix_test
matrix-test-cfg:
stage: test
image: rust:bookworm
script:
- ./maint/matrix_test_cfg
cli-help:
stage: test
image: rust:bookworm
script:
- ./maint/check-cli-help
coverage-aggregated:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
stage: test
image: rust:bookworm
needs: []
script:
- apt update && apt install -y tor python3 python3-pip python3-setuptools curl python3-bs4 python3-lxml
- rustup component add llvm-tools
- cp grcov $CARGO_HOME/bin/ || cargo install grcov
- cp $CARGO_HOME/bin/grcov .
# Generate report
- ./maint/coverage all
after_script:
- cargo clean
cache:
paths:
- cache
artifacts:
paths:
- coverage
tags:
- ipv6
check-targets:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
stage: test
image: rust:bookworm
script:
- ./maint/cargo_check_target -il
after_script:
- cargo clean
pages:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "main"
stage: deploy
image: debian:bookworm-slim
script:
- apt update && apt install -y git
# Export report as website, while keeping the existing public page
- git fetch
- git checkout origin/pages -- public/
- mv coverage public/
artifacts:
paths:
- public