blob: 994589ba2b5c7ddd52753b9d45577b27bcbc1ba9 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
|
variables:
DEBIAN_FRONTEND: noninteractive
DEBIAN_OLD_STABLE: buster
DEBIAN_STABLE: bullseye
REPRODUCIBLE_FLAGS: -trimpath -ldflags=-buildid=
# set up apt for automated use
.apt-template: &apt-template
- export LC_ALL=C.UTF-8
- export DEBIAN_FRONTEND=noninteractive
- echo Etc/UTC > /etc/timezone
- echo 'quiet "1";'
'APT::Install-Recommends "0";'
'APT::Install-Suggests "0";'
'APT::Acquire::Retries "20";'
'APT::Get::Assume-Yes "true";'
'Dpkg::Use-Pty "0";'
> /etc/apt/apt.conf.d/99gitlab
- apt-get update
- apt-get dist-upgrade
# Set things up to use the OS-native packages for Go. Anything that
# is downloaded by go during the `go fmt` stage is not coming from the
# Debian/Ubuntu repo. So those would need to be packaged for this to
# make it into Debian and/or Ubuntu.
.debian-native-template: &debian-native-template
variables:
GOPATH: /usr/share/gocode
before_script:
- apt-get update
- apt-get -qy install --no-install-recommends
build-essential
ca-certificates
git
golang
golang-github-cheekybits-genny-dev
golang-github-jtolds-gls-dev
golang-github-klauspost-reedsolomon-dev
golang-github-lucas-clemente-quic-go-dev
golang-github-smartystreets-assertions-dev
golang-github-smartystreets-goconvey-dev
golang-github-tjfoc-gmsm-dev
golang-github-xtaci-kcp-dev
golang-github-xtaci-smux-dev
golang-golang-x-crypto-dev
golang-golang-x-net-dev
golang-goptlib-dev
golang-golang-x-sys-dev
golang-golang-x-text-dev
golang-golang-x-xerrors-dev
lbzip2
# use Go installed as part of the official, Debian-based Docker images
.golang-docker-debian-template: &golang-docker-debian-template
before_script:
- apt-get update
- apt-get -qy install --no-install-recommends
ca-certificates
git
lbzip2
.go-test: &go-test
- gofmt -d .
- test -z "$(go fmt ./...)"
- go vet ./...
- go test -v -race ./...
- cd $CI_PROJECT_DIR/client/
- go get
- go build $REPRODUCIBLE_FLAGS
.test-template: &test-template
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- client/*.aar
- client/*.jar
- client/client
expire_in: 1 week
when: on_success
after_script:
- echo "Download debug artifacts from https://gitlab.com/${CI_PROJECT_PATH}/-/jobs"
# this file changes every time but should not be cached
- rm -f $GRADLE_USER_HOME/caches/modules-2/modules-2.lock
- rm -rf $GRADLE_USER_HOME/caches/*/plugin-resolution/
# -- jobs ------------------------------------------------------------
android:
image: golang:1.21-$DEBIAN_STABLE
variables:
ANDROID_HOME: /usr/lib/android-sdk
LANG: C.UTF-8
cache:
paths:
- .gradle/wrapper
- .gradle/caches
<<: *test-template
before_script:
- *apt-template
- apt-get install
android-sdk-platform-23
android-sdk-platform-tools
build-essential
curl
default-jdk-headless
git
gnupg
unzip
wget
ca-certificates
lbzip2
- ndk=android-ndk-r21e-linux-x86_64.zip
- wget --continue --no-verbose https://dl.google.com/android/repository/$ndk
- echo "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e $ndk" > $ndk.sha256
- sha256sum -c $ndk.sha256
- unzip -q $ndk
- rm ${ndk}*
- mv android-ndk-* $ANDROID_HOME/ndk-bundle/
- chmod -R a+rX $ANDROID_HOME
script:
- *go-test
- export GRADLE_USER_HOME=$CI_PROJECT_DIR/.gradle
- go version
- go env
- go get golang.org/x/mobile/cmd/gomobile
- go get golang.org/x/mobile/cmd/gobind
- go install golang.org/x/mobile/cmd/gobind
- go install golang.org/x/mobile/cmd/gomobile
- gomobile init
- cd $CI_PROJECT_DIR/client
# gomobile builds a shared library not a CLI executable
- sed -i 's,^package main$,package snowflakeclient,' *.go
- go get golang.org/x/mobile/bind
- gomobile bind -v -target=android $REPRODUCIBLE_FLAGS .
go-1.21:
image: golang:1.21-$DEBIAN_STABLE
<<: *golang-docker-debian-template
<<: *test-template
script:
- *go-test
debian-testing:
image: debian:testing
<<: *debian-native-template
<<: *test-template
script:
- *go-test
shadow-integration:
# Need golang 1.21 from unstable to build snowflake
image: debian:sid-slim
variables:
SHADOW_VERSION: "v3.1.0"
TGEN_VERSION: "v1.1.2"
cache:
key: sf-integration-$SHADOW_VERSION-$TGEN_VERSION
paths:
- /opt/
script:
- apt-get update
- apt-get install -y git tor
- mkdir -p ~/.local/bin
- mkdir -p ~/.local/src
- export PATH=$PATH:$CI_PROJECT_DIR/opt/bin/
# Install shadow and tgen
- pushd ~/.local/src
- |
if [ ! -f opt/shadow/bin/shadow ]
then
echo "The required version of shadow was not cached, building from source"
git clone --branch $SHADOW_VERSION --depth 1 https://github.com/shadow/shadow.git
pushd shadow/
CONTAINER=debian:sid-slim ci/container_scripts/install_deps.sh
CC=gcc CONTAINER=debian:sid-slim ci/container_scripts/install_extra_deps.sh
apt-get install -y zlib1g
export PATH="$HOME/.cargo/bin:${PATH}"
./setup build --jobs $(nproc) --prefix $CI_PROJECT_DIR/opt/
./setup install
popd
fi
- |
if [ ! -f opt/shadow/bin/tgen ]
then
echo "The required version of tgen was not cached, building from source"
git clone --branch $TGEN_VERSION --depth 1 https://github.com/shadow/tgen.git
pushd tgen/
apt-get install -y cmake libglib2.0-dev libigraph-dev
mkdir build && cd build
cmake .. -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/opt/
make
make install
popd
fi
install $CI_PROJECT_DIR/opt/bin/tgen ~/.local/bin/tgen
- popd
# Install snowflake binaries to .local folder
- |
for app in "proxy" "client" "server" "broker" "probetest"; do
pushd $app
go build
install $app ~/.local/bin/snowflake-$app
popd
done
# Install stun server
- GOBIN=~/.local/bin go install github.com/gortc/stund@latest
# Run a minimal snowflake shadow experiment
- git clone --depth 1 https://github.com/cohosh/shadow-snowflake-minimal
- pushd shadow-snowflake-minimal/
- shadow --log-level=debug --model-unblocked-syscall-latency=true snowflake-minimal.yaml > shadow.log
# Check to make sure streams succeeded
- |
if [ $(grep -c "stream-success" shadow.data/hosts/snowflakeclient/tgen.1006.stdout) = 10 ]
then
echo "All streams in shadow completed successfully"
else
echo "Shadow simulation failed"
exit 1
fi
generate_tarball:
stage: deploy
image: golang:1.21-$DEBIAN_STABLE
rules:
- if: $CI_COMMIT_TAG
script:
- go mod vendor
- tar czf ${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz --transform "s,^,${CI_PROJECT_NAME}-${CI_COMMIT_TAG}/," *
after_script:
- echo TAR_JOB_ID=$CI_JOB_ID >> generate_tarball.env
artifacts:
paths:
- ${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz
reports:
dotenv: generate_tarball.env
release-job:
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
rules:
- if: $CI_COMMIT_TAG
needs:
- job: generate_tarball
artifacts: true
script:
- echo "running release_job"
release:
name: 'Release $CI_COMMIT_TAG'
description: 'Created using the release-cli'
tag_name: '$CI_COMMIT_TAG'
ref: '$CI_COMMIT_TAG'
assets:
links:
- name: '${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz'
url: '${CI_PROJECT_URL}/-/jobs/${TAR_JOB_ID}/artifacts/file/${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz'
|
