-
Notifications
You must be signed in to change notification settings - Fork 1
/
.cirrus.yml
509 lines (428 loc) · 16.1 KB
/
.cirrus.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
# TODOs
#
# * publish packages on GitHub Pages when releasing
# * support quaterly/latest ports
#
# known issues
#
# * caching does not work as I wish. do not expect *_cache implementations are
# right.
env:
HOME: /root
PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
USER: root
build_task:
skip: "changesIncludeOnly('**.md')"
matrix:
# XXX use compute_engine_instance, instead of freebsd_instance, which is a
# syntactic sugar. with freebsd_instance, disk parameter is ignored.
# see https://github.com/cirruslabs/cirrus-ci-docs/issues/911
- compute_engine_instance:
image_project: freebsd-org-cloud-dev
image: family/freebsd-13-1
platform: freebsd
cpu: 4
memory: 8
disk: 80
- compute_engine_instance:
image_project: freebsd-org-cloud-dev
image: family/freebsd-12-3
platform: freebsd
cpu: 4
memory: 8
disk: 80
# increase timeout_in. in most cases, build will not finish in 60m, which is
# the default. the default is not documented at
# https://cirrus-ci.org/guide/writing-tasks/, but at https://cirrus-ci.org/faq/
# the max timeout_in is two hours.
timeout_in: 120m
define_common_variables_script: |
# set this to yes to allow MAKE_JOBS (parallel make)
PD_ALLOW_MAKE_JOBS=no
# create a list of all package origins and package names in the repository
PD_ALL_ORIGIN=`echo */* | sed -e 's|/[[:space:]]| |g' -e 's|/$||'`
for D in ${PD_ALL_ORIGIN}; do
PD_FLAVORS=`make -V FLAVORS -C "${D}"`
echo "PD_FLAVORS=${PD_FLAVORS}"
if [ "x${PD_FLAVORS}" != "x" ]; then
for F in ${PD_FLAVORS}; do
echo "FLAVOR=${F}"
P=`make -V PKGNAME -C "${D}" FLAVOR="${F}"`
echo "${P}"
PD_ALL_PACKAGES="${PD_ALL_PACKAGES} ${P}"
echo "${PD_ALL_PACKAGES}"
done
else
P=`make -V PKGNAME -C "${D}"`
echo "${P}"
PD_ALL_PACKAGES="${PD_ALL_PACKAGES} ${P}"
echo "${PD_ALL_PACKAGES}"
fi
done
PD_ALLOW_MAKE_JOBS_PACKAGES=""
if [ "x${PD_ALLOW_MAKE_JOBS}" = "xyes" ]; then
PD_ALLOW_MAKE_JOBS_PACKAGES="${PD_ALL_PACKAGES}"
fi
# define common variavles used in *_script. remember to export variables
# at the end of this script.
PD_WORKDIR="/tmp"
PD_BASEFS="${PD_WORKDIR}/poudriere"
PD_CCACHE_DIR=/var/cache/ccache
PD_REQUIRED_PACKAGES="ports-mgmt/poudriere-devel ports-mgmt/portshaker ports-mgmt/portshaker-config git-lite"
# the branch to use for the FreeBSD ports tree.
# see https://github.com/freebsd/freebsd-ports/branches for available
# branches
PD_FREEBSD_PORTS_BRANCH="2022Q4"
# the branch to use for the FreeBSD package tree. the build fetches
# and installs packages required by PD_ALL_ORIGIN.
# see https://pkg.freebsd.org/FreeBSD:13:amd64/ for available branches
PD_FREEBSD_PACKAGE_BRANCH=quarterly
# e.g. 122
PD_VERSION_SHORT=`uname -r | sed -e 's/-.*//' -e 's/\.//'`
# e.g. amd64
PD_MACHINE=`uname -m`
# e.g. 12.2-RELEASE
PD_RELEASE_NAME=`uname -r | sed -e 's/-p.*//'`
# e.g. 122amd64
PD_JAIL_NAME="${PD_VERSION_SHORT}${PD_MACHINE}"
PD_PORTS_NAME="default"
# e.g 122amd64-default
PD_BUILD_NAME="${PD_JAIL_NAME}-${PD_PORTS_NAME}"
# `export` the variables so that other scripts can use them.
#
# note that the ${CIRRUS_ENV} file is NOT a shell script. empty line is
# not allowed. value should NOT be quoted (i.e. NOT `foo="bar buz"`. use
# `foo=bar buz` instead).
cat << __EOF__ >> ${CIRRUS_ENV}
PD_ALL_ORIGIN=${PD_ALL_ORIGIN}
PD_ALL_PACKAGES=${PD_ALL_PACKAGES}
PD_ALLOW_MAKE_JOBS=${PD_ALLOW_MAKE_JOBS}
PD_ALLOW_MAKE_JOBS_PACKAGES=${PD_ALLOW_MAKE_JOBS_PACKAGES}
PD_WORKDIR=${PD_WORKDIR}
PD_BASEFS=${PD_BASEFS}
PD_CCACHE_DIR=${PD_CCACHE_DIR}
PD_REQUIRED_PACKAGES=${PD_REQUIRED_PACKAGES}
PD_VERSION_SHORT=${PD_VERSION_SHORT}
PD_MACHINE=${PD_MACHINE}
PD_RELEASE_NAME=${PD_RELEASE_NAME}
PD_JAIL_NAME=${PD_JAIL_NAME}
PD_PORTS_NAME=${PD_PORTS_NAME}
PD_BUILD_NAME=${PD_BUILD_NAME}
PD_FREEBSD_PORTS_BRANCH=${PD_FREEBSD_PORTS_BRANCH}
PD_FREEBSD_PACKAGE_BRANCH=${PD_FREEBSD_PACKAGE_BRANCH}
__EOF__
cat ${CIRRUS_ENV}
setup_script: |
uname -msr
sysctl kern.osreldate
sysctl hw.ncpu
top -bP
env | sort
df -h
mount
# configure _merged_ ports tree
cat << __EOF__ > /usr/local/etc/portshaker.conf
mirror_base_dir="/var/cache/portshaker"
ports_trees="${PD_PORTS_NAME}"
default_ports_tree="${PD_BASEFS}/ports/${PD_PORTS_NAME}"
default_merge_from="github:freebsd:freebsd-ports:${PD_FREEBSD_PORTS_BRANCH} github:${CIRRUS_REPO_OWNER}:${CIRRUS_REPO_NAME}:${CIRRUS_BRANCH}"
fail_on_conflict="y"
__EOF__
# configure poudriere
cat << __EOF__ > /usr/local/etc/poudriere.conf
NO_ZFS=yes
BASEFS=${PD_BASEFS}
USE_TMPFS=data
FREEBSD_HOST=https://download.FreeBSD.org
FLAVOR_DEFAULT_ALL=yes
DISTFILES_CACHE=${PD_BASEFS}/ports/${PD_PORTS_NAME}/distfiles
CCACHE_DIR=${PD_CCACHE_DIR}
ALLOW_MAKE_JOBS=${PD_ALLOW_MAKE_JOBS}
ALLOW_MAKE_JOBS_PACKAGES="${PD_ALLOW_MAKE_JOBS_PACKAGES}"
# increase MAX_FILES. crosstool-NG.sh tries to modify ulimit during the
# build, which is not allowed.
#
# crosstool-NG.sh: line 112: ulimit: open files: cannot modify limit: Operation not permitted
MAX_FILES=4096
__EOF__
cat /usr/local/etc/poudriere.conf
required_packages_cache:
folder: /var/cache/pkg
fingerprint_script:
- uname -msr
- echo 20210909
populate_script: |
# do NOT quote PD_REQUIRED_PACKAGES
pkg fetch --dependencies --yes ${PD_REQUIRED_PACKAGES}
required_packages_install_script: |
# do NOT quote PD_REQUIRED_PACKAGES
pkg install --yes ${PD_REQUIRED_PACKAGES}
ports_tree_cache:
folder: /var/cache/portshaker/github_freebsd_freebsd-ports_${PD_FREEBSD_PORTS_BRANCH}
fingerprint_script:
- echo github_freebsd_freebsd-ports_${PD_FREEBSD_PORTS_BRANCH}
- echo 20210909
populate_script: |
time -h portshaker -u github:freebsd:freebsd-ports:${PD_FREEBSD_PORTS_BRANCH}
ports_tree_install_script: |
echo Updating "github:${CIRRUS_REPO_OWNER}:${CIRRUS_REPO_NAME}:${CIRRUS_BRANCH}"
ln -s "${CIRRUS_WORKING_DIR}" "/var/cache/portshaker/github_${CIRRUS_REPO_OWNER}_${CIRRUS_REPO_NAME}_${CIRRUS_BRANCH}"
ls -al /var/cache/portshaker
# create the merged ports tree
time -h portshaker -M
# create the merged ports tree
poudriere ports -c -p "${PD_PORTS_NAME}" -M "${PD_BASEFS}/ports/${PD_PORTS_NAME}" -m null
# ensure DISTFILES_CACHE exists
mkdir -p "${PD_BASEFS}/ports/${PD_PORTS_NAME}/distfiles"
create_jail_script: |
# cannot cache jails at the moment. see:
# https://github.com/freebsd/poudriere/issues/137
#
# also, the size of ${PD_BASEFS}/jails is too big to cache, and
# cirrus-ci refuses to cache
poudriere jail -c -j "${PD_JAIL_NAME}" -m ftp -v "${PD_RELEASE_NAME}"
build_script: |
# create artifacts directory
#
# https://cirrus-ci.org/guide/writing-tasks
# "Right now only storing files under $CIRRUS_WORKING_DIR folder as
# artifacts is supported with a total size limit of 1G for a community
# task"
mkdir -p "${CIRRUS_WORKING_DIR}/logs"
poudriere ports -l
poudriere jail -l
# build PD_ALL_ORIGIN
echo "Building ${PD_ALL_ORIGIN}"
# do NOT quote PD_ALL_ORIGIN
# use -b flag to pre-install binary packages on which ${PD_ALL_ORIGIN}
# depends. it saves time by skipping depended builds. -b flag requires
# ports-mgmt/poudriere-devel.
poudriere bulk -j "${PD_JAIL_NAME}" -p "${PD_PORTS_NAME}" -b "${PD_FREEBSD_PACKAGE_BRANCH}" -t -C ${PD_ALL_ORIGIN}
copy_packages_script: |
__PD_SRC="${PD_BASEFS}/data/packages/${PD_BUILD_NAME}/.latest/All"
__PD_DEST="packages/${PD_JAIL_NAME}/${PD_BUILD_NAME}/All"
ls -alR "${__PD_SRC}"
mkdir -p "${__PD_DEST}"
for P in ${PD_ALL_PACKAGES}; do
ls -al ${__PD_SRC}/${P}*
cp ${__PD_SRC}/${P}* "${__PD_DEST}"
done
ls -alR "${__PD_DEST}"
du -cksh "${__PD_DEST}"
packages_artifacts:
# can be downloaded at:
# https://api.cirrus-ci.com/v1/artifact/build/<CIRRUS_BUILD_ID>/<ARTIFACTS_NAME>.zip
path: "packages/${PD_JAIL_NAME}/${PD_BUILD_NAME}/All/*"
type: application/octet-stream
on_failure:
show_error_log_script: |
# do NOT quote, wildcard
for F in ${PD_BASEFS}/data/logs/bulk/${PD_BUILD_NAME}/latest/logs/errors/*.log; do
echo "${F}"
cat "${F}"
done
always:
copy_log_script: |
# do NOT quote, wildcard
cp ${PD_BASEFS}/data/logs/bulk/${PD_BUILD_NAME}/latest/logs/*.log "${CIRRUS_WORKING_DIR}/logs/"
log_artifacts:
path: "logs/*.log"
type: text/plain
generate_repo_task:
skip: "changesIncludeOnly('**.md')"
environment:
# use encrypted private key on cirrus-ci.
#
# XXX you need to create your own private and public keys, and upload the
# private key to cirrus-ci. see also pkg-repo(8)
PD_REPO_KEY: ENCRYPTED[55405781c279459db5e73e1f85a4b93e8ff65c40750e0a47fc621de48e1c3a84c794a5c7eb867ce0c4c25b3b7d813b1d]
depends_on:
- build
matrix:
- compute_engine_instance:
image_project: freebsd-org-cloud-dev
image: family/freebsd-13-0
platform: freebsd
download_packages_script: |
fetch -o "${CIRRUS_WORKING_DIR}/packages.zip" \
"https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/packages.zip"
(cd ${CIRRUS_WORKING_DIR} && unzip packages.zip)
generete_repo_script: |
# XXX change this to your GitHub account
PD_REPO_OWNER="trombik"
# sign the repo only if CIRRUS_REPO_OWNER is PD_REPO_OWNER
if [ "${CIRRUS_REPO_OWNER}" = "${PD_REPO_OWNER}" ]; then
echo "Sign the repo"
touch repo.key
chmod 0400 repo.key
echo "${PD_REPO_KEY}" > repo.key
else
echo "Not sign the repo"
fi
for D in ${CIRRUS_WORKING_DIR}/packages/*/* ; do
if [ -d "${D}" ]; then
ls -alR "${D}"
if [ -f repo.key ]; then
echo "Create a signed repo"
pkg repo "${D}" repo.key
else
echo "Create a repo"
pkg repo "${D}"
fi
ls -alR "${D}"
fi
done
mv "${CIRRUS_WORKING_DIR}/packages" "${CIRRUS_WORKING_DIR}/repos"
repos_artifacts:
# XXX cirrus-ci does not upload directories
paths:
- "repos/*"
- "repos/*/*"
- "repos/*/*/*"
- "repos/*/*/*/*"
test_task:
skip: "changesIncludeOnly('**.md')"
depends_on:
- generate_repo
matrix:
- compute_engine_instance:
image_project: freebsd-org-cloud-dev
image: family/freebsd-13-0
platform: freebsd
- compute_engine_instance:
image_project: freebsd-org-cloud-dev
image: family/freebsd-12-2
platform: freebsd
define_common_variables_script: |
# e.g. 122
PD_VERSION_SHORT=`uname -r | sed -e 's/-.*//' -e 's/\.//'`
# e.g. amd64
PD_MACHINE=`uname -m`
# e.g. 12.2-RELEASE
PD_RELEASE_NAME=`uname -r | sed -e 's/-p.*//'`
# e.g. 122amd64
PD_JAIL_NAME="${PD_VERSION_SHORT}${PD_MACHINE}"
PD_PORTS_NAME="default"
# e.g 122amd64-default
PD_BUILD_NAME="${PD_JAIL_NAME}-${PD_PORTS_NAME}"
cat << __EOF__ >> ${CIRRUS_ENV}
PD_VERSION_SHORT=${PD_VERSION_SHORT}
PD_MACHINE=${PD_MACHINE}
PD_RELEASE_NAME=${PD_RELEASE_NAME}
PD_JAIL_NAME=${PD_JAIL_NAME}
PD_PORTS_NAME=${PD_PORTS_NAME}
PD_BUILD_NAME=${PD_BUILD_NAME}
__EOF__
cat ${CIRRUS_ENV}
show_env_script: |
env | sort
download_repos_script: |
fetch -o ${CIRRUS_WORKING_DIR}/repos.zip \
"https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/repos.zip"
(cd ${CIRRUS_WORKING_DIR} && unzip repos.zip)
ls -alR "${CIRRUS_WORKING_DIR}/repos"
setup_pkg_conf_script: |
# XXX change this to your GitHub account
PD_REPO_OWNER="trombik"
# create FINGERPRINTS only if repo owner is me.
PD_FINGERPRINTS="NONE"
if [ "${CIRRUS_REPO_OWNER}" = "${PD_REPO_OWNER}" ]; then
echo "Create FINGERPRINTS"
PD_FINGERPRINTS="/usr/local/etc/pkg/fingerprints/local"
mkdir -p "${PD_FINGERPRINTS}"
cat .fingerprint > "${PD_FINGERPRINTS}/trusted"
else
echo "Do NOT create FINGERPRINTS"
fi
mkdir -p /usr/local/etc/pkg/repos
cat << __EOF__ > /usr/local/etc/pkg/repos/local.conf
local: {
URL: "file:///${CIRRUS_WORKING_DIR}/repos/${PD_JAIL_NAME}/${PD_BUILD_NAME}"
ENABLED: yes
FINGERPRINTS: "${PD_FINGERPRINTS}"
}
__EOF__
cat /usr/local/etc/pkg/repos/local.conf
install_packages_script: |
# create a list of all packages in the repository, and install them.
PD_ALL_ORIGIN=`echo */* | sed -e 's|/[[:space:]]| |g' -e 's|/$||'`
echo "PD_ALL_ORIGIN=${PD_ALL_ORIGIN}"
# install ${PD_ALL_ORIGIN}
PD_ALL_PACKAGES=""
for D in ${PD_ALL_ORIGIN}; do
PD_FLAVORS=`make -V FLAVORS -C "${D}"`
echo "PD_FLAVORS=${PD_FLAVORS}"
if [ "x${PD_FLAVORS}" != "x" ]; then
for F in ${PD_FLAVORS}; do
echo "FLAVOR=${F}"
P=`make -V PKGNAME -C "${D}" FLAVOR="${F}"`
echo "${P}"
PD_ALL_PACKAGES="${PD_ALL_PACKAGES} ${P}"
echo "${PD_ALL_PACKAGES}"
done
else
P=`make -V PKGNAME -C "${D}"`
echo "${P}"
PD_ALL_PACKAGES="${PD_ALL_PACKAGES} ${P}"
echo "${PD_ALL_PACKAGES}"
fi
done
pkg install -y ${PD_ALL_PACKAGES}
test_script: |
# here is the test. add your own test here.
#
# a simple test to see if the binaries work
PD_FLAVORS=`make -C devel/esp-quick-toolchain -V FLAVORS`
PD_GCC_FILE_NAME="xtensa-lx106-elf-gcc"
for F in ${PD_FLAVORS}; do
ls -al /usr/local/esp-quick-toolchain-${F}/xtensa-lx106-elf/bin
file /usr/local/esp-quick-toolchain-${F}/xtensa-lx106-elf/bin/${PD_GCC_FILE_NAME}
/usr/local/esp-quick-toolchain-${F}/xtensa-lx106-elf/bin/${PD_GCC_FILE_NAME} --version
done
publish_task:
skip: $CIRRUS_TAG == ""
depends_on:
- test
matrix:
- compute_engine_instance:
image_project: freebsd-org-cloud-dev
image: family/freebsd-13-0
platform: freebsd
environment:
github_api_token: ENCRYPTED[a895bd61e6f00706488c64cda281a25154eafc1c60785a909f770ce58111a6d6c84366b54301595f51c4550435cb2f24]
publish_artifact_script: |
filename="repos.zip"
fetch -o ${CIRRUS_WORKING_DIR}/repos.zip \
"https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/${filename}"
# Author: Stefan Buck
# License: MIT
# https://gist.github.com/stefanbuck/ce788fee19ab6eb0b4447a85fc99f447
#
# upload-github-release-asset.sh github_api_token=TOKEN owner=stefanbuck repo=playground tag=v0.1.0 filename=./build.zip
echo "CIRRUS_TAG=${CIRRUS_TAG}"
if [ "x${CIRRUS_TAG}" != "x" ]; then
GH_API="https://api.github.com"
GH_REPO="$GH_API/repos/${CIRRUS_REPO_OWNER}/${CIRRUS_REPO_NAME}"
GH_RELEASES="$GH_REPO/releases"
GH_TAGS="$GH_RELEASES/tags/${CIRRUS_TAG}"
AUTH="Authorization: token $github_api_token"
if test "$tag" == "LATEST"; then
GH_TAGS="$GH_REPO/releases/latest"
fi
# Validate token.
curl -o /dev/null -sH "$AUTH" $GH_REPO || { echo "Error: Invalid repo, token or network issue!"; exit 1; }
# Create a release from tag if it does not exist.
curl -s -d "{\"tag_name\":\"$tag\"}" -H "$AUTH" "$GH_RELEASES" > /dev/null
# Read asset tags.
response=$(curl -sH "$AUTH" $GH_TAGS)
# Get ID of the asset based on given filename.
eval "$(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=')"
echo "ID=${id}"
[ -n "$id" ] || { echo "Error: Failed to get release id for tag: $tag"; echo "$response" | awk 'length($0)<100' >&2; exit 1; }
# Upload asset
echo "Uploading asset... "
# Construct url
GH_ASSET="https://uploads.github.com/repos/${CIRRUS_REPO_OWNER}/${CIRRUS_REPO_NAME}/releases/$id/assets?name=$(basename $filename)"
echo "GH_ASSET=${GH_ASSET}"
curl "$GITHUB_OAUTH_BASIC" --data-binary @"$filename" -H "$AUTH" -H "Content-Type: application/octet-stream" $GH_ASSET
fi