aglsetup: Override templateconf.cfg in case of -f, --force.
[AGL/meta-agl.git] / scripts / .aglsetup_genconfig.bash
1 #!/bin/bash
2
3 ################################################################################
4 #
5 # The MIT License (MIT)
6 #
7 # Copyright (c) 2016-2019 Stéphane Desneux <sdx@iot.bzh>
8 #           (c) 2016 Jan-Simon Möller <jsmoeller@linuxfoundation.org>
9 #
10 # Permission is hereby granted, free of charge, to any person obtaining a copy
11 # of this software and associated documentation files (the "Software"), to deal
12 # in the Software without restriction, including without limitation the rights
13 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 # copies of the Software, and to permit persons to whom the Software is
15 # furnished to do so, subject to the following conditions:
16 #
17 # The above copyright notice and this permission notice shall be included in
18 # all copies or substantial portions of the Software.
19 #
20 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 # SOFTWARE.
27 #
28 ################################################################################
29
30 # this script shouldn't be called directly, but through aglsetup.sh that will in
31 # turn execute (source) generated instructions back in the parent shell,
32 # whether it's bash, zsh, or any other supported shell
33
34 VERSION=1.2.0
35 DEFAULT_MACHINE=qemux86-64
36 DEFAULT_BUILDDIR=./build
37 VERBOSE=0
38 SHOWVERSION=0
39 : ${DEBUG:=false}
40
41 #SCRIPT=$(basename $BASH_SOURCE)
42 SCRIPT=aglsetup.sh
43 SCRIPTDIR=$(cd $(dirname $BASH_SOURCE) && pwd -P)
44 METADIR=$(cd $(dirname $BASH_SOURCE)/../.. && pwd -P)
45
46 function info() { echo "$@" >&2; }
47 function infon() { echo -n "$@" >&2; }
48 function error() { echo "ERROR: $@" >&2; return 1; }
49 function verbose() { [[ $VERBOSE == 1 ]] && echo "$@" >&2; return 0; }
50 function debug() { $DEBUG && echo "DEBUG: $@" >&2; return 0;}
51
52 debug "------------ $SCRIPT: starting with command line arguments: $@"
53
54 #compute AGL_REPOSITORIES
55 AGL_REPOSITORIES=$(for x in $(ls -d $METADIR/meta-*/templates/{machine,feature} $METADIR/bsp/*/templates/machine 2>/dev/null); do echo $(basename $(dirname $(dirname $x))); done | sort -u)
56
57 function list_machines() {
58         for a in $@; do
59                 for y in $(ls -d $METADIR/{.,bsp}/$a/templates/machine/* 2>/dev/null); do
60                         echo $(basename $y)
61                 done
62         done
63 }
64
65 function list_all_machines() {
66         for x in $AGL_REPOSITORIES; do
67                 list_machines $x
68         done
69 }
70
71 function validate_builddir() {
72         if [[ "$BUILDDIR" =~ [[:space:]] ]]; then
73                 error "Build dir '$BUILDDIR' shouldn't contain any space"
74         fi
75         debug "Build dir is valid"
76 }
77
78 function validate_machines() {
79         list_all_machines | sort | uniq -c | while read cnt machine; do
80                 [[ $cnt == 1 ]] && continue
81                 info "Machine $machine found in the following repositories:"
82                 for x in $(ls -d $METADIR/*/templates/machine/$machine $METADIR/bsp/*/templates/machine/$machine 2>/dev/null); do
83                         info "   - $x"
84                 done
85                 error "Multiple machine templates are not allowed"
86         done
87         debug "Machines list has no duplicate."
88 }
89
90 function list_features() {
91         for x in $@; do
92                 for y in $(ls -d $METADIR/$x/templates/feature/* 2>/dev/null); do
93                         echo $(basename $y)
94                 done
95         done
96 }
97
98 function list_all_features() {
99         for x in $AGL_REPOSITORIES; do
100                 list_features $x
101         done
102 }
103
104 function validate_features() {
105         list_all_features | sort | uniq -c | while read cnt feature; do
106                 [[ $cnt == 1 ]] && continue;
107                 info "Feature $feature found in the following repositories:"
108                 for x in $(ls -d $METADIR/*/templates/feature/$feature 2>/dev/null); do
109                         info "   - $x"
110                 done
111                 error "Multiple feature templates are not allowed"
112         done
113         debug "Features list has no duplicate."
114 }
115
116 function find_machine_dir() {
117         machine=$1
118         for x in $AGL_REPOSITORIES; do
119                 dirs=$(ls -d $METADIR/{.,bsp}/$x/templates/machine/$machine 2>/dev/null)
120                 for dir in $dirs; do
121                     [[ -d $dir ]] && { echo $dir; return 0; }
122                 done
123         done
124         return 1
125 }
126
127 function find_feature_dir() {
128         feature=$1
129         for x in $AGL_REPOSITORIES; do
130                 dir=$METADIR/$x/templates/feature/$feature
131                 [[ -d $dir ]] && { echo $dir; return 0; }
132         done
133         return 1
134 }
135
136 function usage() {
137     cat <<EOF >&2
138 Usage: . $SCRIPT [options] [feature [feature [... ]]]
139
140 Version: $VERSION
141 Compatibility: bash, zsh, ksh
142
143 Options:
144    -m|--machine <machine>
145       what machine to use
146       default: '$DEFAULT_MACHINE'
147    -b|--build <directory>
148       build directory to use
149       default: '$DEFAULT_BUILDDIR'
150    -s|--script <filename>
151       file where setup script is generated
152       default: none (no script)
153    -f|--force
154       flag to force overwriting any existing configuration
155       default: false
156    -r|--rpm-revision <schema>
157       Specify how to handle RPM packages revisions
158       <schema> can be:
159           'prservice[:<address>]' : Use a PR service daemon.
160               if <address> is not specified, the default value 'localhost:0'
161               is used (shortcut for a PR service started by bitbake)
162           'timestamp' : Use a generated time stamp (UTC).
163           'value:<revision>' : Use <revision> explicitly.
164           'none' : Do nothing.
165    -t|--topic <value>
166       Specify an optional topic for this setup.
167       If specified, the topic will be propagated in build manifests:
168          - in deployment dir: tmp/deploy/images/*/build-info
169          - in target image: /etc/platform-info/build
170          - in SDK: tmp/deploy/sdk/*.build-info
171    -v|--verbose
172       verbose mode
173       default: false
174    -V|--version
175       display version, set AGLSETUP_VERSION variable with version value and exit
176    -d|--debug
177       debug mode
178       for early debug, set env variable DEBUG. 
179       for example:
180            DEBUG=true source aglsetup.sh -V
181       default: false
182    -h|--help
183       get some help
184
185 EOF
186         local buf
187
188         echo "Available machines:" >&2
189         for x in $AGL_REPOSITORIES; do
190                 buf=$(list_machines $x)
191                 [[ -z "$buf" ]] && continue
192                 echo "   [$x]"
193                 for y in $buf; do
194                         [[ $y == $DEFAULT_MACHINE ]] && def="* " || def="  "
195                         echo "     $def$y"
196                 done
197         done
198         echo >&2
199
200         echo "Available features:" >&2
201         for x in $AGL_REPOSITORIES; do
202                 buf=$(list_features $x)
203                 [[ -z "$buf" ]] && continue
204                 echo "   [$x]"
205                 for feature in $buf; do
206                         print_feature="$feature"
207                         featuredir=$(find_feature_dir $feature)
208                         if [ -e $featuredir/included.dep ];then
209                                 print_feature="$print_feature :($(find_feature_dependency $feature $feature))"
210                         fi;
211                         echo "       $print_feature"
212                 done
213         done
214         echo >&2
215 }
216
217 function append_fragment() {
218         basefile=$1; shift # output file
219         f=$1; shift # input file
220         label=$(echo "$@")
221
222         debug "adding fragment to $basefile: $f"
223         echo >>$basefile
224         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
225         echo "# fragment { " >>$basefile
226         [[ -f $f ]] && echo "# $f" >>$basefile || true
227         echo "#" >>$basefile
228         [[ -n "$label" ]] && echo "$label" >>$basefile
229         [[ -f $f ]] && cat $f >>$basefile || true
230         echo "" >>$basefile
231         echo "#" >>$basefile
232         echo "# }" >>$basefile
233         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
234         [[ -f $f ]] && echo $f >>$BUILDDIR/conf/fragments.log || true
235 }
236
237 function execute_setup() {
238         script=$1
239         debug "Executing script $script"
240         opts="-e"
241         $DEBUG && opts="$opts -x"
242         pushd $BUILDDIR &>/dev/null
243                 $BASH $opts $script \
244                         && rc=0 \
245                         || { rc=$?; error "Script $script failed"; }
246         popd &>/dev/null
247         return $rc
248 }
249
250
251 # process all fragments
252 FRAGMENTS_BBLAYERS=""
253 FRAGMENTS_LOCALCONF=""
254 FRAGMENTS_SETUP=""
255 function process_fragments() {
256         for dir in "$@"; do
257                 debug "processing fragments in dir $dir"
258
259                 verbose "   Searching fragments: $dir"
260
261                 # lookup for files with priorities specified: something like xx_bblayers.conf.yyyyy.inc
262                 for x in $(ls $dir/??[._]bblayers.conf*.inc 2>/dev/null); do
263                         FRAGMENTS_BBLAYERS="$FRAGMENTS_BBLAYERS $(basename $x):$x"
264                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
265                 done
266
267                 # same for local.conf
268                 for x in $(ls $dir/??[._]local.conf*.inc 2>/dev/null); do
269                         FRAGMENTS_LOCALCONF="$FRAGMENTS_LOCALCONF $(basename $x):$x"
270                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
271                 done
272
273                 # same fot setup.sh
274                 for x in $(ls $dir/??[._]setup*.sh 2>/dev/null); do
275                         FRAGMENTS_SETUP="$FRAGMENTS_SETUP $(basename $x):$x"
276                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
277                 done
278         done
279 }
280
281 function containsFeature () {
282   for feature in $1; do
283     [[ "$feature" == "$2" ]] && return 1;
284   done;
285   return 0;
286 }
287
288 function find_feature_dependency() {
289         res_dep_features=""
290         featuredir=$(find_feature_dir $1)
291         full_feature=$2;
292         if [ -e $featuredir/included.dep ]; then
293                 dep_features="$(cat $featuredir/included.dep)"
294                 for dep_feature in $dep_features; do
295                         full_feature="$full_feature $res_dep_features"
296                         res_dep_features="$res_dep_features $dep_feature"
297                         if  containsFeature $dep_feature $full_feature ; then
298                                 res_dep_features="$res_dep_features $(find_feature_dependency $dep_feature $full_feature)"
299                         fi;
300                 done;
301         fi;
302         echo "$res_dep_features";
303         return 0;
304 }
305
306 GLOBAL_ARGS=( "$@" )
307 debug "Parsing arguments: $@"
308 TEMP=$(getopt -o m:b:r:t:s:fvVdh --long machine:,builddir:,rpm-revision:,topic:,script:,force,verbose,version,debug,help -n $SCRIPT -- "$@")
309 [[ $? != 0 ]] && { usage; exit 1; }
310 eval set -- "$TEMP"
311
312 set -e
313
314 ### default options values
315 MACHINE=$DEFAULT_MACHINE
316 BUILDDIR=$DEFAULT_BUILDDIR
317 SETUPSCRIPT=
318 FORCE=
319 RPMREVISION=
320 TOPIC=
321 SETUP_MANIFEST=aglsetup.manifest
322
323 while true; do
324         case "$1" in
325                 -m|--machine)      MACHINE=$2; shift 2;;
326                 -b|--builddir)     BUILDDIR=$2; shift 2;;
327                 -s|--setupscript)  SETUPSCRIPT=$2; shift 2;;
328                 -f|--force)        FORCE=1; shift;;
329                 -r|--rpm-revision) RPMREVISION=$2; shift 2;;
330                 -t|--topic)        TOPIC=$2; shift 2;;
331                 -v|--verbose)      VERBOSE=1; shift;;
332                 -V|--version)      SHOWVERSION=1; shift;;
333                 -d|--debug)        VERBOSE=1; DEBUG=true; shift;;
334                 -h|--help)         HELP=1; shift;;
335                 --)                shift; break;;
336                 *) error "Arguments parsing error"; exit 1;;
337         esac
338 done
339
340 [[ "$HELP" == 1 ]] && { usage; exit 0; }
341
342 if [[ "$SHOWVERSION" == 1 ]]; then
343         # display version on stdout
344         echo "$VERSION"
345
346         # generate output script if requested by caller
347         if [[ -n "$SETUPSCRIPT" ]]; then
348                 cat <<EOF >$SETUPSCRIPT
349 AGLSETUP_VERSION=$VERSION
350 EOF
351         fi
352
353         # IMPORTANT: exit successfully
354         # older aglsetup scripts with version <1.2.0 will fail with option --version
355         exit 0
356 fi
357
358 info "------------ $SCRIPT: Starting"
359
360 verbose "Command line arguments: ${GLOBAL_ARGS[@]}"
361
362 # the remaining args are the features
363 FEATURES="$@"
364
365 # validate the machine list
366 debug "validating machines list"
367 validate_machines
368
369 # validate the machine
370 debug "validating machine $MACHINE"
371 find_machine_dir $MACHINE >/dev/null || error "Machine '$MACHINE' not found in [ $(list_all_machines)]"
372
373 # validate the features list
374 debug "validating features list"
375 validate_features
376
377 TMP_FEATURES="";
378 for FEATURE in $FEATURES;do
379     TMP_FEATURES="$TMP_FEATURES $FEATURE"
380     TMP_FEATURES="$TMP_FEATURES $(find_feature_dependency $FEATURE $TMP_FEATURES)"
381 done
382 # remove duplicate features if any
383 FEATURES=$(for x in $TMP_FEATURES; do echo $x; done | sort -u | awk '{printf("%s ",$1);}')
384
385 # validate the features
386 for f in $FEATURES; do
387         debug "validating feature $f"
388         find_feature_dir $f >/dev/null || error "Feature '$f' not found in [ $(list_all_features)]"
389 done
390
391 # validate build dir
392 debug "validating builddir $BUILDDIR"
393 BUILDDIR=$(mkdir -p "$BUILDDIR" && cd "$BUILDDIR" && pwd -P)
394 validate_builddir
395
396 ###########################################################################################
397 function dump_log() {
398         info "    ------------ $(basename $1) -----------------"
399         sed 's/^/   | /g' $1
400         info "    ----------------------------------------"
401 }
402
403 function genconfig() {
404         info "Generating configuration files:"
405         info "   Build dir: $BUILDDIR"
406         info "   Machine: $MACHINE"
407         info "   Features: $FEATURES"
408
409         # step 1: run usual OE setup to generate conf dir
410         export TEMPLATECONF=$(cd $SCRIPTDIR/../templates/base && pwd -P)
411         debug "running oe-init-build-env with TEMPLATECONF=$TEMPLATECONF"
412         info "   Running $METADIR/external/poky/oe-init-build-env"
413         info "   Templates dir: $TEMPLATECONF"
414
415         CURDIR=$(pwd -P)
416         . $METADIR/external/poky/oe-init-build-env $BUILDDIR >/dev/null
417         cd $CURDIR
418
419         # step 2: concatenate other remaining fragments coming from base
420         process_fragments $TEMPLATECONF
421
422         # step 3: fragments for machine
423         process_fragments $(find_machine_dir $MACHINE)
424
425         # step 4: fragments for features
426         for feature in $FEATURES; do
427                 process_fragments $(find_feature_dir $feature)
428         done
429
430         # step 5: sort fragments and append them in destination files
431         FRAGMENTS_BBLAYERS=$(sed 's/ /\n/g' <<<$FRAGMENTS_BBLAYERS | sort)
432         debug "bblayer fragments: $FRAGMENTS_BBLAYERS"
433         info "   Config: $BUILDDIR/conf/bblayers.conf"
434         for x in $FRAGMENTS_BBLAYERS; do
435                 file=${x/#*:/}
436                 append_fragment $BUILDDIR/conf/bblayers.conf $file
437                 verbose "      + $file"
438         done
439
440         FRAGMENTS_LOCALCONF=$(sed 's/ /\n/g' <<<$FRAGMENTS_LOCALCONF | sort)
441         debug "localconf fragments: $FRAGMENTS_LOCALCONF"
442         info "   Config: $BUILDDIR/conf/local.conf"
443         for x in $FRAGMENTS_LOCALCONF; do
444                 file=${x/#*:/}
445                 append_fragment $BUILDDIR/conf/local.conf $file
446                 verbose "      + $file"
447         done
448         # special fragment to call distro-manifest-generator.sh from 
449         # meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
450         append_fragment $BUILDDIR/conf/local.conf /dev/stdin "# generated by $(realpath $BASH_SOURCE)" <<-EOF
451                 DISTRO_SETUP_MANIFEST = "$(realpath -Ls $BUILDDIR)/$SETUP_MANIFEST"
452                 DISTRO_MANIFEST_GENERATOR = "$(dirname $(realpath $BASH_SOURCE))/distro-manifest-generator.sh"
453         EOF
454
455         FRAGMENTS_SETUP=$(sed 's/ /\n/g' <<<$FRAGMENTS_SETUP | sort)
456         debug "setup fragments: $FRAGMENTS_SETUP"
457         cat <<EOF >$BUILDDIR/conf/setup.sh
458 #!/bin/bash
459
460 # this script has been generated by $BASH_SOURCE
461
462 export MACHINE="$MACHINE"
463 export FEATURES="$FEATURES"
464 export BUILDDIR="$BUILDDIR"
465 export METADIR="$METADIR"
466 export RPMREVISION="$RPMREVISION"
467 export LOCALCONF="$BUILDDIR/conf/local.conf"
468
469 echo "--- beginning of setup script"
470 EOF
471         info "   Setup script: $BUILDDIR/conf/setup.sh"
472         for x in $FRAGMENTS_SETUP; do
473                 file=${x/#*:/}
474                 append_fragment $BUILDDIR/conf/setup.sh $file "echo '--- fragment $file'"
475                 verbose "      + $file"
476         done
477         append_fragment $BUILDDIR/conf/setup.sh "" "echo '--- end of setup script'"
478
479         infon "   Executing setup script ... "
480         execute_setup $BUILDDIR/conf/setup.sh 2>&1 | tee $BUILDDIR/conf/setup.log
481         [[ ${PIPESTATUS[0]} == 0 ]] && {
482                 info "OK"
483                 [[ $VERBOSE == 1 ]] && dump_log $BUILDDIR/conf/setup.log
484                 rm $BUILDDIR/conf/setup.sh
485         } \
486         || {
487                 info "FAIL: please check $BUILDDIR/conf/setup.log"
488                 dump_log $BUILDDIR/conf/setup.log
489                 return 1
490         }
491 }
492
493 ###########################################################################################
494
495 # check for overwrite
496 [[ $FORCE -eq 1 ]] && rm -f \
497         $BUILDDIR/conf/local.conf \
498         $BUILDDIR/conf/bblayers.conf \
499         $BUILDDIR/conf/templateconf.cfg \
500         $BUILDDIR/conf/setup.* \
501         $BUILDDIR/conf/*.log
502
503 ####### step 1: generate configuration file #######
504
505 if [[ -f $BUILDDIR/conf/local.conf || -f $BUILDDIR/conf/bblayers.conf ]]; then
506         info "Configuration files already exist:"
507         for x in $BUILDDIR/conf/local.conf $BUILDDIR/conf/bblayers.conf; do
508                 [[ -f $x ]] && info "   - $x"
509         done
510         info "Skipping configuration files generation."
511         info "Use option -f|--force to overwrite existing configuration."
512 else
513         genconfig
514 fi
515
516 ####### step 2: generate aglsetup.manifest #######
517
518 infon "Generating setup manifest: $BUILDDIR/$SETUP_MANIFEST ... "
519 for x in /etc/os-release /usr/lib/os-release; do
520         [[ -f $x ]] && . $x
521 done
522 FEATURES_md5=$(echo $FEATURES|md5sum -|awk '{print $1;}')
523 cat <<EOF >$BUILDDIR/$SETUP_MANIFEST
524 # ----------------------------------------------
525 # This fragment has been generated by $SCRIPT at setup time
526
527 # distro name
528 DIST_DISTRO_NAME="AGL"
529
530 # target machine as passed to $SCRIPT
531 DIST_MACHINE="$MACHINE"
532
533 # features as resolved by $SCRIPT
534 DIST_FEATURES="$FEATURES"
535 DIST_FEATURES_MD5="${FEATURES_md5}"
536
537 # build host information deduced from os-release
538 DIST_BUILD_HOST="$(id -un)@$(hostname -f || hostname || hostname -s)"
539 DIST_BUILD_OS="${PRETTY_NAME:-${NAME} ${VERSION} [COMPUTED]}"
540
541 # meta directory
542 DIST_METADIR="$METADIR"
543
544 # timestamp
545 DIST_SETUP_TS="$(date -u +%Y%m%d_%H%M%S_%Z)"
546
547 # topic
548 DIST_SETUP_TOPIC="$TOPIC"
549
550 # ------------ end of $SCRIPT fragment --------
551 EOF
552 info "OK"
553
554 ####### step 3: generate agl-init-build-env #######
555
556 # always generate setup script in builddir: it can be sourced later manually without re-running the setup
557 infon "Generating setup file: $BUILDDIR/agl-init-build-env ... "
558
559 cat <<EOF >$BUILDDIR/agl-init-build-env
560 . $METADIR/external/poky/oe-init-build-env $BUILDDIR
561 if [ -n "\$DL_DIR" ]; then
562         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE DL_DIR"
563 fi
564 if [ -n "\$SSTATE_DIR" ]; then
565         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE SSTATE_DIR"
566 fi
567 export BB_ENV_EXTRAWHITE
568 unset TEMPLATECONF
569 EOF
570 info "OK"
571
572 ####### step 4: generate output script #######
573
574 # finally, generate output script if requested by caller
575 if [[ -n "$SETUPSCRIPT" ]]; then
576         debug "generating setupscript in $SETUPSCRIPT"
577         cat <<EOF >$SETUPSCRIPT
578 . $BUILDDIR/agl-init-build-env
579 EOF
580 fi
581
582 info "------------ $SCRIPT: Done"