Import aglsetup templates and config for BBE
[AGL/meta-agl.git] / scripts / .aglsetup_genconfig.bash
1 #!/bin/bash
2
3 ################################################################################
4 #
5 # The MIT License (MIT)
6 #
7 # Copyright (c) 2016 Stéphane Desneux <sdx@iot.bzh>
8 #           (c) 2016 Jan-Simon Möller <jsmoeller@linuxfoundation.org>
9 #
10 # Permission is hereby granted, free of charge, to any person obtaining a copy
11 # of this software and associated documentation files (the "Software"), to deal
12 # in the Software without restriction, including without limitation the rights
13 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 # copies of the Software, and to permit persons to whom the Software is
15 # furnished to do so, subject to the following conditions:
16 #
17 # The above copyright notice and this permission notice shall be included in
18 # all copies or substantial portions of the Software.
19 #
20 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 # SOFTWARE.
27 #
28 ################################################################################
29
30 # this script shouldn't be called directly, but through aglsetup.sh that will in
31 # turn execute (source) generated instructions back in the parent shell,
32 # whether it's bash, zsh, or any other supported shell
33
34 VERSION=1.1.0
35 DEFAULT_MACHINE=qemux86-64
36 DEFAULT_BUILDDIR=./build
37 VERBOSE=0
38 DEBUG=0
39
40 #SCRIPT=$(basename $BASH_SOURCE)
41 SCRIPT=aglsetup.sh
42 SCRIPTDIR=$(cd $(dirname $BASH_SOURCE) && pwd -P)
43 METADIR=$(cd $(dirname $BASH_SOURCE)/../.. && pwd -P)
44
45 function info() { echo "$@" >&2; }
46 function infon() { echo -n "$@" >&2; }
47 function error() { echo "ERROR: $@" >&2; return 1; }
48 function verbose() { [[ $VERBOSE == 1 ]] && echo "$@" >&2; return 0; }
49 function debug() { [[ $DEBUG == 1 ]] && echo "DEBUG: $@" >&2; return 0;}
50
51 info "------------ $SCRIPT: Starting"
52
53 #compute AGL_REPOSITORIES
54 AGL_REPOSITORIES=$(for x in $(ls -d $METADIR/meta-ag*/templates/{machine,feature} $METADIR/bsp/*/templates/machine); do echo $(basename $(dirname $(dirname $x))); done | sort -u)
55
56 function list_machines() {
57         for a in $@; do
58                 for y in $(ls -d $METADIR/{.,bsp}/$a/templates/machine/* 2>/dev/null); do
59                         echo $(basename $y)
60                 done
61         done
62 }
63
64 function list_all_machines() {
65         for x in $AGL_REPOSITORIES; do
66                 list_machines $x
67         done
68 }
69
70 function validate_builddir() {
71         if [[ "$BUILDDIR" =~ [[:space:]] ]]; then
72                 error "Build dir '$BUILDDIR' shouldn't contain any space"
73         fi
74         debug "Build dir is valid"
75 }
76
77 function validate_machines() {
78         list_all_machines | sort | uniq -c | while read cnt machine; do
79                 [[ $cnt == 1 ]] && continue
80                 info "Machine $machine found in the following repositories:"
81                 for x in $(ls -d $METADIR/*/templates/machine/$machine $METADIR/bsp/*/templates/machine/$machine ); do
82                         info "   - $x"
83                 done
84                 error "Multiple machine templates are not allowed"
85         done
86         debug "Machines list has no duplicate."
87 }
88
89 function list_features() {
90         for x in $@; do
91                 for y in $(ls -d $METADIR/$x/templates/feature/* 2>/dev/null); do
92                         echo $(basename $y)
93                 done
94         done
95 }
96
97 function list_all_features() {
98         for x in $AGL_REPOSITORIES; do
99                 list_features $x
100         done
101 }
102
103 function validate_features() {
104         list_all_features | sort | uniq -c | while read cnt feature; do
105                 [[ $cnt == 1 ]] && continue;
106                 info "Feature $feature found in the following repositories:"
107                 for x in $(ls -d $METADIR/*/templates/feature/$feature ); do
108                         info "   - $x"
109                 done
110                 error "Multiple feature templates are not allowed"
111         done
112         debug "Features list has no duplicate."
113 }
114
115 function find_machine_dir() {
116         machine=$1
117         for x in $AGL_REPOSITORIES; do
118                 dirs=$(ls -d $METADIR/{.,bsp}/$x/templates/machine/$machine)
119                 for dir in $dirs; do
120                     [[ -d $dir ]] && { echo $dir; return 0; }
121                 done
122         done
123         return 1
124 }
125
126 function find_feature_dir() {
127         feature=$1
128         for x in $AGL_REPOSITORIES; do
129                 dir=$METADIR/$x/templates/feature/$feature
130                 [[ -d $dir ]] && { echo $dir; return 0; }
131         done
132         return 1
133 }
134
135 function usage() {
136     cat <<EOF >&2
137 Usage: . $SCRIPT [options] [feature [feature [... ]]]
138
139 Version: $VERSION
140 Compatibility: bash, zsh, ksh
141
142 Options:
143    -m|--machine <machine>
144       what machine to use
145       default: '$DEFAULT_MACHINE'
146    -b|--build <directory>
147       build directory to use
148       default: '$DEFAULT_BUILDDIR'
149    -s|--script <filename>
150       file where setup script is generated
151       default: none (no script)
152    -f|--force
153       flag to force overwriting any existing configuration
154       default: false
155    -r|--rpm-revision <schema>
156       Specify how to handle RPM packages revisions
157       <schema> can be:
158           'prservice[:<address>]' : Use a PR service daemon.
159               if <address> is not specified, the default value 'localhost:0'
160               is used (shortcut for a PR service started by bitbake)
161           'timestamp' : Use a generated time stamp (UTC).
162           'value:<revision>' : Use <revision> explicitly.
163           'none' : Do nothing.
164    -v|--verbose
165       verbose mode
166       default: false
167    -d|--debug
168       debug mode
169       default: false
170    -h|--help
171       get some help
172
173 EOF
174         local buf
175
176         echo "Available machines:" >&2
177         for x in $AGL_REPOSITORIES; do
178                 buf=$(list_machines $x)
179                 [[ -z "$buf" ]] && continue
180                 echo "   [$x]"
181                 for y in $buf; do
182                         [[ $y == $DEFAULT_MACHINE ]] && def="* " || def="  "
183                         echo "     $def$y"
184                 done
185         done
186         echo >&2
187
188         echo "Available features:" >&2
189         for x in $AGL_REPOSITORIES; do
190                 buf=$(list_features $x)
191                 [[ -z "$buf" ]] && continue
192                 echo "   [$x]"
193                 for feature in $buf; do
194                         print_feature="$feature"
195                         featuredir=$(find_feature_dir $feature)
196                         if [ -e $featuredir/included.dep ];then
197                                 print_feature="$print_feature :($(find_feature_dependency $feature $feature))"
198                         fi;
199                         echo "       $print_feature"
200                 done
201         done
202         echo >&2
203 }
204
205 function append_fragment() {
206         basefile=$1; shift # output file
207         f=$1; shift # input file
208         label=$(echo "$@")
209
210         debug "adding fragment to $basefile: $f"
211         echo >>$basefile
212         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
213         echo "# fragment { " >>$basefile
214         [[ -f $f ]] && echo "# $f" >>$basefile || true
215         echo "#" >>$basefile
216         [[ -n "$label" ]] && echo "$label" >>$basefile
217         [[ -f $f ]] && cat $f >>$basefile || true
218         echo "#" >>$basefile
219         echo "# }" >>$basefile
220         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
221         [[ -f $f ]] && echo $f >>$BUILDDIR/conf/fragments.log || true
222 }
223
224 function execute_setup() {
225         script=$1
226         debug "Executing script $script"
227         opts="-e"
228         [[ $DEBUG == 1 ]] && opts="$opts -x"
229         pushd $BUILDDIR &>/dev/null
230                 $BASH $opts $script \
231                         && rc=0 \
232                         || { rc=$?; error "Script $script failed"; }
233         popd &>/dev/null
234         return $rc
235 }
236
237
238 # process all fragments
239 FRAGMENTS_BBLAYERS=""
240 FRAGMENTS_LOCALCONF=""
241 FRAGMENTS_SETUP=""
242 function process_fragments() {
243         for dir in "$@"; do
244                 debug "processing fragments in dir $dir"
245
246                 verbose "   Searching fragments: $dir"
247
248                 # lookup for files with priorities specified: something like xx_bblayers.conf.yyyyy.inc
249                 for x in $(ls $dir/??[._]bblayers.conf*.inc 2>/dev/null); do
250                         FRAGMENTS_BBLAYERS="$FRAGMENTS_BBLAYERS $(basename $x):$x"
251                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
252                 done
253
254                 # same for local.conf
255                 for x in $(ls $dir/??[._]local.conf*.inc 2>/dev/null); do
256                         FRAGMENTS_LOCALCONF="$FRAGMENTS_LOCALCONF $(basename $x):$x"
257                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
258                 done
259
260                 # same fot setup.sh
261                 for x in $(ls $dir/??[._]setup*.sh 2>/dev/null); do
262                         FRAGMENTS_SETUP="$FRAGMENTS_SETUP $(basename $x):$x"
263                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
264                 done
265         done
266 }
267
268 function containsFeature () {
269   for feature in $1; do
270     [[ "$feature" == "$2" ]] && return 1;
271   done;
272   return 0;
273 }
274
275 function find_feature_dependency() {
276         res_dep_features=""
277         featuredir=$(find_feature_dir $1)
278         full_feature=$2;
279         if [ -e $featuredir/included.dep ]; then
280                 dep_features="$(cat $featuredir/included.dep)"
281                 for dep_feature in $dep_features; do
282                         full_feature="$full_feature $res_dep_features"
283                         res_dep_features="$res_dep_features $dep_feature"
284                         if  containsFeature $dep_feature $full_feature ; then
285                                 res_dep_features="$res_dep_features $(find_feature_dependency $dep_feature $full_feature)"
286                         fi;
287                 done;
288         fi;
289         echo "$res_dep_features";
290         return 0;
291 }
292
293 GLOBAL_ARGS=( "$@" )
294 debug "Parsing arguments: $@"
295 TEMP=$(getopt -o m:b:r:s:fvdh --long machine:,builddir:,rpm-revision:,script:,force,verbose,debug,help -n $SCRIPT -- "$@")
296 [[ $? != 0 ]] && { usage; exit 1; }
297 eval set -- "$TEMP"
298
299 set -e
300
301 ### default options values
302 MACHINE=$DEFAULT_MACHINE
303 BUILDDIR=$DEFAULT_BUILDDIR
304 SETUPSCRIPT=
305 FORCE=
306 RPMREVISION=
307 SETUP_MANIFEST=aglsetup.manifest
308
309 while true; do
310         case "$1" in
311                 -m|--machine)      MACHINE=$2; shift 2;;
312                 -b|--builddir)     BUILDDIR=$2; shift 2;;
313                 -s|--setupscript)  SETUPSCRIPT=$2; shift 2;;
314                 -f|--force)        FORCE=1; shift;;
315                 -r|--rpm-revision) RPMREVISION=$2; shift 2;;
316                 -v|--verbose)      VERBOSE=1; shift;;
317                 -d|--debug)        VERBOSE=1; DEBUG=1; shift;;
318                 -h|--help)         HELP=1; shift;;
319                 --)                shift; break;;
320                 *) error "Arguments parsing error"; exit 1;;
321         esac
322 done
323
324 [[ "$HELP" == 1 ]] && { usage; exit 0; }
325
326 verbose "Command line arguments: ${GLOBAL_ARGS[@]}"
327
328 # the remaining args are the features
329 FEATURES="$@"
330
331 # validate the machine list
332 debug "validating machines list"
333 validate_machines
334
335 # validate the machine
336 debug "validating machine $MACHINE"
337 find_machine_dir $MACHINE >/dev/null || error "Machine '$MACHINE' not found in [ $(list_all_machines)]"
338
339 # validate the features list
340 debug "validating features list"
341 validate_features
342
343 TMP_FEATURES="";
344 for FEATURE in $FEATURES;do
345     TMP_FEATURES="$TMP_FEATURES $FEATURE"
346     TMP_FEATURES="$TMP_FEATURES $(find_feature_dependency $FEATURE $TMP_FEATURES)"
347 done
348 # remove duplicate features if any
349 FEATURES=$(for x in $TMP_FEATURES; do echo $x; done | sort -u | awk '{printf("%s ",$1);}')
350
351 # validate the features
352 for f in $FEATURES; do
353         debug "validating feature $f"
354         find_feature_dir $f >/dev/null || error "Feature '$f' not found in [ $(list_all_features)]"
355 done
356
357 # validate build dir
358 debug "validating builddir $BUILDDIR"
359 BUILDDIR=$(mkdir -p "$BUILDDIR" && cd "$BUILDDIR" && pwd -P)
360 validate_builddir
361
362 ###########################################################################################
363 function dump_log() {
364         info "    ------------ $(basename $1) -----------------"
365         sed 's/^/   | /g' $1
366         info "    ----------------------------------------"
367 }
368
369 function genconfig() {
370         info "Generating configuration files:"
371         info "   Build dir: $BUILDDIR"
372         info "   Machine: $MACHINE"
373         info "   Features: $FEATURES"
374
375         # step 1: run usual OE setup to generate conf dir
376         export TEMPLATECONF=$(cd $SCRIPTDIR/../templates/base && pwd -P)
377         debug "running oe-init-build-env with TEMPLATECONF=$TEMPLATECONF"
378         info "   Running $METADIR/external/poky/oe-init-build-env"
379         info "   Templates dir: $TEMPLATECONF"
380
381         CURDIR=$(pwd -P)
382         . $METADIR/external/poky/oe-init-build-env $BUILDDIR >/dev/null
383         cd $CURDIR
384
385         # step 2: concatenate other remaining fragments coming from base
386         process_fragments $TEMPLATECONF
387
388         # step 3: fragments for machine
389         process_fragments $(find_machine_dir $MACHINE)
390
391         # step 4: fragments for features
392         for feature in $FEATURES; do
393                 process_fragments $(find_feature_dir $feature)
394         done
395
396         # step 5: sort fragments and append them in destination files
397         FRAGMENTS_BBLAYERS=$(sed 's/ /\n/g' <<<$FRAGMENTS_BBLAYERS | sort)
398         debug "bblayer fragments: $FRAGMENTS_BBLAYERS"
399         info "   Config: $BUILDDIR/conf/bblayers.conf"
400         for x in $FRAGMENTS_BBLAYERS; do
401                 file=${x/#*:/}
402                 append_fragment $BUILDDIR/conf/bblayers.conf $file
403                 verbose "      + $file"
404         done
405
406         FRAGMENTS_LOCALCONF=$(sed 's/ /\n/g' <<<$FRAGMENTS_LOCALCONF | sort)
407         debug "localconf fragments: $FRAGMENTS_LOCALCONF"
408         info "   Config: $BUILDDIR/conf/local.conf"
409         for x in $FRAGMENTS_LOCALCONF; do
410                 file=${x/#*:/}
411                 append_fragment $BUILDDIR/conf/local.conf $file
412                 verbose "      + $file"
413         done
414         # special fragment to call distro-manifest-generator.sh from 
415         # meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
416         append_fragment $BUILDDIR/conf/local.conf /dev/stdin "# generated by $(realpath $BASH_SOURCE)" <<-EOF
417                 DISTRO_SETUP_MANIFEST = "$(realpath -Ls $BUILDDIR)/$SETUP_MANIFEST"
418                 DISTRO_MANIFEST_GENERATOR = "$(dirname $(realpath $BASH_SOURCE))/distro-manifest-generator.sh"
419         EOF
420
421         FRAGMENTS_SETUP=$(sed 's/ /\n/g' <<<$FRAGMENTS_SETUP | sort)
422         debug "setup fragments: $FRAGMENTS_SETUP"
423         cat <<EOF >$BUILDDIR/conf/setup.sh
424 #!/bin/bash
425
426 # this script has been generated by $BASH_SOURCE
427
428 export MACHINE="$MACHINE"
429 export FEATURES="$FEATURES"
430 export BUILDDIR="$BUILDDIR"
431 export METADIR="$METADIR"
432 export RPMREVISION="$RPMREVISION"
433 export LOCALCONF="$BUILDDIR/conf/local.conf"
434
435 echo "--- beginning of setup script"
436 EOF
437         info "   Setup script: $BUILDDIR/conf/setup.sh"
438         for x in $FRAGMENTS_SETUP; do
439                 file=${x/#*:/}
440                 append_fragment $BUILDDIR/conf/setup.sh $file "echo '--- fragment $file'"
441                 verbose "      + $file"
442         done
443         append_fragment $BUILDDIR/conf/setup.sh "" "echo '--- end of setup script'"
444
445         infon "   Executing setup script ... "
446         execute_setup $BUILDDIR/conf/setup.sh 2>&1 | tee $BUILDDIR/conf/setup.log
447         [[ ${PIPESTATUS[0]} == 0 ]] && {
448                 info "OK"
449                 [[ $VERBOSE == 1 ]] && dump_log $BUILDDIR/conf/setup.log
450                 rm $BUILDDIR/conf/setup.sh
451         } \
452         || {
453                 info "FAIL: please check $BUILDDIR/conf/setup.log"
454                 dump_log $BUILDDIR/conf/setup.log
455                 return 1
456         }
457 }
458
459 ###########################################################################################
460
461 # check for overwrite
462 [[ $FORCE -eq 1 ]] && rm -f \
463         $BUILDDIR/conf/local.conf \
464         $BUILDDIR/conf/bblayers.conf \
465         $BUILDDIR/conf/setup.* \
466         $BUILDDIR/conf/*.log
467
468 ####### step 1: generate configuration file #######
469
470 if [[ -f $BUILDDIR/conf/local.conf || -f $BUILDDIR/conf/bblayers.conf ]]; then
471         info "Configuration files already exist:"
472         for x in $BUILDDIR/conf/local.conf $BUILDDIR/conf/bblayers.conf; do
473                 [[ -f $x ]] && info "   - $x"
474         done
475         info "Skipping configuration files generation."
476         info "Use option -f|--force to overwrite existing configuration."
477 else
478         genconfig
479 fi
480
481 ####### step 2: generate aglsetup.manifest #######
482
483 infon "Generating setup manifest: $BUILDDIR/$SETUP_MANIFEST ... "
484 for x in /etc/os-release /usr/lib/os-release; do
485         [[ -f $x ]] && . $x
486 done
487 FEATURES_md5=$(echo $FEATURES|md5sum -|awk '{print $1;}')
488 cat <<EOF >$BUILDDIR/$SETUP_MANIFEST
489 # ----------------------------------------------
490 # This fragment has been generated by $SCRIPT at setup time
491
492 # distro name
493 DIST_DISTRO_NAME="AGL"
494
495 # target machine as passed to $SCRIPT
496 DIST_MACHINE="$MACHINE"
497
498 # features as resolved by $SCRIPT
499 DIST_FEATURES="$FEATURES"
500 DIST_FEATURES_MD5="${FEATURES_md5}"
501
502 # build host information deduced from os-release
503 DIST_BUILD_HOST="$(id -un)@$(hostname -f || hostname || hostname -s)"
504 DIST_BUILD_OS="${PRETTY_NAME:-${NAME} ${VERSION} [COMPUTED]}"
505
506 # meta directory
507 DIST_METADIR="$METADIR"
508
509 # timestamp
510 DIST_SETUP_TS="$(date -u +%Y%m%d_%H%M%S_%Z)"
511
512 # ------------ end of $SCRIPT fragment --------
513 EOF
514 info "OK"
515
516 ####### step 3: generate agl-init-build-env #######
517
518 # always generate setup script in builddir: it can be sourced later manually without re-running the setup
519 infon "Generating setup file: $BUILDDIR/agl-init-build-env ... "
520
521 cat <<EOF >$BUILDDIR/agl-init-build-env
522 . $METADIR/external/poky/oe-init-build-env $BUILDDIR
523 if [ -n "\$DL_DIR" ]; then
524         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE DL_DIR"
525 fi
526 if [ -n "\$SSTATE_DIR" ]; then
527         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE SSTATE_DIR"
528 fi
529 export BB_ENV_EXTRAWHITE
530 unset TEMPLATECONF
531 EOF
532 info "OK"
533
534 ####### step 4: generate output script #######
535
536 # finally, generate output script if requested by caller
537 if [[ -n "$SETUPSCRIPT" ]]; then
538         debug "generating setupscript in $SETUPSCRIPT"
539         cat <<EOF >$SETUPSCRIPT
540 . $BUILDDIR/agl-init-build-env
541 EOF
542 fi
543
544 info "------------ $SCRIPT: Done"