scripts/.aglsetup_genconfig.bash

   1 #!/bin/bash
   2
   3 ################################################################################
   4 #
   5 # The MIT License (MIT)
   6 #
   7 # Copyright (c) 2016-2019 Stéphane Desneux <sdx@iot.bzh>
   8 #           (c) 2016 Jan-Simon Möller <jsmoeller@linuxfoundation.org>
   9 #
  10 # Permission is hereby granted, free of charge, to any person obtaining a copy
  11 # of this software and associated documentation files (the "Software"), to deal
  12 # in the Software without restriction, including without limitation the rights
  13 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 # copies of the Software, and to permit persons to whom the Software is
  15 # furnished to do so, subject to the following conditions:
  16 #
  17 # The above copyright notice and this permission notice shall be included in
  18 # all copies or substantial portions of the Software.
  19 #
  20 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  23 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  24 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  25 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  26 # SOFTWARE.
  27 #
  28 ################################################################################
  29
  30 # this script shouldn't be called directly, but through aglsetup.sh that will in
  31 # turn execute (source) generated instructions back in the parent shell,
  32 # whether it's bash, zsh, or any other supported shell
  33
  34 VERSION=1.2.0
  35 DEFAULT_MACHINE=qemux86-64
  36 DEFAULT_BUILDDIR=./build
  37 VERBOSE=0
  38 SHOWVERSION=0
  39 : ${DEBUG:=false}
  40
  41 #SCRIPT=$(basename $BASH_SOURCE)
  42 SCRIPT=aglsetup.sh
  43 SCRIPTDIR=$(cd $(dirname $BASH_SOURCE) && pwd -P)
  44 METADIR=$(cd $(dirname $BASH_SOURCE)/../.. && pwd -P)
  45
  46 function info() { echo "$@" >&2; }
  47 function infon() { echo -n "$@" >&2; }
  48 function error() { echo "ERROR: $@" >&2; return 1; }
  49 function verbose() { [[ $VERBOSE == 1 ]] && echo "$@" >&2; return 0; }
  50 function debug() { $DEBUG && echo "DEBUG: $@" >&2; return 0;}
  51
  52 debug "------------ $SCRIPT: starting with command line arguments: $@"
  53
  54 #compute AGL_REPOSITORIES
  55 AGL_REPOSITORIES=$(for x in $(ls -d $METADIR/*/templates/{machine,feature}); do echo $(basename $(dirname $(dirname $x))); done | sort -u)
  56
  57 function list_machines() {
  58         for x in $@; do
  59                 for y in $(ls -d $METADIR/$x/templates/machine/* 2>/dev/null); do
  60                         echo $(basename $y)
  61                 done
  62         done
  63 }
  64
  65 function list_all_machines() {
  66         for x in $AGL_REPOSITORIES; do
  67                 list_machines $x
  68         done
  69 }
  70
  71 function validate_builddir() {
  72         if [[ "$BUILDDIR" =~ [[:space:]] ]]; then
  73                 error "Build dir '$BUILDDIR' shouldn't contain any space"
  74         fi
  75         debug "Build dir is valid"
  76 }
  77
  78 function validate_machines() {
  79         list_all_machines | sort | uniq -c | while read cnt machine; do
  80                 [[ $cnt == 1 ]] && continue
  81                 info "Machine $machine found in the following repositories:"
  82                 for x in $(ls -d $METADIR/*/templates/machine/$machine); do
  83                         info "   - $x"
  84                 done
  85                 error "Multiple machine templates are not allowed"
  86         done
  87         debug "Machines list has no duplicate."
  88 }
  89
  90 function list_features() {
  91         for x in $@; do
  92                 for y in $(ls -d $METADIR/$x/templates/feature/* 2>/dev/null); do
  93                         echo $(basename $y)
  94                 done
  95         done
  96 }
  97
  98 function list_all_features() {
  99         for x in $AGL_REPOSITORIES; do
 100                 list_features $x
 101         done
 102 }
 103
 104 function validate_features() {
 105         list_all_features | sort | uniq -c | while read cnt feature; do
 106                 [[ $cnt == 1 ]] && continue;
 107                 info "Feature $feature found in the following repositories:"
 108                 for x in $(ls -d $METADIR/*/templates/feature/$feature); do
 109                         info "   - $x"
 110                 done
 111                 error "Multiple feature templates are not allowed"
 112         done
 113         debug "Features list has no duplicate."
 114 }
 115
 116 function find_machine_dir() {
 117         machine=$1
 118         for x in $AGL_REPOSITORIES; do
 119                 dir=$METADIR/$x/templates/machine/$machine
 120                 [[ -d $dir ]] && { echo $dir; return 0; }
 121         done
 122         return 1
 123 }
 124
 125 function find_feature_dir() {
 126         feature=$1
 127         for x in $AGL_REPOSITORIES; do
 128                 dir=$METADIR/$x/templates/feature/$feature
 129                 [[ -d $dir ]] && { echo $dir; return 0; }
 130         done
 131         return 1
 132 }
 133
 134 function usage() {
 135     cat <<EOF >&2
 136 Usage: . $SCRIPT [options] [feature [feature [... ]]]
 137
 138 Version: $VERSION
 139 Compatibility: bash, zsh, ksh
 140
 141 Options:
 142    -m|--machine <machine>
 143       what machine to use
 144       default: '$DEFAULT_MACHINE'
 145    -b|--build <directory>
 146       build directory to use
 147       default: '$DEFAULT_BUILDDIR'
 148    -s|--script <filename>
 149       file where setup script is generated
 150       default: none (no script)
 151    -f|--force
 152       flag to force overwriting any existing configuration
 153       default: false
 154    -r|--rpm-revision <schema>
 155       Specify how to handle RPM packages revisions
 156       <schema> can be:
 157           'prservice[:<address>]' : Use a PR service daemon.
 158               if <address> is not specified, the default value 'localhost:0'
 159               is used (shortcut for a PR service started by bitbake)
 160           'timestamp' : Use a generated time stamp (UTC).
 161           'value:<revision>' : Use <revision> explicitly.
 162           'none' : Do nothing.
 163    -t|--topic <value>
 164       Specify an optional topic for this setup.
 165       If specified, the topic will be propagated in build manifests:
 166          - in deployment dir: tmp/deploy/images/*/build-info
 167          - in target image: /etc/platform-info/build
 168          - in SDK: tmp/deploy/sdk/*.build-info
 169    -v|--verbose
 170       verbose mode
 171       default: false
 172    -V|--version
 173       display version, set AGLSETUP_VERSION variable with version value and exit
 174    -d|--debug
 175       debug mode
 176       for early debug, set env variable DEBUG.
 177       for example:
 178            DEBUG=true source aglsetup.sh -V
 179       default: false
 180    -h|--help
 181       get some help
 182
 183 EOF
 184         local buf
 185
 186         echo "Available machines:" >&2
 187         for x in $AGL_REPOSITORIES; do
 188                 buf=$(list_machines $x)
 189                 [[ -z "$buf" ]] && continue
 190                 echo "   [$x]"
 191                 for y in $buf; do
 192                         [[ $y == $DEFAULT_MACHINE ]] && def="* " || def="  "
 193                         echo "     $def$y"
 194                 done
 195         done
 196         echo >&2
 197
 198         echo "Available features:" >&2
 199         for x in $AGL_REPOSITORIES; do
 200                 buf=$(list_features $x)
 201                 [[ -z "$buf" ]] && continue
 202                 echo "   [$x]"
 203                 for feature in $buf; do
 204                         print_feature="$feature"
 205                         featuredir=$(find_feature_dir $feature)
 206                         if [ -e $featuredir/included.dep ];then
 207                                 print_feature="$print_feature :($(find_feature_dependency $feature $feature))"
 208                         fi;
 209                         echo "       $print_feature"
 210                 done
 211         done
 212         echo >&2
 213 }
 214
 215 function append_fragment() {
 216         basefile=$1; shift # output file
 217         f=$1; shift # input file
 218         label=$(echo "$@")
 219
 220         debug "adding fragment to $basefile: $f"
 221         echo >>$basefile
 222         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
 223         echo "# fragment { " >>$basefile
 224         [[ -f $f ]] && echo "# $f" >>$basefile || true
 225         echo "#" >>$basefile
 226         [[ -n "$label" ]] && echo "$label" >>$basefile
 227         [[ -f $f ]] && cat $f >>$basefile || true
 228         echo "" >>$basefile
 229         echo "#" >>$basefile
 230         echo "# }" >>$basefile
 231         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
 232         [[ -f $f ]] && echo $f >>$BUILDDIR/conf/fragments.log || true
 233 }
 234
 235 function execute_setup() {
 236         script=$1
 237         debug "Executing script $script"
 238         opts="-e"
 239         $DEBUG && opts="$opts -x"
 240         pushd $BUILDDIR &>/dev/null
 241                 $BASH $opts $script \
 242                         && rc=0 \
 243                         || { rc=$?; error "Script $script failed"; }
 244         popd &>/dev/null
 245         return $rc
 246 }
 247
 248
 249 # process all fragments
 250 FRAGMENTS_BBLAYERS=""
 251 FRAGMENTS_LOCALCONF=""
 252 FRAGMENTS_SETUP=""
 253 function process_fragments() {
 254         for dir in "$@"; do
 255                 debug "processing fragments in dir $dir"
 256
 257                 verbose "   Searching fragments: $dir"
 258
 259                 # lookup for files with priorities specified: something like xx_bblayers.conf.yyyyy.inc
 260                 for x in $(ls $dir/??[._]bblayers.conf*.inc 2>/dev/null); do
 261                         FRAGMENTS_BBLAYERS="$FRAGMENTS_BBLAYERS $(basename $x):$x"
 262                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
 263                 done
 264
 265                 # same for local.conf
 266                 for x in $(ls $dir/??[._]local.conf*.inc 2>/dev/null); do
 267                         FRAGMENTS_LOCALCONF="$FRAGMENTS_LOCALCONF $(basename $x):$x"
 268                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
 269                 done
 270
 271                 # same fot setup.sh
 272                 for x in $(ls $dir/??[._]setup*.sh 2>/dev/null); do
 273                         FRAGMENTS_SETUP="$FRAGMENTS_SETUP $(basename $x):$x"
 274                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
 275                 done
 276         done
 277 }
 278
 279 function containsFeature () {
 280   for feature in $1; do
 281     [[ "$feature" == "$2" ]] && return 1;
 282   done;
 283   return 0;
 284 }
 285
 286 function find_feature_dependency() {
 287         res_dep_features=""
 288         featuredir=$(find_feature_dir $1)
 289         full_feature=$2;
 290         if [ -e $featuredir/included.dep ]; then
 291                 dep_features="$(cat $featuredir/included.dep)"
 292                 for dep_feature in $dep_features; do
 293                         full_feature="$full_feature $res_dep_features"
 294                         res_dep_features="$res_dep_features $dep_feature"
 295                         if  containsFeature $dep_feature $full_feature ; then
 296                                 res_dep_features="$res_dep_features $(find_feature_dependency $dep_feature $full_feature)"
 297                         fi;
 298                 done;
 299         fi;
 300         echo "$res_dep_features";
 301         return 0;
 302 }
 303
 304 GLOBAL_ARGS=( "$@" )
 305 debug "Parsing arguments: $@"
 306 TEMP=$(getopt -o m:b:r:t:s:fvVdh --long machine:,builddir:,rpm-revision:,topic:,script:,force,verbose,version,debug,help -n $SCRIPT -- "$@")
 307 [[ $? != 0 ]] && { usage; exit 1; }
 308 eval set -- "$TEMP"
 309
 310 set -e
 311
 312 ### default options values
 313 MACHINE=$DEFAULT_MACHINE
 314 BUILDDIR=$DEFAULT_BUILDDIR
 315 SETUPSCRIPT=
 316 FORCE=
 317 RPMREVISION=
 318 TOPIC=
 319 SETUP_MANIFEST=aglsetup.manifest
 320
 321 while true; do
 322         case "$1" in
 323                 -m|--machine)      MACHINE=$2; shift 2;;
 324                 -b|--builddir)     BUILDDIR=$2; shift 2;;
 325                 -s|--setupscript)  SETUPSCRIPT=$2; shift 2;;
 326                 -f|--force)        FORCE=1; shift;;
 327                 -r|--rpm-revision) RPMREVISION=$2; shift 2;;
 328                 -t|--topic)        TOPIC=$2; shift 2;;
 329                 -v|--verbose)      VERBOSE=1; shift;;
 330                 -V|--version)      SHOWVERSION=1; shift;;
 331                 -d|--debug)        VERBOSE=1; DEBUG=true; shift;;
 332                 -h|--help)         HELP=1; shift;;
 333                 --)                shift; break;;
 334                 *) error "Arguments parsing error"; exit 1;;
 335         esac
 336 done
 337
 338 [[ "$HELP" == 1 ]] && { usage; exit 0; }
 339
 340 if [[ "$SHOWVERSION" == 1 ]]; then
 341         # display version on stdout
 342         echo "$VERSION"
 343
 344         # generate output script if requested by caller
 345         if [[ -n "$SETUPSCRIPT" ]]; then
 346                 cat <<EOF >$SETUPSCRIPT
 347 AGLSETUP_VERSION=$VERSION
 348 EOF
 349         fi
 350
 351         # IMPORTANT: exit successfully
 352         # older aglsetup scripts with version <1.2.0 will fail with option --version
 353         exit 0
 354 fi
 355
 356 info "------------ $SCRIPT: Starting"
 357
 358 verbose "Command line arguments: ${GLOBAL_ARGS[@]}"
 359
 360 # the remaining args are the features
 361 FEATURES="$@"
 362
 363 # validate the machine list
 364 debug "validating machines list"
 365 validate_machines
 366
 367 # validate the machine
 368 debug "validating machine $MACHINE"
 369 find_machine_dir $MACHINE >/dev/null || error "Machine '$MACHINE' not found in [ $(list_all_machines)]"
 370
 371 # validate the features list
 372 debug "validating features list"
 373 validate_features
 374
 375 TMP_FEATURES="";
 376 for FEATURE in $FEATURES;do
 377     TMP_FEATURES="$TMP_FEATURES $FEATURE"
 378     TMP_FEATURES="$TMP_FEATURES $(find_feature_dependency $FEATURE $TMP_FEATURES)"
 379 done
 380 # remove duplicate features if any
 381 FEATURES=$(for x in $TMP_FEATURES; do echo $x; done | sort -u | awk '{printf("%s ",$1);}')
 382
 383 # validate the features
 384 for f in $FEATURES; do
 385         debug "validating feature $f"
 386         find_feature_dir $f >/dev/null || error "Feature '$f' not found in [ $(list_all_features)]"
 387 done
 388
 389 # validate build dir
 390 debug "validating builddir $BUILDDIR"
 391 BUILDDIR=$(mkdir -p "$BUILDDIR" && cd "$BUILDDIR" && pwd -P)
 392 validate_builddir
 393
 394 ###########################################################################################
 395 function dump_log() {
 396         info "    ------------ $(basename $1) -----------------"
 397         sed 's/^/   | /g' $1
 398         info "    ----------------------------------------"
 399 }
 400
 401 function genconfig() {
 402         info "Generating configuration files:"
 403         info "   Build dir: $BUILDDIR"
 404         info "   Machine: $MACHINE"
 405         info "   Features: $FEATURES"
 406
 407         # step 1: run usual OE setup to generate conf dir
 408         export TEMPLATECONF=$(cd $SCRIPTDIR/../templates/base && pwd -P)
 409         debug "running oe-init-build-env with TEMPLATECONF=$TEMPLATECONF"
 410         info "   Running $METADIR/poky/oe-init-build-env"
 411         info "   Templates dir: $TEMPLATECONF"
 412
 413         CURDIR=$(pwd -P)
 414         . $METADIR/poky/oe-init-build-env $BUILDDIR >/dev/null
 415         cd $CURDIR
 416
 417         # step 2: concatenate other remaining fragments coming from base
 418         process_fragments $TEMPLATECONF
 419
 420         # step 3: fragments for machine
 421         process_fragments $(find_machine_dir $MACHINE)
 422
 423         # step 4: fragments for features
 424         for feature in $FEATURES; do
 425                 process_fragments $(find_feature_dir $feature)
 426         done
 427
 428         # step 5: sort fragments and append them in destination files
 429         FRAGMENTS_BBLAYERS=$(sed 's/ /\n/g' <<<$FRAGMENTS_BBLAYERS | sort)
 430         debug "bblayer fragments: $FRAGMENTS_BBLAYERS"
 431         info "   Config: $BUILDDIR/conf/bblayers.conf"
 432         for x in $FRAGMENTS_BBLAYERS; do
 433                 file=${x/#*:/}
 434                 append_fragment $BUILDDIR/conf/bblayers.conf $file
 435                 verbose "      + $file"
 436         done
 437
 438         FRAGMENTS_LOCALCONF=$(sed 's/ /\n/g' <<<$FRAGMENTS_LOCALCONF | sort)
 439         debug "localconf fragments: $FRAGMENTS_LOCALCONF"
 440         info "   Config: $BUILDDIR/conf/local.conf"
 441         for x in $FRAGMENTS_LOCALCONF; do
 442                 file=${x/#*:/}
 443                 append_fragment $BUILDDIR/conf/local.conf $file
 444                 verbose "      + $file"
 445         done
 446         # special fragment to call distro-manifest-generator.sh from
 447         # meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
 448         append_fragment $BUILDDIR/conf/local.conf /dev/stdin "# generated by $(realpath $BASH_SOURCE)" <<-EOF
 449                 DISTRO_SETUP_MANIFEST = "$(realpath -Ls $BUILDDIR)/$SETUP_MANIFEST"
 450                 DISTRO_MANIFEST_GENERATOR = "$(dirname $(realpath $BASH_SOURCE))/distro-manifest-generator.sh"
 451         EOF
 452
 453         FRAGMENTS_SETUP=$(sed 's/ /\n/g' <<<$FRAGMENTS_SETUP | sort)
 454         debug "setup fragments: $FRAGMENTS_SETUP"
 455         cat <<EOF >$BUILDDIR/conf/setup.sh
 456 #!/bin/bash
 457
 458 # this script has been generated by $BASH_SOURCE
 459
 460 export MACHINE="$MACHINE"
 461 export FEATURES="$FEATURES"
 462 export BUILDDIR="$BUILDDIR"
 463 export METADIR="$METADIR"
 464 export RPMREVISION="$RPMREVISION"
 465 export LOCALCONF="$BUILDDIR/conf/local.conf"
 466
 467 echo "--- beginning of setup script"
 468 EOF
 469         info "   Setup script: $BUILDDIR/conf/setup.sh"
 470         for x in $FRAGMENTS_SETUP; do
 471                 file=${x/#*:/}
 472                 append_fragment $BUILDDIR/conf/setup.sh $file "echo '--- fragment $file'"
 473                 verbose "      + $file"
 474         done
 475         append_fragment $BUILDDIR/conf/setup.sh "" "echo '--- end of setup script'"
 476
 477         infon "   Executing setup script ... "
 478         execute_setup $BUILDDIR/conf/setup.sh 2>&1 | tee $BUILDDIR/conf/setup.log
 479         [[ ${PIPESTATUS[0]} == 0 ]] && {
 480                 info "OK"
 481                 [[ $VERBOSE == 1 ]] && dump_log $BUILDDIR/conf/setup.log
 482                 rm $BUILDDIR/conf/setup.sh
 483         } \
 484         || {
 485                 info "FAIL: please check $BUILDDIR/conf/setup.log"
 486                 dump_log $BUILDDIR/conf/setup.log
 487                 return 1
 488         }
 489 }
 490
 491 ###########################################################################################
 492
 493 # check for overwrite
 494 [[ $FORCE -eq 1 ]] && rm -f \
 495         $BUILDDIR/conf/local.conf \
 496         $BUILDDIR/conf/bblayers.conf \
 497         $BUILDDIR/conf/setup.* \
 498         $BUILDDIR/conf/*.log
 499
 500 ####### step 1: generate configuration file #######
 501
 502 if [[ -f $BUILDDIR/conf/local.conf || -f $BUILDDIR/conf/bblayers.conf ]]; then
 503         info "Configuration files already exist:"
 504         for x in $BUILDDIR/conf/local.conf $BUILDDIR/conf/bblayers.conf; do
 505                 [[ -f $x ]] && info "   - $x"
 506         done
 507         info "Skipping configuration files generation."
 508         info "Use option -f|--force to overwrite existing configuration."
 509 else
 510         genconfig
 511 fi
 512
 513 ####### step 2: generate aglsetup.manifest #######
 514
 515 infon "Generating setup manifest: $BUILDDIR/$SETUP_MANIFEST ... "
 516 for x in /etc/os-release /usr/lib/os-release; do
 517         [[ -f $x ]] && . $x
 518 done
 519 FEATURES_md5=$(echo $FEATURES|md5sum -|awk '{print $1;}')
 520 cat <<EOF >$BUILDDIR/$SETUP_MANIFEST
 521 # ----------------------------------------------
 522 # This fragment has been generated by $SCRIPT at setup time
 523
 524 # distro name
 525 DIST_DISTRO_NAME="AGL"
 526
 527 # target machine as passed to $SCRIPT
 528 DIST_MACHINE="$MACHINE"
 529
 530 # features as resolved by $SCRIPT
 531 DIST_FEATURES="$FEATURES"
 532 DIST_FEATURES_MD5="${FEATURES_md5}"
 533
 534 # build host information deduced from os-release
 535 DIST_BUILD_HOST="$(id -un)@$(hostname -f || hostname || hostname -s)"
 536 DIST_BUILD_OS="${PRETTY_NAME:-${NAME} ${VERSION} [COMPUTED]}"
 537
 538 # meta directory
 539 DIST_METADIR="$METADIR"
 540
 541 # timestamp
 542 DIST_SETUP_TS="$(date -u +%Y%m%d_%H%M%S_%Z)"
 543
 544 # topic
 545 DIST_SETUP_TOPIC="$TOPIC"
 546
 547 # ------------ end of $SCRIPT fragment --------
 548 EOF
 549 info "OK"
 550
 551 ####### step 3: generate agl-init-build-env #######
 552
 553 # always generate setup script in builddir: it can be sourced later manually without re-running the setup
 554 infon "Generating setup file: $BUILDDIR/agl-init-build-env ... "
 555
 556 cat <<EOF >$BUILDDIR/agl-init-build-env
 557 . $METADIR/poky/oe-init-build-env $BUILDDIR
 558 if [ -n "\$DL_DIR" ]; then
 559         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE DL_DIR"
 560 fi
 561 if [ -n "\$SSTATE_DIR" ]; then
 562         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE SSTATE_DIR"
 563 fi
 564 export BB_ENV_EXTRAWHITE
 565 unset TEMPLATECONF
 566 EOF
 567 info "OK"
 568
 569 ####### step 4: generate output script #######
 570
 571 # finally, generate output script if requested by caller
 572 if [[ -n "$SETUPSCRIPT" ]]; then
 573         debug "generating setupscript in $SETUPSCRIPT"
 574         cat <<EOF >$SETUPSCRIPT
 575 . $BUILDDIR/agl-init-build-env
 576 EOF
 577 fi
 578
 579 info "------------ $SCRIPT: Done"