scripts/.aglsetup_genconfig.bash

   1 #!/bin/bash
   2
   3 ################################################################################
   4 #
   5 # The MIT License (MIT)
   6 #
   7 # Copyright (c) 2016 Stéphane Desneux <sdx@iot.bzh>
   8 #           (c) 2016 Jan-Simon Möller <jsmoeller@linuxfoundation.org>
   9 #
  10 # Permission is hereby granted, free of charge, to any person obtaining a copy
  11 # of this software and associated documentation files (the "Software"), to deal
  12 # in the Software without restriction, including without limitation the rights
  13 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 # copies of the Software, and to permit persons to whom the Software is
  15 # furnished to do so, subject to the following conditions:
  16 #
  17 # The above copyright notice and this permission notice shall be included in
  18 # all copies or substantial portions of the Software.
  19 #
  20 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  23 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  24 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  25 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  26 # SOFTWARE.
  27 #
  28 ################################################################################
  29
  30 # this script shouldn't be called directly, but through aglsetup.sh that will in
  31 # turn execute (source) generated instructions back in the parent shell,
  32 # whether it's bash, zsh, or any other supported shell
  33
  34 VERSION=1.1.0
  35 DEFAULT_MACHINE=qemux86-64
  36 DEFAULT_BUILDDIR=./build
  37 VERBOSE=0
  38 DEBUG=0
  39
  40 #SCRIPT=$(basename $BASH_SOURCE)
  41 SCRIPT=aglsetup.sh
  42 SCRIPTDIR=$(cd $(dirname $BASH_SOURCE) && pwd -P)
  43 METADIR=$(cd $(dirname $BASH_SOURCE)/../.. && pwd -P)
  44
  45 function info() { echo "$@" >&2; }
  46 function infon() { echo -n "$@" >&2; }
  47 function error() { echo "ERROR: $@" >&2; return 1; }
  48 function verbose() { [[ $VERBOSE == 1 ]] && echo "$@" >&2; return 0; }
  49 function debug() { [[ $DEBUG == 1 ]] && echo "DEBUG: $@" >&2; return 0;}
  50
  51 info "------------ $SCRIPT: Starting"
  52
  53 #compute AGL_REPOSITORIES
  54 AGL_REPOSITORIES=$(for x in $(ls -d $METADIR/*/templates/{machine,feature}); do echo $(basename $(dirname $(dirname $x))); done | sort -u)
  55
  56 function list_machines() {
  57         for x in $@; do
  58                 for y in $(ls -d $METADIR/$x/templates/machine/* 2>/dev/null); do
  59                         echo $(basename $y)
  60                 done
  61         done
  62 }
  63
  64 function list_all_machines() {
  65         for x in $AGL_REPOSITORIES; do
  66                 list_machines $x
  67         done
  68 }
  69
  70 function validate_builddir() {
  71         if [[ "$BUILDDIR" =~ [[:space:]] ]]; then
  72                 error "Build dir '$BUILDDIR' shouldn't contain any space"
  73         fi
  74         debug "Build dir is valid"
  75 }
  76
  77 function validate_machines() {
  78         list_all_machines | sort | uniq -c | while read cnt machine; do
  79                 [[ $cnt == 1 ]] && continue
  80                 info "Machine $machine found in the following repositories:"
  81                 for x in $(ls -d $METADIR/*/templates/machine/$machine); do
  82                         info "   - $x"
  83                 done
  84                 error "Multiple machine templates are not allowed"
  85         done
  86         debug "Machines list has no duplicate."
  87 }
  88
  89 function list_features() {
  90         for x in $@; do
  91                 for y in $(ls -d $METADIR/$x/templates/feature/* 2>/dev/null); do
  92                         echo $(basename $y)
  93                 done
  94         done
  95 }
  96
  97 function list_all_features() {
  98         for x in $AGL_REPOSITORIES; do
  99                 list_features $x
 100         done
 101 }
 102
 103 function validate_features() {
 104         list_all_features | sort | uniq -c | while read cnt feature; do
 105                 [[ $cnt == 1 ]] && continue;
 106                 info "Feature $feature found in the following repositories:"
 107                 for x in $(ls -d $METADIR/*/templates/feature/$feature); do
 108                         info "   - $x"
 109                 done
 110                 error "Multiple feature templates are not allowed"
 111         done
 112         debug "Features list has no duplicate."
 113 }
 114
 115 function find_machine_dir() {
 116         machine=$1
 117         for x in $AGL_REPOSITORIES; do
 118                 dir=$METADIR/$x/templates/machine/$machine
 119                 [[ -d $dir ]] && { echo $dir; return 0; }
 120         done
 121         return 1
 122 }
 123
 124 function find_feature_dir() {
 125         feature=$1
 126         for x in $AGL_REPOSITORIES; do
 127                 dir=$METADIR/$x/templates/feature/$feature
 128                 [[ -d $dir ]] && { echo $dir; return 0; }
 129         done
 130         return 1
 131 }
 132
 133 function usage() {
 134     cat <<EOF >&2
 135 Usage: . $SCRIPT [options] [feature [feature [... ]]]
 136
 137 Version: $VERSION
 138 Compatibility: bash, zsh, ksh
 139
 140 Options:
 141    -m|--machine <machine>
 142       what machine to use
 143       default: '$DEFAULT_MACHINE'
 144    -b|--build <directory>
 145       build directory to use
 146       default: '$DEFAULT_BUILDDIR'
 147    -s|--script <filename>
 148       file where setup script is generated
 149       default: none (no script)
 150    -f|--force
 151       flag to force overwriting any existing configuration
 152       default: false
 153    -r|--rpm-revision <schema>
 154       Specify how to handle RPM packages revisions
 155       <schema> can be:
 156           'prservice[:<address>]' : Use a PR service daemon.
 157               if <address> is not specified, the default value 'localhost:0'
 158               is used (shortcut for a PR service started by bitbake)
 159           'timestamp' : Use a generated time stamp (UTC).
 160           'value:<revision>' : Use <revision> explicitly.
 161           'none' : Do nothing.
 162    -v|--verbose
 163       verbose mode
 164       default: false
 165    -d|--debug
 166       debug mode
 167       default: false
 168    -h|--help
 169       get some help
 170
 171 EOF
 172         local buf
 173
 174         echo "Available machines:" >&2
 175         for x in $AGL_REPOSITORIES; do
 176                 buf=$(list_machines $x)
 177                 [[ -z "$buf" ]] && continue
 178                 echo "   [$x]"
 179                 for y in $buf; do
 180                         [[ $y == $DEFAULT_MACHINE ]] && def="* " || def="  "
 181                         echo "     $def$y"
 182                 done
 183         done
 184         echo >&2
 185
 186         echo "Available features:" >&2
 187         for x in $AGL_REPOSITORIES; do
 188                 buf=$(list_features $x)
 189                 [[ -z "$buf" ]] && continue
 190                 echo "   [$x]"
 191                 for feature in $buf; do
 192                         print_feature="$feature"
 193                         featuredir=$(find_feature_dir $feature)
 194                         if [ -e $featuredir/included.dep ];then
 195                                 print_feature="$print_feature :($(find_feature_dependency $feature $feature))"
 196                         fi;
 197                         echo "       $print_feature"
 198                 done
 199         done
 200         echo >&2
 201 }
 202
 203 function append_fragment() {
 204         basefile=$1; shift # output file
 205         f=$1; shift # input file
 206         label=$(echo "$@")
 207
 208         debug "adding fragment to $basefile: $f"
 209         echo >>$basefile
 210         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
 211         echo "# fragment { " >>$basefile
 212         [[ -f $f ]] && echo "# $f" >>$basefile || true
 213         echo "#" >>$basefile
 214         [[ -n "$label" ]] && echo "$label" >>$basefile
 215         [[ -f $f ]] && cat $f >>$basefile || true
 216         echo "#" >>$basefile
 217         echo "# }" >>$basefile
 218         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
 219         [[ -f $f ]] && echo $f >>$BUILDDIR/conf/fragments.log || true
 220 }
 221
 222 function execute_setup() {
 223         script=$1
 224         debug "Executing script $script"
 225         opts="-e"
 226         [[ $DEBUG == 1 ]] && opts="$opts -x"
 227         pushd $BUILDDIR &>/dev/null
 228                 $BASH $opts $script \
 229                         && rc=0 \
 230                         || { rc=$?; error "Script $script failed"; }
 231         popd &>/dev/null
 232         return $rc
 233 }
 234
 235
 236 # process all fragments
 237 FRAGMENTS_BBLAYERS=""
 238 FRAGMENTS_LOCALCONF=""
 239 FRAGMENTS_SETUP=""
 240 function process_fragments() {
 241         for dir in "$@"; do
 242                 debug "processing fragments in dir $dir"
 243
 244                 verbose "   Searching fragments: $dir"
 245
 246                 # lookup for files with priorities specified: something like xx_bblayers.conf.yyyyy.inc
 247                 for x in $(ls $dir/??[._]bblayers.conf*.inc 2>/dev/null); do
 248                         FRAGMENTS_BBLAYERS="$FRAGMENTS_BBLAYERS $(basename $x):$x"
 249                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
 250                 done
 251
 252                 # same for local.conf
 253                 for x in $(ls $dir/??[._]local.conf*.inc 2>/dev/null); do
 254                         FRAGMENTS_LOCALCONF="$FRAGMENTS_LOCALCONF $(basename $x):$x"
 255                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
 256                 done
 257
 258                 # same fot setup.sh
 259                 for x in $(ls $dir/??[._]setup*.sh 2>/dev/null); do
 260                         FRAGMENTS_SETUP="$FRAGMENTS_SETUP $(basename $x):$x"
 261                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
 262                 done
 263         done
 264 }
 265
 266 function containsFeature () {
 267   for feature in $1; do
 268     [[ "$feature" == "$2" ]] && return 1;
 269   done;
 270   return 0;
 271 }
 272
 273 function find_feature_dependency() {
 274         res_dep_features=""
 275         featuredir=$(find_feature_dir $1)
 276         full_feature=$2;
 277         if [ -e $featuredir/included.dep ]; then
 278                 dep_features="$(cat $featuredir/included.dep)"
 279                 for dep_feature in $dep_features; do
 280                         full_feature="$full_feature $res_dep_features"
 281                         res_dep_features="$res_dep_features $dep_feature"
 282                         if  containsFeature $dep_feature $full_feature ; then
 283                                 res_dep_features="$res_dep_features $(find_feature_dependency $dep_feature $full_feature)"
 284                         fi;
 285                 done;
 286         fi;
 287         echo "$res_dep_features";
 288         return 0;
 289 }
 290
 291 GLOBAL_ARGS=( "$@" )
 292 debug "Parsing arguments: $@"
 293 TEMP=$(getopt -o m:b:r:s:fvdh --long machine:,builddir:,rpm-revision:,script:,force,verbose,debug,help -n $SCRIPT -- "$@")
 294 [[ $? != 0 ]] && { usage; exit 1; }
 295 eval set -- "$TEMP"
 296
 297 set -e
 298
 299 ### default options values
 300 MACHINE=$DEFAULT_MACHINE
 301 BUILDDIR=$DEFAULT_BUILDDIR
 302 SETUPSCRIPT=
 303 FORCE=
 304 RPMREVISION=
 305 SETUP_MANIFEST=aglsetup.manifest
 306
 307 while true; do
 308         case "$1" in
 309                 -m|--machine)      MACHINE=$2; shift 2;;
 310                 -b|--builddir)     BUILDDIR=$2; shift 2;;
 311                 -s|--setupscript)  SETUPSCRIPT=$2; shift 2;;
 312                 -f|--force)        FORCE=1; shift;;
 313                 -r|--rpm-revision) RPMREVISION=$2; shift 2;;
 314                 -v|--verbose)      VERBOSE=1; shift;;
 315                 -d|--debug)        VERBOSE=1; DEBUG=1; shift;;
 316                 -h|--help)         HELP=1; shift;;
 317                 --)                shift; break;;
 318                 *) error "Arguments parsing error"; exit 1;;
 319         esac
 320 done
 321
 322 [[ "$HELP" == 1 ]] && { usage; exit 0; }
 323
 324 verbose "Command line arguments: ${GLOBAL_ARGS[@]}"
 325
 326 # the remaining args are the features
 327 FEATURES="$@"
 328
 329 # validate the machine list
 330 debug "validating machines list"
 331 validate_machines
 332
 333 # validate the machine
 334 debug "validating machine $MACHINE"
 335 find_machine_dir $MACHINE >/dev/null || error "Machine '$MACHINE' not found in [ $(list_all_machines)]"
 336
 337 # validate the features list
 338 debug "validating features list"
 339 validate_features
 340
 341 TMP_FEATURES="";
 342 for FEATURE in $FEATURES;do
 343     TMP_FEATURES="$TMP_FEATURES $FEATURE"
 344     TMP_FEATURES="$TMP_FEATURES $(find_feature_dependency $FEATURE $TMP_FEATURES)"
 345 done
 346 # remove duplicate features if any
 347 FEATURES=$(for x in $TMP_FEATURES; do echo $x; done | sort -u | awk '{printf("%s ",$1);}')
 348
 349 # validate the features
 350 for f in $FEATURES; do
 351         debug "validating feature $f"
 352         find_feature_dir $f >/dev/null || error "Feature '$f' not found in [ $(list_all_features)]"
 353 done
 354
 355 # validate build dir
 356 debug "validating builddir $BUILDDIR"
 357 BUILDDIR=$(mkdir -p "$BUILDDIR" && cd "$BUILDDIR" && pwd -P)
 358 validate_builddir
 359
 360 ###########################################################################################
 361 function dump_log() {
 362         info "    ------------ $(basename $1) -----------------"
 363         sed 's/^/   | /g' $1
 364         info "    ----------------------------------------"
 365 }
 366
 367 function genconfig() {
 368         info "Generating configuration files:"
 369         info "   Build dir: $BUILDDIR"
 370         info "   Machine: $MACHINE"
 371         info "   Features: $FEATURES"
 372
 373         # step 1: run usual OE setup to generate conf dir
 374         export TEMPLATECONF=$(cd $SCRIPTDIR/../templates/base && pwd -P)
 375         debug "running oe-init-build-env with TEMPLATECONF=$TEMPLATECONF"
 376         info "   Running $METADIR/poky/oe-init-build-env"
 377         info "   Templates dir: $TEMPLATECONF"
 378
 379         CURDIR=$(pwd -P)
 380         . $METADIR/poky/oe-init-build-env $BUILDDIR >/dev/null
 381         cd $CURDIR
 382
 383         # step 2: concatenate other remaining fragments coming from base
 384         process_fragments $TEMPLATECONF
 385
 386         # step 3: fragments for machine
 387         process_fragments $(find_machine_dir $MACHINE)
 388
 389         # step 4: fragments for features
 390         for feature in $FEATURES; do
 391                 process_fragments $(find_feature_dir $feature)
 392         done
 393
 394         # step 5: sort fragments and append them in destination files
 395         FRAGMENTS_BBLAYERS=$(sed 's/ /\n/g' <<<$FRAGMENTS_BBLAYERS | sort)
 396         debug "bblayer fragments: $FRAGMENTS_BBLAYERS"
 397         info "   Config: $BUILDDIR/conf/bblayers.conf"
 398         for x in $FRAGMENTS_BBLAYERS; do
 399                 file=${x/#*:/}
 400                 append_fragment $BUILDDIR/conf/bblayers.conf $file
 401                 verbose "      + $file"
 402         done
 403
 404         FRAGMENTS_LOCALCONF=$(sed 's/ /\n/g' <<<$FRAGMENTS_LOCALCONF | sort)
 405         debug "localconf fragments: $FRAGMENTS_LOCALCONF"
 406         info "   Config: $BUILDDIR/conf/local.conf"
 407         for x in $FRAGMENTS_LOCALCONF; do
 408                 file=${x/#*:/}
 409                 append_fragment $BUILDDIR/conf/local.conf $file
 410                 verbose "      + $file"
 411         done
 412         # special fragment to call distro-manifest-generator.sh from
 413         # meta-agl-profile-core/recipes-core/distro-build-manifest/distro-build-manifest.bb
 414         append_fragment $BUILDDIR/conf/local.conf /dev/stdin "# generated by $(realpath $BASH_SOURCE)" <<-EOF
 415                 DISTRO_SETUP_MANIFEST = "$(realpath -Ls $BUILDDIR)/$SETUP_MANIFEST"
 416                 DISTRO_MANIFEST_GENERATOR = "$(dirname $(realpath $BASH_SOURCE))/distro-manifest-generator.sh"
 417         EOF
 418
 419         FRAGMENTS_SETUP=$(sed 's/ /\n/g' <<<$FRAGMENTS_SETUP | sort)
 420         debug "setup fragments: $FRAGMENTS_SETUP"
 421         cat <<EOF >$BUILDDIR/conf/setup.sh
 422 #!/bin/bash
 423
 424 # this script has been generated by $BASH_SOURCE
 425
 426 export MACHINE="$MACHINE"
 427 export FEATURES="$FEATURES"
 428 export BUILDDIR="$BUILDDIR"
 429 export METADIR="$METADIR"
 430 export RPMREVISION="$RPMREVISION"
 431 export LOCALCONF="$BUILDDIR/conf/local.conf"
 432
 433 echo "--- beginning of setup script"
 434 EOF
 435         info "   Setup script: $BUILDDIR/conf/setup.sh"
 436         for x in $FRAGMENTS_SETUP; do
 437                 file=${x/#*:/}
 438                 append_fragment $BUILDDIR/conf/setup.sh $file "echo '--- fragment $file'"
 439                 verbose "      + $file"
 440         done
 441         append_fragment $BUILDDIR/conf/setup.sh "" "echo '--- end of setup script'"
 442
 443         infon "   Executing setup script ... "
 444         execute_setup $BUILDDIR/conf/setup.sh 2>&1 | tee $BUILDDIR/conf/setup.log
 445         [[ ${PIPESTATUS[0]} == 0 ]] && {
 446                 info "OK"
 447                 [[ $VERBOSE == 1 ]] && dump_log $BUILDDIR/conf/setup.log
 448                 rm $BUILDDIR/conf/setup.sh
 449         } \
 450         || {
 451                 info "FAIL: please check $BUILDDIR/conf/setup.log"
 452                 dump_log $BUILDDIR/conf/setup.log
 453                 return 1
 454         }
 455 }
 456
 457 ###########################################################################################
 458
 459 # check for overwrite
 460 [[ $FORCE -eq 1 ]] && rm -f \
 461         $BUILDDIR/conf/local.conf \
 462         $BUILDDIR/conf/bblayers.conf \
 463         $BUILDDIR/conf/setup.* \
 464         $BUILDDIR/conf/*.log
 465
 466 ####### step 1: generate configuration file #######
 467
 468 if [[ -f $BUILDDIR/conf/local.conf || -f $BUILDDIR/conf/bblayers.conf ]]; then
 469         info "Configuration files already exist:"
 470         for x in $BUILDDIR/conf/local.conf $BUILDDIR/conf/bblayers.conf; do
 471                 [[ -f $x ]] && info "   - $x"
 472         done
 473         info "Skipping configuration files generation."
 474         info "Use option -f|--force to overwrite existing configuration."
 475 else
 476         genconfig
 477 fi
 478
 479 ####### step 2: generate aglsetup.manifest #######
 480
 481 infon "Generating setup manifest: $BUILDDIR/$SETUP_MANIFEST ... "
 482 for x in /etc/os-release /usr/lib/os-release; do
 483         [[ -f $x ]] && . $x
 484 done
 485 FEATURES_md5=$(echo $FEATURES|md5sum -|awk '{print $1;}')
 486 cat <<EOF >$BUILDDIR/$SETUP_MANIFEST
 487 # ----------------------------------------------
 488 # This fragment has been generated by $SCRIPT at setup time
 489
 490 # distro name
 491 DIST_DISTRO_NAME="AGL"
 492
 493 # target machine as passed to $SCRIPT
 494 DIST_MACHINE="$MACHINE"
 495
 496 # features as resolved by $SCRIPT
 497 DIST_FEATURES="$FEATURES"
 498 DIST_FEATURES_MD5="${FEATURES_md5}"
 499
 500 # build host information deduced from os-release
 501 DIST_BUILD_HOST="$(id -un)@$(hostname -f || hostname || hostname -s)"
 502 DIST_BUILD_OS="${PRETTY_NAME:-${NAME} ${VERSION} [COMPUTED]}"
 503
 504 # meta directory
 505 DIST_METADIR="$METADIR"
 506
 507 # timestamp
 508 DIST_SETUP_TS="$(date -u +%Y%m%d_%H%M%S_%Z)"
 509
 510 # ------------ end of $SCRIPT fragment --------
 511 EOF
 512 info "OK"
 513
 514 ####### step 3: generate agl-init-build-env #######
 515
 516 # always generate setup script in builddir: it can be sourced later manually without re-running the setup
 517 infon "Generating setup file: $BUILDDIR/agl-init-build-env ... "
 518
 519 cat <<EOF >$BUILDDIR/agl-init-build-env
 520 . $METADIR/poky/oe-init-build-env $BUILDDIR
 521 if [ -n "\$DL_DIR" ]; then
 522         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE DL_DIR"
 523 fi
 524 if [ -n "\$SSTATE_DIR" ]; then
 525         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE SSTATE_DIR"
 526 fi
 527 export BB_ENV_EXTRAWHITE
 528 unset TEMPLATECONF
 529 EOF
 530 info "OK"
 531
 532 ####### step 4: generate output script #######
 533
 534 # finally, generate output script if requested by caller
 535 if [[ -n "$SETUPSCRIPT" ]]; then
 536         debug "generating setupscript in $SETUPSCRIPT"
 537         cat <<EOF >$SETUPSCRIPT
 538 . $BUILDDIR/agl-init-build-env
 539 EOF
 540 fi
 541
 542 info "------------ $SCRIPT: Done"