Re-enable security flags proposed in Poky
[AGL/meta-agl.git] / scripts / .aglsetup_genconfig.bash
1 #!/bin/bash
2
3 ################################################################################
4 #
5 # The MIT License (MIT)
6 #
7 # Copyright (c) 2016 Stéphane Desneux <sdx@iot.bzh>
8 #           (c) 2016 Jan-Simon Möller <jsmoeller@linuxfoundation.org>
9 #
10 # Permission is hereby granted, free of charge, to any person obtaining a copy
11 # of this software and associated documentation files (the "Software"), to deal
12 # in the Software without restriction, including without limitation the rights
13 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 # copies of the Software, and to permit persons to whom the Software is
15 # furnished to do so, subject to the following conditions:
16 #
17 # The above copyright notice and this permission notice shall be included in
18 # all copies or substantial portions of the Software.
19 #
20 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 # SOFTWARE.
27 #
28 ################################################################################
29
30 # this script shouldn't be called directly, but through aglsetup.sh that will in 
31 # turn execute (source) generated instructions back in the parent shell, 
32 # whether it's bash, zsh, or any other supported shell
33
34 VERSION=1.1.0
35 DEFAULT_MACHINE=qemux86-64
36 DEFAULT_BUILDDIR=./build
37 VERBOSE=0
38 DEBUG=0
39
40 #SCRIPT=$(basename $BASH_SOURCE)
41 SCRIPT=aglsetup.sh
42 SCRIPTDIR=$(cd $(dirname $BASH_SOURCE) && pwd -P)
43 METADIR=$(cd $(dirname $BASH_SOURCE)/../.. && pwd -P)
44
45 function info() { echo "$@" >&2; }
46 function infon() { echo -n "$@" >&2; }
47 function error() { echo "ERROR: $@" >&2; return 1; }
48 function verbose() { [[ $VERBOSE == 1 ]] && echo "$@" >&2; return 0; }
49 function debug() { [[ $DEBUG == 1 ]] && echo "DEBUG: $@" >&2; return 0;}
50
51 info "------------ $SCRIPT: Starting"
52
53 #compute AGL_REPOSITORIES
54 AGL_REPOSITORIES=$(for x in $(ls -d $METADIR/*/templates/{machine,feature}); do echo $(basename $(dirname $(dirname $x))); done | sort -u)
55
56 function list_machines() {
57         for x in $@; do
58                 for y in $(ls -d $METADIR/$x/templates/machine/* 2>/dev/null); do
59                         echo $(basename $y)
60                 done
61         done
62 }
63
64 function list_all_machines() {
65         for x in $AGL_REPOSITORIES; do
66                 list_machines $x
67         done
68 }
69
70 function validate_builddir() {
71         if [[ "$BUILDDIR" =~ [[:space:]] ]]; then
72                 error "Build dir '$BUILDDIR' shouldn't contain any space"
73         fi
74         debug "Build dir is valid"
75 }
76
77 function validate_machines() {
78         list_all_machines | sort | uniq -c | while read cnt machine; do
79                 [[ $cnt == 1 ]] && continue
80                 info "Machine $machine found in the following repositories:"
81                 for x in $(ls -d $METADIR/*/templates/machine/$machine); do
82                         info "   - $x"
83                 done
84                 error "Multiple machine templates are not allowed"
85         done
86         debug "Machines list has no duplicate."
87 }
88
89 function list_features() {
90         for x in $@; do
91                 for y in $(ls -d $METADIR/$x/templates/feature/* 2>/dev/null); do
92                         echo $(basename $y)
93                 done
94         done
95 }
96
97 function list_all_features() {
98         for x in $AGL_REPOSITORIES; do
99                 list_features $x
100         done
101 }
102
103 function validate_features() {
104         list_all_features | sort | uniq -c | while read cnt feature; do
105                 [[ $cnt == 1 ]] && continue;
106                 info "Feature $feature found in the following repositories:"
107                 for x in $(ls -d $METADIR/*/templates/feature/$feature); do
108                         info "   - $x"
109                 done
110                 error "Multiple feature templates are not allowed"
111         done
112         debug "Features list has no duplicate."
113 }
114
115 function find_machine_dir() {
116         machine=$1
117         for x in $AGL_REPOSITORIES; do
118                 dir=$METADIR/$x/templates/machine/$machine
119                 [[ -d $dir ]] && { echo $dir; return 0; }
120         done
121         return 1
122 }
123
124 function find_feature_dir() {
125         feature=$1
126         for x in $AGL_REPOSITORIES; do
127                 dir=$METADIR/$x/templates/feature/$feature
128                 [[ -d $dir ]] && { echo $dir; return 0; }
129         done
130         return 1
131 }
132
133 function usage() {
134     cat <<EOF >&2
135 Usage: . $SCRIPT [options] [feature [feature [... ]]]
136
137 Version: $VERSION
138 Compatibility: bash, zsh, ksh
139
140 Options:
141    -m|--machine <machine>
142       what machine to use
143       default: '$DEFAULT_MACHINE'
144    -b|--build <directory>
145       build directory to use
146       default: '$DEFAULT_BUILDDIR'
147    -s|--script <filename>
148       file where setup script is generated
149       default: none (no script)
150    -f|--force
151       flag to force overwriting any existing configuration
152       default: false
153    -v|--verbose
154       verbose mode
155       default: false
156    -d|--debug
157       debug mode
158       default: false
159    -h|--help
160       get some help
161
162 EOF
163         local buf
164         
165         echo "Available machines:" >&2
166         for x in $AGL_REPOSITORIES; do
167                 buf=$(list_machines $x)
168                 [[ -z "$buf" ]] && continue
169                 echo "   [$x]"
170                 for y in $buf; do 
171                         [[ $y == $DEFAULT_MACHINE ]] && def="* " || def="  "
172                         echo "     $def$y"
173                 done
174         done
175         echo >&2
176
177         echo "Available features:" >&2
178         for x in $AGL_REPOSITORIES; do
179                 buf=$(list_features $x)
180                 [[ -z "$buf" ]] && continue
181                 echo "   [$x]"
182                 for feature in $buf; do
183                         print_feature="$feature"
184                         featuredir=$(find_feature_dir $feature)
185                         if [ -e $featuredir/included.dep ];then
186                                 print_feature="$print_feature :($(find_feature_dependency $feature $feature))"
187                         fi;
188                         echo "       $print_feature"
189                 done
190         done
191         echo >&2
192 }
193
194 function append_fragment() {
195         basefile=$1; shift # output file
196         f=$1; shift # input file
197         label=$(echo "$@")
198
199         debug "adding fragment to $basefile: $f"
200         echo >>$basefile
201         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
202         echo "# fragment { " >>$basefile
203         [[ -f $f ]] && echo "# $f" >>$basefile || true
204         echo "#" >>$basefile
205         [[ -n "$label" ]] && echo "$label" >>$basefile
206         [[ -f $f ]] && cat $f >>$basefile || true
207         echo "#" >>$basefile
208         echo "# }" >>$basefile
209         echo "# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #" >>$basefile
210         [[ -f $f ]] && echo $f >>$BUILDDIR/conf/fragments.log || true
211 }
212
213 function execute_setup() {
214         script=$1
215         debug "Executing script $script"
216         opts=
217         [[ $DEBUG == 1 ]] && opts="$opts -x"
218         pushd $BUILDDIR &>/dev/null
219                 $BASH $opts $script \
220                         && rc=0 \
221                         || { rc=$?; error "Script $script failed"; }
222         popd &>/dev/null
223         return $rc
224 }
225
226
227 # process all fragments
228 FRAGMENTS_BBLAYERS=""
229 FRAGMENTS_LOCALCONF=""
230 FRAGMENTS_SETUP=""
231 function process_fragments() {
232         for dir in "$@"; do
233                 debug "processing fragments in dir $dir"
234
235                 verbose "   Searching fragments: $dir"
236
237                 # lookup for files with priorities specified: something like xx_bblayers.conf.yyyyy.inc
238                 for x in $(ls $dir/??[._]bblayers.conf*.inc 2>/dev/null); do
239                         FRAGMENTS_BBLAYERS="$FRAGMENTS_BBLAYERS $(basename $x):$x"
240                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
241                 done
242
243                 # same for local.conf
244                 for x in $(ls $dir/??[._]local.conf*.inc 2>/dev/null); do
245                         FRAGMENTS_LOCALCONF="$FRAGMENTS_LOCALCONF $(basename $x):$x"
246                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
247                 done
248
249                 # same fot setup.sh
250                 for x in $(ls $dir/??[._]setup*.sh 2>/dev/null); do
251                         FRAGMENTS_SETUP="$FRAGMENTS_SETUP $(basename $x):$x"
252                         verbose "      priority $(basename $x | cut -c1-2): $(basename $x)"
253                 done
254         done
255 }
256
257 function containsFeature () {
258   for feature in $1; do
259     [[ "$feature" == "$2" ]] && return 1;
260   done;
261   return 0;
262 }
263
264 function find_feature_dependency() {
265         res_dep_features=""
266         featuredir=$(find_feature_dir $1)
267         full_feature=$2;
268         if [ -e $featuredir/included.dep ]; then
269                 dep_features="$(cat $featuredir/included.dep)"
270                 for dep_feature in $dep_features; do
271                         full_feature="$full_feature $res_dep_features"
272                         res_dep_features="$res_dep_features $dep_feature"
273                         if  containsFeature $dep_feature $full_feature ; then
274                                 res_dep_features="$res_dep_features $(find_feature_dependency $dep_feature $full_feature)"
275                         fi;
276                 done;
277         fi;
278         echo "$res_dep_features";
279         return 0;
280 }
281
282 GLOBAL_ARGS=( "$@" )
283 debug "Parsing arguments: $@"
284 TEMP=$(getopt -o m:b:s:fvdh --long machine:,builddir:,script:,force,verbose,debug,help -n $SCRIPT -- "$@")
285 [[ $? != 0 ]] && { usage; exit 1; }
286 eval set -- "$TEMP"
287
288 set -e
289
290 ### default options values
291 MACHINE=$DEFAULT_MACHINE
292 BUILDDIR=$DEFAULT_BUILDDIR
293 SETUPSCRIPT=
294 FORCE=
295
296 while true; do
297         case "$1" in
298                 -m|--machine)  MACHINE=$2; shift 2;;
299                 -b|--builddir) BUILDDIR=$2; shift 2;;
300                 -s|--setupscript) SETUPSCRIPT=$2; shift 2;;
301                 -f|--force) FORCE=1; shift;;
302                 -v|--verbose) VERBOSE=1; shift;;
303                 -d|--debug) VERBOSE=1; DEBUG=1; shift;;
304                 -h|--help)     HELP=1; shift;;
305                 --)            shift; break;;
306                 *) error "Arguments parsing error"; exit 1;;
307         esac
308 done
309
310 [[ "$HELP" == 1 ]] && { usage; exit 0; }
311
312 verbose "Command line arguments: ${GLOBAL_ARGS[@]}"
313
314 # the remaining args are the features
315 FEATURES="$@"
316
317 # validate the machine list
318 debug "validating machines list"
319 validate_machines
320
321 # validate the machine
322 debug "validating machine $MACHINE"
323 find_machine_dir $MACHINE >/dev/null || error "Machine '$MACHINE' not found in [ $(list_all_machines)]"
324
325 # validate the features list
326 debug "validating features list"
327 validate_features
328
329 TMP_FEATURES="";
330 for FEATURE in $FEATURES;do
331     TMP_FEATURES="$TMP_FEATURES $FEATURE"
332     TMP_FEATURES="$TMP_FEATURES $(find_feature_dependency $FEATURE $TMP_FEATURES)"
333 done
334 # remove duplicate features if any
335 FEATURES=$(for x in $TMP_FEATURES; do echo $x; done | sort -u | awk '{printf("%s ",$1);}')
336
337 # validate the features
338 for f in $FEATURES; do
339         debug "validating feature $f"
340         find_feature_dir $f >/dev/null || error "Feature '$f' not found in [ $(list_all_features)]"
341 done
342
343 # validate build dir
344 debug "validating builddir $BUILDDIR"
345 BUILDDIR=$(mkdir -p "$BUILDDIR" && cd "$BUILDDIR" && pwd -P)
346 validate_builddir
347
348 ###########################################################################################
349 function dump_log() {
350         info "    ------------ $(basename $1) -----------------"
351         sed 's/^/   | /g' $1
352         info "    ----------------------------------------"
353 }
354
355 function genconfig() {
356         info "Generating configuration files:"
357         info "   Build dir: $BUILDDIR"
358         info "   Machine: $MACHINE"
359         info "   Features: $FEATURES"
360
361         # step 1: run usual OE setup to generate conf dir
362         export TEMPLATECONF=$(cd $SCRIPTDIR/../templates/base && pwd -P)
363         debug "running oe-init-build-env with TEMPLATECONF=$TEMPLATECONF"
364         info "   Running $METADIR/poky/oe-init-build-env"
365         info "   Templates dir: $TEMPLATECONF"
366
367         CURDIR=$(pwd -P)
368         . $METADIR/poky/oe-init-build-env $BUILDDIR >/dev/null
369         cd $CURDIR
370
371         # step 2: concatenate other remaining fragments coming from base
372         process_fragments $TEMPLATECONF
373
374         # step 3: fragments for machine
375         process_fragments $(find_machine_dir $MACHINE)
376
377         # step 4: fragments for features
378         for feature in $FEATURES; do
379                 process_fragments $(find_feature_dir $feature)
380         done
381
382         # step 5: sort fragments and append them in destination files
383         FRAGMENTS_BBLAYERS=$(sed 's/ /\n/g' <<<$FRAGMENTS_BBLAYERS | sort)
384         debug "bblayer fragments: $FRAGMENTS_BBLAYERS"
385         info "   Config: $BUILDDIR/conf/bblayers.conf"
386         for x in $FRAGMENTS_BBLAYERS; do
387                 file=${x/#*:/}
388                 append_fragment $BUILDDIR/conf/bblayers.conf $file
389                 verbose "      + $file"
390         done
391
392         FRAGMENTS_LOCALCONF=$(sed 's/ /\n/g' <<<$FRAGMENTS_LOCALCONF | sort)
393         debug "localconf fragments: $FRAGMENTS_LOCALCONF"
394         info "   Config: $BUILDDIR/conf/local.conf"
395         for x in $FRAGMENTS_LOCALCONF; do
396                 file=${x/#*:/}
397                 append_fragment $BUILDDIR/conf/local.conf $file
398                 verbose "      + $file"
399         done
400
401         FRAGMENTS_SETUP=$(sed 's/ /\n/g' <<<$FRAGMENTS_SETUP | sort)
402         debug "setup fragments: $FRAGMENTS_SETUP"
403         cat <<EOF >$BUILDDIR/conf/setup.sh
404 #!/bin/bash
405
406 # this script has been generated by $BASH_SOURCE
407
408 export MACHINE="$MACHINE"
409 export FEATURES="$FEATURES"
410 export BUILDDIR="$BUILDDIR"
411 export METADIR="$METADIR"
412
413 echo "--- beginning of setup script"
414 EOF
415         info "   Setup script: $BUILDDIR/conf/setup.sh"
416         for x in $FRAGMENTS_SETUP; do
417                 file=${x/#*:/}
418                 append_fragment $BUILDDIR/conf/setup.sh $file "echo '--- fragment $file'"
419                 verbose "      + $file"
420         done
421         append_fragment $BUILDDIR/conf/setup.sh "" "echo '--- end of setup script'"
422
423         infon "   Executing setup script ... "
424         execute_setup $BUILDDIR/conf/setup.sh 2>&1 | tee $BUILDDIR/conf/setup.log
425         [[ ${PIPESTATUS[0]} == 0 ]] && {
426                 info "OK"
427                 [[ $VERBOSE == 1 ]] && dump_log $BUILDDIR/conf/setup.log
428                 rm $BUILDDIR/conf/setup.sh
429         } \
430         || {
431                 info "FAIL: please check $BUILDDIR/conf/setup.log"
432                 dump_log $BUILDDIR/conf/setup.log
433                 return 1
434         }
435         # NOTE: the setup.sh script is removed if execution succeeded (only the log remains)
436 }
437
438 ###########################################################################################
439
440 # check for overwrite
441 [[ $FORCE -eq 1 ]] && rm -f \
442         $BUILDDIR/conf/local.conf \
443         $BUILDDIR/conf/bblayers.conf \
444         $BUILDDIR/conf/setup.* \
445         $BUILDDIR/conf/*.log
446
447 if [[ -f $BUILDDIR/conf/local.conf || -f $BUILDDIR/conf/bblayers.conf ]]; then
448         info "Configuration files already exist:"
449         for x in $BUILDDIR/conf/local.conf $BUILDDIR/conf/bblayers.conf; do
450                 [[ -f $x ]] && info "   - $x" 
451         done
452         info "Skipping configuration files generation."
453         info "Use option -f|--force to overwrite existing configuration."
454 else
455         genconfig
456 fi
457
458 # always generate setup script in builddir: it can be sourced later manually without re-running the setup
459 infon "Generating setup file: $BUILDDIR/agl-init-build-env ... "
460 cat <<EOF >$BUILDDIR/agl-init-build-env
461 . $METADIR/poky/oe-init-build-env $BUILDDIR
462 if [ -n "\$DL_DIR" ]; then
463         BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE DL_DIR"
464 fi
465 if [ -n "\$SSTATE_DIR" ]; then
466          BB_ENV_EXTRAWHITE="\$BB_ENV_EXTRAWHITE SSTATE_DIR" 
467 fi
468 export BB_ENV_EXTRAWHITE
469 unset TEMPLATECONF
470 EOF
471 info "OK"
472
473 # finally, generate output script if requested by caller
474 if [[ -n "$SETUPSCRIPT" ]]; then
475         debug "generating setupscript in $SETUPSCRIPT"
476         cat <<EOF >$SETUPSCRIPT
477 . $BUILDDIR/agl-init-build-env
478 EOF
479 fi
480
481 info "------------ $SCRIPT: Done"