scripts/sdks/agl/_build-sdks-json.sh

   1 #!/bin/bash
   2 ###########################################################################
   3 # Copyright 2017-2018 IoT.bzh
   4 #
   5 # author: Sebastien Douheret <sebastien@iot.bzh>
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 ###########################################################################
  19
  20 # script_name=$(basename ${0})
  21 CURL="curl -f -s --connect-timeout 10"
  22
  23 ### Define urls where SDKs can be downloaded / AGL part
  24 SDK_AGL_BASEURL="https://download.automotivelinux.org/AGL/"
  25 DOWNLOADABLE_AGL_URLS="
  26     snapshots/master/latest/*
  27
  28     release/dab/4.0.3/*
  29
  30     release/eel/5.1.0/*
  31     release/eel/latest/*
  32
  33     release/flounder/6.0.3/*
  34     release/flounder/latest/*
  35
  36     release/guppy/6.99.2/*
  37     release/guppy/latest/*
  38 "
  39 #     release/dab/4.0.2/*
  40 #
  41 #     release/eel/5.0.0/*
  42 #     release/eel/5.0.1/*
  43 #     release/eel/5.0.2/*
  44 #     release/eel/5.0.3/*
  45 #     release/eel/5.1.0/*
  46 #     release/eel/latest/
  47 #
  48 #     release/flounder/5.99.6/*
  49
  50 SDK_AGL_URL_SUFFIX="/deploy/sdk/"
  51 IMG_AGL_URL_SUFFIX="/deploy/images/"
  52 #######################################
  53
  54 ### Define urls where SDKs can be downloaded / IoT.bzh part
  55 SDK_IOTBZH_BASEURL="http://iot.bzh/download/public/XDS/images/"
  56 DOWNLOADABLE_IOTBZH_URLS="
  57     dab-m3ulcb/*
  58
  59     eel-intel-corei7-64/*
  60     eel-m3ulcb/*
  61
  62     flounder-h3ulcb/*
  63     flounder-m3ulcb/*
  64     flounder-qemux86-64/*
  65
  66     master-m3ulcb/*
  67 "
  68
  69 SDK_IOTBZH_URL_SUFFIX="/sdk/"
  70 IMG_IOTBZH_URL_SUFFIX="/images/"
  71 #######################################
  72
  73
  74 # Compute full urls list  (parse '*' characters)
  75 function expand_url { # url, prefix, suffix
  76     local result=""
  77     url=$2$1
  78     if [[ "$url" = *"*"* ]]; then
  79         bUrl=$(echo $url | cut -d'*' -f 1)
  80         eUrl=$(echo $url | cut -d'*' -f 2)
  81         dirs=$(curl -s ${bUrl} | grep '\[DIR\]' | grep -oP  'href="[^"]*"' | cut -d'"' -f 2)
  82         for dir in $(echo $dirs); do
  83             result="$result ${bUrl::-1}/${dir::-1}${eUrl:1}"
  84         done
  85     else
  86         result="$url"
  87     fi
  88
  89     echo $result
  90 }
  91
  92 function dbg_error()  { >&2 echo "ERROR:" $@; }
  93 function dbg_warning(){ [ $DEBUG -ge 1 ] && { >&2 echo "WARNING:" $@; } ; }
  94 function dbg_info()   { [ $DEBUG -ge 1 ] && { >&2 echo "INFO:" $@; } ; }
  95 function dbg_debug()   { [ $DEBUG -ge 2 ] && { >&2 echo "DEBUG:" $@; } ; }
  96
  97 function count() { echo $#; }
  98
  99 function first() { echo $1; }
 100
 101
 102 function search_image_url {
 103     htmlPage=$(${CURL} ${1}/)
 104     if [[ $? == 0 ]]; then # curl request without error, at least URL is OK
 105         if [ "${2}" = "qemux86-64" ]; then
 106             image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.vmdk(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/')
 107         else
 108             image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.rootfs\.wic(\.xz)?"')) | sed -E 's/.*href="([^"]*)".*/\1/')
 109             # search for tar.xz if no wic found...
 110             if [ $(count $image_name) -eq 0 ]; then
 111                 image_name=$(echo $(first $(echo ${htmlPage} | egrep -o 'href="agl-demo-platform-crosssdk-[^"]*\.tar\.xz"')) | sed -E 's/.*href="([^"]*)".*/\1/')
 112             fi
 113         fi
 114     else
 115         dbg_warning "could not list content of ${1}, HTTP request failed."
 116         image_name="null"
 117     fi
 118
 119     # return string result
 120     if [[ ${image_name} = "null" ]]; then
 121         echo "null" # to be inserted as-is in the JSON output
 122     else
 123         echo "\"${1}/${image_name}\"" # if no image file found, but URL is valid, lets point to the directory
 124     fi
 125 }
 126
 127 # Given an URL, generate the corresponding JSON chunk
 128 function generate_json {
 129     url=$1
 130     SDK_URL=$1$2
 131     IMG_URL=$1$3
 132
 133     arch=""
 134
 135     htmlPage=$(${CURL} "${SDK_URL}")
 136     files=$(echo ${htmlPage} | egrep -o 'href="[^"]*crosssdk[^"]*\.sh"' | cut -d '"' -f 2)
 137     if [ "$?" != "0" ] || [ "${files}" = "" ]; then
 138         dbg_warning " IGNORED ${url}: no valid files found"
 139         return
 140     fi
 141
 142     # assume that sdk name follow this format :
 143     #  _PROFILE_-_COMPILER_ARCH_-_TARGET_-crosssdk-_ARCH_-toolchain-_VERSION_.sh
 144     # for example:
 145     #  poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain-4.0.1.sh
 146     if [ $(count files) -gt 1 ]
 147     then
 148         dbg_warning " IGNORE ${url}: multiple shell script containing 'crosssdk' in their names"
 149         return
 150     fi
 151     sdkFile=${files}
 152     dbg_info "Processing SDK: ${sdkFile} (from ${url})"
 153     profile=$(echo "${sdkFile}" | sed -r 's/(.*)-glibc.*/\1/')
 154     version=$(echo "${sdkFile}" | sed -r 's/.*toolchain-(.*).sh/\1/')
 155     arch=$(echo "${sdkFile}" | sed -r 's/.*crosssdk-(.*)-toolchain.*/\1/')
 156
 157     endUrl=${url#$SDK_AGL_BASEURL}
 158     if [ "${url::14}" = "http://iot.bzh" ]; then
 159         endUrl=${url#$SDK_IOTBZH_BASEURL}
 160         name=$(echo "IoTbzh-$(echo ${endUrl} | sed s:/:-:g)-${version}")
 161         board=""
 162     elif [ "${endUrl::4}" = "http" ]; then
 163         name="${profile}_${arch}_${version}"
 164         board=""
 165     else
 166         name=$(echo "AGL-$(echo ${endUrl} | cut -d'/' -f1,2,3,4)" | sed s:/:-:g)
 167         board=$(echo ${endUrl} | cut -d'/' -f4)
 168         board=${board%-nogfx}
 169     fi
 170
 171     # Distringuish qemux86-64 and corei7-64
 172     if [[ "$name" == *"qemux86-64"* && "$arch" == "corei7-64" ]]; then
 173         arch="qemux86-64"
 174     fi
 175
 176     [ "${profile}" = "" ] && { dbg_error "profile is not set" ; return ; }
 177     [ "${version}" = "" ] && { dbg_error "version is not set" ; return ; }
 178     [ "${arch}" = "" ] && { dbg_error "arch is not set" ; return ; }
 179     [ "${name}" = "" ] && { name=${profile}_${arch}_${version}; }
 180
 181     sdkDate="$(echo "${htmlPage}" |egrep -o ${sdkFile/+/\\+}'</a>.*[0-9\-]+ [0-9]+:[0-9]+' |cut -d'>' -f 4|cut -d' ' -f1,2)"
 182     sdkSize="$(echo "${htmlPage}" |egrep -o  "${sdkFile/+/\\+}.*${sdkDate}.*[0-9\.MG]+</td>" |cut -d'>' -f7 |cut -d'<' -f1)"
 183     md5sum="$(wget -q -O - ${SDK_URL}/${sdkFile/.sh/.md5} |cut -d' ' -f1)"
 184
 185     image_url=$(search_image_url ${IMG_URL}${board} ${arch})
 186
 187
 188     read -r -d '' res <<- EndOfMessage
 189 {
 190     "name":         "${name}",
 191     "description":  "AGL SDK ${arch} (version ${version})",
 192     "profile":      "${profile}",
 193     "version":      "${version}",
 194     "arch":         "${arch}",
 195     "path":         "",
 196     "url":          "${SDK_URL}${sdkFile}",
 197     "image_url":    ${image_url},
 198     "status":       "Not Installed",
 199     "date":         "${sdkDate}",
 200     "size":         "${sdkSize}",
 201     "md5sum":       "${md5sum}",
 202     "setupFile":    ""
 203 },
 204 EndOfMessage
 205
 206     sdksList="${sdksList}${res}"
 207 }
 208
 209 DEBUG=0
 210 if [[ ${1:0:2} = "-v" ]] ; then
 211     verbose=$(echo -s ${1} | sed -E "s/-([v]+).*/\1/")
 212     DEBUG=$(echo ${verbose} | wc -c)
 213     echo "Set verbosity level to : ${DEBUG}"
 214     shift
 215 fi
 216
 217 # Compute list of available/installable SDKs
 218 sdksList=" "
 219 for base_url in $DOWNLOADABLE_AGL_URLS
 220 do
 221     dbg_debug "Look into ${base_url}"
 222     urls="$(expand_url $base_url $SDK_AGL_BASEURL)"
 223     for url in $urls
 224     do
 225         dbg_debug "URL to check: ${url}"
 226         generate_json $url $SDK_AGL_URL_SUFFIX $IMG_AGL_URL_SUFFIX
 227     done
 228 done
 229
 230 for base_url in $DOWNLOADABLE_IOTBZH_URLS
 231 do
 232     dbg_debug "Look into ${base_url}"
 233     urls="$(expand_url $base_url $SDK_IOTBZH_BASEURL)"
 234     for url in $urls
 235     do
 236         dbg_debug "URL to check: ${url}"
 237         generate_json $url $SDK_IOTBZH_URL_SUFFIX $IMG_IOTBZH_URL_SUFFIX
 238     done
 239 done
 240
 241
 242 OUT_FILE=$(dirname "$0")/sdks_$(date +"%F_%H%M").json
 243
 244 echo "[" > ${OUT_FILE}
 245 echo "${sdksList::-1}" >> ${OUT_FILE}
 246 echo "]" >> ${OUT_FILE}
 247
 248 echo "SDKs list successfully saved in ${OUT_FILE}"