Add SDKs support.

Don't allow to install SDKs through XDS for now.
Only probe existing SDKs that have been manually installed using scripts/agl/install-agl-sdks.sh.

Signed-off-by: Sebastien Douheret <sebastien.douheret@iot.bzh>

diff --git a/.vscode/settings.json b/.vscode/settings.json
index a873478..9023297 100644 (file)
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -17,6 +17,7 @@
     "cSpell.words": [
         "apiv", "gonic", "devel", "csrffound", "Syncthing", "STID",
         "ISTCONFIG", "socketio", "ldflags", "SThg", "Intf", "dismissible",
-        "rpath", "WSID", "sess", "IXDS", "xdsconfig", "xdsserver", "mfolder"
+        "rpath", "WSID", "sess", "IXDS", "xdsconfig", "xdsserver", "mfolder", 
+       "inotify", "Inot", "pname", "pkill"
     ]
 }
\ No newline at end of file

diff --git a/README.md b/README.md
index 677ada2..7b60cdc 100644 (file)
--- a/README.md
+++ b/README.md
@@ -65,6 +65,7 @@ Supported fields in configuration file are:
     "webAppDir": "location of client dashboard (default: webapp/dist)",
     "shareRootDir": "root directory where projects will be copied",
     "logsDir": "directory to store logs (eg. syncthing output)",
+    "sdkRootDir": "root directory where cross SDKs are installed",
     "syncthing": {
         "binDir": "syncthing binaries directory (default: executable directory)",
         "home": "syncthing home directory (usually .../syncthing-config)",

diff --git a/config.json.in b/config.json.in
index dd34579..751bb29 100644 (file)
--- a/config.json.in
+++ b/config.json.in
@@ -2,9 +2,10 @@
     "webAppDir": "webapp/dist",
     "shareRootDir": "${ROOT_DIR}/tmp/builder_dev_host/share",
     "logsDir": "/tmp/xds-server/logs",
+    "sdkRootDir": "/xdt/sdk",
     "syncthing": {
         "binDir": "./bin",
-        "home": "${ROOT_DIR}/tmp/local_dev/syncthing-config",
+        "home": "${ROOT_DIR}/tmp/builder_dev_host/syncthing-config",
         "gui-address": "http://localhost:8384"
     }
 }
\ No newline at end of file

diff --git a/lib/apiv1/make.go b/lib/apiv1/make.go
index 0f7561f..1e6d937 100644 (file)
--- a/lib/apiv1/make.go
+++ b/lib/apiv1/make.go
@@ -138,6 +138,10 @@ func (s *APIService) buildMake(c *gin.Context) {
        cmdID := makeCommandID
        makeCommandID++
 
+       /* SEB TODO . /opt/poky-agl/3.90.0+snapshot/environment-setup-aarch64-agl-linux
+       env := os.Environ()
+       */
+
        s.log.Debugf("Execute [Cmd ID %d]: %v", cmdID, cmd)
        err := common.ExecPipeWs(cmd, sop, sess.ID, cmdID, execTmo, s.log, oCB, eCB)
        if err != nil {

diff --git a/lib/common/filepath.go b/lib/common/filepath.go
new file mode 100644 (file)
index 0000000..603c2a2
--- /dev/null
+++ b/lib/common/filepath.go
@@ -0,0 +1,15 @@
+package common
+
+import "os"
+
+// Exists returns whether the given file or directory exists or not
+func Exists(path string) bool {
+       _, err := os.Stat(path)
+       if err == nil {
+               return true
+       }
+       if os.IsNotExist(err) {
+               return false
+       }
+       return true
+}

diff --git a/lib/common/httpclient.go b/lib/common/httpclient.go
index 40d7bc2..72132bf 100644 (file)
--- a/lib/common/httpclient.go
+++ b/lib/common/httpclient.go
@@ -9,6 +9,8 @@ import (
        "io/ioutil"
        "net/http"
        "strings"
+
+       "github.com/Sirupsen/logrus"
 )
 
 type HTTPClient struct {
@@ -20,6 +22,7 @@ type HTTPClient struct {
        id         string
        csrf       string
        conf       HTTPClientConfig
+       logger     *logrus.Logger
 }
 
 type HTTPClientConfig struct {
@@ -30,6 +33,13 @@ type HTTPClientConfig struct {
        CsrfDisable         bool
 }
 
+const (
+       logError   = 1
+       logWarning = 2
+       logInfo    = 3
+       logDebug   = 4
+)
+
 // Inspired by syncthing/cmd/cli
 
 const insecure = false
@@ -64,6 +74,30 @@ func HTTPNewClient(baseURL string, cfg HTTPClientConfig) (*HTTPClient, error) {
        return &client, nil
 }
 
+// SetLogger Define the logger to use
+func (c *HTTPClient) SetLogger(log *logrus.Logger) {
+       c.logger = log
+}
+
+func (c *HTTPClient) log(level int, format string, args ...interface{}) {
+       if c.logger != nil {
+               switch level {
+               case logError:
+                       c.logger.Errorf(format, args...)
+                       break
+               case logWarning:
+                       c.logger.Warningf(format, args...)
+                       break
+               case logInfo:
+                       c.logger.Infof(format, args...)
+                       break
+               default:
+                       c.logger.Debugf(format, args...)
+                       break
+               }
+       }
+}
+
 // Send request to retrieve Client id and/or CSRF token
 func (c *HTTPClient) getCidAndCsrf() error {
        request, err := http.NewRequest("GET", c.endpoint, nil)
@@ -171,6 +205,8 @@ func (c *HTTPClient) handleRequest(request *http.Request) (*http.Response, error
                request.Header.Set("X-CSRF-Token-"+c.id[:5], c.csrf)
        }
 
+       c.log(logDebug, "HTTP %s %v", request.Method, request.URL)
+
        response, err := c.httpClient.Do(request)
        if err != nil {
                return nil, err

diff --git a/lib/crosssdk/sdk.go b/lib/crosssdk/sdk.go
new file mode 100644 (file)
index 0000000..2f22f22
--- /dev/null
+++ b/lib/crosssdk/sdk.go
@@ -0,0 +1,47 @@
+package crosssdk
+
+import (
+       "fmt"
+       "path"
+       "path/filepath"
+)
+
+// SDK Define a cross tool chain used to build application
+type SDK struct {
+       Profile string
+       Version string
+       Arch    string
+       Path    string
+       EnvFile string
+}
+
+// NewCrossSDK creates a new instance of Syncthing
+func NewCrossSDK(path string) (*SDK, error) {
+       // Assume that we have .../<profile>/<version>/<arch>
+       s := SDK{Path: path}
+
+       s.Arch = filepath.Base(path)
+
+       d := filepath.Dir(path)
+       s.Version = filepath.Base(d)
+
+       d = filepath.Dir(d)
+       s.Profile = filepath.Base(d)
+
+       envFile := filepath.Join(path, "environment-setup*")
+       ef, err := filepath.Glob(envFile)
+       if err != nil {
+               return nil, fmt.Errorf("Cannot retrieve environment setup file: %v", err)
+       }
+       if len(ef) != 1 {
+               return nil, fmt.Errorf("No environment setup file found match %s", envFile)
+       }
+       s.EnvFile = ef[0]
+
+       return &s, nil
+}
+
+// GetEnvCmd returns the command to initialized the environment to use a cross SDK
+func (s *SDK) GetEnvCmd() string {
+       return ". " + path.Join(s.Path, s.EnvFile)
+}

diff --git a/lib/crosssdk/sdks.go b/lib/crosssdk/sdks.go
new file mode 100644 (file)
index 0000000..435aae6
--- /dev/null
+++ b/lib/crosssdk/sdks.go
@@ -0,0 +1,39 @@
+package crosssdk
+
+import (
+       "path"
+       "path/filepath"
+
+       "github.com/Sirupsen/logrus"
+       "github.com/iotbzh/xds-server/lib/common"
+       "github.com/iotbzh/xds-server/lib/xdsconfig"
+)
+
+// SDKs List of installed SDK
+type SDKs []*SDK
+
+// Init creates a new instance of Syncthing
+func Init(cfg *xdsconfig.Config, log *logrus.Logger) (*SDKs, error) {
+       s := SDKs{}
+
+       // Retrieve installed sdks
+       sdkRD := cfg.FileConf.SdkRootDir
+
+       if common.Exists(sdkRD) {
+
+               // Assume that SDK install tree is <rootdir>/<profile>/<version>/<arch>
+               dirs, err := filepath.Glob(path.Join(sdkRD, "*", "*", "*"))
+               if err != nil {
+                       log.Debugf("Error while retrieving SDKs: dir=%s, error=%s", sdkRD, err.Error())
+                       return &s, err
+               }
+               for _, d := range dirs {
+                       sdk, err := NewCrossSDK(d)
+                       if err != nil {
+                               log.Debugf("Error while processing SDK dir=%s, err=%s", d, err.Error())
+                       }
+                       s = append(s, sdk)
+               }
+       }
+       return &s, nil
+}

diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go
index 15cab0d..9452fbd 100644 (file)
--- a/lib/syncthing/st.go
+++ b/lib/syncthing/st.go
@@ -27,13 +27,15 @@ type SyncThing struct {
        APIKey  string
        Home    string
        STCmd   *exec.Cmd
+       STICmd  *exec.Cmd
 
        // Private fields
-       binDir     string
-       logsDir    string
-       exitSTChan chan ExitChan
-       client     *common.HTTPClient
-       log        *logrus.Logger
+       binDir      string
+       logsDir     string
+       exitSTChan  chan ExitChan
+       exitSTIChan chan ExitChan
+       client      *common.HTTPClient
+       log         *logrus.Logger
 }
 
 // ExitChan Channel used for process exit
@@ -173,6 +175,28 @@ func (s *SyncThing) Start() (*exec.Cmd, error) {
        return s.STCmd, err
 }
 
+// StartInotify Starts syncthing-inotify process
+func (s *SyncThing) StartInotify() (*exec.Cmd, error) {
+       var err error
+
+       s.log.Infof(" STI home=%s", s.Home)
+       s.log.Infof(" STI  url=%s", s.BaseURL)
+
+       args := []string{
+               "--home=" + s.Home,
+               "-target=" + s.BaseURL,
+       }
+       if s.log.Level == logrus.DebugLevel {
+               args = append(args, "-verbosity=4")
+       }
+
+       env := []string{}
+
+       s.STICmd, err = s.startProc("syncthing-inotify", args, env, &s.exitSTIChan)
+
+       return s.STICmd, err
+}
+
 func (s *SyncThing) stopProc(pname string, proc *os.Process, exit chan ExitChan) {
        if err := proc.Signal(os.Interrupt); err != nil {
                s.log.Infof("Proc interrupt %s error: %s", pname, err.Error())
@@ -199,6 +223,15 @@ func (s *SyncThing) Stop() {
        s.STCmd = nil
 }
 
+// StopInotify Stops syncthing process
+func (s *SyncThing) StopInotify() {
+       if s.STICmd == nil {
+               return
+       }
+       s.stopProc("syncthing-inotify", s.STICmd.Process, s.exitSTIChan)
+       s.STICmd = nil
+}
+
 // Connect Establish HTTP connection with Syncthing
 func (s *SyncThing) Connect() error {
        var err error
@@ -217,6 +250,9 @@ func (s *SyncThing) Connect() error {
        if s.client == nil {
                return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)")
        }
+
+       s.client.SetLogger(s.log)
+
        return nil
 }
 

diff --git a/lib/webserver/server.go b/lib/webserver/server.go
index 7be157a..40ce948 100644 (file)
--- a/lib/webserver/server.go
+++ b/lib/webserver/server.go
@@ -11,13 +11,14 @@ import (
        "github.com/gin-gonic/gin"
        "github.com/googollee/go-socket.io"
        "github.com/iotbzh/xds-server/lib/apiv1"
+       "github.com/iotbzh/xds-server/lib/crosssdk"
        "github.com/iotbzh/xds-server/lib/model"
        "github.com/iotbzh/xds-server/lib/session"
        "github.com/iotbzh/xds-server/lib/xdsconfig"
 )
 
-// ServerService .
-type ServerService struct {
+// Server .
+type Server struct {
        router    *gin.Engine
        api       *apiv1.APIService
        sIOServer *socketio.Server
@@ -25,6 +26,7 @@ type ServerService struct {
        cfg       *xdsconfig.Config
        sessions  *session.Sessions
        mfolder   *model.Folder
+       sdks      *crosssdk.SDKs
        log       *logrus.Logger
        stop      chan struct{} // signals intentional stop
 }
@@ -32,11 +34,11 @@ type ServerService struct {
 const indexFilename = "index.html"
 const cookieMaxAge = "3600"
 
-// NewServer creates an instance of ServerService
-func NewServer(cfg *xdsconfig.Config, mfolder *model.Folder, log *logrus.Logger) *ServerService {
+// New creates an instance of Server
+func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, log *logrus.Logger) *Server {
 
        // Setup logging for gin router
-       if cfg.Log.Level == logrus.DebugLevel {
+       if log.Level == logrus.DebugLevel {
                gin.SetMode(gin.DebugMode)
        } else {
                gin.SetMode(gin.ReleaseMode)
@@ -51,15 +53,16 @@ func NewServer(cfg *xdsconfig.Config, mfolder *model.Folder, log *logrus.Logger)
        // Creates gin router
        r := gin.New()
 
-       svr := &ServerService{
+       svr := &Server{
                router:    r,
                api:       nil,
                sIOServer: nil,
                webApp:    nil,
                cfg:       cfg,
-               log:       log,
                sessions:  nil,
                mfolder:   mfolder,
+               sdks:      sdks,
+               log:       log,
                stop:      make(chan struct{}),
        }
 
@@ -67,7 +70,7 @@ func NewServer(cfg *xdsconfig.Config, mfolder *model.Folder, log *logrus.Logger)
 }
 
 // Serve starts a new instance of the Web Server
-func (s *ServerService) Serve() error {
+func (s *Server) Serve() error {
        var err error
 
        // Setup middlewares
@@ -128,17 +131,17 @@ func (s *ServerService) Serve() error {
 }
 
 // Stop web server
-func (s *ServerService) Stop() {
+func (s *Server) Stop() {
        close(s.stop)
 }
 
 // serveIndexFile provides initial file (eg. index.html) of webapp
-func (s *ServerService) serveIndexFile(c *gin.Context) {
+func (s *Server) serveIndexFile(c *gin.Context) {
        c.HTML(200, indexFilename, gin.H{})
 }
 
 // Add details in Header
-func (s *ServerService) middlewareXDSDetails() gin.HandlerFunc {
+func (s *Server) middlewareXDSDetails() gin.HandlerFunc {
        return func(c *gin.Context) {
                c.Header("XDS-Version", s.cfg.Version)
                c.Header("XDS-API-Version", s.cfg.APIVersion)
@@ -147,7 +150,7 @@ func (s *ServerService) middlewareXDSDetails() gin.HandlerFunc {
 }
 
 // CORS middleware
-func (s *ServerService) middlewareCORS() gin.HandlerFunc {
+func (s *Server) middlewareCORS() gin.HandlerFunc {
        return func(c *gin.Context) {
 
                if c.Request.Method == "OPTIONS" {
@@ -165,7 +168,7 @@ func (s *ServerService) middlewareCORS() gin.HandlerFunc {
 }
 
 // socketHandler is the handler for the "main" websocket connection
-func (s *ServerService) socketHandler(c *gin.Context) {
+func (s *Server) socketHandler(c *gin.Context) {
 
        // Retrieve user session
        sess := s.sessions.Get(c)

diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go
index 3f8a91d..465620b 100644 (file)
--- a/lib/xdsconfig/config.go
+++ b/lib/xdsconfig/config.go
@@ -7,6 +7,7 @@ import (
 
        "github.com/Sirupsen/logrus"
        "github.com/codegangsta/cli"
+       "github.com/iotbzh/xds-server/lib/common"
 )
 
 // Config parameters (json format) of /config command
@@ -57,14 +58,14 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) {
        }
 
        // Update location of shared dir if needed
-       if !dirExists(c.ShareRootDir) {
+       if !common.Exists(c.ShareRootDir) {
                if err := os.MkdirAll(c.ShareRootDir, 0770); err != nil {
                        return nil, fmt.Errorf("No valid shared directory found: %v", err)
                }
        }
        c.Log.Infoln("Share root directory: ", c.ShareRootDir)
 
-       if c.FileConf.LogsDir != "" && !dirExists(c.FileConf.LogsDir) {
+       if c.FileConf.LogsDir != "" && !common.Exists(c.FileConf.LogsDir) {
                if err := os.MkdirAll(c.FileConf.LogsDir, 0770); err != nil {
                        return nil, fmt.Errorf("Cannot create logs dir: %v", err)
                }
@@ -73,11 +74,3 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) {
 
        return &c, nil
 }
-
-func dirExists(path string) bool {
-       _, err := os.Stat(path)
-       if os.IsNotExist(err) {
-               return false
-       }
-       return true
-}

diff --git a/lib/xdsconfig/fileconfig.go b/lib/xdsconfig/fileconfig.go
index 3daf77c..776eb78 100644 (file)
--- a/lib/xdsconfig/fileconfig.go
+++ b/lib/xdsconfig/fileconfig.go
@@ -9,6 +9,8 @@ import (
        "path/filepath"
        "regexp"
        "strings"
+
+       "github.com/iotbzh/xds-server/lib/common"
 )
 
 type SyncThingConf struct {
@@ -21,6 +23,7 @@ type SyncThingConf struct {
 type FileConfig struct {
        WebAppDir    string         `json:"webAppDir"`
        ShareRootDir string         `json:"shareRootDir"`
+       SdkRootDir   string        `json:"sdkRootDir"`
        HTTPPort     string         `json:"httpPort"`
        SThgConf     *SyncThingConf `json:"syncthing"`
        LogsDir      string         `json:"logsDir"`
@@ -78,23 +81,20 @@ func updateConfigFromFile(c *Config, confFile string) error {
        c.FileConf = fCfg
 
        // Support environment variables (IOW ${MY_ENV_VAR} syntax) in config.json
-       // TODO: better to use reflect package to iterate on fields and be more generic
-       var rep string
-       if rep, err = resolveEnvVar(fCfg.WebAppDir); err != nil {
-               return err
-       }
-       fCfg.WebAppDir = path.Clean(rep)
-
-       if rep, err = resolveEnvVar(fCfg.ShareRootDir); err != nil {
-               return err
-       }
-       fCfg.ShareRootDir = path.Clean(rep)
-
-       if rep, err = resolveEnvVar(fCfg.SThgConf.Home); err != nil {
-               return err
+       for _, field := range []*string{
+               &fCfg.WebAppDir,
+               &fCfg.ShareRootDir,
+               &fCfg.SdkRootDir,
+               &fCfg.LogsDir,
+               &fCfg.SThgConf.Home} {
+
+               rep, err := resolveEnvVar(*field)
+               if err != nil {
+                       return err
+               }
+               *field = path.Clean(rep)
        }
-       fCfg.SThgConf.Home = path.Clean(rep)
-
+       
        // Config file settings overwrite default config
 
        if fCfg.WebAppDir != "" {
@@ -105,7 +105,7 @@ func updateConfigFromFile(c *Config, confFile string) error {
                // Check first from current directory
                for _, rootD := range []string{cwd, exePath} {
                        ff := path.Join(rootD, c.WebAppDir, "index.html")
-                       if exists(ff) {
+                       if common.Exists(ff) {
                                c.WebAppDir = path.Join(rootD, c.WebAppDir)
                                break
                        }
@@ -140,15 +140,3 @@ func resolveEnvVar(s string) (string, error) {
 
        return res, nil
 }
-
-// exists returns whether the given file or directory exists or not
-func exists(path string) bool {
-       _, err := os.Stat(path)
-       if err == nil {
-               return true
-       }
-       if os.IsNotExist(err) {
-               return false
-       }
-       return true
-}

diff --git a/main.go b/main.go
index ba445f5..36586cf 100644 (file)
--- a/main.go
+++ b/main.go
@@ -13,6 +13,7 @@ import (
 
        "github.com/Sirupsen/logrus"
        "github.com/codegangsta/cli"
+       "github.com/iotbzh/xds-server/lib/crosssdk"
        "github.com/iotbzh/xds-server/lib/model"
        "github.com/iotbzh/xds-server/lib/syncthing"
        "github.com/iotbzh/xds-server/lib/webserver"
@@ -39,15 +40,17 @@ var AppSubVersion = "unknown-dev"
 
 // Context holds the XDS server context
 type Context struct {
-       ProgName  string
-       Cli       *cli.Context
-       Config    *xdsconfig.Config
-       Log       *logrus.Logger
-       SThg      *st.SyncThing
-       SThgCmd   *exec.Cmd
-       MFolder   *model.Folder
-       WWWServer *webserver.ServerService
-       Exit      chan os.Signal
+       ProgName    string
+       Cli         *cli.Context
+       Config      *xdsconfig.Config
+       Log         *logrus.Logger
+       SThg        *st.SyncThing
+       SThgCmd     *exec.Cmd
+       SThgInotCmd *exec.Cmd
+       MFolder     *model.Folder
+       SDKs        *crosssdk.SDKs
+       WWWServer   *webserver.Server
+       Exit        chan os.Signal
 }
 
 // NewContext Create a new instance of XDS server
@@ -86,9 +89,10 @@ func NewContext(cliCtx *cli.Context) *Context {
 func handlerSigTerm(ctx *Context) {
        <-ctx.Exit
        if ctx.SThg != nil {
-               ctx.Log.Infof("Stopping Syncthing... (PID %d)",
-                       ctx.SThgCmd.Process.Pid)
+               ctx.Log.Infof("Stoping Syncthing... (PID %d)", ctx.SThgCmd.Process.Pid)
                ctx.SThg.Stop()
+               ctx.Log.Infof("Stoping Syncthing-inotify... (PID %d)", ctx.SThgInotCmd.Process.Pid)
+               ctx.SThg.StopInotify()
        }
        if ctx.WWWServer != nil {
                ctx.Log.Infof("Stoping Web server...")
@@ -97,7 +101,7 @@ func handlerSigTerm(ctx *Context) {
        os.Exit(1)
 }
 
-// xdsServer main routine
+// XDS Server application main routine
 func xdsApp(cliCtx *cli.Context) error {
        var err error
 
@@ -130,7 +134,15 @@ func xdsApp(cliCtx *cli.Context) error {
                }
                ctx.Log.Infof("Syncthing started (PID %d)", ctx.SThgCmd.Process.Pid)
 
+               ctx.Log.Infof("Starting Syncthing-inotify...")
+               ctx.SThgInotCmd, err = ctx.SThg.StartInotify()
+               if err != nil {
+                       return cli.NewExitError(err, 2)
+               }
+               ctx.Log.Infof("Syncthing-inotify started (PID %d)", ctx.SThgInotCmd.Process.Pid)
+
                // Establish connection with local Syncthing (retry if connection fail)
+               time.Sleep(2 * time.Second)
                retry := 10
                err = nil
                for retry > 0 {
@@ -177,8 +189,14 @@ func xdsApp(cliCtx *cli.Context) error {
                return cli.NewExitError(err, 3)
        }
 
+       // Init cross SDKs
+       ctx.SDKs, err = crosssdk.Init(ctx.Config, ctx.Log)
+       if err != nil {
+               return cli.NewExitError(err, 2)
+       }
+
        // Create and start Web Server
-       ctx.WWWServer = webserver.NewServer(ctx.Config, ctx.MFolder, ctx.Log)
+       ctx.WWWServer = webserver.New(ctx.Config, ctx.MFolder, ctx.SDKs, ctx.Log)
        if err = ctx.WWWServer.Serve(); err != nil {
                ctx.Log.Println(err)
                return cli.NewExitError(err, 3)

diff --git a/webapp/src/app/common/syncthing.service.ts b/webapp/src/app/common/syncthing.service.ts
index 2b4c609..28b19a9 100644 (file)
--- a/webapp/src/app/common/syncthing.service.ts
+++ b/webapp/src/app/common/syncthing.service.ts
@@ -29,7 +29,7 @@ export interface ISyncThingStatus {
 }
 
 // Private interfaces of Syncthing
-const ISTCONFIG_VERSION = 19;
+const ISTCONFIG_VERSION = 20;
 
 interface ISTFolderDeviceConfiguration {
     deviceID: string;