From 40a7183f3b4aa32379aa8b4949f5f9c5e32f79f6 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Wed, 17 May 2017 11:28:57 +0200 Subject: [PATCH] Add SDKs support. Don't allow to install SDKs through XDS for now. Only probe existing SDKs that have been manually installed using scripts/agl/install-agl-sdks.sh. Signed-off-by: Sebastien Douheret --- .vscode/settings.json | 3 +- README.md | 1 + config.json.in | 3 +- lib/apiv1/make.go | 4 +++ lib/common/filepath.go | 15 ++++++++++ lib/common/httpclient.go | 36 +++++++++++++++++++++++ lib/crosssdk/sdk.go | 47 ++++++++++++++++++++++++++++++ lib/crosssdk/sdks.go | 39 +++++++++++++++++++++++++ lib/syncthing/st.go | 46 +++++++++++++++++++++++++---- lib/webserver/server.go | 29 +++++++++--------- lib/xdsconfig/config.go | 13 ++------- lib/xdsconfig/fileconfig.go | 46 +++++++++++------------------ main.go | 44 +++++++++++++++++++--------- webapp/src/app/common/syncthing.service.ts | 2 +- 14 files changed, 255 insertions(+), 73 deletions(-) create mode 100644 lib/common/filepath.go create mode 100644 lib/crosssdk/sdk.go create mode 100644 lib/crosssdk/sdks.go diff --git a/.vscode/settings.json b/.vscode/settings.json index a873478..9023297 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,7 @@ "cSpell.words": [ "apiv", "gonic", "devel", "csrffound", "Syncthing", "STID", "ISTCONFIG", "socketio", "ldflags", "SThg", "Intf", "dismissible", - "rpath", "WSID", "sess", "IXDS", "xdsconfig", "xdsserver", "mfolder" + "rpath", "WSID", "sess", "IXDS", "xdsconfig", "xdsserver", "mfolder", + "inotify", "Inot", "pname", "pkill" ] } \ No newline at end of file diff --git a/README.md b/README.md index 677ada2..7b60cdc 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Supported fields in configuration file are: "webAppDir": "location of client dashboard (default: webapp/dist)", "shareRootDir": "root directory where projects will be copied", "logsDir": "directory to store logs (eg. syncthing output)", + "sdkRootDir": "root directory where cross SDKs are installed", "syncthing": { "binDir": "syncthing binaries directory (default: executable directory)", "home": "syncthing home directory (usually .../syncthing-config)", diff --git a/config.json.in b/config.json.in index dd34579..751bb29 100644 --- a/config.json.in +++ b/config.json.in @@ -2,9 +2,10 @@ "webAppDir": "webapp/dist", "shareRootDir": "${ROOT_DIR}/tmp/builder_dev_host/share", "logsDir": "/tmp/xds-server/logs", + "sdkRootDir": "/xdt/sdk", "syncthing": { "binDir": "./bin", - "home": "${ROOT_DIR}/tmp/local_dev/syncthing-config", + "home": "${ROOT_DIR}/tmp/builder_dev_host/syncthing-config", "gui-address": "http://localhost:8384" } } \ No newline at end of file diff --git a/lib/apiv1/make.go b/lib/apiv1/make.go index 0f7561f..1e6d937 100644 --- a/lib/apiv1/make.go +++ b/lib/apiv1/make.go @@ -138,6 +138,10 @@ func (s *APIService) buildMake(c *gin.Context) { cmdID := makeCommandID makeCommandID++ + /* SEB TODO . /opt/poky-agl/3.90.0+snapshot/environment-setup-aarch64-agl-linux + env := os.Environ() + */ + s.log.Debugf("Execute [Cmd ID %d]: %v", cmdID, cmd) err := common.ExecPipeWs(cmd, sop, sess.ID, cmdID, execTmo, s.log, oCB, eCB) if err != nil { diff --git a/lib/common/filepath.go b/lib/common/filepath.go new file mode 100644 index 0000000..603c2a2 --- /dev/null +++ b/lib/common/filepath.go @@ -0,0 +1,15 @@ +package common + +import "os" + +// Exists returns whether the given file or directory exists or not +func Exists(path string) bool { + _, err := os.Stat(path) + if err == nil { + return true + } + if os.IsNotExist(err) { + return false + } + return true +} diff --git a/lib/common/httpclient.go b/lib/common/httpclient.go index 40d7bc2..72132bf 100644 --- a/lib/common/httpclient.go +++ b/lib/common/httpclient.go @@ -9,6 +9,8 @@ import ( "io/ioutil" "net/http" "strings" + + "github.com/Sirupsen/logrus" ) type HTTPClient struct { @@ -20,6 +22,7 @@ type HTTPClient struct { id string csrf string conf HTTPClientConfig + logger *logrus.Logger } type HTTPClientConfig struct { @@ -30,6 +33,13 @@ type HTTPClientConfig struct { CsrfDisable bool } +const ( + logError = 1 + logWarning = 2 + logInfo = 3 + logDebug = 4 +) + // Inspired by syncthing/cmd/cli const insecure = false @@ -64,6 +74,30 @@ func HTTPNewClient(baseURL string, cfg HTTPClientConfig) (*HTTPClient, error) { return &client, nil } +// SetLogger Define the logger to use +func (c *HTTPClient) SetLogger(log *logrus.Logger) { + c.logger = log +} + +func (c *HTTPClient) log(level int, format string, args ...interface{}) { + if c.logger != nil { + switch level { + case logError: + c.logger.Errorf(format, args...) + break + case logWarning: + c.logger.Warningf(format, args...) + break + case logInfo: + c.logger.Infof(format, args...) + break + default: + c.logger.Debugf(format, args...) + break + } + } +} + // Send request to retrieve Client id and/or CSRF token func (c *HTTPClient) getCidAndCsrf() error { request, err := http.NewRequest("GET", c.endpoint, nil) @@ -171,6 +205,8 @@ func (c *HTTPClient) handleRequest(request *http.Request) (*http.Response, error request.Header.Set("X-CSRF-Token-"+c.id[:5], c.csrf) } + c.log(logDebug, "HTTP %s %v", request.Method, request.URL) + response, err := c.httpClient.Do(request) if err != nil { return nil, err diff --git a/lib/crosssdk/sdk.go b/lib/crosssdk/sdk.go new file mode 100644 index 0000000..2f22f22 --- /dev/null +++ b/lib/crosssdk/sdk.go @@ -0,0 +1,47 @@ +package crosssdk + +import ( + "fmt" + "path" + "path/filepath" +) + +// SDK Define a cross tool chain used to build application +type SDK struct { + Profile string + Version string + Arch string + Path string + EnvFile string +} + +// NewCrossSDK creates a new instance of Syncthing +func NewCrossSDK(path string) (*SDK, error) { + // Assume that we have .../// + s := SDK{Path: path} + + s.Arch = filepath.Base(path) + + d := filepath.Dir(path) + s.Version = filepath.Base(d) + + d = filepath.Dir(d) + s.Profile = filepath.Base(d) + + envFile := filepath.Join(path, "environment-setup*") + ef, err := filepath.Glob(envFile) + if err != nil { + return nil, fmt.Errorf("Cannot retrieve environment setup file: %v", err) + } + if len(ef) != 1 { + return nil, fmt.Errorf("No environment setup file found match %s", envFile) + } + s.EnvFile = ef[0] + + return &s, nil +} + +// GetEnvCmd returns the command to initialized the environment to use a cross SDK +func (s *SDK) GetEnvCmd() string { + return ". " + path.Join(s.Path, s.EnvFile) +} diff --git a/lib/crosssdk/sdks.go b/lib/crosssdk/sdks.go new file mode 100644 index 0000000..435aae6 --- /dev/null +++ b/lib/crosssdk/sdks.go @@ -0,0 +1,39 @@ +package crosssdk + +import ( + "path" + "path/filepath" + + "github.com/Sirupsen/logrus" + "github.com/iotbzh/xds-server/lib/common" + "github.com/iotbzh/xds-server/lib/xdsconfig" +) + +// SDKs List of installed SDK +type SDKs []*SDK + +// Init creates a new instance of Syncthing +func Init(cfg *xdsconfig.Config, log *logrus.Logger) (*SDKs, error) { + s := SDKs{} + + // Retrieve installed sdks + sdkRD := cfg.FileConf.SdkRootDir + + if common.Exists(sdkRD) { + + // Assume that SDK install tree is /// + dirs, err := filepath.Glob(path.Join(sdkRD, "*", "*", "*")) + if err != nil { + log.Debugf("Error while retrieving SDKs: dir=%s, error=%s", sdkRD, err.Error()) + return &s, err + } + for _, d := range dirs { + sdk, err := NewCrossSDK(d) + if err != nil { + log.Debugf("Error while processing SDK dir=%s, err=%s", d, err.Error()) + } + s = append(s, sdk) + } + } + return &s, nil +} diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 15cab0d..9452fbd 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -27,13 +27,15 @@ type SyncThing struct { APIKey string Home string STCmd *exec.Cmd + STICmd *exec.Cmd // Private fields - binDir string - logsDir string - exitSTChan chan ExitChan - client *common.HTTPClient - log *logrus.Logger + binDir string + logsDir string + exitSTChan chan ExitChan + exitSTIChan chan ExitChan + client *common.HTTPClient + log *logrus.Logger } // ExitChan Channel used for process exit @@ -173,6 +175,28 @@ func (s *SyncThing) Start() (*exec.Cmd, error) { return s.STCmd, err } +// StartInotify Starts syncthing-inotify process +func (s *SyncThing) StartInotify() (*exec.Cmd, error) { + var err error + + s.log.Infof(" STI home=%s", s.Home) + s.log.Infof(" STI url=%s", s.BaseURL) + + args := []string{ + "--home=" + s.Home, + "-target=" + s.BaseURL, + } + if s.log.Level == logrus.DebugLevel { + args = append(args, "-verbosity=4") + } + + env := []string{} + + s.STICmd, err = s.startProc("syncthing-inotify", args, env, &s.exitSTIChan) + + return s.STICmd, err +} + func (s *SyncThing) stopProc(pname string, proc *os.Process, exit chan ExitChan) { if err := proc.Signal(os.Interrupt); err != nil { s.log.Infof("Proc interrupt %s error: %s", pname, err.Error()) @@ -199,6 +223,15 @@ func (s *SyncThing) Stop() { s.STCmd = nil } +// StopInotify Stops syncthing process +func (s *SyncThing) StopInotify() { + if s.STICmd == nil { + return + } + s.stopProc("syncthing-inotify", s.STICmd.Process, s.exitSTIChan) + s.STICmd = nil +} + // Connect Establish HTTP connection with Syncthing func (s *SyncThing) Connect() error { var err error @@ -217,6 +250,9 @@ func (s *SyncThing) Connect() error { if s.client == nil { return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)") } + + s.client.SetLogger(s.log) + return nil } diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 7be157a..40ce948 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -11,13 +11,14 @@ import ( "github.com/gin-gonic/gin" "github.com/googollee/go-socket.io" "github.com/iotbzh/xds-server/lib/apiv1" + "github.com/iotbzh/xds-server/lib/crosssdk" "github.com/iotbzh/xds-server/lib/model" "github.com/iotbzh/xds-server/lib/session" "github.com/iotbzh/xds-server/lib/xdsconfig" ) -// ServerService . -type ServerService struct { +// Server . +type Server struct { router *gin.Engine api *apiv1.APIService sIOServer *socketio.Server @@ -25,6 +26,7 @@ type ServerService struct { cfg *xdsconfig.Config sessions *session.Sessions mfolder *model.Folder + sdks *crosssdk.SDKs log *logrus.Logger stop chan struct{} // signals intentional stop } @@ -32,11 +34,11 @@ type ServerService struct { const indexFilename = "index.html" const cookieMaxAge = "3600" -// NewServer creates an instance of ServerService -func NewServer(cfg *xdsconfig.Config, mfolder *model.Folder, log *logrus.Logger) *ServerService { +// New creates an instance of Server +func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, log *logrus.Logger) *Server { // Setup logging for gin router - if cfg.Log.Level == logrus.DebugLevel { + if log.Level == logrus.DebugLevel { gin.SetMode(gin.DebugMode) } else { gin.SetMode(gin.ReleaseMode) @@ -51,15 +53,16 @@ func NewServer(cfg *xdsconfig.Config, mfolder *model.Folder, log *logrus.Logger) // Creates gin router r := gin.New() - svr := &ServerService{ + svr := &Server{ router: r, api: nil, sIOServer: nil, webApp: nil, cfg: cfg, - log: log, sessions: nil, mfolder: mfolder, + sdks: sdks, + log: log, stop: make(chan struct{}), } @@ -67,7 +70,7 @@ func NewServer(cfg *xdsconfig.Config, mfolder *model.Folder, log *logrus.Logger) } // Serve starts a new instance of the Web Server -func (s *ServerService) Serve() error { +func (s *Server) Serve() error { var err error // Setup middlewares @@ -128,17 +131,17 @@ func (s *ServerService) Serve() error { } // Stop web server -func (s *ServerService) Stop() { +func (s *Server) Stop() { close(s.stop) } // serveIndexFile provides initial file (eg. index.html) of webapp -func (s *ServerService) serveIndexFile(c *gin.Context) { +func (s *Server) serveIndexFile(c *gin.Context) { c.HTML(200, indexFilename, gin.H{}) } // Add details in Header -func (s *ServerService) middlewareXDSDetails() gin.HandlerFunc { +func (s *Server) middlewareXDSDetails() gin.HandlerFunc { return func(c *gin.Context) { c.Header("XDS-Version", s.cfg.Version) c.Header("XDS-API-Version", s.cfg.APIVersion) @@ -147,7 +150,7 @@ func (s *ServerService) middlewareXDSDetails() gin.HandlerFunc { } // CORS middleware -func (s *ServerService) middlewareCORS() gin.HandlerFunc { +func (s *Server) middlewareCORS() gin.HandlerFunc { return func(c *gin.Context) { if c.Request.Method == "OPTIONS" { @@ -165,7 +168,7 @@ func (s *ServerService) middlewareCORS() gin.HandlerFunc { } // socketHandler is the handler for the "main" websocket connection -func (s *ServerService) socketHandler(c *gin.Context) { +func (s *Server) socketHandler(c *gin.Context) { // Retrieve user session sess := s.sessions.Get(c) diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go index 3f8a91d..465620b 100644 --- a/lib/xdsconfig/config.go +++ b/lib/xdsconfig/config.go @@ -7,6 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" + "github.com/iotbzh/xds-server/lib/common" ) // Config parameters (json format) of /config command @@ -57,14 +58,14 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) { } // Update location of shared dir if needed - if !dirExists(c.ShareRootDir) { + if !common.Exists(c.ShareRootDir) { if err := os.MkdirAll(c.ShareRootDir, 0770); err != nil { return nil, fmt.Errorf("No valid shared directory found: %v", err) } } c.Log.Infoln("Share root directory: ", c.ShareRootDir) - if c.FileConf.LogsDir != "" && !dirExists(c.FileConf.LogsDir) { + if c.FileConf.LogsDir != "" && !common.Exists(c.FileConf.LogsDir) { if err := os.MkdirAll(c.FileConf.LogsDir, 0770); err != nil { return nil, fmt.Errorf("Cannot create logs dir: %v", err) } @@ -73,11 +74,3 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) { return &c, nil } - -func dirExists(path string) bool { - _, err := os.Stat(path) - if os.IsNotExist(err) { - return false - } - return true -} diff --git a/lib/xdsconfig/fileconfig.go b/lib/xdsconfig/fileconfig.go index 3daf77c..776eb78 100644 --- a/lib/xdsconfig/fileconfig.go +++ b/lib/xdsconfig/fileconfig.go @@ -9,6 +9,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/iotbzh/xds-server/lib/common" ) type SyncThingConf struct { @@ -21,6 +23,7 @@ type SyncThingConf struct { type FileConfig struct { WebAppDir string `json:"webAppDir"` ShareRootDir string `json:"shareRootDir"` + SdkRootDir string `json:"sdkRootDir"` HTTPPort string `json:"httpPort"` SThgConf *SyncThingConf `json:"syncthing"` LogsDir string `json:"logsDir"` @@ -78,23 +81,20 @@ func updateConfigFromFile(c *Config, confFile string) error { c.FileConf = fCfg // Support environment variables (IOW ${MY_ENV_VAR} syntax) in config.json - // TODO: better to use reflect package to iterate on fields and be more generic - var rep string - if rep, err = resolveEnvVar(fCfg.WebAppDir); err != nil { - return err - } - fCfg.WebAppDir = path.Clean(rep) - - if rep, err = resolveEnvVar(fCfg.ShareRootDir); err != nil { - return err - } - fCfg.ShareRootDir = path.Clean(rep) - - if rep, err = resolveEnvVar(fCfg.SThgConf.Home); err != nil { - return err + for _, field := range []*string{ + &fCfg.WebAppDir, + &fCfg.ShareRootDir, + &fCfg.SdkRootDir, + &fCfg.LogsDir, + &fCfg.SThgConf.Home} { + + rep, err := resolveEnvVar(*field) + if err != nil { + return err + } + *field = path.Clean(rep) } - fCfg.SThgConf.Home = path.Clean(rep) - + // Config file settings overwrite default config if fCfg.WebAppDir != "" { @@ -105,7 +105,7 @@ func updateConfigFromFile(c *Config, confFile string) error { // Check first from current directory for _, rootD := range []string{cwd, exePath} { ff := path.Join(rootD, c.WebAppDir, "index.html") - if exists(ff) { + if common.Exists(ff) { c.WebAppDir = path.Join(rootD, c.WebAppDir) break } @@ -140,15 +140,3 @@ func resolveEnvVar(s string) (string, error) { return res, nil } - -// exists returns whether the given file or directory exists or not -func exists(path string) bool { - _, err := os.Stat(path) - if err == nil { - return true - } - if os.IsNotExist(err) { - return false - } - return true -} diff --git a/main.go b/main.go index ba445f5..36586cf 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" + "github.com/iotbzh/xds-server/lib/crosssdk" "github.com/iotbzh/xds-server/lib/model" "github.com/iotbzh/xds-server/lib/syncthing" "github.com/iotbzh/xds-server/lib/webserver" @@ -39,15 +40,17 @@ var AppSubVersion = "unknown-dev" // Context holds the XDS server context type Context struct { - ProgName string - Cli *cli.Context - Config *xdsconfig.Config - Log *logrus.Logger - SThg *st.SyncThing - SThgCmd *exec.Cmd - MFolder *model.Folder - WWWServer *webserver.ServerService - Exit chan os.Signal + ProgName string + Cli *cli.Context + Config *xdsconfig.Config + Log *logrus.Logger + SThg *st.SyncThing + SThgCmd *exec.Cmd + SThgInotCmd *exec.Cmd + MFolder *model.Folder + SDKs *crosssdk.SDKs + WWWServer *webserver.Server + Exit chan os.Signal } // NewContext Create a new instance of XDS server @@ -86,9 +89,10 @@ func NewContext(cliCtx *cli.Context) *Context { func handlerSigTerm(ctx *Context) { <-ctx.Exit if ctx.SThg != nil { - ctx.Log.Infof("Stopping Syncthing... (PID %d)", - ctx.SThgCmd.Process.Pid) + ctx.Log.Infof("Stoping Syncthing... (PID %d)", ctx.SThgCmd.Process.Pid) ctx.SThg.Stop() + ctx.Log.Infof("Stoping Syncthing-inotify... (PID %d)", ctx.SThgInotCmd.Process.Pid) + ctx.SThg.StopInotify() } if ctx.WWWServer != nil { ctx.Log.Infof("Stoping Web server...") @@ -97,7 +101,7 @@ func handlerSigTerm(ctx *Context) { os.Exit(1) } -// xdsServer main routine +// XDS Server application main routine func xdsApp(cliCtx *cli.Context) error { var err error @@ -130,7 +134,15 @@ func xdsApp(cliCtx *cli.Context) error { } ctx.Log.Infof("Syncthing started (PID %d)", ctx.SThgCmd.Process.Pid) + ctx.Log.Infof("Starting Syncthing-inotify...") + ctx.SThgInotCmd, err = ctx.SThg.StartInotify() + if err != nil { + return cli.NewExitError(err, 2) + } + ctx.Log.Infof("Syncthing-inotify started (PID %d)", ctx.SThgInotCmd.Process.Pid) + // Establish connection with local Syncthing (retry if connection fail) + time.Sleep(2 * time.Second) retry := 10 err = nil for retry > 0 { @@ -177,8 +189,14 @@ func xdsApp(cliCtx *cli.Context) error { return cli.NewExitError(err, 3) } + // Init cross SDKs + ctx.SDKs, err = crosssdk.Init(ctx.Config, ctx.Log) + if err != nil { + return cli.NewExitError(err, 2) + } + // Create and start Web Server - ctx.WWWServer = webserver.NewServer(ctx.Config, ctx.MFolder, ctx.Log) + ctx.WWWServer = webserver.New(ctx.Config, ctx.MFolder, ctx.SDKs, ctx.Log) if err = ctx.WWWServer.Serve(); err != nil { ctx.Log.Println(err) return cli.NewExitError(err, 3) diff --git a/webapp/src/app/common/syncthing.service.ts b/webapp/src/app/common/syncthing.service.ts index 2b4c609..28b19a9 100644 --- a/webapp/src/app/common/syncthing.service.ts +++ b/webapp/src/app/common/syncthing.service.ts @@ -29,7 +29,7 @@ export interface ISyncThingStatus { } // Private interfaces of Syncthing -const ISTCONFIG_VERSION = 19; +const ISTCONFIG_VERSION = 20; interface ISTFolderDeviceConfiguration { deviceID: string; -- 2.16.6