From a50baa7c309f7eb55fe87c71f4c688ace325b6ac Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Wed, 17 May 2017 17:10:45 +0200 Subject: [PATCH] Add logsDir setting and more - add logsDir setting in config.json - redirect Syncthing and Syncthing-inotify into log files - Use autogenerated Syncthing apikey if gui-apikey not set in config.json --- README.md | 4 +- agent-config.json.in | 1 + lib/common/filepath.go | 15 ++++++ lib/common/httpclient.go | 36 +++++++++++++ lib/session/session.go | 2 +- lib/syncthing/st.go | 126 +++++++++++++++++++++++++++++++++----------- lib/xdsconfig/config.go | 30 +++++------ lib/xdsconfig/fileconfig.go | 37 ++++++------- main.go | 13 ++--- scripts/get-syncthing.sh | 2 +- 10 files changed, 187 insertions(+), 79 deletions(-) create mode 100644 lib/common/filepath.go diff --git a/README.md b/README.md index 524ad14..ad8e6bb 100644 --- a/README.md +++ b/README.md @@ -51,10 +51,12 @@ Supported fields in configuration file are: ```json { "httpPort": "http port of agent REST interface", + "logsDir": "directory to store logs (eg. syncthing output)", "syncthing": { "binDir": "syncthing binaries directory (use xds-agent executable dir when not set)", "home": "syncthing home directory (usually .../syncthing-config)", - "gui-address": "syncthing gui url (default http://localhost:8384)" + "gui-address": "syncthing gui url (default http://localhost:8384)", + "gui-apikey": "syncthing api-key to use (default auto-generated)" } } ``` diff --git a/agent-config.json.in b/agent-config.json.in index a0c0ad5..3ee736d 100644 --- a/agent-config.json.in +++ b/agent-config.json.in @@ -1,4 +1,5 @@ { + "logsDir": "/tmp/xds-agent/logs", "syncthing": { "binDir": "./bin", "home": "${ROOT_DIR}/tmp/local_dev/syncthing-config", diff --git a/lib/common/filepath.go b/lib/common/filepath.go new file mode 100644 index 0000000..603c2a2 --- /dev/null +++ b/lib/common/filepath.go @@ -0,0 +1,15 @@ +package common + +import "os" + +// Exists returns whether the given file or directory exists or not +func Exists(path string) bool { + _, err := os.Stat(path) + if err == nil { + return true + } + if os.IsNotExist(err) { + return false + } + return true +} diff --git a/lib/common/httpclient.go b/lib/common/httpclient.go index 40d7bc2..72132bf 100644 --- a/lib/common/httpclient.go +++ b/lib/common/httpclient.go @@ -9,6 +9,8 @@ import ( "io/ioutil" "net/http" "strings" + + "github.com/Sirupsen/logrus" ) type HTTPClient struct { @@ -20,6 +22,7 @@ type HTTPClient struct { id string csrf string conf HTTPClientConfig + logger *logrus.Logger } type HTTPClientConfig struct { @@ -30,6 +33,13 @@ type HTTPClientConfig struct { CsrfDisable bool } +const ( + logError = 1 + logWarning = 2 + logInfo = 3 + logDebug = 4 +) + // Inspired by syncthing/cmd/cli const insecure = false @@ -64,6 +74,30 @@ func HTTPNewClient(baseURL string, cfg HTTPClientConfig) (*HTTPClient, error) { return &client, nil } +// SetLogger Define the logger to use +func (c *HTTPClient) SetLogger(log *logrus.Logger) { + c.logger = log +} + +func (c *HTTPClient) log(level int, format string, args ...interface{}) { + if c.logger != nil { + switch level { + case logError: + c.logger.Errorf(format, args...) + break + case logWarning: + c.logger.Warningf(format, args...) + break + case logInfo: + c.logger.Infof(format, args...) + break + default: + c.logger.Debugf(format, args...) + break + } + } +} + // Send request to retrieve Client id and/or CSRF token func (c *HTTPClient) getCidAndCsrf() error { request, err := http.NewRequest("GET", c.endpoint, nil) @@ -171,6 +205,8 @@ func (c *HTTPClient) handleRequest(request *http.Request) (*http.Response, error request.Header.Set("X-CSRF-Token-"+c.id[:5], c.csrf) } + c.log(logDebug, "HTTP %s %v", request.Method, request.URL) + response, err := c.httpClient.Do(request) if err != nil { return nil, err diff --git a/lib/session/session.go b/lib/session/session.go index af05daa..b56f9ff 100644 --- a/lib/session/session.go +++ b/lib/session/session.go @@ -205,8 +205,8 @@ func (s *Sessions) monitorSessMap() { s.log.Debugln("Stop monitorSessMap") return case <-time.After(sessionMonitorTime * time.Second): - s.log.Debugf("Sessions Map size: %d", len(s.sessMap)) if dbgFullTrace { + s.log.Debugf("Sessions Map size: %d", len(s.sessMap)) s.log.Debugf("Sessions Map : %v", s.sessMap) } diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index e513876..5976e2f 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -2,9 +2,12 @@ package st import ( "encoding/json" + "io" "io/ioutil" "os" "path" + "path/filepath" + "regexp" "strings" "syscall" "time" @@ -22,47 +25,65 @@ import ( // SyncThing . type SyncThing struct { BaseURL string - ApiKey string + APIKey string Home string STCmd *exec.Cmd STICmd *exec.Cmd // Private fields binDir string + logsDir string exitSTChan chan ExitChan exitSTIChan chan ExitChan client *common.HTTPClient log *logrus.Logger } -// Monitor process exit +// ExitChan Channel used for process exit type ExitChan struct { status int err error } // NewSyncThing creates a new instance of Syncthing -//func NewSyncThing(url string, apiKey string, home string, log *logrus.Logger) *SyncThing { -func NewSyncThing(conf *xdsconfig.SyncThingConf, log *logrus.Logger) *SyncThing { - url := conf.GuiAddress - apiKey := conf.GuiAPIKey - home := conf.Home +func NewSyncThing(conf *xdsconfig.Config, log *logrus.Logger) *SyncThing { + var url, apiKey, home, binDir string + var err error + + stCfg := conf.FileConf.SThgConf + if stCfg != nil { + url = stCfg.GuiAddress + apiKey = stCfg.GuiAPIKey + home = stCfg.Home + binDir = stCfg.BinDir + } + + if url == "" { + url = "http://localhost:8384" + } + if url[0:7] != "http://" { + url = "http://" + url + } + + if home == "" { + home = "/mnt/share" + } + + if binDir == "" { + if binDir, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil { + binDir = "/usr/local/bin" + } + } s := SyncThing{ BaseURL: url, - ApiKey: apiKey, + APIKey: apiKey, Home: home, - binDir: conf.BinDir, + binDir: binDir, + logsDir: conf.FileConf.LogsDir, log: log, } - if s.BaseURL == "" { - s.BaseURL = "http://localhost:8384" - } - if s.BaseURL[0:7] != "http://" { - s.BaseURL = "http://" + s.BaseURL - } - return &s } @@ -84,28 +105,43 @@ func (s *SyncThing) startProc(exeName string, args []string, env []string, eChan cmd.Env = append(cmd.Env, ev) } + // open log file + var outfile *os.File + logFilename := filepath.Join(s.logsDir, exeName+".log") + if s.logsDir != "" { + outfile, err := os.Create(logFilename) + if err != nil { + return nil, fmt.Errorf("Cannot create log file %s", logFilename) + } + + cmdOut, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("Pipe stdout error for : %s", err) + } + + go io.Copy(outfile, cmdOut) + } + err = cmd.Start() if err != nil { return nil, err } *eChan = make(chan ExitChan, 1) - go func(c *exec.Cmd) { + go func(c *exec.Cmd, oF *os.File) { status := 0 - cmdOut, err := c.StdoutPipe() - if err == nil { - s.log.Errorf("Pipe stdout error for : %s", err) - } else if cmdOut != nil { - stdOutput, _ := ioutil.ReadAll(cmdOut) - fmt.Printf("STDOUT: %s\n", stdOutput) - } sts, err := c.Process.Wait() if !sts.Success() { s := sts.Sys().(syscall.WaitStatus) status = s.ExitStatus() } + if oF != nil { + oF.Close() + } + s.log.Debugf("%s exited with status %d, err %v", exeName, status, err) + *eChan <- ExitChan{status, err} - }(cmd) + }(cmd, outfile) return cmd, nil } @@ -113,14 +149,19 @@ func (s *SyncThing) startProc(exeName string, args []string, env []string, eChan // Start Starts syncthing process func (s *SyncThing) Start() (*exec.Cmd, error) { var err error + + s.log.Infof(" ST home=%s", s.Home) + s.log.Infof(" ST url=%s", s.BaseURL) + args := []string{ "--home=" + s.Home, "-no-browser", "--gui-address=" + s.BaseURL, } - if s.ApiKey != "" { - args = append(args, "-gui-apikey=\""+s.ApiKey+"\"") + if s.APIKey != "" { + args = append(args, "-gui-apikey=\""+s.APIKey+"\"") + s.log.Infof(" ST apikey=%s", s.APIKey) } if s.log.Level == logrus.DebugLevel { args = append(args, "-verbose") @@ -132,38 +173,58 @@ func (s *SyncThing) Start() (*exec.Cmd, error) { s.STCmd, err = s.startProc("syncthing", args, env, &s.exitSTChan) + // Use autogenerated apikey if not set by config.json + if s.APIKey == "" { + if fd, err := os.Open(filepath.Join(s.Home, "config.xml")); err == nil { + defer fd.Close() + if b, err := ioutil.ReadAll(fd); err == nil { + re := regexp.MustCompile("(.*)") + key := re.FindStringSubmatch(string(b)) + if len(key) >= 1 { + s.APIKey = key[1] + } + } + } + } + return s.STCmd, err } // StartInotify Starts syncthing-inotify process func (s *SyncThing) StartInotify() (*exec.Cmd, error) { var err error + exeName := "syncthing-inotify" + + s.log.Infof(" STI url=%s", s.BaseURL) args := []string{ - "--home=" + s.Home, "-target=" + s.BaseURL, } + if s.APIKey != "" { + args = append(args, "-api="+s.APIKey) + s.log.Infof("%s uses apikey=%s", exeName, s.APIKey) + } if s.log.Level == logrus.DebugLevel { args = append(args, "-verbosity=4") } env := []string{} - s.STICmd, err = s.startProc("syncthing-inotify", args, env, &s.exitSTIChan) + s.STICmd, err = s.startProc(exeName, args, env, &s.exitSTIChan) return s.STICmd, err } func (s *SyncThing) stopProc(pname string, proc *os.Process, exit chan ExitChan) { if err := proc.Signal(os.Interrupt); err != nil { - s.log.Errorf("Proc interrupt %s error: %s", pname, err.Error()) + s.log.Infof("Proc interrupt %s error: %s", pname, err.Error()) select { case <-exit: case <-time.After(time.Second): // A bigger bonk on the head. if err := proc.Signal(os.Kill); err != nil { - s.log.Errorf("Proc term %s error: %s", pname, err.Error()) + s.log.Infof("Proc term %s error: %s", pname, err.Error()) } <-exit } @@ -207,6 +268,9 @@ func (s *SyncThing) Connect() error { if s.client == nil { return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)") } + + s.client.SetLogger(s.log) + return nil } diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go index 1f53cbd..efea5ba 100644 --- a/lib/xdsconfig/config.go +++ b/lib/xdsconfig/config.go @@ -7,6 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" + "github.com/iotbzh/xds-server/lib/common" ) // Config parameters (json format) of /config command @@ -16,9 +17,9 @@ type Config struct { VersionGitTag string `json:"gitTag"` // Private / un-exported fields - HTTPPort string `json:"-"` - FileConf *FileConfig - log *logrus.Logger + HTTPPort string `json:"-"` + FileConf *FileConfig `json:"-"` + Log *logrus.Logger `json:"-"` } // Config default values @@ -29,7 +30,7 @@ const ( ) // Init loads the configuration on start-up -func Init(ctx *cli.Context, log *logrus.Logger) (Config, error) { +func Init(ctx *cli.Context, log *logrus.Logger) (*Config, error) { var err error // Define default configuration @@ -39,27 +40,26 @@ func Init(ctx *cli.Context, log *logrus.Logger) (Config, error) { VersionGitTag: ctx.App.Metadata["git-tag"].(string), HTTPPort: DefaultPort, - log: log, + Log: log, } // config file settings overwrite default config c.FileConf, err = updateConfigFromFile(&c, ctx.GlobalString("config")) if err != nil { - return Config{}, err + return nil, err } - return c, nil + if c.FileConf.LogsDir != "" && !common.Exists(c.FileConf.LogsDir) { + if err := os.MkdirAll(c.FileConf.LogsDir, 0770); err != nil { + return nil, fmt.Errorf("Cannot create logs dir: %v", err) + } + } + c.Log.Infoln("Logs directory: ", c.FileConf.LogsDir) + + return &c, nil } // UpdateAll Update the current configuration func (c *Config) UpdateAll(newCfg Config) error { return fmt.Errorf("Not Supported") } - -func dirExists(path string) bool { - _, err := os.Stat(path) - if os.IsNotExist(err) { - return false - } - return true -} diff --git a/lib/xdsconfig/fileconfig.go b/lib/xdsconfig/fileconfig.go index 0c4828c..5cf8db2 100644 --- a/lib/xdsconfig/fileconfig.go +++ b/lib/xdsconfig/fileconfig.go @@ -19,6 +19,7 @@ type SyncThingConf struct { type FileConfig struct { HTTPPort string `json:"httpPort"` + LogsDir string `json:"logsDir"` SThgConf *SyncThingConf `json:"syncthing"` } @@ -60,7 +61,7 @@ func updateConfigFromFile(c *Config, confFile string) (*FileConfig, error) { return &fCfg, nil } - c.log.Infof("Use config file: %s", *cFile) + c.Log.Infof("Use config file: %s", *cFile) // TODO move on viper package to support comments in JSON and also // bind with flags (command line options) @@ -73,18 +74,22 @@ func updateConfigFromFile(c *Config, confFile string) (*FileConfig, error) { } // Support environment variables (IOW ${MY_ENV_VAR} syntax) in agent-config.json - // TODO: better to use reflect package to iterate on fields and be more generic - var rep string - - if rep, err = resolveEnvVar(fCfg.SThgConf.BinDir); err != nil { - return nil, err + for _, field := range []*string{ + &fCfg.LogsDir, + &fCfg.SThgConf.Home, + &fCfg.SThgConf.BinDir} { + + rep, err := resolveEnvVar(*field) + if err != nil { + return nil, err + } + *field = path.Clean(rep) } - fCfg.SThgConf.BinDir = path.Clean(rep) - if rep, err = resolveEnvVar(fCfg.SThgConf.Home); err != nil { - return nil, err + // Config file settings overwrite default config + if fCfg.HTTPPort != "" { + c.HTTPPort = fCfg.HTTPPort } - fCfg.SThgConf.Home = path.Clean(rep) return &fCfg, nil } @@ -106,15 +111,3 @@ func resolveEnvVar(s string) (string, error) { return res, nil } - -// exists returns whether the given file or directory exists or not -func exists(path string) bool { - _, err := os.Stat(path) - if err == nil { - return true - } - if os.IsNotExist(err) { - return false - } - return true -} diff --git a/main.go b/main.go index 93d13a2..0a0ad0a 100644 --- a/main.go +++ b/main.go @@ -7,8 +7,6 @@ import ( "os" "time" - "fmt" - "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" "github.com/iotbzh/xds-agent/lib/agent" @@ -37,19 +35,19 @@ var AppSubVersion = "unknown-dev" // xdsAgent main routine func xdsAgent(cliCtx *cli.Context) error { + var err error // Create Agent context ctx := agent.NewAgent(cliCtx) // Load config - cfg, err := xdsconfig.Init(cliCtx, ctx.Log) + ctx.Config, err = xdsconfig.Init(cliCtx, ctx.Log) if err != nil { return cli.NewExitError(err, 2) } - ctx.Config = &cfg // Start local instance of Syncthing and Syncthing-notify - ctx.SThg = st.NewSyncThing(ctx.Config.FileConf.SThgConf, ctx.Log) + ctx.SThg = st.NewSyncThing(ctx.Config, ctx.Log) ctx.Log.Infof("Starting Syncthing...") ctx.SThgCmd, err = ctx.SThg.Start() @@ -72,12 +70,11 @@ func xdsAgent(cliCtx *cli.Context) error { if err := ctx.SThg.Connect(); err == nil { break } - ctx.Log.Infof("Establishing connection to Syncthing (retry %d/5)", retry) + ctx.Log.Infof("Establishing connection to Syncthing (retry %d/10)", retry) time.Sleep(time.Second) retry-- } - if ctx.SThg == nil { - err = fmt.Errorf("ERROR: cannot connect to Syncthing (url: %s)", ctx.SThg.BaseURL) + if err != nil || retry == 0 { return cli.NewExitError(err, 2) } diff --git a/scripts/get-syncthing.sh b/scripts/get-syncthing.sh index 8bc5346..6eb9c5b 100755 --- a/scripts/get-syncthing.sh +++ b/scripts/get-syncthing.sh @@ -31,7 +31,7 @@ tarball="syncthing-linux-amd64-v${SYNCTHING_VERSION}.tar.gz" \ && grep -E " ${tarball}\$" sha1sum.txt.asc | sha1sum -c - \ && rm sha1sum.txt.asc \ && tar -xvf "$tarball" --strip-components=1 "$(basename "$tarball" .tar.gz)"/syncthing \ - && mv syncthing ${DESTDIR}/syncthing + && mv syncthing ${DESTDIR}/syncthing || exit 1 echo "Get Syncthing-inotify..." -- 2.16.6