From: Sebastien Douheret <sebastien.douheret@iot.bzh>
Date: Fri, 20 Oct 2017 12:30:33 +0000 (+0200)
Subject: Merge branch 'master' into wip
X-Git-Tag: v0.3.0-rc1^0
X-Git-Url: https://gerrit.automotivelinux.org/gerrit/gitweb?a=commitdiff_plain;h=2d90eac319979dba64371258b30e61e77a15db7d;hp=-c;p=src%2Fxds%2Fxds-agent.git

Merge branch 'master' into wip

Signed-off-by: Sebastien Douheret <sebastien.douheret@iot.bzh>
---

2d90eac319979dba64371258b30e61e77a15db7d
diff --combined Makefile
index 3329ead,7e8bd6c..db4d874
--- a/Makefile
+++ b/Makefile
@@@ -4,8 -4,9 +4,8 @@@
  VERSION := 0.2.0
  
  # Syncthing version to install
 -SYNCTHING_VERSION = 0.14.28
 -SYNCTHING_INOTIFY_VERSION = 0.8.6
 -
 +SYNCTHING_VERSION = 0.14.38
 +SYNCTHING_INOTIFY_VERSION = 0.8.7
  
  
  # Retrieve git tag/commit to set sub-version string
@@@ -23,13 -24,13 +23,13 @@@ ifeq ($(origin SUB_VERSION), undefined
  	endif
  endif
  
 -# for backward compatibility
 -DESTDIR := $(INSTALL_DIR)
 -
 -# Configurable variables for installation (default /opt/AGL/xds/agent)
 +# Configurable variables for installation (default /opt/AGL/...)
  ifeq ($(origin DESTDIR), undefined)
  	DESTDIR := /opt/AGL/xds/agent
  endif
 +ifeq ($(origin DESTDIR_WWW), undefined)
 +	DESTDIR_WWW := $(DESTDIR)/www
 +endif
  
  HOST_GOOS=$(shell go env GOOS)
  HOST_GOARCH=$(shell go env GOARCH)
@@@ -74,15 -75,28 +74,15 @@@ els
  endif
  
  
 -all: tools/syncthing vendor build
 +all: tools/syncthing build
  
 -build: tools/syncthing/copytobin
 +.PHONY: build
 +build: vendor xds webapp
 +
 +xds: scripts tools/syncthing/copytobin
  	@echo "### Build XDS agent (version $(VERSION), subversion $(SUB_VERSION)) - $(BUILD_MODE)";
- 	@cd $(ROOT_SRCDIR); $(BUILD_ENV_FLAGS) go build $(VERBOSE_$(V)) -i -o $(LOCAL_BINDIR)/xds-agent$(EXT) -ldflags "$(GORELEASE) -X main.AppVersion=$(VERSION) -X main.AppSubVersion=$(SUB_VERSION)" -gcflags "$(GO_GCFLAGS)" .
+ 	@cd $(ROOT_SRCDIR); $(BUILD_ENV_FLAGS) go build $(VERBOSE_$(V)) -i -o $(LOCAL_BINDIR)/xds-agent$(EXT) -ldflags "$(GO_LDFLAGS) -X main.AppVersion=$(VERSION) -X main.AppSubVersion=$(SUB_VERSION)" -gcflags "$(GO_GCFLAGS)" .
  
 -package: clean tools/syncthing vendor build
 -	@mkdir -p $(PACKAGE_DIR)/xds-agent $(PACKAGE_DIR)/scripts
 -	@cp -a $(LOCAL_BINDIR)/* $(PACKAGE_DIR)/xds-agent
 -	@cp -r $(ROOT_SRCDIR)/conf.d $(ROOT_SRCDIR)/scripts $(PACKAGE_DIR)/xds-agent
 -	cd $(PACKAGE_DIR) && zip -r $(ROOT_SRCDIR)/$(PACKAGE_ZIPFILE) ./xds-agent
 -
 -.PHONY: package-all
 -package-all:
 -	@echo "# Build linux amd64..."
 -	GOOS=linux GOARCH=amd64 RELEASE=1 make -f $(ROOT_SRCDIR)/Makefile package
 -	@echo "# Build windows amd64..."
 -	GOOS=windows GOARCH=amd64 RELEASE=1 make -f $(ROOT_SRCDIR)/Makefile package
 -	@echo "# Build darwin amd64..."
 -	GOOS=darwin GOARCH=amd64 RELEASE=1 make -f $(ROOT_SRCDIR)/Makefile package
 -	make -f $(ROOT_SRCDIR)/Makefile clean
 -
  test: tools/glide
  	go test --race $(shell $(LOCAL_TOOLSDIR)/glide novendor)
  
@@@ -100,50 -114,25 +100,55 @@@ debug: build/xds tools/syncthing/copyto
  
  .PHONY: clean
  clean:
 -	rm -rf $(LOCAL_BINDIR)/* debug $(ROOT_GOPRJ)/pkg/*/$(REPOPATH) $(PACKAGE_DIR)
 +	rm -rf $(LOCAL_BINDIR)/* $(ROOT_SRCDIR)/debug $(ROOT_GOPRJ)/pkg/*/$(REPOPATH) $(PACKAGE_DIR)
  
  .PHONY: distclean
  distclean: clean
 -	rm -rf $(LOCAL_BINDIR) tools glide.lock vendor $(ROOT_SRCDIR)/*.zip
 +	cd $(ROOT_SRCDIR) && rm -rf $(LOCAL_BINDIR) ./tools ./glide.lock ./vendor ./*.zip ./webapp/node_modules ./webapp/dist
 +
 +webapp: webapp/install
 +	(cd webapp && gulp build)
 +
 +webapp/debug:
 +	(cd webapp && gulp watch &)
 +
 +webapp/install:
 +	(cd webapp && npm install)
 +	@if [ -d ${DESTDIR}/usr/local/etc ]; then rm -rf ${DESTDIR}/usr; fi
  
  .PHONY: install
- install: all
- 	mkdir -p $(DESTDIR) && cp $(LOCAL_BINDIR)/* $(DESTDIR)
- 	mkdir -p $(DESTDIR_WWW) && cp -a webapp/dist/* $(DESTDIR_WWW)
+ install:
+ 	@test -e $(LOCAL_BINDIR)/xds-agent$(EXT) || { echo "Please execute first: make all\n"; exit 1; }
 -	@test -e $(LOCAL_BINDIR)/syncthing$(EXT) -a -e $(LOCAL_BINDIR)/syncthing-inotify$(EXT) || { echo 	"Please execute first: make all\n"; exit 1; }
 -	export DESTDIR=$(DESTDIR) && $(ROOT_SRCDIR)/scripts/install.sh
++	@test -e $(LOCAL_BINDIR)/syncthing$(EXT) -a -e $(LOCAL_BINDIR)/syncthing-inotify$(EXT) || { echo "Please execute first: make all\n"; exit 1; }
++	export DESTDIR=$(DESTDIR) && export DESTDIR_WWW=$(DESTDIR_WWW) && $(ROOT_SRCDIR)/scripts/install.sh
+ 
+ .PHONY: uninstall
+ uninstall:
 -	export DESTDIR=$(DESTDIR) && $(ROOT_SRCDIR)/scripts/install.sh uninstall
++	export DESTDIR=$(DESTDIR) && export DESTDIR_WWW=$(DESTDIR_WWW) && $(ROOT_SRCDIR)/scripts/install.sh uninstall
 +
 +package: clean tools/syncthing vendor build
- 	@mkdir -p $(PACKAGE_DIR)/xds-agent
- 	@cp agent-config.json.in $(PACKAGE_DIR)/xds-agent/agent-config.json
++	@mkdir -p $(PACKAGE_DIR)/xds-agent $(PACKAGE_DIR)/scripts
 +	@cp -a $(LOCAL_BINDIR)/* $(PACKAGE_DIR)/xds-agent
++	@cp -r $(ROOT_SRCDIR)/conf.d $(ROOT_SRCDIR)/scripts $(PACKAGE_DIR)/xds-agent
 +	cd $(PACKAGE_DIR) && zip -r $(ROOT_SRCDIR)/$(PACKAGE_ZIPFILE) ./xds-agent
 +
 +.PHONY: package-all
 +package-all:
 +	@echo "# Build linux amd64..."
 +	GOOS=linux GOARCH=amd64 RELEASE=1 make -f $(ROOT_SRCDIR)/Makefile package
 +	@echo "# Build windows amd64..."
 +	GOOS=windows GOARCH=amd64 RELEASE=1 make -f $(ROOT_SRCDIR)/Makefile package
 +	@echo "# Build darwin amd64..."
 +	GOOS=darwin GOARCH=amd64 RELEASE=1 make -f $(ROOT_SRCDIR)/Makefile package
 +	make -f $(ROOT_SRCDIR)/Makefile clean
  
  vendor: tools/glide glide.yaml
  	$(LOCAL_TOOLSDIR)/glide install --strip-vendor
  
 +vendor/debug: vendor
 +	(cd vendor/github.com/iotbzh && \
 +		rm -rf xds-common && ln -s ../../../../xds-common )
 +
  .PHONY: tools/glide
  tools/glide:
  	@test -f $(LOCAL_TOOLSDIR)/glide || { \
@@@ -161,7 -150,7 +166,7 @@@ tools/syncthing
  	SYNCTHING_INOTIFY_VERSION=$(SYNCTHING_INOTIFY_VERSION) \
  	./scripts/get-syncthing.sh; }
  
 -.PHONY:
 +.PHONY: tools/syncthing/copytobin
  tools/syncthing/copytobin:
  	@test -e $(LOCAL_TOOLSDIR)/syncthing$(EXT) -a -e $(LOCAL_TOOLSDIR)/syncthing-inotify$(EXT) || { echo "Please execute first: make tools/syncthing\n"; exit 1; }
  	@mkdir -p $(LOCAL_BINDIR)
@@@ -174,6 -163,7 +179,7 @@@ help
  	@echo "  build"
  	@echo "  package"
  	@echo "  install"
+ 	@echo "  uninstall"
  	@echo "  clean"
  	@echo "  distclean"
  	@echo ""
diff --combined conf.d/etc/xds-agent/config.json
index e8599cb,35d702d..7c7d7cb
--- a/conf.d/etc/xds-agent/config.json
+++ b/conf.d/etc/xds-agent/config.json
@@@ -1,15 -1,9 +1,15 @@@
  {
 +    "httpPort": "8000",
 +    "webAppDir": "./www",
      "logsDir": "${HOME}/.xds/agent/logs",
 -    "xds-apikey": "1234abcezam",
 +    "xdsServers": [
 +        {
 +          "url": "http://localhost:8810"
 +        }
 +    ],
      "syncthing": {
          "home": "${HOME}/.xds/agent/syncthing-config",
-         "gui-address": "http://localhost:8384",
+         "gui-address": "http://localhost:8386",
          "gui-apikey": "1234abcezam"
      }
  }
diff --combined lib/syncthing/st.go
index 924f407,6199a8f..031a2ac
--- a/lib/syncthing/st.go
+++ b/lib/syncthing/st.go
@@@ -24,14 -24,11 +24,14 @@@ import 
  
  // SyncThing .
  type SyncThing struct {
 -	BaseURL string
 -	APIKey  string
 -	Home    string
 -	STCmd   *exec.Cmd
 -	STICmd  *exec.Cmd
 +	BaseURL   string
 +	APIKey    string
 +	Home      string
 +	STCmd     *exec.Cmd
 +	STICmd    *exec.Cmd
 +	MyID      string
 +	Connected bool
 +	Events    *Events
  
  	// Private fields
  	binDir      string
@@@ -40,7 -37,6 +40,7 @@@
  	exitSTIChan chan ExitChan
  	client      *common.HTTPClient
  	log         *logrus.Logger
 +	conf        *xdsconfig.Config
  }
  
  // ExitChan Channel used for process exit
@@@ -49,42 -45,6 +49,42 @@@ type ExitChan struct 
  	err    error
  }
  
 +// ConfigInSync Check whether if Syncthing configuration is in sync
 +type configInSync struct {
 +	ConfigInSync bool `json:"configInSync"`
 +}
 +
 +// FolderStatus Information about the current status of a folder.
 +type FolderStatus struct {
 +	GlobalFiles       int   `json:"globalFiles"`
 +	GlobalDirectories int   `json:"globalDirectories"`
 +	GlobalSymlinks    int   `json:"globalSymlinks"`
 +	GlobalDeleted     int   `json:"globalDeleted"`
 +	GlobalBytes       int64 `json:"globalBytes"`
 +
 +	LocalFiles       int   `json:"localFiles"`
 +	LocalDirectories int   `json:"localDirectories"`
 +	LocalSymlinks    int   `json:"localSymlinks"`
 +	LocalDeleted     int   `json:"localDeleted"`
 +	LocalBytes       int64 `json:"localBytes"`
 +
 +	NeedFiles       int   `json:"needFiles"`
 +	NeedDirectories int   `json:"needDirectories"`
 +	NeedSymlinks    int   `json:"needSymlinks"`
 +	NeedDeletes     int   `json:"needDeletes"`
 +	NeedBytes       int64 `json:"needBytes"`
 +
 +	InSyncFiles int   `json:"inSyncFiles"`
 +	InSyncBytes int64 `json:"inSyncBytes"`
 +
 +	State        string    `json:"state"`
 +	StateChanged time.Time `json:"stateChanged"`
 +
 +	Sequence int64 `json:"sequence"`
 +
 +	IgnorePatterns bool `json:"ignorePatterns"`
 +}
 +
  // NewSyncThing creates a new instance of Syncthing
  func NewSyncThing(conf *xdsconfig.Config, log *logrus.Logger) *SyncThing {
  	var url, apiKey, home, binDir string
@@@ -98,7 -58,7 +98,7 @@@
  	}
  
  	if url == "" {
- 		url = "http://localhost:8384"
+ 		url = "http://localhost:8386"
  	}
  	if url[0:7] != "http://" {
  		url = "http://" + url
@@@ -115,12 -75,8 +115,12 @@@
  		binDir:  binDir,
  		logsDir: conf.FileConf.LogsDir,
  		log:     log,
 +		conf:    conf,
  	}
  
 +	// Create Events monitoring
 +	s.Events = s.NewEventListener()
 +
  	return &s
  }
  
@@@ -130,9 -86,8 +130,9 @@@ func (s *SyncThing) startProc(exeName s
  	var exePath string
  
  	// Kill existing process (useful for debug ;-) )
 -	if os.Getenv("DEBUG_MODE") != "" {
 -		exec.Command("bash", "-c", "pkill -9 "+exeName).Output()
 +	if _, dbg := os.LookupEnv("XDS_DEBUG_MODE"); dbg {
 +		fmt.Printf("\n!!! DEBUG_MODE set: KILL existing %s process(es) !!!\n", exeName)
 +		exec.Command("bash", "-c", "ps -ax |grep "+exeName+" |grep "+s.BaseURL+" |cut  -d' ' -f 1|xargs -I{} kill -9 {}").Output()
  	}
  
  	// When not set (or set to '.') set bin to path of xds-agent executable
@@@ -227,8 -182,6 +227,8 @@@ func (s *SyncThing) Start() (*exec.Cmd
  		"STNOUPGRADE=1",
  	}
  
 +	/* FIXME - STILL NEEDED ?, if not SUP code
 +
  	// XXX - temporary hack because -gui-apikey seems to correctly handle by
  	// syncthing the early first time
  	stConfigFile := filepath.Join(s.Home, "config.xml")
@@@ -258,12 -211,12 +258,12 @@@
  			return nil, fmt.Errorf("Cannot write Syncthing config file to set apikey")
  		}
  	}
 -
 +	*/
  	s.STCmd, err = s.startProc("syncthing", args, env, &s.exitSTChan)
  
  	// Use autogenerated apikey if not set by config.json
 -	if s.APIKey == "" {
 -		if fd, err := os.Open(stConfigFile); err == nil {
 +	if err == nil && s.APIKey == "" {
 +		if fd, err := os.Open(filepath.Join(s.Home, "config.xml")); err == nil {
  			defer fd.Close()
  			if b, err := ioutil.ReadAll(fd); err == nil {
  				re := regexp.MustCompile("<apikey>(.*)</apikey>")
@@@ -341,17 -294,11 +341,17 @@@ func (s *SyncThing) StopInotify() 
  // Connect Establish HTTP connection with Syncthing
  func (s *SyncThing) Connect() error {
  	var err error
 +	s.Connected = false
  	s.client, err = common.HTTPNewClient(s.BaseURL,
  		common.HTTPClientConfig{
  			URLPrefix:           "/rest",
  			HeaderClientKeyName: "X-Syncthing-ID",
 +			LogOut:              s.conf.LogVerboseOut,
 +			LogPrefix:           "SYNCTHING: ",
 +			LogLevel:            common.HTTPLogLevelWarning,
  		})
 +	s.client.SetLogLevel(s.log.Level.String())
 +
  	if err != nil {
  		msg := ": " + err.Error()
  		if strings.Contains(err.Error(), "connection refused") {
@@@ -363,17 -310,11 +363,17 @@@
  		return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)")
  	}
  
 -	s.client.SetLogLevel(s.log.Level.String())
 -	s.client.LoggerPrefix = "SYNCTHING: "
 -	s.client.LoggerOut = s.log.Out
 +	s.MyID, err = s.IDGet()
 +	if err != nil {
 +		return fmt.Errorf("ERROR: cannot retrieve ID")
 +	}
 +
 +	s.Connected = true
  
 -	return nil
 +	// Start events monitoring
 +	err = s.Events.Start()
 +
 +	return err
  }
  
  // IDGet returns the Syncthing ID of Syncthing instance running locally
@@@ -406,16 -347,3 +406,16 @@@ func (s *SyncThing) ConfigSet(cfg confi
  	}
  	return s.client.HTTPPost("system/config", string(body))
  }
 +
 +// IsConfigInSync Returns true if configuration is in sync
 +func (s *SyncThing) IsConfigInSync() (bool, error) {
 +	var data []byte
 +	var d configInSync
 +	if err := s.client.HTTPGet("system/config/insync", &data); err != nil {
 +		return false, err
 +	}
 +	if err := json.Unmarshal(data, &d); err != nil {
 +		return false, err
 +	}
 +	return d.ConfigInSync, nil
 +}
diff --combined lib/xdsconfig/configfile.go
index 6eaaf6a,0000000..e3737f4
mode 100644,000000..100644
--- a/lib/xdsconfig/configfile.go
+++ b/lib/xdsconfig/configfile.go
@@@ -1,112 -1,0 +1,109 @@@
 +package xdsconfig
 +
 +import (
 +	"encoding/json"
 +	"os"
 +	"path"
 +
 +	common "github.com/iotbzh/xds-common/golib"
 +)
 +
 +type SyncThingConf struct {
 +	BinDir     string `json:"binDir"`
 +	Home       string `json:"home"`
 +	GuiAddress string `json:"gui-address"`
 +	GuiAPIKey  string `json:"gui-apikey"`
 +}
 +
 +type XDSServerConf struct {
 +	URL       string `json:"url"`
 +	ConnRetry int    `json:"connRetry"`
 +
 +	// private/not exported fields
 +	ID            string `json:"-"`
 +	APIBaseURL    string `json:"-"`
 +	APIPartialURL string `json:"-"`
 +}
 +
 +type FileConfig struct {
 +	HTTPPort    string          `json:"httpPort"`
 +	WebAppDir   string          `json:"webAppDir"`
 +	LogsDir     string          `json:"logsDir"`
 +	XDSAPIKey   string          `json:"xds-apikey"`
 +	ServersConf []XDSServerConf `json:"xdsServers"`
 +	SThgConf    *SyncThingConf  `json:"syncthing"`
 +}
 +
 +// readGlobalConfig reads configuration from a config file.
 +// Order to determine which config file is used:
 +//  1/ from command line option: "--config myConfig.json"
 +//  2/ $HOME/.xds/agent/agent-config.json file
- //  3/ <current_dir>/agent-config.json file
- //  4/ <executable dir>/agent-config.json file
++//  3/ /etc/xds-agent/config.json file
 +
 +func readGlobalConfig(c *Config, confFile string) error {
 +
 +	searchIn := make([]string, 0, 3)
 +	if confFile != "" {
 +		searchIn = append(searchIn, confFile)
 +	}
 +	if homeDir := common.GetUserHome(); homeDir != "" {
 +		searchIn = append(searchIn, path.Join(homeDir, ".xds", "agent", "agent-config.json"))
 +	}
 +
 +	searchIn = append(searchIn, "/etc/xds-agent/agent-config.json")
 +
- 	searchIn = append(searchIn, path.Join(common.GetExePath(), "agent-config.json"))
- 
 +	var cFile *string
 +	for _, p := range searchIn {
 +		if _, err := os.Stat(p); err == nil {
 +			cFile = &p
 +			break
 +		}
 +	}
 +	if cFile == nil {
 +		c.Log.Infof("No config file found")
 +		return nil
 +	}
 +
 +	c.Log.Infof("Use config file: %s", *cFile)
 +
 +	// TODO move on viper package to support comments in JSON and also
 +	// bind with flags (command line options)
 +	// see https://github.com/spf13/viper#working-with-flags
 +
 +	fd, _ := os.Open(*cFile)
 +	defer fd.Close()
 +
 +	// Decode config file content and save it in a first variable
 +	fCfg := FileConfig{}
 +	if err := json.NewDecoder(fd).Decode(&fCfg); err != nil {
 +		return err
 +	}
 +
 +	// Decode config file content and overwrite default settings
 +	fd.Seek(0, 0)
 +	json.NewDecoder(fd).Decode(&c.FileConf)
 +
 +	// Disable Syncthing support when there is no syncthing field in config
 +	if fCfg.SThgConf == nil {
 +		c.FileConf.SThgConf = nil
 +	}
 +
 +	// Support environment variables (IOW ${MY_ENV_VAR} syntax) in agent-config.json
 +	vars := []*string{
 +		&c.FileConf.LogsDir,
 +		&c.FileConf.WebAppDir,
 +	}
 +	if c.FileConf.SThgConf != nil {
 +		vars = append(vars, &c.FileConf.SThgConf.Home,
 +			&c.FileConf.SThgConf.BinDir)
 +	}
 +	for _, field := range vars {
 +		var err error
 +		*field, err = common.ResolveEnvVar(*field)
 +		if err != nil {
 +			return err
 +		}
 +	}
 +
 +	return nil
 +}
diff --combined scripts/install.sh
index 0000000,6f93f2b..357c5e8
mode 000000,100755..100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@@ -1,0 -1,42 +1,44 @@@
+ #!/bin/bash
+ 
+ # Install XDS agent as a user systemd service
+ 
+ DESTDIR=${DESTDIR:-/opt/AGL/xds/agent}
++DESTDIR_WWW=${DESTDIR_WWW:-${DESTDIR}/www}
+ 
+ ROOT_SRCDIR=$(cd $(dirname "$0")/.. && pwd)
+ 
+ install() {
+     mkdir -p ${DESTDIR} && cp ${ROOT_SRCDIR}/bin/* ${DESTDIR} || exit 1
++    mkdir -p ${DESTDIR_WWW} && cp -a ${ROOT_SRCDIR}/webapp/dist/* ${DESTDIR_WWW} || exit 1
+ 
+     cp ${ROOT_SRCDIR}/conf.d/etc/xds-agent /etc/ || exit 1
+     cp ${ROOT_SRCDIR}/conf.d/etc/default/xds-agent /etc/default/ || exit 1
+ 
+     FILE=/etc/profile.d/xds-agent.sh
+     sed -e "s;%%XDS_INSTALL_BIN_DIR%%;${DESTDIR};g" ${ROOT_SRCDIR}/conf.d/${FILE} > ${FILE} || exit 1
+ 
+     FILE=/usr/lib/systemd/user/xds-agent.service
+     sed -e "s;/opt/AGL/xds/agent;${DESTDIR};g" ${ROOT_SRCDIR}/conf.d/${FILE} > ${FILE} || exit 1
+ 
+     echo ""
+     echo "To enable xds-agent service, execute:      systemctl --user enable xds-agent"
+     echo "and to start xds-agent service, execute:   systemctl --user start xds-agent"
+ }
+ 
+ uninstall() {
+     rm -rf "${DESTDIR}"
+     rm -f /etc/xds-agent /etc/profile.d/xds-agent.sh /usr/lib/systemd/user/xds-agent.service
+ }
+ 
+ if [ "$1" == "uninstall" ]; then
+     echo -n "Are-you sure you want to remove ${DESTDIR} [y/n]? "
+     read answer
+     if [ "${answer}" = "y" ]; then
+         uninstall
+         echo "xds-agent sucessfully uninstalled."
+     else
+         echo "Uninstall canceled."
+     fi
+ else
+     install
+ fi