X-Git-Url: https://gerrit.automotivelinux.org/gerrit/gitweb?a=blobdiff_plain;f=test%2Ftarget_test.go;h=010411db34b57621755761d988b5ca8cb1756c9c;hb=5dc2ff003106f0ced38caadb06033f24c792f9b9;hp=696c8fcf8ad9adbb6b33086b3bc6f2ced876c9a1;hpb=4c01e5e3fad9455b97519380babdf7e8801d8643;p=src%2Fxds%2Fxds-server.git diff --git a/test/target_test.go b/test/target_test.go index 696c8fc..010411d 100644 --- a/test/target_test.go +++ b/test/target_test.go @@ -17,82 +17,349 @@ package xdsservertest import ( + "bytes" + "fmt" + "log" "os" + "os/exec" "path" + "regexp" "strconv" "strings" "testing" "time" - "gerrit.automotivelinux.org/gerrit/src/xds/xds-server/lib/xsapiv1" - "github.com/stretchr/testify/assert" + common "gerrit.automotivelinux.org/gerrit/src/xds/xds-common.git" + "gerrit.automotivelinux.org/gerrit/src/xds/xds-server.git/lib/xsapiv1" + "github.com/stretchr/testify/require" ) -/*flush channel with timeout*/ -func flushChannelTerm(channel chan xsapiv1.TerminalOutMsg, ms time.Duration) { - timeoutB := false - for !timeoutB { +func launchSSHd(sshDir string, proc **os.Process, sshdCmd string) (*os.File, string) { + port := "22222" + argsProcessSSHd := []string{ + sshdCmd, + "-f", + sshDir + "/sshd_config", + "-D", + "-h", + sshDir + "/ssh_host_rsa_key", + "-o", + "AuthorizedKeysFile=" + sshDir + "/authorized_keys", + "-p", + port, + } + logFile := logDir + logFileSSHd + file, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatal(err) + } + defer file.Close() + tmpProc, err := os.StartProcess(argsProcessSSHd[0], argsProcessSSHd, &os.ProcAttr{ + Files: []*os.File{os.Stdin, file, file}, + }) + if err != nil { + log.Fatal(err) + } + *proc = tmpProc + return file, port +} + +func InitSSH(t *testing.T, procSSHd **os.Process) (string, string) { + Debug(t, "Initialise ssh with local user") + sshDir := path.Join(os.Getenv(envRootCfgDir), "ssh") + cmd := exec.Command("cp", "-r", sshFixturesDir, sshDir) + var out bytes.Buffer + cmd.Stdout = &out + require.Nil(t, cmd.Run()) + + cmd = exec.Command("ls", sshDir) + cmd.Stdout = &out + require.Nil(t, cmd.Run()) + + files := strings.Split(fmt.Sprint(cmd.Stdout), "\n") + + for _, f := range files { + if f != "" { + file := sshDir + "/" + f + cmd = exec.Command("chmod", "600", file) + cmd.Stdout = &out + require.Nil(t, cmd.Run()) + } + } + + var outSSHd bytes.Buffer + var sshdCmd string + cmd = exec.Command("which", "sshd") + cmd.Stdout = &outSSHd + if cmd.Run() != nil { + if common.Exists("/usr/sbin/sshd") { + sshdCmd = "/usr/sbin/sshd" + } else if common.Exists("/usr/bin/sshd") { + sshdCmd = "/usr/sbin/sshd" + } else { + require.FailNow(t, "Cannot find sshd command, please install it or set in your PATH") + } + } else { + sshdCmd = strings.TrimSpace(fmt.Sprint(cmd.Stdout)) + } + + var port string + _, port = launchSSHd(sshDir, procSSHd, sshdCmd) + go func(p *os.Process) { + Debug(t, "sshd is launching") + if status, err := p.Wait(); err != nil { + log.Fatalf("status=%v\n err=%v\n", status, err) + } + }(*procSSHd) + return sshDir, port +} + +/*wait for terminal prompt*/ +func waitForPrompt(t *testing.T, channel chan xsapiv1.TerminalOutMsg, prompt string) string { + step := 1 * time.Millisecond + timeout := 10 * time.Second + current := 0 * time.Second + out := "" + re := regexp.MustCompile("^" + prompt) + + for { select { - case <-channel: - case <-time.After(ms * time.Millisecond): - timeoutB = true + case outMsg := <-channel: + out += string(outMsg.Stdout) + if string(outMsg.Stderr) != "" { + out += string(outMsg.Stderr) + } + for _, line := range strings.Split(out, "\n") { + if re.MatchString(line) { + return line + } + } + case <-time.After(step): + current = current + step + if current >= timeout { + require.FailNow(t, "Never received prompt message from terminal (output:"+out+")") + } } } } -func TestTarget(t *testing.T) { - var targetArray []xsapiv1.TargetConfig - assert.Nil(t, HTTPCli.Get("/targets", &targetArray)) - assert.Equal(t, len(targetArray), 0) +func ConnectTargetEvents(t *testing.T, channel chan xsapiv1.TargetConfig) { + sCli.Conn.On(xsapiv1.EVTTargetAdd, func(e xsapiv1.EventMsg) { + target, _ := e.DecodeTargetEvent() + channel <- target + }) + + args := xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetAdd} + require.Nil(t, HTTPCli.Post("/events/register", args, nil)) + + sCli.Conn.On(xsapiv1.EVTTargetRemove, func(e xsapiv1.EventMsg) { + target, _ := e.DecodeTargetEvent() + channel <- target + }) + + args = xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetRemove} + require.Nil(t, HTTPCli.Post("/events/register", args, nil)) +} + +func DisconnectTargetEvents(t *testing.T) { + args := xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetAdd} + require.Nil(t, HTTPCli.Post("/events/unregister", args, nil)) + args = xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetRemove} + require.Nil(t, HTTPCli.Post("/events/unregister", args, nil)) +} + +func ConnectTermEvents(t *testing.T, channel chan xsapiv1.TerminalConfig) { + sCli.Conn.On(xsapiv1.EVTTargetTerminalAdd, func(e xsapiv1.EventMsg) { + termEvt, _ := e.DecodeTerminalEvent() + channel <- termEvt + }) + + args := xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetTerminalAdd} + require.Nil(t, HTTPCli.Post("/events/register", args, nil)) + + sCli.Conn.On(xsapiv1.EVTTargetTerminalStateChange, func(e xsapiv1.EventMsg) { + termEvt, _ := e.DecodeTerminalEvent() + channel <- termEvt + }) + + args = xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetTerminalStateChange} + require.Nil(t, HTTPCli.Post("/events/register", args, nil)) - target := xsapiv1.TargetConfig{ - Name: "fakeTarget", - Type: xsapiv1.TypeTgtStandard, - IP: "127.0.0.1", + sCli.Conn.On(xsapiv1.EVTTargetTerminalRemove, func(e xsapiv1.EventMsg) { + termEvt, _ := e.DecodeTerminalEvent() + channel <- termEvt + }) + + args = xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetTerminalRemove} + require.Nil(t, HTTPCli.Post("/events/register", args, nil)) +} + +func DisconnectTermEvents(t *testing.T) { + args := xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetTerminalAdd} + require.Nil(t, HTTPCli.Post("/events/unregister", args, nil)) + args = xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetTerminalStateChange} + require.Nil(t, HTTPCli.Post("/events/unregister", args, nil)) + args = xsapiv1.EventRegisterArgs{Name: xsapiv1.EVTTargetTerminalRemove} + require.Nil(t, HTTPCli.Post("/events/unregister", args, nil)) +} + +func AddTargets(t *testing.T, nbTargets int, chTarget chan xsapiv1.TargetConfig) []string { + listID := make([]string, nbTargets) + for i := 0; i < nbTargets; i++ { + /*target is local*/ + target := xsapiv1.TargetConfig{ + Name: "fakeTarget" + strconv.Itoa(i), + Type: xsapiv1.TypeTgtStandard, + IP: "127.0.0.1", + } + /*add target*/ + require.Nil(t, HTTPCli.Post("/targets", target, &target)) + Debugf(t, "add target %v", target.Name) + targetEvt := <-chTarget //waiting for event targetAdd + require.Equal(t, target.ID, targetEvt.ID) + listID[i] = target.ID + } + for i := 0; i < nbTargets; i++ { + var target xsapiv1.TargetConfig + require.Nil(t, HTTPCli.Get("/targets/"+listID[i], &target)) + require.Equal(t, target.Status, "Enable") + } + return listID +} + +func AddTerms(t *testing.T, nbTerms int, listID []string, chTermEvt chan xsapiv1.TerminalConfig, sshDir string, port string) { + for j := 0; j < len(listID); j++ { + listTermsID := make([]string, nbTerms) + for i := 0; i < nbTerms; i++ { + term := xsapiv1.TerminalConfig{ + Name: "terminal" + strconv.Itoa(i), + Type: xsapiv1.TypeTermSSH, + Options: []string{ + "-p", + port, + "-i", + sshDir + "/ssh", + "-o", + "StrictHostKeyChecking=no", + }, + } + /*add terminal on target*/ + require.Nil(t, HTTPCli.Post("/targets/"+listID[j]+"/terminals", term, &term)) + Debugf(t, "add terminal %v", term.Name) + termEvt := <-chTermEvt //waiting for event terminalAdd*/ + require.Equal(t, term.ID, termEvt.ID) + listTermsID[i] = term.ID + } + require.Equal(t, len(listTermsID), nbTerms) + for i := 0; i < nbTerms; i++ { + var term xsapiv1.TerminalConfig + require.Nil(t, HTTPCli.Get("/targets/"+listID[j]+"/terminals/"+listTermsID[i], &term)) + require.Equal(t, term.Status, "Close") + } } - var targetRes, targetResBis xsapiv1.TargetConfig - assert.Nil(t, HTTPCli.Post("/targets", target, &targetRes)) - target.Name = "fakeTargetBis" - assert.Nil(t, HTTPCli.Post("/targets", target, &targetResBis)) - assert.Equal(t, targetRes.Status, "Enable") - assert.Equal(t, targetResBis.Status, "Enable") - - term := xsapiv1.TerminalConfig{ - Name: "terminal", - Type: xsapiv1.TypeTermSSH, +} + +func PostTerms(t *testing.T, post string, chTermEvt chan xsapiv1.TerminalConfig, chTerm chan xsapiv1.TerminalOutMsg, + prompt string) { + var status string + switch post { + case "open": + status = "Open" + case "close": + status = "Closing" + } + var targets []xsapiv1.TargetConfig + require.Nil(t, HTTPCli.Get("/targets", &targets)) + for i := 0; i < len(targets); i++ { + var terms []xsapiv1.TerminalConfig + require.Nil(t, HTTPCli.Get("/targets/"+targets[i].ID+"/terminals", &terms)) + listTermsID := make([]string, len(terms)) + for j := 0; j < len(terms); j++ { + var term xsapiv1.TerminalConfig + /*post action on term*/ + require.Nil(t, HTTPCli.Post("/targets/"+targets[i].ID+"/terminals/"+terms[j].ID+"/"+post, terms[j], &term)) + Debugf(t, "%v terminal %v", post, term.Name) + termEvt := <-chTermEvt //waiting for event terminalStateChange + if post == "open" { + data := []byte("PS1=" + prompt + " bash -norc\n") + require.Nil(t, sCli.Conn.Emit(xsapiv1.TerminalInEvent, data)) + waitForPrompt(t, chTerm, prompt) + } + require.Equal(t, term.ID, termEvt.ID) + require.Equal(t, term.Status, status) + require.Equal(t, termEvt.Status, status) + listTermsID[i] = term.ID + } + time.Sleep(10 * time.Millisecond) + for j := 0; j < len(listTermsID); j++ { + var term xsapiv1.TerminalConfig + require.Nil(t, HTTPCli.Get("/targets/"+targets[i].ID+"/terminals/"+listTermsID[i], &term)) + require.True(t, strings.EqualFold(term.Status, post)) + Debugf(t, "check that term status %v is %v", term.Name, post) + } } - var termRes, termResBis xsapiv1.TerminalConfig - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) - term.Name = "terminalBis" - assert.Nil(t, HTTPCli.Post("/targets/"+targetResBis.ID+"/terminals", term, &termResBis)) - assert.Equal(t, termRes.Status, "Enable") - assert.Equal(t, termResBis.Status, "Enable") +} +func RemoveTermsTargets(t *testing.T, chTarget chan xsapiv1.TargetConfig, chTermEvt chan xsapiv1.TerminalConfig) { + var targets []xsapiv1.TargetConfig + require.Nil(t, HTTPCli.Get("/targets", &targets)) + for i := 0; i < len(targets); i++ { + var terms []xsapiv1.TerminalConfig + require.Nil(t, HTTPCli.Get("/targets/"+targets[i].ID+"/terminals", &terms)) + for j := 0; j < len(terms); j++ { + var term xsapiv1.TerminalConfig + require.Nil(t, HTTPCli.Delete("/targets/"+targets[i].ID+"/terminals/"+terms[j].ID, &term)) + termEvt := <-chTermEvt + require.Equal(t, term.ID, termEvt.ID) + require.NotNil(t, HTTPCli.Delete("/targets/"+targets[i].ID+"/terminals/"+terms[j].ID, &term)) + Debugf(t, "remove terminal %v", term.Name) + } + var tgtRes xsapiv1.TargetConfig + require.Nil(t, HTTPCli.Delete("/targets/"+targets[i].ID, &tgtRes)) + targetEvt := <-chTarget //waiting for remove terminal event + require.Equal(t, tgtRes.ID, targetEvt.ID) + require.Equal(t, targets[i].ID, tgtRes.ID) + } +} +func TestTarget(t *testing.T) { + prompt := "--PROMPT--" + var procSSHd *os.Process + sshDir, port := InitSSH(t, &procSSHd) + defer procSSHd.Kill() + + nbTargets := 2 + nbTermsByTarget := 2 + /*channel for target events*/ + chTarget := make(chan xsapiv1.TargetConfig) + defer close(chTarget) + ConnectTargetEvents(t, chTarget) + + /*channel for terminal events*/ + chTermEvt := make(chan xsapiv1.TerminalConfig) + defer close(chTermEvt) + ConnectTermEvents(t, chTermEvt) + + /*check that targetArray is empty at startup*/ + var targetArray []xsapiv1.TargetConfig + require.Nil(t, HTTPCli.Get("/targets", &targetArray)) + require.Equal(t, len(targetArray), 0) + + listID := AddTargets(t, nbTargets, chTarget) + AddTerms(t, nbTermsByTarget, listID, chTermEvt, sshDir, port) + + /*channel for TerminalOutMsg*/ chTerm := make(chan xsapiv1.TerminalOutMsg) defer close(chTerm) - sCli, err := NewIoSocketClient(prefixURL, HTTPCli.GetClientID()) - if err != nil { - t.Fatal(err) - } + + /*connect on terminalOutMsg event*/ sCli.Conn.On(xsapiv1.TerminalOutEvent, func(ev xsapiv1.TerminalOutMsg) { chTerm <- ev }) - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals/"+termRes.ID+"/open", termRes, &termRes)) - assert.Nil(t, HTTPCli.Post("/targets/"+targetResBis.ID+"/terminals/"+termResBis.ID+"/open", termResBis, &termResBis)) - assert.Equal(t, termRes.Status, "Open") - assert.Equal(t, termResBis.Status, "Open") - - termOut := <-chTerm - flushChannelTerm(chTerm, 50) - stdoutMsg := string(termOut.Stdout) - if strings.Contains(stdoutMsg, "Connection refused") { - t.Fatalf("%vYou may have to launch ssh server", stdoutMsg) - } else if strings.Contains(stdoutMsg, "password") { - t.Fatalf("%vcopy your pub key in authorized_keys\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", stdoutMsg) - } - assert.True(t, strings.Contains(stdoutMsg, "Last login")) + /*open terminals*/ + PostTerms(t, "open", chTermEvt, chTerm, prompt) + /*create toto file through terminals*/ rootCfgDir := os.Getenv(envRootCfgDir) totoFile := path.Join(rootCfgDir, "toto") @@ -101,75 +368,80 @@ func TestTarget(t *testing.T) { totoFileCurrent := totoFile + strconv.Itoa(i) /*send cmd though term*/ data := []byte("echo helloWorld" + strconv.Itoa(i) + " >> " + totoFileCurrent + "\n") - - assert.Nil(t, sCli.Conn.Emit(xsapiv1.TerminalInEvent, data)) - flushChannelTerm(chTerm, 50) + Debugf(t, "send following command through terminal: %v", string(data)) + require.Nil(t, sCli.Conn.Emit(xsapiv1.TerminalInEvent, data)) + waitForPrompt(t, chTerm, prompt) //waiting for terminal prompt /*check that toto file is created*/ - _, err = os.Stat(totoFileCurrent) - assert.Nil(t, err) + _, err := os.Stat(totoFileCurrent) + require.Nil(t, err) + /*send cmd though term*/ data = []byte("cat " + totoFileCurrent + "\n") - assert.Nil(t, sCli.Conn.Emit(xsapiv1.TerminalInEvent, data)) + Debugf(t, "send following command through terminal: %v", string(data)) + require.Nil(t, sCli.Conn.Emit(xsapiv1.TerminalInEvent, data)) - <-chTerm //cmd sent - termOut = <-chTerm //result of cat cmd - flushChannelTerm(chTerm, 50) - assert.Equal(t, string(termOut.Stdout), "helloWorld"+strconv.Itoa(i)+"\r\n") + <-chTerm //cmd sent + termOut := <-chTerm //result of cat cmd + waitForPrompt(t, chTerm, prompt) //wait for terminal prompt + /*check that terminal msg is what was written before*/ + require.Equal(t, string(termOut.Stdout), "helloWorld"+strconv.Itoa(i)+"\r\n") + Debugf(t, "check terminal output msg: %v", string(termOut.Stdout)) } - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals/"+termRes.ID+"/close", termRes, &termRes)) - assert.Nil(t, HTTPCli.Post("/targets/"+targetResBis.ID+"/terminals/"+termResBis.ID+"/close", termResBis, &termResBis)) - assert.Equal(t, termRes.Status, "Close") - assert.Equal(t, termResBis.Status, "Close") + PostTerms(t, "close", chTermEvt, nil, prompt) /*remove targets and terms*/ - assert.Nil(t, HTTPCli.Get("/targets", &targetArray)) - for i := 0; i < len(targetArray); i++ { - var termArray []xsapiv1.TerminalConfig - assert.Nil(t, HTTPCli.Get("/targets/"+targetArray[i].ID+"/terminals", &termArray)) - for j := 0; j < len(termArray); j++ { - assert.Nil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID+"/terminals/"+termArray[j].ID, &termRes)) - assert.NotNil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID+"/terminals/"+termArray[j].ID, &termRes)) - } - var tgtRes xsapiv1.TargetConfig - assert.Nil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID, &tgtRes)) - assert.Equal(t, targetArray[i].ID, tgtRes.ID) - } + RemoveTermsTargets(t, chTarget, chTermEvt) + DisconnectTargetEvents(t) + DisconnectTermEvents(t) } func TestTargetErrors(t *testing.T) { + /*cannot create empty target*/ target := xsapiv1.TargetConfig{} var targetRes xsapiv1.TargetConfig - assert.NotNil(t, HTTPCli.Post("/targets", target, &targetRes)) + require.NotNil(t, HTTPCli.Post("/targets", target, &targetRes)) + Debugf(t, "error while creating empty target") + /*check cannot create target with no IP*/ target.Type = xsapiv1.TypeTgtStandard - assert.NotNil(t, HTTPCli.Post("/targets", target, &targetRes)) + require.NotNil(t, HTTPCli.Post("/targets", target, &targetRes)) + Debugf(t, "error while creating target without IP") target.IP = "127.0.0.1" - assert.Nil(t, HTTPCli.Post("/targets", target, &targetRes)) + require.Nil(t, HTTPCli.Post("/targets", target, &targetRes)) + Debugf(t, "create target %v", targetRes.Name) + /*cannot create empty terminal*/ term := xsapiv1.TerminalConfig{} var termRes xsapiv1.TerminalConfig - assert.NotNil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + require.NotNil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + Debugf(t, "error while creating empty terminal") term.Type = xsapiv1.TypeTermSSH - assert.NotNil(t, HTTPCli.Post("/targets/"+"1010"+"/terminals", term, &termRes)) - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) - assert.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + require.NotNil(t, HTTPCli.Post("/targets/"+"1010"+"/terminals", term, &termRes)) + Debugf(t, "error while creating terminal on an non existing target") + require.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + require.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + require.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + require.Nil(t, HTTPCli.Post("/targets/"+targetRes.ID+"/terminals", term, &termRes)) + Debugf(t, "create several terminals") /*remove targets and terms*/ var targetArray []xsapiv1.TargetConfig - assert.Nil(t, HTTPCli.Get("/targets", &targetArray)) + require.Nil(t, HTTPCli.Get("/targets", &targetArray)) for i := 0; i < len(targetArray); i++ { var termArray []xsapiv1.TerminalConfig - assert.Nil(t, HTTPCli.Get("/targets/"+targetArray[i].ID+"/terminals", &termArray)) + require.Nil(t, HTTPCli.Get("/targets/"+targetArray[i].ID+"/terminals", &termArray)) for j := 0; j < len(termArray); j++ { - assert.Nil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID+"/terminals/"+termArray[j].ID, &termRes)) - assert.NotNil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID+"/terminals/"+termArray[j].ID, &termRes)) + require.Nil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID+"/terminals/"+termArray[j].ID, &termRes)) + Debugf(t, "delete terminal %v", termRes.Name) + require.NotNil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID+"/terminals/"+termArray[j].ID, &termRes)) + Debugf(t, "error while deleting an already deleted terminal %v", termRes.Name) } var tgtRes xsapiv1.TargetConfig - assert.Nil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID, &tgtRes)) - assert.Equal(t, targetArray[i].ID, tgtRes.ID) - assert.NotNil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID, &tgtRes)) + require.Nil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID, &tgtRes)) + Debugf(t, "delete target %v", tgtRes.Name) + require.Equal(t, targetArray[i].ID, tgtRes.ID) + require.NotNil(t, HTTPCli.Delete("/targets/"+targetArray[i].ID, &tgtRes)) + Debugf(t, "error while deleting an already deleted target %v", tgtRes.Name) } }