INTERNAL: improve symptomdata

- Optimized file handling
- Added unzip capability
- Added config file and environment into symptom default data

Signed-off-by: Juha Hyttinen <juha.hyttinen@nokia.com>
Change-Id: I1dfa951c28d1102581bf4b7839e665fd37549da6

diff --git a/pkg/xapp/Utils.go b/pkg/xapp/Utils.go
index 90bc093..532b876 100755 (executable)
--- a/pkg/xapp/Utils.go
+++ b/pkg/xapp/Utils.go
@@ -21,6 +21,7 @@ package xapp
 
 import (
        "archive/zip"
+       "fmt"
        "io"
        "io/ioutil"
        "os"
@@ -82,10 +83,7 @@ func (u *Utils) AddFileToZip(zipWriter *zip.Writer, filePath string, filename st
                return err
        }
 
-       if strings.HasPrefix(filename, filePath) {
-               filename = strings.TrimPrefix(filename, filePath)
-       }
-       header.Name = filename
+       header.Name = strings.TrimPrefix(filename, filePath)
        header.Method = zip.Deflate
 
        writer, err := zipWriter.CreateHeader(header)
@@ -104,7 +102,6 @@ func (u *Utils) ZipFiles(newZipFile *os.File, filePath string, files []string) e
        defer zipWriter.Close()
        for _, file := range files {
                if err := u.AddFileToZip(zipWriter, filePath, file); err != nil {
-                       Logger.Error("AddFileToZip() failed: %+v", err.Error())
                        return err
                }
        }
@@ -112,25 +109,83 @@ func (u *Utils) ZipFiles(newZipFile *os.File, filePath string, files []string) e
        return nil
 }
 
+func (u *Utils) ZipFilesToTmpFile(baseDir string, tmpfilename string, fileList []string) (string, error) {
+       //Generate zip file
+       tmpFile, err := ioutil.TempFile("", tmpfilename)
+       if err != nil {
+               return "", fmt.Errorf("Failed to create a tmp file: %w", err)
+       }
+       err = u.ZipFiles(tmpFile, baseDir, fileList)
+       if err != nil {
+               os.Remove(tmpFile.Name())
+               return "", fmt.Errorf("Failed to zip the files: %w", err)
+       }
+       return tmpFile.Name(), nil
+}
+
+func (u *Utils) GetFileFromZip(file *zip.File, filePath string) (string, error) {
+       filename := filepath.Join(filePath, file.Name)
+
+       if file.FileInfo().IsDir() {
+               os.MkdirAll(filename, os.ModePerm)
+               return "", nil
+       }
+
+       if err := os.MkdirAll(filepath.Dir(filename), os.ModePerm); err != nil {
+               return "", fmt.Errorf("mkdir failed %s", filepath.Dir(filename))
+       }
+
+       dstFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+       if err != nil {
+               return "", fmt.Errorf("openfile failed %s", filename)
+       }
+       defer dstFile.Close()
+
+       fileInArchive, err := file.Open()
+       if err != nil {
+               return "", fmt.Errorf("zip file open failed %s", file.Name)
+       }
+       defer fileInArchive.Close()
+
+       if _, err := io.Copy(dstFile, fileInArchive); err != nil {
+               return "", fmt.Errorf("copy failed %s -> %s", file.Name, filename)
+       }
+       return filename, nil
+
+}
+
+func (u *Utils) UnZipFiles(zippedFile string, filePath string) ([]string, error) {
+       retval := []string{}
+       zipReader, err := zip.OpenReader(zippedFile)
+       if err != nil {
+               return retval, fmt.Errorf("Failed to open zip reader: %w", err)
+       }
+       defer zipReader.Close()
+
+       //fmt.Printf("Reading zipfile: %s\n", zippedFile)
+       for _, file := range zipReader.File {
+               fname, err := u.GetFileFromZip(file, filePath)
+               if err != nil {
+                       return retval, fmt.Errorf("Failed to unzip the files: %w", err)
+               }
+               if len(fname) > 0 {
+                       retval = append(retval, fname)
+               }
+       }
+       return retval, nil
+}
+
 func (u *Utils) FetchFiles(filePath string, fileList []string) []string {
        files, err := ioutil.ReadDir(filePath)
        if err != nil {
-               Logger.Error("ioutil.ReadDir failed: %+v", err)
+               fmt.Printf("ioutil.ReadDir failed: %+v\n", err)
                return nil
        }
        for _, file := range files {
                if !file.IsDir() {
                        fileList = append(fileList, filepath.Join(filePath, file.Name()))
                } else {
-                       subPath := filepath.Join(filePath, file.Name())
-                       subFiles, _ := ioutil.ReadDir(subPath)
-                       for _, subFile := range subFiles {
-                               if !subFile.IsDir() {
-                                       fileList = append(fileList, filepath.Join(subPath, subFile.Name()))
-                               } else {
-                                       fileList = u.FetchFiles(filepath.Join(subPath, subFile.Name()), fileList)
-                               }
-                       }
+                       fileList = u.FetchFiles(filepath.Join(filePath, file.Name()), fileList)
                }
        }
        return fileList

diff --git a/pkg/xapp/restapi.go b/pkg/xapp/restapi.go
index 14b194a..00f8aba 100755 (executable)
--- a/pkg/xapp/restapi.go
+++ b/pkg/xapp/restapi.go
@@ -25,6 +25,8 @@ import (
        "io/ioutil"
        "net/http"
        "os"
+       "path"
+       "strings"
 
        "github.com/gorilla/mux"
        "github.com/spf13/viper"
@@ -134,30 +136,43 @@ func (r *Router) CollectDefaultSymptomData(fileName string, data interface{}) st
                return ""
        }
 
+       //
        if metrics, err := r.GetLocalMetrics(GetPortData("http").Port); err == nil {
                if err := Util.WriteToFile(baseDir+"metrics.json", metrics); err != nil {
                        Logger.Error("writeToFile failed for metrics.json: %v", err)
                }
        }
 
+       //
        if data != nil {
                if b, err := json.MarshalIndent(data, "", "  "); err == nil {
                        Util.WriteToFile(baseDir+fileName, string(b))
                }
        }
 
-       rtPath := os.Getenv("RMR_STASH_RT")
-       if rtPath == "" {
-               return baseDir
+       //
+       cfile := viper.ConfigFileUsed()
+       input, err := ioutil.ReadFile(cfile)
+       if err == nil {
+               Util.WriteToFile(baseDir+path.Base(cfile), string(input))
+       } else {
+               Logger.Error("ioutil.ReadFile failed: %v", err)
        }
 
-       input, err := ioutil.ReadFile(rtPath)
-       if err != nil {
-               Logger.Error("ioutil.ReadFile failed: %v", err)
-               return baseDir
+       //
+       Util.WriteToFile(baseDir+"environment", strings.Join(os.Environ(), "\n"))
+
+       //
+       rtPath := os.Getenv("RMR_STASH_RT")
+       if rtPath != "" {
+               input, err = ioutil.ReadFile(rtPath)
+               if err == nil {
+                       Util.WriteToFile(baseDir+"rttable.txt", string(input))
+               } else {
+                       Logger.Error("ioutil.ReadFile failed: %v", err)
+               }
        }
 
-       Util.WriteToFile(baseDir+"rttable.txt", string(input))
        return baseDir
 }
 
@@ -172,24 +187,18 @@ func (r *Router) SendSymptomDataJson(w http.ResponseWriter, req *http.Request, d
 }
 
 func (r *Router) SendSymptomDataFile(w http.ResponseWriter, req *http.Request, baseDir, zipFile string) {
-       // Compress and reply with attachment
-       tmpFile, err := ioutil.TempFile("", "symptom")
-       if err != nil {
-               r.SendSymptomDataError(w, req, "Failed to create a tmp file: "+err.Error())
-               return
-       }
-       defer os.Remove(tmpFile.Name())
 
        var fileList []string
        fileList = Util.FetchFiles(baseDir, fileList)
-       err = Util.ZipFiles(tmpFile, baseDir, fileList)
+       tmpFileName, err := Util.ZipFilesToTmpFile(baseDir, "symptom", fileList)
        if err != nil {
-               r.SendSymptomDataError(w, req, "Failed to zip the files: "+err.Error())
+               r.SendSymptomDataError(w, req, err.Error())
                return
        }
+       defer os.Remove(tmpFileName)
 
        w.Header().Set("Content-Disposition", "attachment; filename="+zipFile)
-       http.ServeFile(w, req, tmpFile.Name())
+       http.ServeFile(w, req, tmpFileName)
 }
 
 func (r *Router) SendSymptomDataError(w http.ResponseWriter, req *http.Request, message string) {

diff --git a/pkg/xapp/utils_test.go b/pkg/xapp/utils_test.go
index 21ae8b9..3bfcff5 100644 (file)
--- a/pkg/xapp/utils_test.go
+++ b/pkg/xapp/utils_test.go
@@ -20,7 +20,6 @@
 package xapp
 
 import (
-       "io/ioutil"
        "net/http"
        "os"
        "testing"
@@ -38,14 +37,16 @@ func TestNewUtils(t *testing.T) {
        utils.FetchFiles("./", []string{"go.mod"})
        utils.FetchFiles("./", []string{"go.mod"})
 
-       tmpFile, err := ioutil.TempFile("", "symptom")
+       tmpFileName, err := utils.ZipFilesToTmpFile("/tmp/abcd", "symptom", []string{"/tmp/abcd/file.txt"})
        assert.Equal(t, err, nil)
-       defer os.Remove(tmpFile.Name())
+       defer os.Remove(tmpFileName)
 
-       err = utils.ZipFiles(tmpFile, "/tmp/abcd", []string{"/tmp/abcd/file.txt"})
+       assert.Equal(t, utils.CreateDir("/tmp/dcba"), nil)
+       _, err = utils.UnZipFiles(tmpFileName, "/tmp/dcba")
        assert.Equal(t, err, nil)
 
        utils.DeleteFile("/tmp/abcd")
+       utils.DeleteFile("/tmp/dcba")
 }
 
 func TestSymptomdata(t *testing.T) {

diff --git a/pkg/xapp/xapp.go b/pkg/xapp/xapp.go
index 583d288..7aa8bb2 100755 (executable)
--- a/pkg/xapp/xapp.go
+++ b/pkg/xapp/xapp.go
@@ -156,6 +156,7 @@ func doRegister() error {
        xappversion := viper.GetString("version")
        pltNs := getPltNamespace("PLT_NAMESPACE", DEFAULT_PLT_NS)
 
+       //httpEp, rmrEp := getService(xappname, SERVICE_HTTP), getService(xappname, SERVICE_RMR)
        httpEp, rmrEp := getService(host, SERVICE_HTTP), getService(host, SERVICE_RMR)
        if httpEp == "" || rmrEp == "" {
                Logger.Warn("Couldn't resolve service endpoints: httpEp=%s rmrEp=%s", httpEp, rmrEp)