Code Review
/
nonrtric
/
plt
/
ranpm.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
Make it easier to build doc & subsites
[nonrtric/plt/ranpm.git]
/
pm-file-converter
/
components
/
miniocollector
/
miniocollector.go
diff --git
a/pm-file-converter/components/miniocollector/miniocollector.go
b/pm-file-converter/components/miniocollector/miniocollector.go
index
1663194
..
24ff41f
100644
(file)
--- a/
pm-file-converter/components/miniocollector/miniocollector.go
+++ b/
pm-file-converter/components/miniocollector/miniocollector.go
@@
-24,10
+24,6
@@
import (
"compress/gzip"
"context"
"fmt"
"compress/gzip"
"context"
"fmt"
- jsoniter "github.com/json-iterator/go"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
- log "github.com/sirupsen/logrus"
"io"
"main/common/dataTypes"
"main/components/xmltransform"
"io"
"main/common/dataTypes"
"main/components/xmltransform"
@@
-35,9
+31,15
@@
import (
"os"
"strings"
"time"
"os"
"strings"
"time"
+
+ jsoniter "github.com/json-iterator/go"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ log "github.com/sirupsen/logrus"
)
)
-func Xml_to_json_conv(evt_data *dataTypes.XmlFileEventHeader) string {
+// nolint
+func XmlToJsonConv(evtData *dataTypes.XmlFileEventHeader) string {
filestoreUser := os.Getenv("FILESTORE_USER")
filestorePwd := os.Getenv("FILESTORE_PWD")
filestoreServer := os.Getenv("FILESTORE_SERVER")
filestoreUser := os.Getenv("FILESTORE_USER")
filestorePwd := os.Getenv("FILESTORE_PWD")
filestoreServer := os.Getenv("FILESTORE_SERVER")
@@
-50,12
+52,12
@@
func Xml_to_json_conv(evt_data *dataTypes.XmlFileEventHeader) string {
log.Fatalln(err)
}
expiry := time.Second * 24 * 60 * 60 // 1 day.
log.Fatalln(err)
}
expiry := time.Second * 24 * 60 * 60 // 1 day.
- objectName := evt
_d
ata.Name
- bucketName := evt
_d
ata.ObjectStoreBucket
- compresion := evt
_d
ata.Compression
+ objectName := evt
D
ata.Name
+ bucketName := evt
D
ata.ObjectStoreBucket
+ compresion := evt
D
ata.Compression
reqParams := make(url.Values)
reqParams := make(url.Values)
- xmlh, err := jsoniter.Marshal(evt
_d
ata)
+ xmlh, err := jsoniter.Marshal(evt
D
ata)
if err != nil {
fmt.Printf("Error: %s", err)
return ""
if err != nil {
fmt.Printf("Error: %s", err)
return ""
@@
-66,17
+68,18
@@
func Xml_to_json_conv(evt_data *dataTypes.XmlFileEventHeader) string {
if err != nil {
log.Fatalln(err)
}
if err != nil {
log.Fatalln(err)
}
- file
_b
ytes := xmltransform.Convert(presignedURL.String(), compresion, string(xmlh))
+ file
B
ytes := xmltransform.Convert(presignedURL.String(), compresion, string(xmlh))
newObjectName := objectName + "kafka-producer-pm-xml2json-0.json.gz"
var buf bytes.Buffer
newObjectName := objectName + "kafka-producer-pm-xml2json-0.json.gz"
var buf bytes.Buffer
- err = gzipWrite(&buf, &file
_b
ytes)
- upload
_o
bject(s3Client, buf.Bytes(), newObjectName, "pm-files-json")
+ err = gzipWrite(&buf, &file
B
ytes)
+ upload
O
bject(s3Client, buf.Bytes(), newObjectName, "pm-files-json")
fmt.Println("")
return newObjectName
}
fmt.Println("")
return newObjectName
}
-func upload_object(mc *minio.Client, b []byte, objectName string, fsbucket string) {
+// nolint
+func uploadObject(mc *minio.Client, b []byte, objectName string, fsbucket string) {
contentType := "application/json"
if strings.HasSuffix(objectName, ".gz") {
contentType = "application/gzip"
contentType := "application/json"
if strings.HasSuffix(objectName, ".gz") {
contentType = "application/gzip"
@@
-85,8
+88,8
@@
func upload_object(mc *minio.Client, b []byte, objectName string, fsbucket strin
// Upload the xml file with PutObject
r := bytes.NewReader(b)
tctx := context.Background()
// Upload the xml file with PutObject
r := bytes.NewReader(b)
tctx := context.Background()
- if check
_minio_b
ucket(mc, fsbucket) == false {
- err := create
_minio_b
ucket(mc, fsbucket)
+ if check
MinioB
ucket(mc, fsbucket) == false {
+ err := create
MinioB
ucket(mc, fsbucket)
if err != nil {
log.Error("Cannot create bucket: ", fsbucket, ", ", err)
return
if err != nil {
log.Error("Cannot create bucket: ", fsbucket, ", ", err)
return
@@
-109,7
+112,8
@@
func upload_object(mc *minio.Client, b []byte, objectName string, fsbucket strin
}
}
}
}
-func create_minio_bucket(mc *minio.Client, bucket string) error {
+// nolint
+func createMinioBucket(mc *minio.Client, bucket string) error {
tctx := context.Background()
err := mc.MakeBucket(tctx, bucket, minio.MakeBucketOptions{})
if err != nil {
tctx := context.Background()
err := mc.MakeBucket(tctx, bucket, minio.MakeBucketOptions{})
if err != nil {
@@
-127,7
+131,8
@@
func create_minio_bucket(mc *minio.Client, bucket string) error {
return nil
}
return nil
}
-func check_minio_bucket(mc *minio.Client, bucket string) bool {
+// nolint
+func checkMinioBucket(mc *minio.Client, bucket string) bool {
tctx := context.Background()
exists, err := mc.BucketExists(tctx, bucket)
if err == nil && exists {
tctx := context.Background()
exists, err := mc.BucketExists(tctx, bucket)
if err == nil && exists {
@@
-139,6
+144,7
@@
func check_minio_bucket(mc *minio.Client, bucket string) bool {
}
// Write gzipped data to a Writer
}
// Write gzipped data to a Writer
+// nolint
func gzipWrite(w io.Writer, data *[]byte) error {
gw, err1 := gzip.NewWriterLevel(w, gzip.BestSpeed)
func gzipWrite(w io.Writer, data *[]byte) error {
gw, err1 := gzip.NewWriterLevel(w, gzip.BestSpeed)