优化日志,增加依赖

This commit is contained in:
chejiulong 2023-02-21 14:36:25 +08:00
parent f603577888
commit 7e5bd5923c
5 changed files with 261 additions and 264 deletions

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/sirupsen/logrus v1.9.0 github.com/sirupsen/logrus v1.9.0
golang.org/x/crypto v0.6.0 golang.org/x/crypto v0.6.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gorm.io/driver/sqlserver v1.4.2 gorm.io/driver/sqlserver v1.4.2
gorm.io/gorm v1.24.5 gorm.io/gorm v1.24.5
) )

2
go.sum
View File

@ -159,6 +159,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=

Binary file not shown.

Binary file not shown.

252
main.go
View File

@ -10,7 +10,6 @@ import (
"io" "io"
"log" "log"
"os" "os"
"os/signal"
"path" "path"
"path/filepath" "path/filepath"
"strconv" "strconv"
@ -24,6 +23,7 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
"gopkg.in/gomail.v2" "gopkg.in/gomail.v2"
"gopkg.in/natefinch/lumberjack.v2"
"gorm.io/driver/sqlserver" "gorm.io/driver/sqlserver"
_ "gorm.io/driver/sqlserver" _ "gorm.io/driver/sqlserver"
"gorm.io/gorm" "gorm.io/gorm"
@ -32,7 +32,6 @@ import (
/* /*
var ( var (
applogger *logrus.Logger applogger *logrus.Logger
redisClient *redis.Client redisClient *redis.Client
executableDir string executableDir string
@ -50,23 +49,21 @@ var (
zipPath = "RawData/Zip/" zipPath = "RawData/Zip/"
txtPath = "RawData/Txt/" txtPath = "RawData/Txt/"
logPath = "logs/" logPath = "logs/"
batchSize = 5000 //提交数据
insertSize = 500 //一次性入库
cSize = 10 //入库协程数
) )
*/ */
var ( //正式环境 var ( //正式环境
applogger *logrus.Logger applogger *logrus.Logger
redisClient *redis.Client redisClient *redis.Client
executableDir string executableDir string
redisAddress = "r-bp11564d96842414128.redis.rds.aliyuncs.com:6379" redisAddress = "r-bp11564d96842414128.redis.rds.aliyuncs.com:6379"
redisPassword = "3Nsb4Pmsl9bcLs24mL12l" redisPassword = "3Nsb4Pmsl9bcLs24mL12l"
redisDB = 233 redisDB = 233
sftpAddress = "10.11.0.63:22" sftpAddress = "esftp.sephora.com.cn:20981"
sftpUser = "wyeth2018" sftpUser = "CHN-CRMTOWemedia-wemedia"
sftpPassword = "nYydNHOu" sftpPassword = "uoWdMHEv39ZFjiOg"
sftpDir = "/wyeth2018/sephora_sms" sftpDir = "/CN-CRMTOWemedia/SMS"
dbAddress = "rm-bp16l424ln96q1ouk.sqlserver.rds.aliyuncs.com:3433" dbAddress = "rm-bp16l424ln96q1ouk.sqlserver.rds.aliyuncs.com:3433"
dbUser = "sephora_sms" dbUser = "sephora_sms"
dbPassword = "5ORiiLmgkniC0EqF" dbPassword = "5ORiiLmgkniC0EqF"
@ -74,10 +71,6 @@ var ( //正式环境
zipPath = "RawData/Zip/" zipPath = "RawData/Zip/"
txtPath = "RawData/Txt/" txtPath = "RawData/Txt/"
logPath = "logs/" logPath = "logs/"
batchSize = 5000 //提交数据
insertSize = 500 //一次性入库
cSize = 10 //入库协程数
) )
type Batch struct { type Batch struct {
@ -143,20 +136,6 @@ func init() {
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
applogger = logrus.New()
logFile := filepath.Join(executableDir, logPath, "download.log")
_, err = os.Stat(logFile)
if os.IsNotExist(err) {
if _, err := os.Create(logFile); err != nil {
log.Fatal(err)
}
}
f, err := os.OpenFile(logFile, os.O_APPEND|os.O_WRONLY, 0666)
if err != nil {
log.Fatal(err)
}
//defer f.Close()
applogger.SetOutput(f)
redisClient = redis.NewClient(&redis.Options{ redisClient = redis.NewClient(&redis.Options{
Addr: redisAddress, Addr: redisAddress,
@ -166,25 +145,30 @@ func init() {
} }
func main() { func main() {
// 创建一个channel用于接收信号 // 打开一个文件作为锁
signalChan := make(chan os.Signal, 1) lockFile, err := os.OpenFile(filepath.Join(executableDir, ".lock"), os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
// 注册SIGINT, SIGTERM信号处理函数 fmt.Println("打开锁文件失败:", err)
signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM)
// 启动goroutine等待信号的到来
go func() {
for {
select {
case <-signalChan:
// 当信号到来时,执行特定的代码
fmt.Println("程序已经在运行,本程序无法同时运行多个")
os.Exit(1) os.Exit(1)
} }
defer lockFile.Close()
// 尝试获取文件的独占锁
err = syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil {
fmt.Println("程序已经在运行,本程序无法同时运行多个")
os.Exit(1)
} }
}() applogger = logrus.New()
logPath := filepath.Join(executableDir, "logs")
logFileName := "sms_processing_" + time.Now().Format("2006-01-02") + ".log"
logFileHook := &lumberjack.Logger{
Filename: filepath.Join(logPath, logFileName),
}
applogger.SetOutput(logFileHook)
ticker := time.NewTicker(time.Minute) // 创建一个定时器,每隔一分钟触发一次任务函数 go downloadDecompression() // 启动立即执行一次
ticker := time.NewTicker(60 * time.Minute) // 创建一个定时器,每隔一分钟触发一次任务函数
tickCount := 1 //记录循环次数 tickCount := 1 //记录循环次数
defer ticker.Stop() // 延迟关闭定时器,确保所有的定时器事件都被处理完毕 defer ticker.Stop() // 延迟关闭定时器,确保所有的定时器事件都被处理完毕
// 循环处理任务 // 循环处理任务
@ -192,7 +176,10 @@ func main() {
select { select {
case <-ticker.C: case <-ticker.C:
// 定时器触发时执行的任务函数 // 定时器触发时执行的任务函数
fmt.Printf("尝试第%d执行....\n", tickCount) logFileName := "sms_processing_" + time.Now().Format("2006-01-02") + ".log"
logFileHook.Filename = filepath.Join(logPath, logFileName)
fmt.Printf("尝试第%d此执行....\n", tickCount)
applogger.Info(fmt.Sprintf("尝试第%d此执行....\n", tickCount))
go downloadDecompression() // 在新协程中异步执行 go downloadDecompression() // 在新协程中异步执行
tickCount++ tickCount++
} }
@ -201,12 +188,13 @@ func main() {
func downloadDecompression() { func downloadDecompression() {
if exists, _ := redisClient.Exists("iniDataStatus").Result(); exists == 1 { if exists, _ := redisClient.Exists("iniDataStatus").Result(); exists == 1 {
fmt.Println("批次执行中,跳过本次") fmt.Print("上一批次执行中,跳过本次\n")
} else { } else {
// 写入执行中标记 // 写入执行中标记
err := redisClient.Set("iniDataStatus", 1, 0).Err() err := redisClient.Set("iniDataStatus", 1, 0).Err()
if err != nil { if err != nil {
fmt.Println("写入标记失败:%d", err) fmt.Printf("写入任务执行中标记失败:%s\n", err.Error())
applogger.Info("写入任务执行中标记失败:%v", err)
} }
// Connect to SFTP server // Connect to SFTP server
sshConfig := &ssh.ClientConfig{ sshConfig := &ssh.ClientConfig{
@ -218,25 +206,29 @@ func downloadDecompression() {
} }
sshClient, err := ssh.Dial("tcp", sftpAddress, sshConfig) sshClient, err := ssh.Dial("tcp", sftpAddress, sshConfig)
if err != nil { if err != nil {
panic(err) fmt.Printf("写入任务执行中标记失败:%s\n", err.Error())
applogger.Info("写入任务执行中标记失败:%v", err)
} }
sftpClient, err := sftp.NewClient(sshClient) sftpClient, err := sftp.NewClient(sshClient)
if err != nil { if err != nil {
panic(err) fmt.Printf("写入任务执行中标记失败:%s\n", err.Error())
applogger.Info("写入任务执行中标记失败:%v", err)
} }
defer sftpClient.Close() defer sftpClient.Close()
files, err := sftpClient.ReadDir(sftpDir) files, err := sftpClient.ReadDir(sftpDir)
if err != nil { if err != nil {
applogger.Fatalf("sftp目录不存在: %v", err) fmt.Printf("sftp目录不存在: 错误信息:%s\n", err.Error())
applogger.Info("sftp目录不存在: 错误信息:%v", err)
} }
it := 1 it := 1
fmt.Printf("共%d个文件\n", len(files)) fmt.Printf("共%d个文件\n", len(files))
for _, file := range files { for _, file := range files {
fmt.Printf("第%d个文件\n", it) fmt.Printf("第%d个文件处理中\n", it)
it++ it++
// Check if file has been downloaded before // Check if file has been downloaded before
fileKey := fmt.Sprintf("downloaded:%s", file.Name()) fileKey := fmt.Sprintf("downloaded:%s", file.Name())
if exists, _ := redisClient.Exists(fileKey).Result(); exists == 1 { if exists, _ := redisClient.Exists(fileKey).Result(); exists == 1 {
fmt.Println("跳过已处理过的文件:" + file.Name())
continue continue
} }
@ -245,27 +237,27 @@ func downloadDecompression() {
// Download file // Download file
srcFile, err := sftpClient.Open(path.Join(sftpDir, file.Name())) srcFile, err := sftpClient.Open(path.Join(sftpDir, file.Name()))
if err != nil { if err != nil {
applogger.Fatalf("Failed to download file: %v", err) applogger.Info("打开sftp文件失败文件名:%s错误信息 %v", file.Name(), err)
continue continue
} }
defer srcFile.Close() defer srcFile.Close()
dstFile, err := os.Create(path.Join(executableDir, zipPath, file.Name())) dstFile, err := os.Create(path.Join(executableDir, zipPath, file.Name()))
if err != nil { if err != nil {
applogger.Fatalf("Failed to download file: %v", err) applogger.Info("创建本地文件失败,文件名: %s错误信息%v", file.Name(), err)
continue continue
} }
defer dstFile.Close() defer dstFile.Close()
if _, err := io.Copy(dstFile, srcFile); err != nil { if _, err := io.Copy(dstFile, srcFile); err != nil {
applogger.Fatalf("Failed to download file: %v", err) applogger.Info("下载文件失败,文件名: %s错误信息%v", file.Name(), err)
continue continue
} }
fmt.Println("下载完成(数据包):" + file.Name()) fmt.Printf("%s数据包下载完成\n", file.Name())
applogger.Info("下载完成(数据包):%d", file.Name()) applogger.Info(fmt.Sprintf("%s数据包下载完成", file.Name()))
// Unzip file // Unzip file
zipReader, err := zip.OpenReader(path.Join(executableDir, zipPath, file.Name())) zipReader, err := zip.OpenReader(path.Join(executableDir, zipPath, file.Name()))
if err != nil { if err != nil {
applogger.Fatalf("Failed to download file: %v", err) applogger.Info("解压文件失败: 文件名:%s错误信息%v", file.Name(), err)
continue continue
} }
defer zipReader.Close() defer zipReader.Close()
@ -280,7 +272,7 @@ func downloadDecompression() {
continue continue
} }
if err != nil || zipFileReader == nil { if err != nil || zipFileReader == nil {
applogger.Fatalf("Failed to open zip file: %v", err) applogger.Info("Failed to open zip file: %v", err)
fmt.Print("压缩文件处理结束") fmt.Print("压缩文件处理结束")
continue continue
} }
@ -288,50 +280,46 @@ func downloadDecompression() {
// Create the file // Create the file
unzipFile, err := os.Create(path.Join(executableDir, txtPath, zipFile.Name)) unzipFile, err := os.Create(path.Join(executableDir, txtPath, zipFile.Name))
if err != nil { if err != nil {
applogger.Info("创建压缩后的文件失败,文件名:%s错误信息 %v", zipFile.Name, err)
applogger.Fatalf("Failed to create unzip file: %v", err)
continue continue
} }
defer unzipFile.Close() defer unzipFile.Close()
// Write the unzip data to the file // Write the unzip data to the file
_, err = io.Copy(unzipFile, zipFileReader) _, err = io.Copy(unzipFile, zipFileReader)
if err != nil { if err != nil {
applogger.Fatalf("Failed to write unzip data to file %v", err) applogger.Info("文件解压失败,文件名:%s错误信息 %v", zipFileReader, err)
continue continue
} }
applogger.Info("解压完成(数据包):%d", zipFile.Name) applogger.Info(fmt.Sprintf("%s数据包解压完成", zipFile.Name))
fmt.Print("解压完成(数据包):" + zipFile.Name) fmt.Printf("%s数据包解压完成\n", zipFile.Name)
batchDataInsert(zipFile.Name) batchDataInsert(zipFile.Name)
applogger.Info("入库完成(数据包):%d", zipFile.Name)
fmt.Print("入库完成(数据包)::" + zipFile.Name)
} }
} else if filepath.Ext(file.Name()) == ".txt" { } else if filepath.Ext(file.Name()) == ".txt" {
srcFile, err := sftpClient.Open(path.Join(sftpDir, file.Name())) srcFile, err := sftpClient.Open(path.Join(sftpDir, file.Name()))
if err != nil { if err != nil {
applogger.Fatalf("sftp目录不存在: %v", err) applogger.Info("打开sftp文件失败文件名:%s错误信息 %v", file.Name(), err)
continue continue
} }
defer srcFile.Close() defer srcFile.Close()
dstFile, err := os.Create(path.Join(executableDir, txtPath, file.Name())) dstFile, err := os.Create(path.Join(executableDir, txtPath, file.Name()))
if err != nil { if err != nil {
applogger.Fatalf("Failed to download file: %v", err) applogger.Info("创建本地文件失败,文件名: %s错误信息%v", file.Name(), err)
continue continue
} }
defer dstFile.Close() defer dstFile.Close()
if _, err := io.Copy(dstFile, srcFile); err != nil { if _, err := io.Copy(dstFile, srcFile); err != nil {
applogger.Fatalf("Failed to download file: %v", err) applogger.Info("下载文件失败,文件名: %s错误信息%v", file.Name(), err)
continue continue
} }
fmt.Printf("下载完成(批次文件):%s \n", file.Name()) fmt.Printf("%s批次文件下载完成 \n", file.Name())
applogger.Info("下载完成(批次文件):%d", file.Name()) applogger.Info(fmt.Sprintf("%s批次文件下载完成", file.Name()))
batchInsert(file.Name()) batchInsert(file.Name())
fmt.Printf("入库完成(批次文件)%s \n", file.Name())
applogger.Info("入库完成(批次文件):%d\n", file.Name())
} }
err = redisClient.Set(fileKey, 1, 0).Err() //写入下载、解压、入库完成标记 err = redisClient.Set(fileKey, 1, 0).Err() //写入下载、解压、入库完成标记
if err != nil { if err != nil {
fmt.Println("写入标记失败:%", err) fmt.Printf("写入文件处理完成标记失败文件名:%s错误信息%v\n", file.Name(), err)
applogger.Info("写入文件处理完成标记失败文件名:%s错误信息v%\n", file.Name(), err)
} }
} }
redisClient.Del("iniDataStatus") //删除任务执行中标记 redisClient.Del("iniDataStatus") //删除任务执行中标记
@ -343,16 +331,13 @@ func downloadDecompression() {
func batchInsert(fileName string) { func batchInsert(fileName string) {
//fmt.Print("批次处理开始") //fmt.Print("批次处理开始")
start := time.Now() start := time.Now()
db, err := connectToDB() db, _ := connectToDB()
if err != nil {
panic("failed to connect database")
}
file, err := os.Open(path.Join(executableDir, txtPath, fileName)) file, err := os.Open(path.Join(executableDir, txtPath, fileName))
defer file.Close()
if err != nil { if err != nil {
panic("failed to open file") fmt.Printf("文件打开失败文件名:%s错误信息%v\n", fileName, err)
applogger.Fatalf("ailed to open file: %v", fileName) applogger.Info("文件打开失败文件名:%s错误信息%v", fileName, err)
} } else {
defer file.Close()
reader := csv.NewReader(bufio.NewReader(file)) reader := csv.NewReader(bufio.NewReader(file))
reader.Read() reader.Read()
batchRows := 0 batchRows := 0
@ -379,29 +364,35 @@ func batchInsert(fileName string) {
batchRows++ batchRows++
} }
time.Sleep(time.Second) time.Sleep(time.Second)
elapsed := time.Since(start) elapsed := time.Since(start)
subject := "丝芙兰批次文件处理完成" subject := "丝芙兰批次文件处理完成"
body := "批次文件:" + fileName + ";\n批次数" + strconv.Itoa(batchRows) + ";\n处理完成请前往管理平台查看处理。" body := "批次文件:" + fileName + ";\n批次数" + strconv.Itoa(batchRows) + ";\n处理完成请前往管理平台查看处理。"
err = SendEmail(subject, body) //发送邮件 SendEmail(subject, body) //发送邮件
if err != nil { fmt.Printf("%s批次文件入库完成 \n", fileName)
applogger.Info("邮件发送失:%d", err) applogger.Info(fmt.Sprintf("%s批次文件入库完成", fileName))
fmt.Print(err) fmt.Printf("执行时间%s 插入批次次数:%d\n\n\n", elapsed, batchRows)
applogger.Info(fmt.Sprintf("执行时间%s 插入批次次数:%d", elapsed, batchRows))
} }
fmt.Printf("执行时间%s\n插入批次次数%d\n", elapsed, batchRows)
} }
func batchDataInsert(fileName string) { func batchDataInsert(fileName string) {
//fmt.Print("批次数据处理开始")
start := time.Now() start := time.Now()
db, err := connectToDB() // Open file
file, err := os.Open(path.Join(executableDir, txtPath, fileName))
if err != nil { if err != nil {
panic("failed to connect database") fmt.Printf("文件打开失败,文件名:%s错误信息%v\n", fileName, err)
} applogger.Info("文件打开失败,文件名:%s错误信息%v", fileName, err)
} else {
defer file.Close()
db, _ := connectToDB()
// Insert data in batches using multiple goroutines // Insert data in batches using multiple goroutines
batchSize := 5000 //提交数据
insertSize := 500 //一次性入库
insertChanSize := 50 //通道缓冲数
goSize := 150 //协程数
var wg sync.WaitGroup var wg sync.WaitGroup
dataBatchChan := make(chan []BatcheData, cSize) dataBatchChan := make(chan []BatcheData, insertChanSize)
for i := 0; i < cSize; i++ { for i := 0; i < goSize; i++ {
wg.Add(1) wg.Add(1)
go func() { go func() {
for batch := range dataBatchChan { for batch := range dataBatchChan {
@ -417,19 +408,13 @@ func batchDataInsert(fileName string) {
} else { } else {
break break
} }
} }
} }
wg.Done() wg.Done()
}() }()
} }
// Open file
file, err := os.Open(path.Join(executableDir, txtPath, fileName))
if err != nil {
panic(err)
}
defer file.Close()
// Create and initialize hashset // Create and initialize hashset
hs := make(map[string]bool) hs := make(map[string]bool)
// Prepare batch data // Prepare batch data
@ -443,14 +428,15 @@ func batchDataInsert(fileName string) {
duplicateCount := make(map[string]int) duplicateCount := make(map[string]int)
insertsCount := make(map[string]int) insertsCount := make(map[string]int)
var count int var count int
//ci := 1
for scanner.Scan() { for scanner.Scan() {
line := scanner.Text() line := scanner.Text()
row, err := csv.NewReader(strings.NewReader(line)).Read() row, err := csv.NewReader(strings.NewReader(line)).Read()
if err != nil { if err != nil {
panic(err) fmt.Printf("文件按行读取失败,文件名:%s错误信息%v\n", fileName, err)
applogger.Info("文件按行读取失败,文件名:%s错误信息%v", fileName, err)
continue
} }
reservedFields := map[string]string{ //合并个性化字段 reservedFields := map[string]string{ //合并个性化字段
"ReservedField1": row[5], "ReservedField1": row[5],
"ReservedField2": row[6], "ReservedField2": row[6],
@ -461,37 +447,37 @@ func batchDataInsert(fileName string) {
} }
reservedFieldsJson, err := json.Marshal(reservedFields) //个性化字段转json reservedFieldsJson, err := json.Marshal(reservedFields) //个性化字段转json
if err != nil { if err != nil {
panic(err) fmt.Printf("个性化字段合并失败,文件名:%s错误信息%v\n", fileName, err)
applogger.Info("个性化字段合并失败,文件名:%s错误信息%v", fileName, err)
continue
} }
// Check if record exists in hashset // Check if record exists in hashset
key := fmt.Sprintf("%s-%s", row[2], row[3]) key := fmt.Sprintf("%s-%s", row[2], row[3])
if _, exists := hs[key]; exists { if _, exists := hs[key]; exists { //如果批次数据重复
bi++ bi++
// Increment duplicate count // Increment duplicate count
if _, ok := duplicateCount[row[2]]; !ok { if _, ok := duplicateCount[row[2]]; !ok {
duplicateCount[row[2]] = 0 duplicateCount[row[2]] = 0
} }
duplicateCount[row[2]]++ duplicateCount[row[2]]++
dataBatchDuplicate = append(dataBatchDuplicate, BatchDataDuplicateLog{ dataBatchDuplicate = append(dataBatchDuplicate, BatchDataDuplicateLog{
CommunicationChannelID: row[2], CommunicationChannelID: row[2],
Mobile: row[3], Mobile: row[3],
FullName: row[4], FullName: row[4],
ReservedField: string(reservedFieldsJson), ReservedField: string(reservedFieldsJson),
}) })
if len(dataBatchDuplicate) >= batchSize { if len(dataBatchDuplicate) >= batchSize {
err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error
if err != nil { if err != nil {
panic(err) fmt.Printf("插入重复数据失败,文件名:%s错误信息%v\n", fileName, err)
} applogger.Info("插入重复数据失败,文件名:%s错误信息%v", fileName, err)
} else {
dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize) dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize)
} }
}
continue continue
} }
// Add record to hashset // Add record to hashset
hs[key] = true hs[key] = true
@ -505,6 +491,8 @@ func batchDataInsert(fileName string) {
if len(dataBatch) >= batchSize { if len(dataBatch) >= batchSize {
dataBatchChan <- dataBatch dataBatchChan <- dataBatch
dataBatch = make([]BatcheData, 0, batchSize) dataBatch = make([]BatcheData, 0, batchSize)
//fmt.Print("提交通次数" + strconv.Itoa(ci))
//ci++
} }
if _, ok := insertsCount[row[2]]; !ok { if _, ok := insertsCount[row[2]]; !ok {
insertsCount[row[2]] = 0 insertsCount[row[2]] = 0
@ -516,7 +504,7 @@ func batchDataInsert(fileName string) {
if len(dataBatch) > 0 { if len(dataBatch) > 0 {
dataBatchChan <- dataBatch dataBatchChan <- dataBatch
dataBatch = make([]BatcheData, 0, batchSize) dataBatch = make([]BatcheData, 0, batchSize)
fmt.Print("文件读取完成,最后一批提交至通道") //fmt.Print("文件读取完成,最后一批提交至通道")
} }
close(dataBatchChan) close(dataBatchChan)
@ -535,30 +523,32 @@ func batchDataInsert(fileName string) {
} }
err = db.CreateInBatches(bpi, insertSize).Error err = db.CreateInBatches(bpi, insertSize).Error
if err != nil { if err != nil {
panic(err) fmt.Printf("插入批次处理信息失败,文件名:%s错误信息%v\n", fileName, err)
applogger.Info("插入批次处理信息失败,文件名:%s错误信息%v", fileName, err)
} }
//插入批此重复数据 //插入批此重复数据
if len(dataBatchDuplicate) > 0 { if len(dataBatchDuplicate) > 0 {
err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error
if err != nil { if err != nil {
panic(err) fmt.Printf("插入重复数据失败,文件名:%s错误信息%v\n", fileName, err)
applogger.Info("插入重复数据失败,文件名:%s错误信息%v", fileName, err)
} else {
dataBatchDuplicate = nil
} }
dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize)
}
//发送提醒邮件
subject := "丝芙兰数据包处理完成"
body := "数据包:" + fileName + ";\n总数" + strconv.Itoa(count+bi) + ";\n过滤重复数" + strconv.Itoa(bi) + ";\n过滤后总数" + strconv.Itoa(count) + ";\n处理完成请前往管理平台查看处理。"
err = SendEmail(subject, body) //发送邮件
if err != nil {
applogger.Info("邮件发送失:%d", err)
fmt.Print(err)
} }
elapsed := time.Since(start) elapsed := time.Since(start)
fmt.Printf("批次数据入库函数执行时间:%s;\n插入数:%d条\n过滤数%d条\n", elapsed, count, bi) //发送提醒邮件
subject := "丝芙兰数据包处理完成"
body := "数据包:" + fileName + ";\n总数" + strconv.Itoa(count+bi) + ";\n过滤重复数" + strconv.Itoa(bi) + ";\n过滤后总数" + strconv.Itoa(count) + ";\n处理完成请前往管理平台查看处理。"
SendEmail(subject, body) //发送邮件
applogger.Info(fmt.Sprintf("%s数据包 ,入库完成", fileName))
fmt.Printf("%s数据包 入库完成\n", fileName)
fmt.Printf("执行时间:%s 插入数据:%d 过滤数:%d\n\n\n", elapsed, count, bi)
applogger.Info(fmt.Sprintf("执行时间:%s 插入数据:%d条 过滤数数:%d条", elapsed, count, bi))
}
} }
func connectToDB() (*gorm.DB, error) { func connectToDB() (*gorm.DB, error) {
@ -575,6 +565,8 @@ func connectToDB() (*gorm.DB, error) {
break break
} }
if attempt >= maxAttempts { if attempt >= maxAttempts {
fmt.Printf("数据库连接失败,错误信息%v\n", err)
applogger.Info("数据库连接失败,错误信息%v", err)
return nil, err return nil, err
} }
time.Sleep(backoff) time.Sleep(backoff)
@ -588,9 +580,11 @@ func SendEmail(subject string, body string) error {
// 邮箱认证信息 // 邮箱认证信息
smtpHost := "smtp.exmail.qq.com" smtpHost := "smtp.exmail.qq.com"
smtpPort := 465 smtpPort := 465
from := "chejiulong@wemediacn.com" from := "auto_system@wemediacn.com"
password := "hdQfpav4x8LwbJPH" password := "yJJYYcy5NKx2y69r"
//to := []string{"chejiulong@wemediacn.com", "xiayujuan@wemediacn.com", "wangyuanbing@wemediacn.com", "tangweiqi@wemediacn.com"}
to := []string{"chejiulong@wemediacn.com"} to := []string{"chejiulong@wemediacn.com"}
// 邮件内容 // 邮件内容
m := gomail.NewMessage() m := gomail.NewMessage()
m.SetHeader("From", from) m.SetHeader("From", from)
@ -602,8 +596,8 @@ func SendEmail(subject string, body string) error {
d.TLSConfig = &tls.Config{InsecureSkipVerify: true} d.TLSConfig = &tls.Config{InsecureSkipVerify: true}
err := d.DialAndSend(m) err := d.DialAndSend(m)
if err != nil { if err != nil {
panic(err) fmt.Printf("邮件发送失败,错误信息%v\n", err)
//return applogger.Info("邮件发送失败,错误信息%v", err)
} }
return nil return nil
} }