sephora/GO
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dataFileName 切掉时间，以便关联批次和数据

Browse Source
This commit is contained in:
chejiulong 2023-03-08 17:48:33 +08:00
parent 0eb1fa02f2
commit 64a2082dff
3 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
iniDataForLinux
View File

Binary file not shown.

BIN
iniDataForMacOs
View File

Binary file not shown.

11
main.go
View File

@ -464,7 +464,7 @@ func batchInsert(fileName string, isLastCall bool, excludedFilename string) int
dataFileName = fmt.Sprintf("lastCall-%s", excludedFilename) dataFileName = fmt.Sprintf("lastCall-%s", excludedFilename)
} else { } else {
batchName = fmt.Sprintf("%s-%s", record[1], record[0]) batchName = fmt.Sprintf("%s-%s", record[1], record[0])
dataFileName = fileName dataFileName = fileName[:len(fileName)-10]
} }
batchParams := BatchParams{ batchParams := BatchParams{
@ -646,6 +646,7 @@ func batchDataInsert(fileName string, lastCallKeys map[string]bool, excludedFile
Order("b1.created_at desc"). Order("b1.created_at desc").
Find(&batches) Find(&batches)
dataFileName = strings.Replace(fileName, "targets", "definition", -1) dataFileName = strings.Replace(fileName, "targets", "definition", -1)
dataFileName = dataFileName[:len(dataFileName)-10]
} }
batchCount := len(batches) batchCount := len(batches)
@ -680,7 +681,6 @@ func batchDataInsert(fileName string, lastCallKeys map[string]bool, excludedFile
"ReservedField3": row[7], "ReservedField3": row[7],
"ReservedField4": row[8], "ReservedField4": row[8],
"ReservedField5": row[9], "ReservedField5": row[9],
"DataFileName": dataFileName,
"FullName": row[4], "FullName": row[4],
} }
reservedFieldsJson, err := json.Marshal(reservedFields) //个性化字段转json reservedFieldsJson, err := json.Marshal(reservedFields) //个性化字段转json
@ -729,6 +729,7 @@ func batchDataInsert(fileName string, lastCallKeys map[string]bool, excludedFile
CommunicationChannelID: row[2], CommunicationChannelID: row[2],
Mobile: row[3], Mobile: row[3],
ReservedField: string(reservedFieldsJson), ReservedField: string(reservedFieldsJson),
DataFileName: dataFileName,
}) })
if len(dataBatch) >= batchSize { if len(dataBatch) >= batchSize {
@ -989,7 +990,7 @@ func iniConfi() {
txtPath = "RawData/Txt/" txtPath = "RawData/Txt/"
logPath = "logs/" logPath = "logs/"
batchSize = 5000 //提交数据 batchSize = 5000 //提交数据
insertSize = 660 //一次性入库 insertSize = 500 //一次性入库
insertChanSize = 10 //通道缓冲数 insertChanSize = 10 //通道缓冲数
goSize = 10 //协程数 goSize = 10 //协程数
taskTime = 1 taskTime = 1
@ -1015,7 +1016,7 @@ func iniConfi() {
txtPath = "RawData/Txt/" txtPath = "RawData/Txt/"
logPath = "logs/" logPath = "logs/"
batchSize = 5000 //提交数据 batchSize = 5000 //提交数据
insertSize = 660 //一次性入库 insertSize = 500 //一次性入库
insertChanSize = 100 //通道缓冲数 insertChanSize = 100 //通道缓冲数
goSize = 50 //协程数 goSize = 50 //协程数
taskTime = 60 taskTime = 60
@ -1129,6 +1130,7 @@ type BatcheData struct {
CommunicationChannelID string `gorm:"column:communication_channel_id"` CommunicationChannelID string `gorm:"column:communication_channel_id"`
Mobile string `gorm:"column:mobile"` Mobile string `gorm:"column:mobile"`
ReservedField string `gorm:"column:reserved_field"` ReservedField string `gorm:"column:reserved_field"`
DataFileName string `gorm:"column:data_file_name"`
} }
func (BatcheData) TableName() string { func (BatcheData) TableName() string {
@ -1153,6 +1155,7 @@ type BatchDataDuplicateLog struct {
CommunicationChannelID string `gorm:"column:communication_channel_id"` CommunicationChannelID string `gorm:"column:communication_channel_id"`
Mobile string `gorm:"column:mobile"` Mobile string `gorm:"column:mobile"`
ReservedField string `gorm:"column:reserved_field"` ReservedField string `gorm:"column:reserved_field"`
DataFileName string `gorm:"column:data_file_name"`
} }
func (BatchDataDuplicateLog) TableName() string { func (BatchDataDuplicateLog) TableName() string {