From 6806d7a5f30761004a1ad5a4ac316499552b457c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=BD=A6=E4=B9=85=E9=BE=99?= Date: Sat, 18 Feb 2023 16:32:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E6=AC=A1=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 28 +++ go.sum | 182 +++++++++++++++++++ main.go | 545 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 755 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f5d101f --- /dev/null +++ b/go.mod @@ -0,0 +1,28 @@ +module SEPHORA + +go 1.19 + +require ( + github.com/go-redis/redis v6.15.9+incompatible + github.com/pkg/sftp v1.13.5 + github.com/sirupsen/logrus v1.9.0 + github.com/willf/bloom v2.0.3+incompatible + golang.org/x/crypto v0.6.0 + gorm.io/driver/sqlserver v1.4.2 + gorm.io/gorm v1.24.5 +) + +require ( + github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect + github.com/golang-sql/sqlexp v0.1.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/microsoft/go-mssqldb v0.20.0 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.26.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.8.1 // indirect + github.com/willf/bitset v1.1.11 // indirect + golang.org/x/sys v0.5.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..37bd84c --- /dev/null +++ b/go.sum @@ -0,0 +1,182 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/microsoft/go-mssqldb v0.19.0/go.mod h1:ukJCBnnzLzpVF0qYRT+eg1e+eSwjeQ7IvenUv8QPook= +github.com/microsoft/go-mssqldb v0.20.0 h1:jAfvs5TIR311fEuNgohFzMtUO0IcCHyI1kWAVws4ES8= +github.com/microsoft/go-mssqldb v0.20.0/go.mod h1:ukJCBnnzLzpVF0qYRT+eg1e+eSwjeQ7IvenUv8QPook= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlserver v1.4.2 h1:nMtEeKqv2R/vv9FoHUFWfXfP6SskAgRar0TPlZV1stk= +gorm.io/driver/sqlserver v1.4.2/go.mod h1:XHwBuB4Tlh7DqO0x7Ema8dmyWsQW7wi38VQOAFkrbXY= +gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= +gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..5984b21 --- /dev/null +++ b/main.go @@ -0,0 +1,545 @@ +package main + +import ( + "archive/zip" + "bufio" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/go-redis/redis" + "github.com/pkg/sftp" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + "gorm.io/driver/sqlserver" + _ "gorm.io/driver/sqlserver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var ( + applogger *logrus.Logger + redisClient *redis.Client + executableDir string + redisAddress = "mysql5.weu.me:6379" + redisPassword = "" + redisDB = 1 + sftpAddress = "192.168.10.86:49156" + sftpUser = "demo" + sftpPassword = "demo" + dbAddress = "192.168.10.18:1433" + dbUser = "sa" + dbPassword = "Aa123123" + dbName = "sephora" + zipPath = "RawData/Zip/" + txtPath = "RawData/Txt/" + logPath = "logs/" + batchSize = 5000 //提交数据 + insertSize = 500 //一次性入库 + cSize = 10 //入库协程数 + +) + +type Batch struct { + ID uint `gorm:"primary_key"` + CommunicationChannelID uint + CommunicationName string `gorm:"type:varchar(255)"` + TargetsMember uint `gorm:"type:int"` + TemplateID uint + Content string `gorm:"type:text"` + CreatedAt time.Time `gorm:"default:getdate()"` + UpdatedAt time.Time `gorm:"default:getdate()"` + Status uint `gorm:"type:int"` + DataFileName string `gorm:"type:text"` +} + +type BatcheData struct { + ID uint `gorm:"primary_key"` + CommunicationChannelID string `gorm:"column:communication_channel_id"` + Mobile string `gorm:"column:mobile"` + FullName string `gorm:"column:full_name"` + ReservedField string `gorm:"column:reserved_field"` +} + +func (BatcheData) TableName() string { + return "batche_datas" +} + +type BatchProcessingInformation struct { + ID uint `gorm:"primaryKey;autoIncrement"` + CommunicationChannelID string `gorm:"column:communication_channel_id"` + RepeatTargetsMember int `gorm:"column:repeat_targets_member"` + DataFileName string `gorm:"column:data_file_name"` +} + +func (BatchProcessingInformation) TableName() string { + return "batches_processing_informations" +} + +type BatchDataDuplicateLog struct { + ID int `gorm:"primaryKey;autoIncrement"` + CommunicationChannelID string `gorm:"column:communication_channel_id"` + Mobile string `gorm:"column:mobile"` + FullName string `gorm:"column:full_name"` + ReservedField string `gorm:"column:reserved_field"` +} + +func (BatchDataDuplicateLog) TableName() string { + return "batche_data_duplicate_logs" +} + +func init() { + + // 获取可执行文件所在的路径 + executablePath, err := os.Executable() + if err != nil { + log.Fatal("failed to get executable path: ", err) + } + executableDir = filepath.Dir(executablePath) + //判断目录是否存在，不存在则创建 + err = os.MkdirAll(filepath.Join(executableDir, zipPath), 0755) + err = os.MkdirAll(filepath.Join(executableDir, txtPath), 0755) + err = os.MkdirAll(filepath.Join(executableDir, logPath), 0755) + if err != nil { + log.Fatal(err) + } + applogger = logrus.New() + logFile := filepath.Join(executableDir, logPath, "download.log") + _, err = os.Stat(logFile) + if os.IsNotExist(err) { + if _, err := os.Create(logFile); err != nil { + log.Fatal(err) + } + } + f, err := os.OpenFile(logFile, os.O_APPEND|os.O_WRONLY, 0666) + if err != nil { + log.Fatal(err) + } + applogger.SetOutput(f) + + redisClient = redis.NewClient(&redis.Options{ + Addr: redisAddress, + Password: redisPassword, + DB: redisDB, + }) +} + +func main() { + // 打开一个文件作为锁 + lockFile, err := os.OpenFile(".lock", os.O_CREATE|os.O_RDWR, 0666) + if err != nil { + fmt.Println("打开锁文件失败:", err) + os.Exit(1) + } + defer lockFile.Close() + // 尝试获取文件的独占锁 + err = syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err != nil { + fmt.Println("程序已经在运行,本程序无法同时运行多个") + os.Exit(1) + } + + ticker := time.NewTicker(time.Minute) // 创建一个定时器，每隔一分钟触发一次任务函数 + //tickCount := 1 //记录循环次数 + defer ticker.Stop() // 延迟关闭定时器，确保所有的定时器事件都被处理完毕 + // 循环处理任务 + for { + select { + case <-ticker.C: + // 定时器触发时执行的任务函数 + //fmt.Printf("尝试第%d执行....\n", tickCount) + go downloadDecompression() // 在新协程中异步执行 + //tickCount++ + } + } +} + +func downloadDecompression() { + if exists, _ := redisClient.Exists("iniDataStatus").Result(); exists == 1 { + //fmt.Println("批次执行中，跳过本次") + } else { + // 写入执行中标记 + err := redisClient.Set("iniDataStatus", 1, 0).Err() + if err != nil { + fmt.Println("写入标记失败：%d", err) + } + // Connect to SFTP server + sshConfig := &ssh.ClientConfig{ + User: sftpUser, + Auth: []ssh.AuthMethod{ + ssh.Password(sftpPassword), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + sshClient, err := ssh.Dial("tcp", sftpAddress, sshConfig) + if err != nil { + panic(err) + } + sftpClient, err := sftp.NewClient(sshClient) + if err != nil { + panic(err) + } + defer sftpClient.Close() + files, err := sftpClient.ReadDir("/sftp/test") + if err != nil { + applogger.Fatalf("Failed to read SFTP directory: %v", err) + } + it := 1 + fmt.Printf("共%d个文件\n", len(files)) + for _, file := range files { + fmt.Printf("第%d个文件\n", it) + it++ + // Check if file has been downloaded before + fileKey := fmt.Sprintf("downloaded:%s", file.Name()) + if exists, _ := redisClient.Exists(fileKey).Result(); exists == 1 { + applogger.Info("跳过已处理文件：: %v", file.Name()) + fmt.Println("跳过已处理文件：" + file.Name()) + continue + } + + if filepath.Ext(file.Name()) == ".zip" { + //fmt.Println("下载开始（数据包）：" + file.Name()) + // Download file + srcFile, err := sftpClient.Open(path.Join("/sftp/test", file.Name())) + if err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + defer srcFile.Close() + dstFile, err := os.Create(path.Join(executableDir, zipPath, file.Name())) + + if err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + defer dstFile.Close() + if _, err := io.Copy(dstFile, srcFile); err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + fmt.Println("下载完成（数据包）：" + file.Name()) + // Unzip file + zipReader, err := zip.OpenReader(path.Join(executableDir, zipPath, file.Name())) + if err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + defer zipReader.Close() + //fmt.Println("压缩报文件数量：", len(zipReader.File)) + for _, zipFile := range zipReader.File { + zipFileReader, err := zipFile.Open() + if strings.Contains(zipFile.Name, "__MACOSX/._") { + //fmt.Print("系统文件.DS_Store,跳过处理", zipFile.Name) + continue + } else if filepath.Ext(zipFile.Name) != ".txt" { + fmt.Print("文件类型不正确,跳过处理", zipFile.Name) + continue + } + if err != nil || zipFileReader == nil { + applogger.Fatalf("Failed to open zip file: %v", err) + fmt.Print("压缩文件处理结束") + continue + } + defer zipFileReader.Close() + // Create the file + unzipFile, err := os.Create(path.Join(executableDir, txtPath, zipFile.Name)) + if err != nil { + + applogger.Fatalf("Failed to create unzip file: %v", err) + continue + } + defer unzipFile.Close() + // Write the unzip data to the file + _, err = io.Copy(unzipFile, zipFileReader) + if err != nil { + applogger.Fatalf("Failed to write unzip data to file %v", err) + continue + } + applogger.Info("Successfully unzipped file %v", zipFile.Name) + batchDataInsert(zipFile.Name) + } + } else if filepath.Ext(file.Name()) == ".txt" { + srcFile, err := sftpClient.Open(path.Join("/sftp/test", file.Name())) + if err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + defer srcFile.Close() + dstFile, err := os.Create(path.Join(executableDir, txtPath, file.Name())) + if err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + defer dstFile.Close() + if _, err := io.Copy(dstFile, srcFile); err != nil { + applogger.Fatalf("Failed to download file: %v", err) + continue + } + fmt.Println("下载完成（批次文件）：" + file.Name()) + batchInsert(file.Name()) + fmt.Println("入库完成（批次文件）：" + file.Name()) + } + err = redisClient.Set(fileKey, 1, 0).Err() //写入下载、解压、入库完成标记 + if err != nil { + fmt.Println("写入标记失败：%", err) + } + + } + redisClient.Del("iniDataStatus") //删除任务执行中标记 + } + +} + +// 批次入库 +func batchInsert(fileName string) { + //fmt.Print("批次处理开始") + start := time.Now() + db, err := connectToDB() + if err != nil { + panic("failed to connect database") + } + file, err := os.Open(path.Join(executableDir, txtPath, fileName)) + if err != nil { + panic("failed to open file") + applogger.Fatalf("ailed to open file: %v", fileName) + } + reader := csv.NewReader(bufio.NewReader(file)) + reader.Read() + batchRows := 0 + for { + record, err := reader.Read() + if err != nil { + break + } + communicationChannelID, _ := strconv.ParseUint(record[0], 10, 32) + TargetsMember, _ := strconv.ParseUint(record[2], 10, 32) + templateID, _ := strconv.ParseUint(record[3], 10, 32) + status := uint(1) + + batch := Batch{ + CommunicationChannelID: uint(communicationChannelID), + CommunicationName: record[1], + TargetsMember: uint(TargetsMember), + TemplateID: uint(templateID), + Content: record[4], + Status: status, + DataFileName: fileName, + } + db.Create(&batch) + batchRows++ + } + time.Sleep(time.Second) + elapsed := time.Since(start) + fmt.Printf("执行时间%s\n插入批次数%d", elapsed, batchRows) + fmt.Print("批次处理完成") + +} + +func batchDataInsert(fileName string) { + //fmt.Print("批次数据处理开始") + start := time.Now() + db, err := connectToDB() + if err != nil { + panic("failed to connect database") + } + // Insert data in batches using multiple goroutines + var wg sync.WaitGroup + dataBatchChan := make(chan []BatcheData, cSize) + for i := 0; i < cSize; i++ { + wg.Add(1) + go func() { + for batch := range dataBatchChan { + retryCount := 0 + for { + if err := db.CreateInBatches(batch, insertSize).Error; err != nil { + if retryCount >= 5 { + panic(err) + } + fmt.Printf("Insert failed, retrying in %v seconds...\n", 2*retryCount) + time.Sleep(time.Duration(2*retryCount) * time.Second) + retryCount++ + } else { + break + } + } + } + wg.Done() + }() + } + + // Open file + file, err := os.Open(path.Join(executableDir, txtPath, fileName)) + if err != nil { + panic(err) + } + defer file.Close() + // Create and initialize hashset + hs := make(map[string]bool) + // Prepare batch data + dataBatch := make([]BatcheData, 0, batchSize) + dataBatchDuplicate := make([]BatchDataDuplicateLog, 0, batchSize) + + // Parse file line by line and insert data in batches + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + scanner.Scan() // skip first line + bi := 0 + bi2 := 0 + //var filtered []string + duplicateCount := make(map[string]int) + //duplicateData := make(map[string]map[string][]BatcheData) + // Define counter to keep track of how many records are inserted + var count int64 + + for scanner.Scan() { + bi2++ + line := scanner.Text() + row, err := csv.NewReader(strings.NewReader(line)).Read() + if err != nil { + panic(err) + } + + reservedFields := map[string]string{ //合并个性化字段 + "ReservedField1": row[5], + "ReservedField2": row[6], + "ReservedField3": row[7], + "ReservedField4": row[8], + "ReservedField5": row[9], + "DataFileName": fileName, + } + reservedFieldsJson, err := json.Marshal(reservedFields) //个性化字段转json + if err != nil { + panic(err) + } + // Check if record exists in hashset + key := fmt.Sprintf("%s-%s", row[2], row[3]) + if _, exists := hs[key]; exists { + bi++ + //filtered = append(filtered, key) + // Increment duplicate count + if _, ok := duplicateCount[row[2]]; !ok { + duplicateCount[row[2]] = 0 + } + duplicateCount[row[2]]++ + + // 按批次记录重复数据 + /* + if _, ok := duplicateData[row[2]]; !ok { + duplicateData[row[2]] = make(map[string][]BatcheData) + } + + if _, ok := duplicateData[row[2]][row[3]]; !ok { + duplicateData[row[2]][row[3]] = []BatcheData{} + } + duplicateData[row[2]][row[3]] = append(duplicateData[row[2]][row[3]], BatcheData{ + CommunicationChannelID: row[2], + Mobile: row[3], + FullName: row[4], + ReservedField: string(reservedFieldsJson), + })*/ + dataBatchDuplicate = append(dataBatchDuplicate, BatchDataDuplicateLog{ + CommunicationChannelID: row[2], + Mobile: row[3], + FullName: row[4], + ReservedField: string(reservedFieldsJson), + }) + + if len(dataBatchDuplicate) >= batchSize { + err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error + if err != nil { + panic(err) + } + + dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize) + + } + continue + } + + // Add record to hashset + hs[key] = true + + dataBatch = append(dataBatch, BatcheData{ + CommunicationChannelID: row[2], + Mobile: row[3], + FullName: row[4], + ReservedField: string(reservedFieldsJson), + }) + + if len(dataBatch) >= batchSize { + dataBatchChan <- dataBatch + dataBatch = make([]BatcheData, 0, batchSize) + } + count++ + } + + if len(dataBatch) > 0 { + dataBatchChan <- dataBatch + dataBatch = make([]BatcheData, 0, batchSize) + } + + close(dataBatchChan) + + wg.Wait() + + //插入批次处理信息 + bpi := []BatchProcessingInformation{} + for key, value := range duplicateCount { + bpi = append(bpi, BatchProcessingInformation{ + CommunicationChannelID: key, + RepeatTargetsMember: value, + DataFileName: fileName, + }) + } + err = db.CreateInBatches(bpi, insertSize).Error + if err != nil { + panic(err) + } + + //插入批此重复信息 + if len(dataBatchDuplicate) > 0 { + err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error + if err != nil { + panic(err) + } + dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize) + } + + elapsed := time.Since(start) + + fmt.Printf("批次数据入库函数执行时间:%s;\n插入数:%d条\n过滤数：%d条\n", elapsed, count, bi) + fmt.Println("入库完成（数据包）：" + file.Name()) +} + +func connectToDB() (*gorm.DB, error) { + dsn := "sqlserver://" + dbUser + ":" + dbPassword + "@" + dbAddress + "?database=" + dbName + "&charset=utf8mb4" + var db *gorm.DB + var err error + attempt := 1 + maxAttempts := 5 + backoff := time.Second + logger := logger.Default.LogMode(logger.Silent) + for { + db, err = gorm.Open(sqlserver.Open(dsn), &gorm.Config{Logger: logger}) + if err == nil { + break + } + if attempt >= maxAttempts { + return nil, err + } + time.Sleep(backoff) + backoff *= 2 + attempt++ + } + return db, nil +}