初次提交
This commit is contained in:
commit
6806d7a5f3
28
go.mod
Normal file
28
go.mod
Normal file
@ -0,0 +1,28 @@
|
||||
module SEPHORA
|
||||
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
github.com/pkg/sftp v1.13.5
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/willf/bloom v2.0.3+incompatible
|
||||
golang.org/x/crypto v0.6.0
|
||||
gorm.io/driver/sqlserver v1.4.2
|
||||
gorm.io/gorm v1.24.5
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
|
||||
github.com/golang-sql/sqlexp v0.1.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/microsoft/go-mssqldb v0.20.0 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/onsi/gomega v1.26.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/stretchr/testify v1.8.1 // indirect
|
||||
github.com/willf/bitset v1.1.11 // indirect
|
||||
golang.org/x/sys v0.5.0 // indirect
|
||||
)
|
||||
182
go.sum
Normal file
182
go.sum
Normal file
@ -0,0 +1,182 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
|
||||
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
|
||||
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/microsoft/go-mssqldb v0.19.0/go.mod h1:ukJCBnnzLzpVF0qYRT+eg1e+eSwjeQ7IvenUv8QPook=
|
||||
github.com/microsoft/go-mssqldb v0.20.0 h1:jAfvs5TIR311fEuNgohFzMtUO0IcCHyI1kWAVws4ES8=
|
||||
github.com/microsoft/go-mssqldb v0.20.0/go.mod h1:ukJCBnnzLzpVF0qYRT+eg1e+eSwjeQ7IvenUv8QPook=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q=
|
||||
github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
|
||||
github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
|
||||
github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
|
||||
github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
|
||||
github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/sqlserver v1.4.2 h1:nMtEeKqv2R/vv9FoHUFWfXfP6SskAgRar0TPlZV1stk=
|
||||
gorm.io/driver/sqlserver v1.4.2/go.mod h1:XHwBuB4Tlh7DqO0x7Ema8dmyWsQW7wi38VQOAFkrbXY=
|
||||
gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
|
||||
gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE=
|
||||
gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
|
||||
545
main.go
Normal file
545
main.go
Normal file
@ -0,0 +1,545 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/go-redis/redis"
|
||||
"github.com/pkg/sftp"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"gorm.io/driver/sqlserver"
|
||||
_ "gorm.io/driver/sqlserver"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
var (
|
||||
applogger *logrus.Logger
|
||||
redisClient *redis.Client
|
||||
executableDir string
|
||||
redisAddress = "mysql5.weu.me:6379"
|
||||
redisPassword = ""
|
||||
redisDB = 1
|
||||
sftpAddress = "192.168.10.86:49156"
|
||||
sftpUser = "demo"
|
||||
sftpPassword = "demo"
|
||||
dbAddress = "192.168.10.18:1433"
|
||||
dbUser = "sa"
|
||||
dbPassword = "Aa123123"
|
||||
dbName = "sephora"
|
||||
zipPath = "RawData/Zip/"
|
||||
txtPath = "RawData/Txt/"
|
||||
logPath = "logs/"
|
||||
batchSize = 5000 //提交数据
|
||||
insertSize = 500 //一次性入库
|
||||
cSize = 10 //入库协程数
|
||||
|
||||
)
|
||||
|
||||
type Batch struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
CommunicationChannelID uint
|
||||
CommunicationName string `gorm:"type:varchar(255)"`
|
||||
TargetsMember uint `gorm:"type:int"`
|
||||
TemplateID uint
|
||||
Content string `gorm:"type:text"`
|
||||
CreatedAt time.Time `gorm:"default:getdate()"`
|
||||
UpdatedAt time.Time `gorm:"default:getdate()"`
|
||||
Status uint `gorm:"type:int"`
|
||||
DataFileName string `gorm:"type:text"`
|
||||
}
|
||||
|
||||
type BatcheData struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
CommunicationChannelID string `gorm:"column:communication_channel_id"`
|
||||
Mobile string `gorm:"column:mobile"`
|
||||
FullName string `gorm:"column:full_name"`
|
||||
ReservedField string `gorm:"column:reserved_field"`
|
||||
}
|
||||
|
||||
func (BatcheData) TableName() string {
|
||||
return "batche_datas"
|
||||
}
|
||||
|
||||
type BatchProcessingInformation struct {
|
||||
ID uint `gorm:"primaryKey;autoIncrement"`
|
||||
CommunicationChannelID string `gorm:"column:communication_channel_id"`
|
||||
RepeatTargetsMember int `gorm:"column:repeat_targets_member"`
|
||||
DataFileName string `gorm:"column:data_file_name"`
|
||||
}
|
||||
|
||||
func (BatchProcessingInformation) TableName() string {
|
||||
return "batches_processing_informations"
|
||||
}
|
||||
|
||||
type BatchDataDuplicateLog struct {
|
||||
ID int `gorm:"primaryKey;autoIncrement"`
|
||||
CommunicationChannelID string `gorm:"column:communication_channel_id"`
|
||||
Mobile string `gorm:"column:mobile"`
|
||||
FullName string `gorm:"column:full_name"`
|
||||
ReservedField string `gorm:"column:reserved_field"`
|
||||
}
|
||||
|
||||
func (BatchDataDuplicateLog) TableName() string {
|
||||
return "batche_data_duplicate_logs"
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
// 获取可执行文件所在的路径
|
||||
executablePath, err := os.Executable()
|
||||
if err != nil {
|
||||
log.Fatal("failed to get executable path: ", err)
|
||||
}
|
||||
executableDir = filepath.Dir(executablePath)
|
||||
//判断目录是否存在，不存在则创建
|
||||
err = os.MkdirAll(filepath.Join(executableDir, zipPath), 0755)
|
||||
err = os.MkdirAll(filepath.Join(executableDir, txtPath), 0755)
|
||||
err = os.MkdirAll(filepath.Join(executableDir, logPath), 0755)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
applogger = logrus.New()
|
||||
logFile := filepath.Join(executableDir, logPath, "download.log")
|
||||
_, err = os.Stat(logFile)
|
||||
if os.IsNotExist(err) {
|
||||
if _, err := os.Create(logFile); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
f, err := os.OpenFile(logFile, os.O_APPEND|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
applogger.SetOutput(f)
|
||||
|
||||
redisClient = redis.NewClient(&redis.Options{
|
||||
Addr: redisAddress,
|
||||
Password: redisPassword,
|
||||
DB: redisDB,
|
||||
})
|
||||
}
|
||||
|
||||
func main() {
|
||||
// 打开一个文件作为锁
|
||||
lockFile, err := os.OpenFile(".lock", os.O_CREATE|os.O_RDWR, 0666)
|
||||
if err != nil {
|
||||
fmt.Println("打开锁文件失败:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer lockFile.Close()
|
||||
// 尝试获取文件的独占锁
|
||||
err = syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if err != nil {
|
||||
fmt.Println("程序已经在运行,本程序无法同时运行多个")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(time.Minute) // 创建一个定时器，每隔一分钟触发一次任务函数
|
||||
//tickCount := 1 //记录循环次数
|
||||
defer ticker.Stop() // 延迟关闭定时器，确保所有的定时器事件都被处理完毕
|
||||
// 循环处理任务
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
// 定时器触发时执行的任务函数
|
||||
//fmt.Printf("尝试第%d执行....\n", tickCount)
|
||||
go downloadDecompression() // 在新协程中异步执行
|
||||
//tickCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func downloadDecompression() {
|
||||
if exists, _ := redisClient.Exists("iniDataStatus").Result(); exists == 1 {
|
||||
//fmt.Println("批次执行中，跳过本次")
|
||||
} else {
|
||||
// 写入执行中标记
|
||||
err := redisClient.Set("iniDataStatus", 1, 0).Err()
|
||||
if err != nil {
|
||||
fmt.Println("写入标记失败：%d", err)
|
||||
}
|
||||
// Connect to SFTP server
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
User: sftpUser,
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.Password(sftpPassword),
|
||||
},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
}
|
||||
sshClient, err := ssh.Dial("tcp", sftpAddress, sshConfig)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
sftpClient, err := sftp.NewClient(sshClient)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer sftpClient.Close()
|
||||
files, err := sftpClient.ReadDir("/sftp/test")
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to read SFTP directory: %v", err)
|
||||
}
|
||||
it := 1
|
||||
fmt.Printf("共%d个文件\n", len(files))
|
||||
for _, file := range files {
|
||||
fmt.Printf("第%d个文件\n", it)
|
||||
it++
|
||||
// Check if file has been downloaded before
|
||||
fileKey := fmt.Sprintf("downloaded:%s", file.Name())
|
||||
if exists, _ := redisClient.Exists(fileKey).Result(); exists == 1 {
|
||||
applogger.Info("跳过已处理文件：: %v", file.Name())
|
||||
fmt.Println("跳过已处理文件：" + file.Name())
|
||||
continue
|
||||
}
|
||||
|
||||
if filepath.Ext(file.Name()) == ".zip" {
|
||||
//fmt.Println("下载开始（数据包）：" + file.Name())
|
||||
// Download file
|
||||
srcFile, err := sftpClient.Open(path.Join("/sftp/test", file.Name()))
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer srcFile.Close()
|
||||
dstFile, err := os.Create(path.Join(executableDir, zipPath, file.Name()))
|
||||
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer dstFile.Close()
|
||||
if _, err := io.Copy(dstFile, srcFile); err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
fmt.Println("下载完成（数据包）：" + file.Name())
|
||||
// Unzip file
|
||||
zipReader, err := zip.OpenReader(path.Join(executableDir, zipPath, file.Name()))
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer zipReader.Close()
|
||||
//fmt.Println("压缩报文件数量：", len(zipReader.File))
|
||||
for _, zipFile := range zipReader.File {
|
||||
zipFileReader, err := zipFile.Open()
|
||||
if strings.Contains(zipFile.Name, "__MACOSX/._") {
|
||||
//fmt.Print("系统文件.DS_Store,跳过处理", zipFile.Name)
|
||||
continue
|
||||
} else if filepath.Ext(zipFile.Name) != ".txt" {
|
||||
fmt.Print("文件类型不正确,跳过处理", zipFile.Name)
|
||||
continue
|
||||
}
|
||||
if err != nil || zipFileReader == nil {
|
||||
applogger.Fatalf("Failed to open zip file: %v", err)
|
||||
fmt.Print("压缩文件处理结束")
|
||||
continue
|
||||
}
|
||||
defer zipFileReader.Close()
|
||||
// Create the file
|
||||
unzipFile, err := os.Create(path.Join(executableDir, txtPath, zipFile.Name))
|
||||
if err != nil {
|
||||
|
||||
applogger.Fatalf("Failed to create unzip file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer unzipFile.Close()
|
||||
// Write the unzip data to the file
|
||||
_, err = io.Copy(unzipFile, zipFileReader)
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to write unzip data to file %v", err)
|
||||
continue
|
||||
}
|
||||
applogger.Info("Successfully unzipped file %v", zipFile.Name)
|
||||
batchDataInsert(zipFile.Name)
|
||||
}
|
||||
} else if filepath.Ext(file.Name()) == ".txt" {
|
||||
srcFile, err := sftpClient.Open(path.Join("/sftp/test", file.Name()))
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer srcFile.Close()
|
||||
dstFile, err := os.Create(path.Join(executableDir, txtPath, file.Name()))
|
||||
if err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
defer dstFile.Close()
|
||||
if _, err := io.Copy(dstFile, srcFile); err != nil {
|
||||
applogger.Fatalf("Failed to download file: %v", err)
|
||||
continue
|
||||
}
|
||||
fmt.Println("下载完成（批次文件）：" + file.Name())
|
||||
batchInsert(file.Name())
|
||||
fmt.Println("入库完成（批次文件）：" + file.Name())
|
||||
}
|
||||
err = redisClient.Set(fileKey, 1, 0).Err() //写入下载、解压、入库完成标记
|
||||
if err != nil {
|
||||
fmt.Println("写入标记失败：%", err)
|
||||
}
|
||||
|
||||
}
|
||||
redisClient.Del("iniDataStatus") //删除任务执行中标记
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 批次入库
|
||||
func batchInsert(fileName string) {
|
||||
//fmt.Print("批次处理开始")
|
||||
start := time.Now()
|
||||
db, err := connectToDB()
|
||||
if err != nil {
|
||||
panic("failed to connect database")
|
||||
}
|
||||
file, err := os.Open(path.Join(executableDir, txtPath, fileName))
|
||||
if err != nil {
|
||||
panic("failed to open file")
|
||||
applogger.Fatalf("ailed to open file: %v", fileName)
|
||||
}
|
||||
reader := csv.NewReader(bufio.NewReader(file))
|
||||
reader.Read()
|
||||
batchRows := 0
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
communicationChannelID, _ := strconv.ParseUint(record[0], 10, 32)
|
||||
TargetsMember, _ := strconv.ParseUint(record[2], 10, 32)
|
||||
templateID, _ := strconv.ParseUint(record[3], 10, 32)
|
||||
status := uint(1)
|
||||
|
||||
batch := Batch{
|
||||
CommunicationChannelID: uint(communicationChannelID),
|
||||
CommunicationName: record[1],
|
||||
TargetsMember: uint(TargetsMember),
|
||||
TemplateID: uint(templateID),
|
||||
Content: record[4],
|
||||
Status: status,
|
||||
DataFileName: fileName,
|
||||
}
|
||||
db.Create(&batch)
|
||||
batchRows++
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
elapsed := time.Since(start)
|
||||
fmt.Printf("执行时间%s\n插入批次数%d", elapsed, batchRows)
|
||||
fmt.Print("批次处理完成")
|
||||
|
||||
}
|
||||
|
||||
func batchDataInsert(fileName string) {
|
||||
//fmt.Print("批次数据处理开始")
|
||||
start := time.Now()
|
||||
db, err := connectToDB()
|
||||
if err != nil {
|
||||
panic("failed to connect database")
|
||||
}
|
||||
// Insert data in batches using multiple goroutines
|
||||
var wg sync.WaitGroup
|
||||
dataBatchChan := make(chan []BatcheData, cSize)
|
||||
for i := 0; i < cSize; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
for batch := range dataBatchChan {
|
||||
retryCount := 0
|
||||
for {
|
||||
if err := db.CreateInBatches(batch, insertSize).Error; err != nil {
|
||||
if retryCount >= 5 {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Printf("Insert failed, retrying in %v seconds...\n", 2*retryCount)
|
||||
time.Sleep(time.Duration(2*retryCount) * time.Second)
|
||||
retryCount++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
// Open file
|
||||
file, err := os.Open(path.Join(executableDir, txtPath, fileName))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
// Create and initialize hashset
|
||||
hs := make(map[string]bool)
|
||||
// Prepare batch data
|
||||
dataBatch := make([]BatcheData, 0, batchSize)
|
||||
dataBatchDuplicate := make([]BatchDataDuplicateLog, 0, batchSize)
|
||||
|
||||
// Parse file line by line and insert data in batches
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
scanner.Scan() // skip first line
|
||||
bi := 0
|
||||
bi2 := 0
|
||||
//var filtered []string
|
||||
duplicateCount := make(map[string]int)
|
||||
//duplicateData := make(map[string]map[string][]BatcheData)
|
||||
// Define counter to keep track of how many records are inserted
|
||||
var count int64
|
||||
|
||||
for scanner.Scan() {
|
||||
bi2++
|
||||
line := scanner.Text()
|
||||
row, err := csv.NewReader(strings.NewReader(line)).Read()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
reservedFields := map[string]string{ //合并个性化字段
|
||||
"ReservedField1": row[5],
|
||||
"ReservedField2": row[6],
|
||||
"ReservedField3": row[7],
|
||||
"ReservedField4": row[8],
|
||||
"ReservedField5": row[9],
|
||||
"DataFileName": fileName,
|
||||
}
|
||||
reservedFieldsJson, err := json.Marshal(reservedFields) //个性化字段转json
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// Check if record exists in hashset
|
||||
key := fmt.Sprintf("%s-%s", row[2], row[3])
|
||||
if _, exists := hs[key]; exists {
|
||||
bi++
|
||||
//filtered = append(filtered, key)
|
||||
// Increment duplicate count
|
||||
if _, ok := duplicateCount[row[2]]; !ok {
|
||||
duplicateCount[row[2]] = 0
|
||||
}
|
||||
duplicateCount[row[2]]++
|
||||
|
||||
// 按批次记录重复数据
|
||||
/*
|
||||
if _, ok := duplicateData[row[2]]; !ok {
|
||||
duplicateData[row[2]] = make(map[string][]BatcheData)
|
||||
}
|
||||
|
||||
if _, ok := duplicateData[row[2]][row[3]]; !ok {
|
||||
duplicateData[row[2]][row[3]] = []BatcheData{}
|
||||
}
|
||||
duplicateData[row[2]][row[3]] = append(duplicateData[row[2]][row[3]], BatcheData{
|
||||
CommunicationChannelID: row[2],
|
||||
Mobile: row[3],
|
||||
FullName: row[4],
|
||||
ReservedField: string(reservedFieldsJson),
|
||||
})*/
|
||||
dataBatchDuplicate = append(dataBatchDuplicate, BatchDataDuplicateLog{
|
||||
CommunicationChannelID: row[2],
|
||||
Mobile: row[3],
|
||||
FullName: row[4],
|
||||
ReservedField: string(reservedFieldsJson),
|
||||
})
|
||||
|
||||
if len(dataBatchDuplicate) >= batchSize {
|
||||
err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize)
|
||||
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Add record to hashset
|
||||
hs[key] = true
|
||||
|
||||
dataBatch = append(dataBatch, BatcheData{
|
||||
CommunicationChannelID: row[2],
|
||||
Mobile: row[3],
|
||||
FullName: row[4],
|
||||
ReservedField: string(reservedFieldsJson),
|
||||
})
|
||||
|
||||
if len(dataBatch) >= batchSize {
|
||||
dataBatchChan <- dataBatch
|
||||
dataBatch = make([]BatcheData, 0, batchSize)
|
||||
}
|
||||
count++
|
||||
}
|
||||
|
||||
if len(dataBatch) > 0 {
|
||||
dataBatchChan <- dataBatch
|
||||
dataBatch = make([]BatcheData, 0, batchSize)
|
||||
}
|
||||
|
||||
close(dataBatchChan)
|
||||
|
||||
wg.Wait()
|
||||
|
||||
//插入批次处理信息
|
||||
bpi := []BatchProcessingInformation{}
|
||||
for key, value := range duplicateCount {
|
||||
bpi = append(bpi, BatchProcessingInformation{
|
||||
CommunicationChannelID: key,
|
||||
RepeatTargetsMember: value,
|
||||
DataFileName: fileName,
|
||||
})
|
||||
}
|
||||
err = db.CreateInBatches(bpi, insertSize).Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
//插入批此重复信息
|
||||
if len(dataBatchDuplicate) > 0 {
|
||||
err = db.CreateInBatches(dataBatchDuplicate, insertSize).Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
dataBatchDuplicate = make([]BatchDataDuplicateLog, 0, batchSize)
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
|
||||
fmt.Printf("批次数据入库函数执行时间:%s;\n插入数:%d条\n过滤数：%d条\n", elapsed, count, bi)
|
||||
fmt.Println("入库完成（数据包）：" + file.Name())
|
||||
}
|
||||
|
||||
func connectToDB() (*gorm.DB, error) {
|
||||
dsn := "sqlserver://" + dbUser + ":" + dbPassword + "@" + dbAddress + "?database=" + dbName + "&charset=utf8mb4"
|
||||
var db *gorm.DB
|
||||
var err error
|
||||
attempt := 1
|
||||
maxAttempts := 5
|
||||
backoff := time.Second
|
||||
logger := logger.Default.LogMode(logger.Silent)
|
||||
for {
|
||||
db, err = gorm.Open(sqlserver.Open(dsn), &gorm.Config{Logger: logger})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if attempt >= maxAttempts {
|
||||
return nil, err
|
||||
}
|
||||
time.Sleep(backoff)
|
||||
backoff *= 2
|
||||
attempt++
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user