init_db

global/global.go

@@ -0,0 +1,16 @@
+package global
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	"gorm.io/gorm"
+)
+
+var (
+	DbMap      map[string]*gorm.DB //数据库连接配置
+	DEFAULT_DB *gorm.DB            //默认数据库连接配置
+)
+
+func init() {
+	// 初始化数据库连接
+	InitDb()
+}

global/initDb.go

@@ -0,0 +1,142 @@
+package global
+
+import (
+	"database/sql/driver"
+	_ "dm"
+	dm "dmgorm2"
+	"eta/eta_task/utils"
+	"fmt"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"gorm.io/gorm/schema"
+	"io"
+	"log"
+	"os"
+	"time"
+)
+
+type LocalTime time.Time
+
+// InitDb
+// @Description: 数据库初始化
+func InitDb() {
+	dbMap := make(map[string]*gorm.DB)
+
+	//开启日志
+	logWriter := io.MultiWriter(utils.Binlog) //binlog日志，记录到文件中去
+	if utils.RunMode == `dev` {               // 测试环境，默认输出在控制台，不需要的话，可以注释if里面下面的这行
+		logWriter = io.MultiWriter(utils.Binlog, os.Stdout)
+	}
+	newLogger := logger.New(log.New(logWriter, "\r\n", log.LstdFlags), logger.Config{
+		SlowThreshold:             200 * time.Millisecond, //慢sql ：200ms
+		LogLevel:                  logger.Info,            //记录的日志类型，info代表所有信息都记录
+		IgnoreRecordNotFoundError: true,                   //是否忽略找不到数据错误信息（只是日志记录记录成err还是普通的输出的区别，并不影响业务代码中的：找不到数据行error）
+		Colorful:                  false,                  //是否颜色输出
+	})
+
+	// 默认库
+	connectDb(utils.MYSQL_URL, utils.DbNameIndex, newLogger, dbMap, true)
+
+	// 报告库
+	connectDb(utils.MYSQL_URL_RDDP, utils.DbNameReport, newLogger, dbMap, false)
+	// 手工数据库
+	connectDb(utils.MYSQL_URL_EDB, utils.DbNameManualIndex, newLogger, dbMap, false)
+	// 指标库
+	connectDb(utils.MYSQL_URL_DATA, utils.DbNameIndex, newLogger, dbMap, false)
+	// 钢联库
+	connectDb(utils.MYSQL_URL_GL, utils.DbNameGL, newLogger, dbMap, false)
+	// master库
+	connectDb(utils.MYSQL_URL_ETA, utils.DbNameMaster, newLogger, dbMap, false)
+	// 用户主库
+	if utils.MYSQL_WEEKLY_URL != `` && (utils.BusinessCode == utils.BusinessCodeRelease || utils.BusinessCode == utils.BusinessCodeSandbox || utils.BusinessCode == utils.BusinessCodeDebug) {
+		connectDb(utils.MYSQL_WEEKLY_URL, utils.DbNameWeekly, newLogger, dbMap, false)
+	}
+
+	//全局赋值数据库链接
+	DbMap = dbMap
+
+}
+
+// connectDb
+// @Description: 达梦数据库连接
+// @param dsn
+// @param aliasName
+// @param newLogger
+// @param dbMap
+// @param isDefault
+func connectDb(dsn, aliasName string, newLogger logger.Interface, dbMap map[string]*gorm.DB, isDefault bool) {
+	//fmt.Println("dsn:", dsn, "  ==  ;aliasName:", aliasName)
+	if dsn == `` {
+		return
+	}
+
+	var dialector gorm.Dialector
+	switch utils.DbDriverName {
+	case utils.DbDriverByMysql:
+		dialector = mysql.Open(dsn)
+	case utils.DbDriverByDm:
+		dialector = dm.Open(dsn)
+	default:
+		panic(fmt.Errorf("数据库 链接异常，错误的数据库类型，数据库:%s", utils.DbDriverName))
+	}
+	db, err := gorm.Open(dialector, &gorm.Config{
+		Logger: newLogger,
+		NamingStrategy: schema.NamingStrategy{
+			SingularTable: true, // 表示使用单数表名，启用该选项后，GORM 将不会对表名进行复数化处理
+		},
+	})
+	if err != nil {
+		//global.LOG.Errorf("mysql 启动异常，数据库:default;Err：", err)
+		panic(fmt.Errorf("数据库 链接异常，数据库:%s;Err:%s", aliasName, err))
+	}
+	//创建连接池
+	sqlDB, err := db.DB()
+	if err != nil {
+		//global.LOG.Errorf("达梦 创建连接池失败，数据库:default;Err：", err)
+		panic(fmt.Errorf("数据库 创建连接池失败，数据库:%s;Err:%s", aliasName, err))
+	}
+
+	dbMap[aliasName] = db
+
+	//默认数据库连接
+	if isDefault {
+		DEFAULT_DB = db
+	}
+
+	// SetMaxIdleConns 设置空闲连接池中连接的最大数量
+	sqlDB.SetMaxIdleConns(50)
+
+	// SetMaxOpenConns 设置打开数据库连接的最大数量。
+	sqlDB.SetMaxOpenConns(100)
+
+	// SetConnMaxLifetime 设置了连接可复用的最大时间。
+	sqlDB.SetConnMaxLifetime(10 * time.Minute)
+
+}
+
+func (t *LocalTime) MarshalJSON() ([]byte, error) {
+	tTime := time.Time(*t)
+	if tTime.IsZero() {
+		return []byte("\"\""), nil
+	}
+	return []byte(fmt.Sprintf("\"%v\"", tTime.Format("2006-01-02 15:04:05"))), nil
+}
+
+func (t LocalTime) Value() (driver.Value, error) {
+	var zeroTime time.Time
+	tlt := time.Time(t)
+	//判断给定时间是否和默认零时间的时间戳相同
+	if tlt.UnixNano() == zeroTime.UnixNano() {
+		return nil, nil
+	}
+	return tlt, nil
+}
+
+func (t *LocalTime) Scan(v interface{}) error {
+	if value, ok := v.(time.Time); ok {
+		*t = LocalTime(value)
+		return nil
+	}
+	return fmt.Errorf("can not convert %v to timestamp", v)
+}

go.mod

@@ -14,9 +14,10 @@ require (
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/rdlucklib/rdluck_tools v1.0.3
 	github.com/shopspring/decimal v1.3.1
-	github.com/tealeg/xlsx v1.0.5
 	golang.org/x/net v0.21.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
+	gorm.io/driver/mysql v1.5.7
+	gorm.io/gorm v1.25.12
 )
 
 require (
@@ -25,7 +26,10 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -78,6 +78,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -92,6 +93,10 @@ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZH
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -191,8 +196,6 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
-github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE=
-github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM=
 github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
 github.com/ylywyn/jpush-api-go-client v0.0.0-20190906031852-8c4466c6e369/go.mod h1:Nv7wKD2/bCdKUFNKcJRa99a+1+aSLlCRJFriFYdjz/I=
@@ -293,3 +296,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"eta/eta_task/models"
 	_ "eta/eta_task/routers"
 	"eta/eta_task/services"
 	"eta/eta_task/services/alarm_msg"
@@ -20,6 +21,9 @@ func main() {
 		web.BConfig.WebConfig.StaticDir["/swagger"] = "swagger"
 	}
 	go services.Task()
+	// 初始化表数据入库
+	models.AfterInitTable()
+
 	//services.TaskInit()
 	web.BConfig.RecoverFunc = Recover
 	web.Run()

models/business_conf.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_task/global"
 	"eta/eta_task/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -50,6 +51,8 @@ func (m *BusinessConf) Create() (err error) {
 	if err != nil {
 		return
 	}
+
+	err = global.DbMap[utils.DbNameMaster].Create(m).Error
 	m.Id = int(id)
 	return
 }
@@ -58,47 +61,40 @@ func (m *BusinessConf) CreateMulti(items []*BusinessConf) (err error) {
 	if len(items) == 0 {
 		return
 	}
-	o := orm.NewOrmUsingDB("eta")
-	_, err = o.InsertMulti(len(items), items)
+	err = global.DbMap[utils.DbNameMaster].CreateInBatches(items, utils.MultiAddNum).Error
 	return
 }
 
 func (m *BusinessConf) Update(cols []string) (err error) {
-	o := orm.NewOrmUsingDB("eta")
-	_, err = o.Update(m, cols...)
+	err = global.DbMap[utils.DbNameMaster].Select(cols).Updates(m).Error
 	return
 }
 
 func (m *BusinessConf) Del() (err error) {
-	o := orm.NewOrmUsingDB("eta")
 	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
-	_, err = o.Raw(sql, m.Id).Exec()
+	err = global.DbMap[utils.DbNameMaster].Exec(sql, m.Id).Error
 	return
 }
 
 func (m *BusinessConf) GetItemById(id int) (item *BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("eta")
 	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
-	err = o.Raw(sql, id).QueryRow(&item)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, id).First(&item).Error
 	return
 }
 
 func (m *BusinessConf) GetItemByCondition(condition string, pars []interface{}) (item *BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("eta")
 	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s LIMIT 1`, m.TableName(), condition)
-	err = o.Raw(sql, pars).QueryRow(&item)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).First(&item).Error
 	return
 }
 
 func (m *BusinessConf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
-	o := orm.NewOrmUsingDB("eta")
 	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
-	err = o.Raw(sql, pars).QueryRow(&count)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).Scan(&count).Error
 	return
 }
 
 func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("eta")
 	fields := strings.Join(fieldArr, ",")
 	if len(fieldArr) == 0 {
 		fields = `*`
@@ -108,12 +104,11 @@ func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{},
 		order = ` ORDER BY ` + orderRule
 	}
 	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
-	_, err = o.Raw(sql, pars).QueryRows(&items)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).Find(&items).Error
 	return
 }
 
 func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("eta")
 	fields := strings.Join(fieldArr, ",")
 	if len(fieldArr) == 0 {
 		fields = `*`
@@ -123,7 +118,8 @@ func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interfac
 		order = ` ORDER BY ` + orderRule
 	}
 	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
-	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).Find(&items).Error
+
 	return
 }
 
@@ -132,9 +128,8 @@ func GetBusinessConf() (list map[string]string, err error) {
 	list = make(map[string]string)
 
 	var items []*BusinessConf
-	o := orm.NewOrmUsingDB("eta")
 	sql := `SELECT * FROM business_conf`
-	_, err = o.Raw(sql).QueryRows(&items)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql).Find(&items).Error
 	if err != nil {
 		return
 	}

models/data_manage/edb_source.go

@@ -1,9 +1,9 @@
 package data_manage
 
 import (
+	"eta/eta_task/global"
 	"eta/eta_task/utils"
 	"fmt"
-	"github.com/beego/beego/v2/client/orm"
 	"strings"
 )
 
@@ -30,7 +30,6 @@ type EdbSource struct {
 
 // GetEdbSourceItemsByCondition 获取指标来源列表
 func GetEdbSourceItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*EdbSource, err error) {
-	o := orm.NewOrmUsingDB("data")
 	fields := strings.Join(fieldArr, ",")
 	if len(fieldArr) == 0 {
 		fields = `*`
@@ -40,15 +39,14 @@ func GetEdbSourceItemsByCondition(condition string, pars []interface{}, fieldArr
 		order = ` ORDER BY ` + orderRule
 	}
 	sql := fmt.Sprintf(`SELECT %s FROM edb_source WHERE 1=1 %s %s`, fields, condition, order)
-	_, err = o.Raw(sql, pars).QueryRows(&items)
+	err = global.DbMap[utils.DbNameIndex].Raw(sql, pars).Find(&items).Error
 	return
 }
 
 // GetEdbSourceItemByCondition 获取指标来源
 func GetEdbSourceItemByCondition(condition string, pars []interface{}) (item *EdbSource, err error) {
-	o := orm.NewOrmUsingDB("data")
 	sql := fmt.Sprintf(`SELECT * FROM edb_source WHERE 1=1 %s`, condition)
-	err = o.Raw(sql, pars).QueryRow(&item)
+	err = global.DbMap[utils.DbNameIndex].Raw(sql, pars).First(&item).Error
 	return
 }
 

models/db.go

@@ -13,7 +13,7 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 )
 
-func init() {
+func init2() {
 
 	_ = orm.RegisterDataBase("default", "mysql", utils.MYSQL_URL)
 	orm.SetMaxIdleConns("default", 50)
@@ -99,7 +99,7 @@ func init() {
 	initEdbRefresh()
 
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	afterInitTable()
+	AfterInitTable()
 }
 
 // initEdbDataTable 注册Edb指标 数据表
@@ -205,11 +205,11 @@ func initEdbRefresh() {
 	)
 }
 
-// afterInitTable
+// AfterInitTable
 // @Description: 初始化表结构的的后置操作
 // @author: Roc
 // @datetime 2024-07-01 13:31:09
-func afterInitTable() {
+func AfterInitTable() {
 	// 初始化指标来源配置
 	data_manage.InitEdbSourceVar()
 

utils/config.go

@@ -17,6 +17,7 @@ var (
 	MYSQL_URL_GL     string
 	MYSQL_URL_ETA    string
 	MYSQL_WEEKLY_URL string //用户主库
+	DbDriverName     string // 数据库驱动名称
 
 	REDIS_CACHE string //缓存地址
 	//Rc          *cache.Cache //redis缓存
@@ -198,6 +199,12 @@ func init() {
 	// 用户主库
 	MYSQL_WEEKLY_URL = config["mysql_url_weekly"]
 
+	// 数据库驱动名称
+	DbDriverName = config["db_driver_name"]
+	if DbDriverName == "" {
+		DbDriverName = "mysql"
+	}
+
 	REDIS_CACHE = config["beego_cache"]
 	if len(REDIS_CACHE) <= 0 {
 		panic(any("redis链接参数没有配置"))

utils/constants.go

@@ -211,3 +211,21 @@ const (
 	TRADE_ANALYSIS_TABLE             = 6 // 持仓分析-多空分析表格
 	TRADE_ANALYSIS_CORRELATION_TABLE = 7 // 持仓分析-相关性表格
 )
+
+// MultiAddNum 批量插入的数据量
+const MultiAddNum = 500
+
+const (
+	DbDriverByMysql = "mysql"
+	DbDriverByDm    = "dm"
+)
+
+const (
+	DbNameMaster      = "eta_master"
+	DbNameReport      = "eta_report"
+	DbNameManualIndex = "eta_manual_index"
+	DbNameIndex       = "eta_index"
+	DbNameGL          = "eta_gl"
+	DbNameAI          = "eta_ai"
+	DbNameWeekly      = "weekly_report"
+)