Roc 2 months ago
parent
commit
6759f5d224
10 changed files with 231 additions and 45 deletions
  1. 16 0
      global/global.go
  2. 135 0
      global/initDb.go
  3. 4 0
      go.mod
  4. 10 0
      go.sum
  5. 5 0
      main.go
  6. 27 36
      models/business_conf.go
  7. 4 4
      models/db.go
  8. 5 5
      models/edb_source.go
  9. 7 0
      utils/config.go
  10. 18 0
      utils/constants.go

+ 16 - 0
global/global.go

@@ -0,0 +1,16 @@
+package global
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	"gorm.io/gorm"
+)
+
+var (
+	DbMap      map[string]*gorm.DB //数据库连接配置
+	DEFAULT_DB *gorm.DB            //默认数据库连接配置
+)
+
+func init() {
+	// 初始化数据库连接
+	InitDb()
+}

+ 135 - 0
global/initDb.go

@@ -0,0 +1,135 @@
+package global
+
+import (
+	"database/sql/driver"
+	_ "dm"
+	dm "dmgorm2"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"gorm.io/gorm/schema"
+	"io"
+	"log"
+	"os"
+	"time"
+)
+
+type LocalTime time.Time
+
+// InitDb
+// @Description: 数据库初始化
+func InitDb() {
+	dbMap := make(map[string]*gorm.DB)
+
+	//开启日志
+	logWriter := io.MultiWriter(utils.Binlog) //binlog日志,记录到文件中去
+	if utils.RunMode == `dev` {               // 测试环境,默认输出在控制台,不需要的话,可以注释if里面下面的这行
+		logWriter = io.MultiWriter(utils.Binlog, os.Stdout)
+	}
+	newLogger := logger.New(log.New(logWriter, "\r\n", log.LstdFlags), logger.Config{
+		SlowThreshold:             200 * time.Millisecond, //慢sql :200ms
+		LogLevel:                  logger.Info,            //记录的日志类型,info代表所有信息都记录
+		IgnoreRecordNotFoundError: true,                   //是否忽略找不到数据错误信息(只是日志记录记录成err还是普通的输出的区别,并不影响业务代码中的:找不到数据行error)
+		Colorful:                  false,                  //是否颜色输出
+	})
+
+	// 默认库
+	// 指标库
+	connectDb(utils.MYSQL_URL, utils.DbNameIndex, newLogger, dbMap, true)
+
+	// 手工数据库
+	connectDb(utils.MYSQL_URL_EDB, utils.DbNameManualIndex, newLogger, dbMap, false)
+	// 钢联库
+	connectDb(utils.MYSQL_URL_GL, utils.DbNameGL, newLogger, dbMap, false)
+	// master库
+	connectDb(utils.MYSQL_URL_MASTER, utils.DbNameMaster, newLogger, dbMap, false)
+
+	//全局赋值数据库链接
+	DbMap = dbMap
+
+}
+
+// connectDb
+// @Description: 达梦数据库连接
+// @param dsn
+// @param aliasName
+// @param newLogger
+// @param dbMap
+// @param isDefault
+func connectDb(dsn, aliasName string, newLogger logger.Interface, dbMap map[string]*gorm.DB, isDefault bool) {
+	//fmt.Println("dsn:", dsn, "  ==  ;aliasName:", aliasName)
+	if dsn == `` {
+		return
+	}
+
+	var dialector gorm.Dialector
+	switch utils.DbDriverName {
+	case utils.DbDriverByMysql:
+		dialector = mysql.Open(dsn)
+	case utils.DbDriverByDm:
+		dialector = dm.Open(dsn)
+	default:
+		panic(fmt.Errorf("数据库 链接异常,错误的数据库类型,数据库:%s", utils.DbDriverName))
+	}
+	db, err := gorm.Open(dialector, &gorm.Config{
+		Logger: newLogger,
+		NamingStrategy: schema.NamingStrategy{
+			SingularTable: true, // 表示使用单数表名,启用该选项后,GORM 将不会对表名进行复数化处理
+		},
+	})
+	if err != nil {
+		//global.LOG.Errorf("mysql 启动异常,数据库:default;Err:", err)
+		panic(fmt.Errorf("数据库 链接异常,数据库:%s;Err:%s", aliasName, err))
+	}
+	//创建连接池
+	sqlDB, err := db.DB()
+	if err != nil {
+		//global.LOG.Errorf("达梦 创建连接池失败,数据库:default;Err:", err)
+		panic(fmt.Errorf("数据库 创建连接池失败,数据库:%s;Err:%s", aliasName, err))
+	}
+
+	dbMap[aliasName] = db
+
+	//默认数据库连接
+	if isDefault {
+		DEFAULT_DB = db
+	}
+
+	// SetMaxIdleConns 设置空闲连接池中连接的最大数量
+	sqlDB.SetMaxIdleConns(50)
+
+	// SetMaxOpenConns 设置打开数据库连接的最大数量。
+	sqlDB.SetMaxOpenConns(100)
+
+	// SetConnMaxLifetime 设置了连接可复用的最大时间。
+	sqlDB.SetConnMaxLifetime(10 * time.Minute)
+
+}
+
+func (t *LocalTime) MarshalJSON() ([]byte, error) {
+	tTime := time.Time(*t)
+	if tTime.IsZero() {
+		return []byte("\"\""), nil
+	}
+	return []byte(fmt.Sprintf("\"%v\"", tTime.Format("2006-01-02 15:04:05"))), nil
+}
+
+func (t LocalTime) Value() (driver.Value, error) {
+	var zeroTime time.Time
+	tlt := time.Time(t)
+	//判断给定时间是否和默认零时间的时间戳相同
+	if tlt.UnixNano() == zeroTime.UnixNano() {
+		return nil, nil
+	}
+	return tlt, nil
+}
+
+func (t *LocalTime) Scan(v interface{}) error {
+	if value, ok := v.(time.Time); ok {
+		*t = LocalTime(value)
+		return nil
+	}
+	return fmt.Errorf("can not convert %v to timestamp", v)
+}

+ 4 - 0
go.mod

@@ -19,6 +19,8 @@ require (
 	github.com/tealeg/xlsx v1.0.5
 	go.mongodb.org/mongo-driver v1.16.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
+	gorm.io/driver/mysql v1.5.7
+	gorm.io/gorm v1.25.12
 )
 
 require (
@@ -37,6 +39,8 @@ require (
 	github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 // indirect
 	github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect

+ 10 - 0
go.sum

@@ -67,6 +67,7 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -110,6 +111,10 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -334,3 +339,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=

+ 5 - 0
main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/models"
 	_ "eta/eta_index_lib/routers"
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
@@ -18,6 +19,10 @@ func main() {
 		web.BConfig.WebConfig.DirectoryIndex = true
 		web.BConfig.WebConfig.StaticDir["/swagger"] = "swagger"
 	}
+
+	// 初始化表数据入库
+	models.AfterInitTable()
+
 	// 异常处理
 	web.ErrorController(&controllers.ErrorController{})
 

+ 27 - 36
models/business_conf.go

@@ -1,9 +1,9 @@
 package models
 
 import (
+	"eta/eta_index_lib/global"
 	"eta/eta_index_lib/utils"
 	"fmt"
-	"github.com/beego/beego/v2/client/orm"
 	"html"
 	"strings"
 	"time"
@@ -86,12 +86,8 @@ func (m *BusinessConf) PrimaryId() string {
 }
 
 func (m *BusinessConf) Create() (err error) {
-	o := orm.NewOrmUsingDB("master")
-	id, err := o.Insert(m)
-	if err != nil {
-		return
-	}
-	m.Id = int(id)
+	err = global.DbMap[utils.DbNameMaster].Create(m).Error
+
 	return
 }
 
@@ -99,47 +95,46 @@ func (m *BusinessConf) CreateMulti(items []*BusinessConf) (err error) {
 	if len(items) == 0 {
 		return
 	}
-	o := orm.NewOrmUsingDB("master")
-	_, err = o.InsertMulti(len(items), items)
+	err = global.DbMap[utils.DbNameMaster].CreateInBatches(items, utils.MultiAddNum).Error
+
 	return
 }
 
 func (m *BusinessConf) Update(cols []string) (err error) {
-	o := orm.NewOrmUsingDB("master")
-	_, err = o.Update(m, cols...)
+	err = global.DbMap[utils.DbNameMaster].Select(cols).Updates(m).Error
+
 	return
 }
 
 func (m *BusinessConf) Del() (err error) {
-	o := orm.NewOrmUsingDB("master")
 	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
-	_, err = o.Raw(sql, m.Id).Exec()
+	err = global.DbMap[utils.DbNameMaster].Exec(sql, m.Id).Error
+
 	return
 }
 
 func (m *BusinessConf) GetItemById(id int) (item *BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("master")
 	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
-	err = o.Raw(sql, id).QueryRow(&item)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, id).First(&item).Error
+
 	return
 }
 
 func (m *BusinessConf) GetItemByCondition(condition string, pars []interface{}) (item *BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("master")
 	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s LIMIT 1`, m.TableName(), condition)
-	err = o.Raw(sql, pars).QueryRow(&item)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).First(&item).Error
+
 	return
 }
 
 func (m *BusinessConf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
-	o := orm.NewOrmUsingDB("master")
 	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
-	err = o.Raw(sql, pars).QueryRow(&count)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).Scan(&count).Error
+
 	return
 }
 
 func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("master")
 	fields := strings.Join(fieldArr, ",")
 	if len(fieldArr) == 0 {
 		fields = `*`
@@ -149,12 +144,12 @@ func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{},
 		order = ` ORDER BY ` + orderRule
 	}
 	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
-	_, err = o.Raw(sql, pars).QueryRows(&items)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).Find(&items).Error
+
 	return
 }
 
 func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("master")
 	fields := strings.Join(fieldArr, ",")
 	if len(fieldArr) == 0 {
 		fields = `*`
@@ -164,7 +159,9 @@ func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interfac
 		order = ` ORDER BY ` + orderRule
 	}
 	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
-	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	pars = append(pars, startSize, pageSize)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, pars...).Find(&items).Error
+
 	return
 }
 
@@ -173,9 +170,8 @@ func GetBusinessConf() (list map[string]string, err error) {
 	list = make(map[string]string)
 
 	var items []*BusinessConf
-	o := orm.NewOrmUsingDB("master")
 	sql := `SELECT * FROM business_conf`
-	_, err = o.Raw(sql).QueryRows(&items)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql).Find(&items).Error
 	if err != nil {
 		return
 	}
@@ -198,27 +194,22 @@ type BusinessConfUpdate struct {
 
 // UpdateBusinessConfMulti 批量修改配置
 func UpdateBusinessConfMulti(items []BusinessConfUpdate) (err error) {
-	o := orm.NewOrmUsingDB("master")
-	p, err := o.Raw("UPDATE business_conf SET conf_val = ? WHERE conf_key = ?").Prepare()
-	if err != nil {
-		return
-	}
-	defer func() {
-		_ = p.Close()
-	}()
+	tx := global.DbMap[utils.DbNameMaster].Begin()
 	for _, v := range items {
-		_, err = p.Exec(v.ConfVal, v.ConfKey)
+		err = tx.Exec("UPDATE business_conf SET conf_val = ? WHERE conf_key = ?", v.ConfVal, v.ConfKey).Error
 		if err != nil {
+			tx.Rollback()
 			return
 		}
 	}
+	tx.Commit()
 	return
 }
 
 func GetBusinessConfByKey(key string) (item *BusinessConf, err error) {
-	o := orm.NewOrmUsingDB("master")
 	sql := fmt.Sprintf(`SELECT * FROM business_conf WHERE conf_key = ? LIMIT 1`)
-	err = o.Raw(sql, key).QueryRow(&item)
+	err = global.DbMap[utils.DbNameMaster].Raw(sql, key).First(&item).Error
+
 	return
 }
 

+ 4 - 4
models/db.go

@@ -13,7 +13,7 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 )
 
-func init() {
+func init2() {
 	_ = orm.RegisterDataBase("default", "mysql", utils.MYSQL_URL)
 	orm.SetMaxIdleConns("default", 50)
 	orm.SetMaxOpenConns("default", 100)
@@ -100,7 +100,7 @@ func init() {
 	initFactorEdbSeries()
 
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	afterInitTable()
+	AfterInitTable()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -232,11 +232,11 @@ func initFactorEdbSeries() {
 	)
 }
 
-// afterInitTable
+// AfterInitTable
 // @Description: 初始化表结构的的后置操作
 // @author: Roc
 // @datetime 2024-07-01 13:31:09
-func afterInitTable() {
+func AfterInitTable() {
 	// 初始化指标来源配置
 	InitEdbSource()
 

+ 5 - 5
models/edb_source.go

@@ -1,9 +1,9 @@
 package models
 
 import (
+	"eta/eta_index_lib/global"
 	"eta/eta_index_lib/utils"
 	"fmt"
-	"github.com/beego/beego/v2/client/orm"
 	"strings"
 )
 
@@ -28,7 +28,6 @@ type EdbSource struct {
 
 // GetEdbSourceItemsByCondition 获取指标来源列表
 func GetEdbSourceItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*EdbSource, err error) {
-	o := orm.NewOrm()
 	fields := strings.Join(fieldArr, ",")
 	if len(fieldArr) == 0 {
 		fields = `*`
@@ -38,15 +37,16 @@ func GetEdbSourceItemsByCondition(condition string, pars []interface{}, fieldArr
 		order = ` ORDER BY ` + orderRule
 	}
 	sql := fmt.Sprintf(`SELECT %s FROM edb_source WHERE 1=1 %s %s`, fields, condition, order)
-	_, err = o.Raw(sql, pars).QueryRows(&items)
+	err = global.DEFAULT_DB.Raw(sql, pars).Find(&items).Error
+
 	return
 }
 
 // GetEdbSourceItemByCondition 获取指标来源
 func GetEdbSourceItemByCondition(condition string, pars []interface{}) (item *EdbSource, err error) {
-	o := orm.NewOrm()
 	sql := fmt.Sprintf(`SELECT * FROM edb_source WHERE 1=1 %s`, condition)
-	err = o.Raw(sql, pars).QueryRow(&item)
+	err = global.DEFAULT_DB.Raw(sql, pars).First(&item).Error
+
 	return
 }
 

+ 7 - 0
utils/config.go

@@ -16,6 +16,7 @@ var (
 	MYSQL_URL_EDB    string
 	MYSQL_URL_GL     string
 	MYSQL_URL_MASTER string
+	DbDriverName     string // 数据库驱动名称
 	MgoUrlData       string // mongodb数据库连接配置
 
 	PYTHON_MYSQL_HOST           string // python数据库链接主机地址
@@ -146,6 +147,12 @@ func init() {
 	MYSQL_URL_GL = config["mysql_url_gl"]
 	MYSQL_URL_MASTER = config["mysql_url_master"]
 
+	// 数据库驱动名称
+	DbDriverName = config["db_driver_name"]
+	if DbDriverName == "" {
+		DbDriverName = "mysql"
+	}
+
 	// mongodb数据库连接配置
 	MgoUrlData = config["mgo_url_data"]
 

+ 18 - 0
utils/constants.go

@@ -355,3 +355,21 @@ const (
 	EdbBaseCalculateExponentialSmoothing = 15 // 指数修匀->15
 	EdbBaseCalculateRjz                  = 16 // 日均值->16
 )
+
+// MultiAddNum 批量插入的数据量
+const MultiAddNum = 500
+
+const (
+	DbDriverByMysql = "mysql"
+	DbDriverByDm    = "dm"
+)
+
+const (
+	DbNameMaster      = "eta_master"
+	DbNameReport      = "eta_report"
+	DbNameManualIndex = "eta_manual_index"
+	DbNameIndex       = "eta_index"
+	DbNameGL          = "eta_gl"
+	DbNameAI          = "eta_ai"
+	DbNameWeekly      = "weekly_report"
+)