Merge branch 'ETA_2.0.6' into debug

# Conflicts:
#	go.sum
#	models/db.go
#	utils/constants.go

controllers/base_from_oilchem.go

@@ -0,0 +1,169 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// OilchemController 隆众资讯
+type OilchemController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增隆众资讯指标接口
+// @Description 新增隆众资讯指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *OilchemController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+
+	source := utils.DATA_SOURCE_OILCHEM
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		err = models.AddEdbDataFromOilchem(req.EdbCode)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromCoal，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// Refresh
+// @Title 刷新隆众资讯指标接口
+// @Description 刷新隆众资讯指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *OilchemController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_OILCHEM
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+
+	err = models.RefreshEdbDataFromOilchem(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromCoal，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+
+// HandleEdbData
+// @Title 处理隆众资讯指标的接口
+// @Description 处理隆众资讯指标的接口
+// @Success 200 string "操作成功"
+// @router /handle/edb_data [post]
+func (this *OilchemController) HandleEdbData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleOielchemEdbDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	if err = services.HandleOilchemIndex(req.List); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

go.mod

@@ -16,14 +16,12 @@ require (
 	github.com/qiniu/qmgo v1.1.8
 	github.com/rdlucklib/rdluck_tools v1.0.3
 	github.com/shopspring/decimal v1.4.0
-	github.com/yidane/formula v0.0.0-20210902154546-0782e1736717
 	go.mongodb.org/mongo-driver v1.16.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211223021540-a6a1e1173105 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect

go.sum

@@ -234,8 +234,6 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/yidane/formula v0.0.0-20210902154546-0782e1736717 h1:9CTJJpdISGxMAELfVlprj5kZEsJEaNAWiobv8ZAd72U=
-github.com/yidane/formula v0.0.0-20210902154546-0782e1736717/go.mod h1:9/dQiKiN04yPMdgsuFmKGuI2Hdp6OmFV9gSWS1col6g=
 github.com/ylywyn/jpush-api-go-client v0.0.0-20190906031852-8c4466c6e369/go.mod h1:Nv7wKD2/bCdKUFNKcJRa99a+1+aSLlCRJFriFYdjz/I=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=

logic/profit_chart_info.go

@@ -7,8 +7,8 @@ import (
 	"eta/eta_index_lib/models/future_good"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/dengsgo/math-engine/engine"
 	"github.com/shopspring/decimal"
-	"github.com/yidane/formula"
 	"sort"
 	"strconv"
 	"strings"
@@ -503,28 +503,36 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 				continue
 			}
 
-			expression := formula.NewExpression(formulaFormStr)
-			calResult, evaluateErr := expression.Evaluate()
-			if evaluateErr != nil {
-				// 分母为0的报错
-				if strings.Contains(evaluateErr.Error(), "divide by zero") {
-					//removeDateList = append(removeDateList, sk)
-					continue
-				}
-				err = errors.New("计算失败：Err:" + evaluateErr.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				continue
-			}
-			// 如果计算结果是NAN，那么就退出当前循环
-			if calResult.IsNan() {
-				continue
-			}
-			calVal, tmpErr := calResult.Float64()
-			if tmpErr != nil {
-				err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+			//expression := formula.NewExpression(formulaFormStr)
+			//calResult, evaluateErr := expression.Evaluate()
+			//if evaluateErr != nil {
+			//	// 分母为0的报错
+			//	if strings.Contains(evaluateErr.Error(), "divide by zero") {
+			//		//removeDateList = append(removeDateList, sk)
+			//		continue
+			//	}
+			//	err = errors.New("计算失败：Err:" + evaluateErr.Error() + ";formulaStr:" + formulaFormStr)
+			//	fmt.Println(err)
+			//	continue
+			//}
+			//// 如果计算结果是NAN，那么就退出当前循环
+			//if calResult.IsNan() {
+			//	continue
+			//}
+			//calVal, tmpErr := calResult.Float64()
+			//if tmpErr != nil {
+			//	err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+			//	fmt.Println(err)
+			//	continue
+			//}
+			calVal, err := engine.ParseAndExec(formulaFormStr)
+			//calVal, err := calResult.Float64()
+			if err != nil {
+				err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
 				fmt.Println(err)
-				continue
+				return nil, nil, err
 			}
+			//nanCheck := fmt.Sprintf("%0.f", calVal)
 			//yDataMap[n] = calVal
 			//xEdbInfoIdList = append(xEdbInfoIdList, n)
 			nanCheck := fmt.Sprintf("%0.f", calVal)

models/base_from_calculate.go

@@ -8,7 +8,6 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/dengsgo/math-engine/engine"
 	"github.com/shopspring/decimal"
-	//"github.com/yidane/formula"
 	"strconv"
 	"strings"
 	"time"

models/base_from_oilchem.go

@@ -0,0 +1,278 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type BaseFromOilchemIndex struct {
+	BaseFromOilchemIndexId int       // 主键ID
+	IndexCode              string    // 指标编码
+	IndexName              string    // 指标名称
+	ClassifyId             uint      // 分类ID
+	Unit                   string    // 单位
+	Frequency              string    // 频度
+	Describe               string    // 指标描述
+	Sort                   int       // 排序
+	CreateTime             time.Time // 创建时间
+	ModifyTime             time.Time // 修改时间
+}
+
+type BaseFromOilchemData struct {
+	BaseFromOilchemDataId  int    // 数据表ID
+	BaseFromOilchemIndexId int    // 指标ID
+	IndexCode              string // 指标编码
+	DataTime               string
+	Value                  string
+	CreateTime             time.Time // 创建时间
+	ModifyTime             time.Time // 修改时间
+}
+
+
+//添加数据
+func AddBaseFromOilchemIndexMuti(items []*BaseFromOilchemIndex) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+func AddBaseFromOilchemIndex(item *BaseFromOilchemIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromOilchemData(item *BaseFromOilchemData) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddEdbDataFromOilchem(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	oilchemBaseDataAll, err := GetOilchemDataByCode(edbCode)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_oilchem(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range oilchemBaseDataAll {
+		eDate := sv.DataTime
+		var timeStr string
+		var dataTime time.Time
+		var sDataTime string
+		var timestamp int64
+
+		sDataTime = eDate
+		dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			fmt.Println("time.Parse Err:" + eDate)
+			return err
+		}
+		timestamp = dataTime.UnixNano() / 1e6
+		timeStr = fmt.Sprintf("%d", timestamp)
+
+		value := strings.Replace(sv.Value, "%", "", -1)
+		if _, ok := existMap[sDataTime]; !ok {
+			addSql += GetAddSql("0", edbCode, sDataTime, timeStr, value)
+			fmt.Println("edbCode：", edbCode)
+			fmt.Println("sDataTime：", sDataTime)
+			fmt.Println("timeStr：", timeStr)
+			fmt.Println("value：", value)
+			isAdd = true
+		}
+		existMap[eDate] = value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// GetOilchemDataByCode
+func GetOilchemDataByCode(indexCode string) (items []*BaseFromOilchemData, err error) {
+	o := orm.NewOrm()
+	sql := "SELECT * FROM base_from_oilchem_data WHERE index_code=? "
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+	return
+}
+
+
+// RefreshEdbDataFromOilchem 刷新隆众资讯
+func RefreshEdbDataFromOilchem(edbInfoId int, edbCode, startDate string) (err error) {
+	source := utils.DATA_SOURCE_OILCHEM
+	subSource := utils.DATA_SUB_SOURCE_EDB
+
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	oilchemDataList, err := GetBaseFromOilchemDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_oilchem(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range oilchemDataList {
+		item := v
+
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshEdbDataFromSci add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+func GetBaseFromOilchemDataByCondition(condition string, pars []interface{}) (list []*BaseFromOilchemData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_oilchem_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+	return
+}
+
+
+type BaseFromOilchemIndexReq struct {
+	BaseFromOilchemIndexId int       // 主键ID
+	IndexCode              string    // 指标编码
+	IndexName              string    // 指标名称
+	ClassifyId             uint      // 分类ID
+	Unit                   string    // 单位
+	Source                 string    // 数据来源
+	Frequency              string    // 频度
+	Describe               string    // 指标描述
+	DataTime               string    // 数据日期
+	Value                  string   // 数据值
+	Sort                   int       // 排序
+	CreateTime             time.Time // 创建时间
+	ModifyTime             time.Time // 修改时间
+	IndexNameStr           string    // 指标名称字符串
+	MarketName             string    // 市场名称
+}
+
+
+type HandleOielchemEdbDataReq struct {
+	List []*BaseFromOilchemIndexReq
+}
+
+func GetBaseFromOilchemIndex() (list []*BaseFromOilchemIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_oilchem_index group by index_name `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetBaseFromOilchemData(indexCode, dataTime string) (item *BaseFromOilchemData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_oilchem_data where index_code=? And data_time = ? `
+	err = o.Raw(sql, indexCode, dataTime).QueryRow(&item)
+	return
+}
+
+// UpdateBaseFromSci99Data
+func UpdateBaseFromOilchemData(value , indexCode, dataTime string) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
+	_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
+	return
+}

models/base_predict_from_calculate.go

@@ -6,8 +6,8 @@ import (
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"github.com/dengsgo/math-engine/engine"
 	"github.com/shopspring/decimal"
-	"github.com/yidane/formula"
 	"strconv"
 	"strings"
 	"time"
@@ -463,24 +463,33 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 		if formulaFormStr != "" {
 			//utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
 			fmt.Println(fmt.Sprintf("date %s:formulaFormStr:%s", sk, formulaFormStr))
-			expression := formula.NewExpression(formulaFormStr)
-			calResult, tmpErr := expression.Evaluate()
-			if tmpErr != nil {
-				// 分母为0的报错
-				if strings.Contains(err.Error(), "divide by zero") {
-					removeDateList = append(removeDateList, sk)
-					continue
-				}
-				err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return
-			}
-			calVal, tmpErr := calResult.Float64()
-			if tmpErr != nil {
-				err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return
+			//expression := formula.NewExpression(formulaFormStr)
+			//calResult, tmpErr := expression.Evaluate()
+
+			calVal, err := engine.ParseAndExec(formulaFormStr)
+			//calVal, err := calResult.Float64()
+			if err != nil {
+				err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
+				return "", 0, err
 			}
+			//nanCheck := fmt.Sprintf("%0.f", calVal)
+
+			//if tmpErr != nil {
+			//	// 分母为0的报错
+			//	if strings.Contains(err.Error(), "divide by zero") {
+			//		removeDateList = append(removeDateList, sk)
+			//		continue
+			//	}
+			//	err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+			//	fmt.Println(err)
+			//	return
+			//}
+			//calVal, tmpErr := calResult.Float64()
+			//if tmpErr != nil {
+			//	err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+			//	fmt.Println(err)
+			//	return
+			//}
 			nanCheck := fmt.Sprintf("%0.f", calVal)
 			// 分母为0.0的报错
 			if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" {
@@ -502,7 +511,7 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 				existValDecimal, tmpErr := decimal.NewFromString(existVal)
 				if tmpErr != nil {
 					err = tmpErr
-					return
+					return "", 0, err
 				}
 				existStr := existValDecimal.String()
 				if existStr != saveValue {
@@ -510,7 +519,7 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 					sql = fmt.Sprintf(sql, dataTableName)
 					_, err = to.Raw(sql, saveValue, edbInfoId, sk).Exec()
 					if err != nil {
-						return
+						return "", 0, err
 					}
 				}
 			}

models/db.go

@@ -166,6 +166,9 @@ func initBaseIndex() {
 		new(BaseFromThsHfData),
 		new(BaseFromEdbMapping),
 		new(EdbDataThsHf),
+		new(BaseFromBusinessData), // 数据源中自有数据的明细数据表
+		new(BaseFromOilchemIndex),
+		new(BaseFromOilchemData),
 	)
 }
 

models/predict_edb.go

@@ -6,8 +6,8 @@ import (
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"github.com/dengsgo/math-engine/engine"
 	"github.com/shopspring/decimal"
-	"github.com/yidane/formula"
 	"strings"
 	"time"
 )
@@ -189,23 +189,32 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 		}
 
 		//utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
-		expression := formula.NewExpression(formulaFormStr)
-		calResult, tmpErr := expression.Evaluate()
-		if tmpErr != nil {
-			// 分母为0的报错
-			if strings.Contains(tmpErr.Error(), "divide by zero") {
-				continue
-			}
-			err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
-			//fmt.Println(err)
-			return
-		}
-		calVal, tmpErr := calResult.Float64()
-		if tmpErr != nil {
-			err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
-			//fmt.Println(err)
-			return
+		//expression := formula.NewExpression(formulaFormStr)
+		//calResult, tmpErr := expression.Evaluate()
+		//if tmpErr != nil {
+		//	// 分母为0的报错
+		//	if strings.Contains(tmpErr.Error(), "divide by zero") {
+		//		continue
+		//	}
+		//	err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+		//	//fmt.Println(err)
+		//	return
+		//}
+		//calVal, tmpErr := calResult.Float64()
+		//if tmpErr != nil {
+		//	err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+		//	//fmt.Println(err)
+		//	return
+		//}
+
+		calVal, err := engine.ParseAndExec(formulaFormStr)
+		//calVal, err := calResult.Float64()
+		if err != nil {
+			err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
+			fmt.Println(err)
+			return nil, err
 		}
+
 		nanCheck := fmt.Sprintf("%0.f", calVal)
 		if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" {
 			continue
@@ -238,7 +247,7 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 			existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value)
 			if tmpErr != nil {
 				err = tmpErr
-				return
+				return nil, tmpErr
 			}
 			existStr := existValDecimal.String()
 			if existStr != saveValue {
@@ -246,7 +255,7 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 				existPredictEdbRuleData.ModifyTime = time.Now()
 				_, err = to.Update(existPredictEdbRuleData, "Value", "ModifyTime")
 				if err != nil {
-					return
+					return nil, err
 				}
 			}
 		}

routers/commentsRouter.go

@@ -1078,6 +1078,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:OilchemController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:OilchemController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:OilchemController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:OilchemController"],
+        beego.ControllerComments{
+            Method: "HandleEdbData",
+            Router: `/handle/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:OilchemController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:OilchemController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:PbController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:PbController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -297,6 +297,11 @@ func init() {
 				&factor_edb_series.FactorEdbSeriesController{},
 			),
 		),
+		beego.NSNamespace("/oilchem",
+			beego.NSInclude(
+				&controllers.OilchemController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_oilchem.go

@@ -0,0 +1,188 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"strings"
+)
+
+var IndexCodeMap = make(map[string]string)
+var IndexMap = make(map[string]*models.BaseFromOilchemIndex)
+func HandleOilchemIndex(list []*models.BaseFromOilchemIndexReq) (err error) {
+	allCode, e := models.GetBaseFromOilchemIndex()
+	if e != nil {
+		err = e
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromOilchemIndex err:", err)
+		return
+	}
+
+	for _, item := range allCode {
+		IndexCodeMap[item.IndexName] = item.IndexCode
+		IndexMap[item.IndexName] = item
+	}
+
+	for _, v := range list {
+		indexCode, needAdd := OilchemIndexCodeGenerator(v.IndexName, v.IndexNameStr, v.MarketName)
+		if needAdd {
+			item := models.BaseFromOilchemIndex{
+				IndexCode:              indexCode,
+				IndexName:              v.IndexName,
+				ClassifyId:             v.ClassifyId,
+				Unit:                   v.Unit,
+				Frequency:              v.Frequency,
+				Describe:               v.Describe,
+				Sort:                   v.Sort,
+				CreateTime:             v.CreateTime,
+				ModifyTime:             v.ModifyTime,
+			}
+			id, e := models.AddBaseFromOilchemIndex(&item)
+			if e != nil {
+				err = e
+				fmt.Println("AddBaseFromOilchemIndexMuti err:", err)
+				utils.FileLog.Info("AddBaseFromOilchemIndexMuti err:", err)
+				return
+			}
+			item.BaseFromOilchemIndexId = int(id)
+			IndexMap[item.IndexName] = &item
+		}
+
+
+		//获取指标数据信息
+		data, e := models.GetBaseFromOilchemData(indexCode, v.DataTime)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = e
+			fmt.Println("select err:", err)
+			utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
+		}
+
+		if data != nil {
+			if data.Value != v.Value {
+				// 更新
+				fmt.Println("更新指标：", indexCode+v.DataTime)
+				utils.FileLog.Info("更新指标：", indexCode+v.DataTime)
+				e = models.UpdateBaseFromOilchemData(v.Value, indexCode, v.DataTime)
+				if e != nil {
+					err = e
+					fmt.Println("Error update into database:", err)
+					utils.FileLog.Info("Error update into database:", err)
+					return
+				}
+			}
+		} else {
+			// 新增
+			dataItem := models.BaseFromOilchemData{
+				BaseFromOilchemIndexId: IndexMap[v.IndexName].BaseFromOilchemIndexId,
+				IndexCode:              indexCode,
+				DataTime:               v.DataTime,
+				Value:                  v.Value,
+				CreateTime:             v.CreateTime,
+				ModifyTime:             v.ModifyTime,
+			}
+			fmt.Println("新增数据：", indexCode+v.DataTime)
+			utils.FileLog.Info("新增数据：", indexCode+v.DataTime)
+
+			_,e = models.AddBaseFromOilchemData(&dataItem)
+			if e != nil {
+				err = e
+				fmt.Println("Error inserting into database:", err)
+				utils.FileLog.Info("Error inserting into database:", err)
+				return
+			}
+		}
+	}
+
+	return
+}
+
+
+func OilchemIndexCodeGenerator(indexName, indexCodeStr, marketSampleName string) (indexCode string, needAdd bool) {
+	strResult := ""
+	indexCode, _ = IndexCodeMap[indexName]
+	if indexCode == "" {
+		//首字母
+		a := pinyin.NewArgs()
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
+		for i := 0; i < len(rows); i++ {
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理括号内名称
+		if marketSampleName != "" {
+			if province, ok := ProvinceMap[marketSampleName]; ok {
+				strResult += province
+			} else {
+				a := pinyin.NewArgs()
+				rows := pinyin.LazyPinyin(marketSampleName, a)
+				for i := 0; i < len(rows); i++ {
+					strResult += rows[i]
+				}
+				if len(rows) == 0 {
+					strResult += marketSampleName
+				}
+			}
+		}
+
+
+		// 去除特殊符号
+		strResult = strings.Replace(strResult, " ", "", -1)
+		strResult = strings.Replace(strResult, "-", "", -1)
+		strResult = strings.Replace(strResult, "/", "", -1)
+		strResult = strings.Replace(strResult, "#", "", -1)
+		strResult = strings.Replace(strResult, "：", "", -1)
+		strResult = strings.Replace(strResult, "(", "", -1)
+		strResult = strings.Replace(strResult, ")", "", -1)
+
+		needAdd = true
+		indexCode = strings.Replace(strResult, " ", "", -1)
+		IndexCodeMap[indexName] = indexCode
+	}
+	return
+}
+
+var ProvinceMap = map[string]string{
+	"上海":  "shanghai",
+	"云南":  "yunnan",
+	"内蒙古": "innermongolia",
+	"北京":  "beijing",
+	"台湾":  "taiwan",
+	"吉林":  "jilin",
+	"四川":  "sichuan",
+	"天津":  "tianjin",
+	"宁夏":  "ningxia",
+	"安徽":  "anhui",
+	"山东":  "shandong",
+	"山西":  "shanxi",
+	"广东":  "guangdong",
+	"广西":  "guangxi",
+	"新疆":  "xinjiang",
+	"江苏":  "jiangsu",
+	"江西":  "jiangxi",
+	"河北":  "hebei",
+	"河南":  "henan",
+	"浙江":  "zhejiang",
+	"海南":  "hainan",
+	"湖北":  "hubei",
+	"湖南":  "hunan",
+	"澳门":  "macao",
+	"甘肃":  "gansu",
+	"福建":  "fujian",
+	"西藏":  "tibet",
+	"贵州":  "guizhou",
+	"辽宁":  "liaoning",
+	"重庆":  "chongqing",
+	"陕西":  "shaanxi",
+	"青海":  "qinhai",
+	"香港":  "hongkong",
+	"黑龙江": "heilongjiang",
+}

services/base_from_predict.go

@@ -5,8 +5,8 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/dengsgo/math-engine/engine"
 	"github.com/shopspring/decimal"
-	"github.com/yidane/formula"
 	"strconv"
 	"strings"
 	"time"
@@ -219,22 +219,32 @@ func CalculateByRuleByNine(formulaStr string, edbInfoList []*models.EdbInfo, edb
 
 		//fmt.Println(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
 		fmt.Println(fmt.Sprintf("%s:formulaFormStr:%s", date, formulaFormStr))
-		expression := formula.NewExpression(formulaFormStr)
-		calResult, tmpErr := expression.Evaluate()
-		if tmpErr != nil {
-			// 分母为0的报错
-			if strings.Contains(tmpErr.Error(), "divide by zero") {
-				continue
-			}
-			err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
-			return
-		}
-		calVal, tmpErr := calResult.Float64()
-		if tmpErr != nil {
-			err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+		//expression := formula.NewExpression(formulaFormStr)
+		//calResult, tmpErr := expression.Evaluate()
+		//if tmpErr != nil {
+		//	// 分母为0的报错
+		//	if strings.Contains(tmpErr.Error(), "divide by zero") {
+		//		continue
+		//	}
+		//	err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+		//	return
+		//}
+
+		calVal, err := engine.ParseAndExec(formulaFormStr)
+		//calVal, err := calResult.Float64()
+		if err != nil {
+			err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
 			fmt.Println(err)
-			return
+			return nil, err
 		}
+		//nanCheck := fmt.Sprintf("%0.f", calVal)
+
+		//calVal, tmpErr := calResult.Float64()
+		//if tmpErr != nil {
+		//	err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
+		//	fmt.Println(err)
+		//	return
+		//}
 
 		saveValue, _ := decimal.NewFromFloat(calVal).RoundCeil(4).Float64() //utils.SubFloatToString(calVal, 4)
 		dataTime, _ := time.Parse(utils.FormatDate, date)

utils/constants.go

@@ -110,6 +110,7 @@ const (
 	DATA_SOURCE_SCI99                                = 85 // 卓创资讯
 	DATA_SOURCE_CCF                                  = 86 // CCF化纤信息
 	DATA_SOURCE_SCI_HQ                               = 88 // 卓创红期->88
+	DATA_SOURCE_OILCHEM       						 = 89 // 隆众资讯 -> 76
 )
 
 // 指标来源的中文展示