package models import ( "eta/eta_index_lib/models/mgo" "eta/eta_index_lib/utils" "fmt" "github.com/beego/beego/v2/client/orm" "github.com/shopspring/decimal" "go.mongodb.org/mongo-driver/bson" "strconv" "strings" "time" ) //弘则手工数据 type ManualEdbdata struct { TradeCode string `orm:"column(TRADE_CODE);pk" description:"指标编码"` Dt string `orm:"column(DT)" description:"日期"` Close string `orm:"column(CLOSE)" description:"值"` ModifyTime time.Time `orm:"column(modify_time)" description:"修改时间"` } func GetEdbdataManualByCondition(condition string, pars []interface{}) (item []*ManualEdbdata, err error) { sql := ` SELECT * FROM edbdata WHERE 1=1 ` if condition != "" { sql += condition } sql += ` ORDER BY DT DESC ` o := orm.NewOrmUsingDB("edb") _, err = o.Raw(sql, pars).QueryRows(&item) return } // AddEdbDataFromManual 新增弘则手工指标数据 func AddEdbDataFromManual(edbCode string) (err error) { o := orm.NewOrm() var condition string var pars []interface{} if edbCode != "" { condition += " AND TRADE_CODE=? " pars = append(pars, edbCode) } manualDataList, err := GetEdbdataManualByCondition(condition, pars) if err != nil { return } dataLen := len(manualDataList) if dataLen > 0 { var isAdd bool addSql := ` INSERT INTO edb_data_manual(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values ` for i := 0; i < dataLen; i++ { item := manualDataList[i] eDate := item.Dt sValue := item.Close tmpDecimal, err := decimal.NewFromString(sValue) if err != nil { return err } sValue = tmpDecimal.Round(4).String() dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local) if err != nil { return err } timestamp := dataTime.UnixNano() / 1e6 timeStr := fmt.Sprintf("%d", timestamp) addSql += GetAddSql("0", edbCode, eDate, timeStr, sValue) isAdd = true } if isAdd { addSql = strings.TrimRight(addSql, ",") _, err = o.Raw(addSql).Exec() if err != nil { return } } } return } // RefreshEdbDataFromManual 刷新手工指标数据 func RefreshEdbDataFromManual(edbInfoId int, edbCode, startDate string) (err error) { source := utils.DATA_SOURCE_MANUAL subSource := utils.DATA_SUB_SOURCE_EDB o := orm.NewOrm() if err != nil { return } edbInfoIdStr := strconv.Itoa(edbInfoId) //计算数据 var condition string var pars []interface{} if edbCode != "" { condition += " AND TRADE_CODE=? " pars = append(pars, edbCode) } if startDate != "" { condition += " AND DT>=? " pars = append(pars, startDate) } else { condition += " AND DT != ? " pars = append(pars, `0000-00-00`) } manualDataList, err := GetEdbdataManualByCondition(condition, pars) if err != nil { return } // 真实数据的最大日期 , 插入规则配置的日期 var realDataMaxDate, edbDataInsertConfigDate time.Time var edbDataInsertConfig *EdbDataInsertConfig var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值 edbDataInsertConfigDateStr := `` { edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId) if err != nil && err.Error() != utils.ErrNoRow() { return } if edbDataInsertConfig != nil { edbDataInsertConfigDate = edbDataInsertConfig.Date edbDataInsertConfigDateStr = edbDataInsertConfig.Date.Format(utils.FormatDate) } } var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfoId) if startDate != "" { existCondition += " AND data_time>=? " existPars = append(existPars, startDate) } existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars) if err != nil { return err } existMap := make(map[string]*EdbInfoSearchData) for _, v := range existList { existMap[v.DataTime] = v } addSql := ` INSERT INTO edb_data_manual (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values ` var isAdd bool manualMap := make(map[string]*ManualEdbdata) //fmt.Println("manualDataList:", len(manualDataList)) for _, v := range manualDataList { item := v eDate := item.Dt sValue := item.Close tmpDecimal, err := decimal.NewFromString(sValue) if err != nil { return err } sValue = tmpDecimal.Round(4).String() dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local) if err != nil { return err } //fmt.Println("Item:", item.Dt, item.Close, item.TradeCode, item.ModifyTime) if findItem, ok := existMap[v.Dt]; !ok { timestamp := dataTime.UnixNano() / 1e6 timeStr := fmt.Sprintf("%d", timestamp) addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue) isAdd = true } else { if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Close { err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, item.Close) if err != nil { return err } } } manualMap[v.Dt] = v // 下面代码主要目的是处理掉手动插入的数据判断 { if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) { realDataMaxDate = dataTime } if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) { isFindConfigDateRealData = true } } } for _, v := range existList { if _, ok := manualMap[v.DataTime]; !ok { // 正常来讲,如果来源数据移除了该日期的数据,也需要删除ETA指标的值 // 但是由于引入手动插入最新值,那么需要判断该日期值,是否等于,如果等于,则不删除该日期值 if edbDataInsertConfigDateStr == `` || edbDataInsertConfigDateStr != v.DataTime { DeleteEdbDataById(utils.DATA_SOURCE_MANUAL, subSource, v.EdbDataId) } } } if isAdd { addSql = strings.TrimRight(addSql, ",") _, err = o.Raw(addSql).Exec() if err != nil { fmt.Println("RefreshAllEdbDataByManual add Err", err.Error()) return } } // 处理手工数据补充的配置 HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData) return } // HandleConfigInsertEdbData // @Description: 处理人工数据补充的配置(mysql) // @author: Roc // @datetime 2024-05-09 13:07:32 // @param realDataMaxDate time.Time // @param edbDataInsertConfig *EdbDataInsertConfig // @param edbInfoId int // @param source int // @param subSource int // @param existMap map[string]*EdbInfoSearchData // @param isFindConfigDateRealData bool func HandleConfigInsertEdbData(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*EdbInfoSearchData, isFindConfigDateRealData bool) { if edbDataInsertConfig == nil { return } var err error defer func() { if err != nil { fmt.Println("处理手工数据补充的配置失败,err:", err) } }() edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期 // 如果存在真实数据的最大日期 && 存在配置插入数据的最大日期 && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期 if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) { go DeleteEdbDataInsertConfigByEdbId(edbInfoId) edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate) // 如果没有找到找到配置日期的实际数据,那么就直接删除 if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData { DeleteEdbDataById(source, subSource, item.EdbDataId) } } else { o := orm.NewOrm() edbDataInsertConfig.RealDate = realDataMaxDate _, err = o.Update(edbDataInsertConfig, "RealDate") } return } // HandleConfigInsertEdbDataByMongo // @Description: 处理人工数据补充的配置(mongo) // @author: Roc // @datetime 2024-05-09 13:07:12 // @param realDataMaxDate time.Time // @param edbDataInsertConfig *EdbDataInsertConfig // @param edbInfoId int // @param source int // @param subSource int // @param existMap map[string]*mgo.EdbDataBusiness // @param isFindConfigDateRealData bool func HandleConfigInsertEdbDataByMongo(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*mgo.EdbDataBusiness, isFindConfigDateRealData bool) { if edbDataInsertConfig == nil { return } var err error defer func() { if err != nil { fmt.Println("处理手工数据补充的配置失败,err:", err) } }() edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期 // 如果存在真实数据的最大日期 && 存在配置插入数据的最大日期 && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期 if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) { go DeleteEdbDataInsertConfigByEdbId(edbInfoId) mogDataObj := mgo.EdbDataBusiness{} coll := mogDataObj.GetCollection() edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate) // 如果没有找到找到配置日期的实际数据,那么就直接删除 if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData { mogDataObj.RemoveManyByColl(coll, bson.M{"_id": item.ID}) } } else { o := orm.NewOrm() edbDataInsertConfig.RealDate = realDataMaxDate _, err = o.Update(edbDataInsertConfig, "RealDate") } return }