package controllers import ( "encoding/json" "eta/eta_index_lib/logic" "eta/eta_index_lib/models" shanghaismm "eta/eta_index_lib/services/shanghai_smm" "eta/eta_index_lib/utils" "fmt" "strconv" "strings" "time" "github.com/mozillazg/go-pinyin" ) type ShanghaiSmmController struct { BaseAuthController } var ShanghaiSmmNameToIndexMap = make(map[string]*models.BaseFromSmmIndex) var ShanghaiSmmCodeToNameMap = make(map[string]string) var ShanghaiSmmNameToCodeMap = make(map[string]string) // @Title 刷新数据 // @Description 刷新数据接口 // @Param request body models.AddEdbClassifyReq true "type json string" // @Success 200 {object} models.EdbClassify // @router /refresh/list [post] func (this *ShanghaiSmmController) RefreshData() { br := new(models.BaseResponse).Init() var err error defer func() { if err != nil { utils.FileLog.Error("SMM-RefreshData err:", err) } this.Data["json"] = br this.ServeJSON() }() var req []*shanghaismm.EdbInfoData if err = json.Unmarshal(this.Ctx.Input.RequestBody, &req); err != nil { br.Msg = "参数解析异常!" br.ErrMsg = "参数解析失败,Err:" + err.Error() return } allIndex, err := models.GetBaseFromSmmIndex() if err != nil { fmt.Println("select Code err:", err) utils.FileLog.Info("GetBaseFromSci99Index err:", err) return } for _, item := range allIndex { ShanghaiSmmNameToIndexMap[item.IndexName] = item ShanghaiSmmNameToCodeMap[item.IndexName] = item.IndexCode } // 检查指标是否存在,如果不存在添加指标 addIndexList := make([]*models.BaseFromSmmIndex, 0) updateIndexList := make([]*models.BaseFromSmmIndex, 0) addDateList := make([]*models.BaseFromSmmData, 0) readyDateList := make([]*models.BaseFromSmmData, 0) updateIndexCode := make([]string, 0) for _, v := range req { indexInfo := ShanghaiSmmNameToIndexMap[v.IndexName] if indexInfo == nil { // 指标不存在,需要添加指标索引 indexCodeStr := "smm" + v.IndexName indexCodeStr, needAdd := ShanghaiSmmIndexCodeGenerator(v.IndexName, indexCodeStr) // 添加指标 if needAdd { fmt.Printf("Name:%s, indexCode:%s \n", v.IndexName, indexCodeStr) // 添加指标 item := new(models.BaseFromSmmIndex) item.IndexCode = indexCodeStr item.IndexName = v.IndexName item.Frequency = v.Frequency item.StartDate = v.RenewDate item.EndDate = v.RenewDate item.CreateTime = time.Now() item.Unit = v.Unit item.ModifyTime = time.Now() addIndexList = append(addIndexList, item) // 添加指标数据 itemDate := new(models.BaseFromSmmData) itemDate.IndexCode = indexCodeStr itemDate.DataTime = v.RenewDate if v.Value == "highs" { itemDate.Value = strconv.FormatFloat(v.Highs, 'f', -1, 64) } else if v.Value == "low" { itemDate.Value = strconv.FormatFloat(v.Low, 'f', -1, 64) } else { itemDate.Value = strconv.FormatFloat(v.Average, 'f', -1, 64) } itemDate.CreateTime = time.Now() itemDate.ModifyTime = time.Now() itemDate.DataTimestamp = time.Now().UnixMilli() readyDateList = append(readyDateList, itemDate) } else { fmt.Printf("有重复的指标或指标名称 Name:%s, indexCode:%s \n", v.IndexName, indexCodeStr) return } } else { //存在指标,检查是否需要更新指标索引,及指标数据 if indexInfo.EndDate != v.RenewDate { // 指标日期更新 t := new(models.BaseFromSmmIndex) t.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId t.EndDate = v.RenewDate updateIndexList = append(updateIndexList, t) // 指标数据更新 item := new(models.BaseFromSmmData) item.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId) item.IndexCode = indexInfo.IndexCode item.DataTime = v.RenewDate if v.Value == "highs" { item.Value = strconv.FormatFloat(v.Highs, 'f', -1, 64) } else if v.Value == "low" { item.Value = strconv.FormatFloat(v.Low, 'f', -1, 64) } else { item.Value = strconv.FormatFloat(v.Average, 'f', -1, 64) } item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTimestamp = time.Now().UnixMilli() addDateList = append(addDateList, item) updateIndexCode = append(updateIndexCode, indexInfo.IndexCode) } else { smmData, err := models.GetBaseFromSmmDataByCodeAndDate(indexInfo.IndexCode, v.RenewDate) if err != nil { fmt.Println("刷新数据失败,Err:", err.Error()) utils.FileLog.Info("刷新数据失败,Err:", err.Error()) continue } var smmDataValue string if v.Value == "highs" { smmDataValue = strconv.FormatFloat(v.Highs, 'f', -1, 64) } else if v.Value == "low" { smmDataValue = strconv.FormatFloat(v.Low, 'f', -1, 64) } else { smmDataValue = strconv.FormatFloat(v.Average, 'f', -1, 64) } if smmDataValue != "" && smmData.Value != smmDataValue { smmData.Value = smmDataValue smmData.ModifyTime = time.Now() err = smmData.Update([]string{"Value", "ModifyTime"}) if err != nil { utils.FileLog.Info("indexCode: %s,更新指标数据失败,Err:%s", v.IndexCode, err.Error()) } updateIndexCode = append(updateIndexCode, indexInfo.IndexCode) } } } } if len(addIndexList) > 0 { _, err = models.AddBaseFromSmmIndex(addIndexList) if err != nil { utils.FileLog.Info("Error 插入指标库失败:", err) br.Msg = "插入指标库失败" br.ErrMsg = "插入指标库失败,Err:" + err.Error() return } } if len(updateIndexList) > 0 { for _, v := range updateIndexList { err = models.ModifyBaseFromSmmIndexDate(v) if err != nil { br.Msg = "更新指标数据失败" br.ErrMsg = "更新指标失败,Err:" + err.Error() return } } } if len(addDateList) > 0 { err = models.AddBaseFromSmmData(addDateList) if err != nil { utils.FileLog.Info("Error 插入指标数据失败:", err) br.Msg = "插入指标数据失败" br.ErrMsg = "插入指标数据失败,Err:" + err.Error() return } } allIndex, err = models.GetBaseFromSmmIndex() if err != nil { fmt.Println("select Code err:", err) utils.FileLog.Info("GetBaseFromSci99Index err:", err) return } // 获得更新后的数据 for _, item := range allIndex { ShanghaiSmmNameToIndexMap[item.IndexName] = item ShanghaiSmmNameToCodeMap[item.IndexName] = item.IndexCode } for _, v := range readyDateList { indexName := ShanghaiSmmCodeToNameMap[v.IndexCode] baseSmmIndex := ShanghaiSmmNameToIndexMap[indexName] v.BaseFromSmmIndexId = int(baseSmmIndex.BaseFromSmmIndexId) } if len(readyDateList) > 0 { err = models.AddBaseFromSmmData(readyDateList) if err != nil { utils.FileLog.Info("Error 插入指标数据失败:", err) br.Msg = "插入指标数据失败" br.ErrMsg = "插入指标数据失败,Err:" + err.Error() return } } var messages []string defer func() { if len(messages) > 0 { errMsg := strings.Join(messages, ",") utils.FileLog.Info("SMM RefreshData errMsg: %s", errMsg) } }() for _, v := range updateIndexCode { itemInfo, err := models.GetSmmIndexInfoMaxAndMinInfo(v) if err == nil && itemInfo != nil { e := models.ModifySmmIndexMaxAndMinInfo(v, itemInfo) if e != nil { messages = append(messages, fmt.Sprintf("更新指标最大最小值失败, indexCode: %s, err: %s", v, e.Error())) return } } // 同步刷新ETA图库有色的指标 { // 获取指标详情 edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_YS, v) if e != nil && e.Error() != utils.ErrNoRow() { messages = append(messages, fmt.Sprintf("刷新ETA指标异常, indexCode: %s, err: %s", v, e.Error())) } // 已经加入到指标库的话,那么就去更新ETA指标库吧 if edbInfo != nil { go logic.RefreshBaseEdbInfo(edbInfo, ``) } } } br.Msg = "数据刷新成功" br.Success = true br.Ret = 200 } // @Title 刷新数据 // @Description 刷新数据接口 // @Param request body models.AddEdbClassifyReq true "type json string" // @Success 200 {object} models.EdbClassify // @router /refresh/excel [post] func (this *ShanghaiSmmController) RefreshExcel() { br := new(models.BaseResponse).Init() defer func() { this.Data["json"] = br this.ServeJSON() }() var req []*shanghaismm.EdbInfoData if err := json.Unmarshal(this.Ctx.Input.RequestBody, &req); err != nil { br.Msg = "参数解析异常!" br.ErrMsg = "参数解析失败,Err:" + err.Error() return } allIndex, err := models.GetBaseFromSmmIndex() if err != nil { fmt.Println("select Code err:", err) utils.FileLog.Info("GetBaseFromSci99Index err:", err) return } ShanghaiSmmCodeToIndexMap := make(map[string]*models.BaseFromSmmIndex) ShanghaiSmmCodeToIndexCount := make(map[string]int) for _, item := range allIndex { ShanghaiSmmCodeToIndexMap[item.IndexCode] = item ShanghaiSmmCodeToIndexCount[item.IndexCode] = 0 ShanghaiSmmNameToCodeMap[item.IndexCode] = item.IndexName } updateIndexList := make([]*models.BaseFromSmmIndex, 0) addDataList := make([]*models.BaseFromSmmData, 0) updateDataList := make([]*models.BaseFromSmmData, 0) queryIndexIds := make([]int, 0) for _, v := range req { indexCode := strings.Replace(v.IndexCode, " ", "", -1) indexInfo := ShanghaiSmmCodeToIndexMap[indexCode] if indexInfo == nil { fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode) return } ShanghaiSmmCodeToIndexCount[v.IndexCode] += 1 if ShanghaiSmmCodeToIndexCount[v.IndexCode] > 1 { fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode) return } queryIndexIds = append(queryIndexIds, int(indexInfo.BaseFromSmmIndexId)) } list, err := models.GetBaseFromSmmDataByIds(queryIndexIds) if err != nil { br.Msg = "查询指标数据失败" br.ErrMsg = "查询指标数据失败,Err:" + err.Error() return } dateDataMap := make(map[int]map[string]string) for _, v := range list { dataMap, ok := dateDataMap[v.BaseFromSmmIndexId] if ok { dataMap[v.DataTime] = v.Value } else { dateDataMap[v.BaseFromSmmIndexId] = map[string]string{ v.DataTime: v.Value, } } } for _, v := range req { indexCode := strings.Replace(v.IndexCode, " ", "", -1) indexInfo := ShanghaiSmmCodeToIndexMap[indexCode] isAdd := false indexItem := new(models.BaseFromSmmIndex) indexItem.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId startDate, _ := time.Parse(utils.FormatDate, indexInfo.StartDate) endDate, _ := time.Parse(utils.FormatDate, indexInfo.EndDate) if v.LastDate.After(endDate) { isAdd = true indexItem.EndDate = v.LastDate.Format(utils.FormatDate) } else { indexItem.EndDate = indexInfo.EndDate } if v.OldDate.Before(startDate) { isAdd = true indexItem.StartDate = v.OldDate.Format(utils.FormatDate) } else { indexItem.StartDate = indexInfo.StartDate } if isAdd { indexItem.IndexName = v.IndexName indexItem.ModifyTime = time.Now() updateIndexList = append(updateIndexList, indexItem) } for k, dv := range v.Data { valueStr := strconv.FormatFloat(dv, 'f', -1, 64) valueRaw := dateDataMap[int(indexInfo.BaseFromSmmIndexId)][k] if valueRaw == "" { dataItem := new(models.BaseFromSmmData) dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId) dataItem.IndexCode = indexInfo.IndexCode dataItem.DataTime = k dataItem.Value = valueStr dataItem.CreateTime = time.Now() dataItem.ModifyTime = time.Now() dataItem.DataTimestamp = time.Now().UnixMilli() addDataList = append(addDataList, dataItem) } else if valueStr != "" && valueRaw != valueStr { dataItem := new(models.BaseFromSmmData) dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId) dataItem.IndexCode = indexInfo.IndexCode dataItem.DataTime = k dataItem.Value = valueStr dataItem.ModifyTime = time.Now() dataItem.DataTimestamp = time.Now().UnixMilli() updateDataList = append(updateDataList, dataItem) } } } if len(addDataList) > 0 { err = models.AddBaseFromSmmData(addDataList) if err != nil { br.Msg = "插入指标数据失败" br.ErrMsg = "插入指标数据失败,Err:" + err.Error() return } } if len(updateIndexList) > 0 { for _, v := range updateIndexList { err = v.UpdateCols([]string{"end_date", "start_date", "modify_time"}) if err != nil { br.Msg = "更新指标索引失败" br.ErrMsg = "更新指标索引失败,Err:" + err.Error() return } } } // 不修改 fmt.Println(len(updateDataList)) var messages []string defer func() { if len(messages) > 0 { errMsg := strings.Join(messages, ",") utils.FileLog.Info("SMM RefreshExcel errMsg: %s", errMsg) } }() for _, v := range updateDataList { itemInfo, err := models.GetSmmIndexInfoMaxAndMinInfo(v.IndexCode) if err == nil && itemInfo != nil { e := models.ModifySmmIndexMaxAndMinInfo(v.IndexCode, itemInfo) if e != nil { messages = append(messages, fmt.Sprintf("更新指标最大最小值失败, indexCode: %s, err: %s", v.IndexCode, e.Error())) return } } // 同步刷新ETA图库有色的指标 { // 获取指标详情 edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_YS, v.IndexCode) if e != nil && e.Error() != utils.ErrNoRow() { messages = append(messages, fmt.Sprintf("刷新ETA指标异常, indexCode: %s, err: %s", v.IndexCode, e.Error())) } // 已经加入到指标库的话,那么就去更新ETA指标库吧 if edbInfo != nil { go logic.RefreshBaseEdbInfo(edbInfo, ``) } } } // if len(updateDataList) > 0 { // err = models.UpdateBaseFromSmmDataListByIndexCode(updateDataList) // if err != nil { // br.Msg = "更新指标数据失败" // br.ErrMsg = "更新指标失败,Err:" + err.Error() // return // } // } br.Ret = 200 br.Success = true br.Msg = "数据刷新成功" } func ShanghaiSmmIndexCodeGenerator(indexName, indexCodeStr string) (indexCode string, needAdd bool) { strResult := "" //首字母 a := pinyin.NewArgs() a.Fallback = func(r rune, a pinyin.Args) []string { return []string{string(r)} } rows := pinyin.Pinyin(indexCodeStr, a) for i := 0; i < len(rows); i++ { //strResult += rows[i][0] if len(rows[i]) != 0 { str := rows[i][0] pi := str[0:1] strResult += pi } } // 去除特殊符号 strResult = strings.Replace(strResult, " ", "", -1) strResult = strings.Replace(strResult, "-", "", -1) strResult = strings.Replace(strResult, "/", "", -1) strResult = strings.Replace(strResult, "#", "", -1) strResult = strings.Replace(strResult, ":", "", -1) strResult = strings.Replace(strResult, "(", "", -1) strResult = strings.Replace(strResult, ")", "", -1) strResult = strings.Replace(strResult, "%", "", -1) strResult = strings.Replace(strResult, "<", "1", -1) strResult = strings.Replace(strResult, "\xe2", "2", -1) // ≥ strResult = strings.Replace(strResult, ".", "", -1) strResult = strings.Replace(strResult, ",", "", -1) strResult = strings.Replace(strResult, ":", "", -1) needAdd = true indexCode = strings.Replace(strResult, " ", "", -1) indexCode = strings.ToLower(indexCode) if _, ok := ShanghaiSmmCodeToNameMap[indexCode]; !ok { ShanghaiSmmCodeToNameMap[indexCode] = indexName ShanghaiSmmNameToCodeMap[indexName] = indexCode } else { needAdd = false } return }