Merge branch 'custom' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/mysteel_chemical2_custom

controllers/shanghai_smm.go

@@ -216,6 +216,9 @@ func (this *ShanghaiSmmController) RefreshExcel() {
 	}
 	updateIndexList := make([]*models.BaseFromSmmIndex, 0)
 	addDataList := make([]*models.BaseFromSmmData, 0)
+	updateDataList := make([]*models.BaseFromSmmData, 0)
+
+	queryIndexIds := make([]int, 0)
 	for _, v := range req {
 		indexCode := strings.Replace(v.IndexCode, " ", "", -1)
 		indexInfo := ShanghaiSmmCodeToIndexMap[indexCode]
@@ -228,6 +231,29 @@ func (this *ShanghaiSmmController) RefreshExcel() {
 			fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode)
 			return
 		}
+		queryIndexIds = append(queryIndexIds, int(indexInfo.BaseFromSmmIndexId))
+	}
+	list, err := models.GetBaseFromSmmDataByIds(queryIndexIds)
+	if err != nil {
+		br.Msg = "查询指标数据失败"
+		br.ErrMsg = "查询指标数据失败,Err:" + err.Error()
+		return
+	}
+	dateDataMap := make(map[int]map[string]string)
+	for _, v := range list {
+		dataMap, ok := dateDataMap[v.BaseFromSmmIndexId]
+		if ok {
+			dataMap[v.DataTime] = v.Value
+		} else {
+			dateDataMap[v.BaseFromSmmIndexId] = map[string]string{
+				v.DataTime: v.Value,
+			}
+		}
+	}
+	for _, v := range req {
+		indexCode := strings.Replace(v.IndexCode, " ", "", -1)
+		indexInfo := ShanghaiSmmCodeToIndexMap[indexCode]
+
 		isAdd := false
 		indexItem := new(models.BaseFromSmmIndex)
 		indexItem.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId
@@ -245,21 +271,33 @@ func (this *ShanghaiSmmController) RefreshExcel() {
 		} else {
 			indexItem.StartDate = indexInfo.StartDate
 		}
-		// 如果指标index有更新，那么需要添加指标数据
 		if isAdd {
 			indexItem.IndexName = v.IndexName
 			indexItem.ModifyTime = time.Now()
 			updateIndexList = append(updateIndexList, indexItem)
-			for k, dv := range v.Data {
+		}
+		for k, dv := range v.Data {
+			valueStr := strconv.FormatFloat(dv, 'f', -1, 64)
+			valueRaw := dateDataMap[int(indexInfo.BaseFromSmmIndexId)][k]
+			if valueRaw == "" {
 				dataItem := new(models.BaseFromSmmData)
 				dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
 				dataItem.IndexCode = indexInfo.IndexCode
 				dataItem.DataTime = k
-				dataItem.Value = strconv.FormatFloat(dv, 'f', -1, 64)
+				dataItem.Value = valueStr
 				dataItem.CreateTime = time.Now()
 				dataItem.ModifyTime = time.Now()
 				dataItem.DataTimestamp = time.Now().UnixMilli()
 				addDataList = append(addDataList, dataItem)
+			} else if valueStr != "" && valueRaw != valueStr {
+				dataItem := new(models.BaseFromSmmData)
+				dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
+				dataItem.IndexCode = indexInfo.IndexCode
+				dataItem.DataTime = k
+				dataItem.Value = valueStr
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = time.Now().UnixMilli()
+				updateDataList = append(updateDataList, dataItem)
 			}
 		}
 	}
@@ -275,12 +313,22 @@ func (this *ShanghaiSmmController) RefreshExcel() {
 		for _, v := range updateIndexList {
 			err = v.UpdateCols([]string{"end_date", "start_date", "modify_time"})
 			if err != nil {
-				br.Msg = "更新指标数据失败"
-				br.ErrMsg = "更新指标失败,Err:" + err.Error()
+				br.Msg = "更新指标索引失败"
+				br.ErrMsg = "更新指标索引失败,Err:" + err.Error()
 				return
 			}
 		}
 	}
+	// 不修改
+	fmt.Println(len(updateDataList))
+	// if len(updateDataList) > 0 {
+	// 	err = models.UpdateBaseFromSmmDataListByIndexCode(updateDataList)
+	// 	if err != nil {
+	// 		br.Msg = "更新指标数据失败"
+	// 		br.ErrMsg = "更新指标失败,Err:" + err.Error()
+	// 		return
+	// 	}
+	// }
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "数据刷新成功"

models/base_calculate.go

@@ -304,6 +304,7 @@ func (obj BaseCalculate) Tbz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData) // 避免因为时间戳导致的日期不对，还是用string来表示比较合适
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -416,6 +417,7 @@ func (obj BaseCalculate) Tcz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -587,6 +589,7 @@ func (obj BaseCalculate) Hbz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
@@ -662,11 +665,12 @@ func (obj BaseCalculate) Hcz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
 	dataLen := len(dataList)
-	fmt.Println("dataLen:", dataLen)
+	//fmt.Println("dataLen:", dataLen)
 	for i := 0; i < dataLen; i++ {
 		j := i + formulaInt
 		if j < dataLen {
@@ -722,6 +726,7 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -733,8 +738,12 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		return
 	}
 
+	nowDay := time.Now()
+	nowDay = time.Date(nowDay.Year(), nowDay.Month(), nowDay.Day(), 0, 0, 0, 0, nowDay.Location())
+
 	// 数据计算
 	dataLen := len(dataList)
+	lastI := dataLen - 1
 	for i := 0; i < dataLen; i++ {
 		//当期
 		currentItem := dataList[i]
@@ -742,22 +751,39 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		var day int
 		var preItem *EdbInfoData
 		var preDate time.Time
+
+		// 如果是第一期的数据，那么直接赋值
 		if i == 0 {
-			day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-			preDate = time.Now()
-		} else {
-			j := i - 1
-			if j < dataLen {
-				preItem = dataList[j]
-				day = int(preItem.DataTime.Sub(currentDate).Hours() / float64(24))
-				utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
-			}
+			dateDataMap[currentItem.DataTime] = currentItem.Value
+			continue
 		}
-		for k := 0; k <= day; k++ {
-			needDay := preDate.AddDate(0, 0, -k)
-			dateDataMap[needDay] = currentItem.Value
+
+		// 上一期
+		j := i - 1
+
+		preItem = dataList[j]
+		preDate = preItem.DataTime
+		day = int(currentDate.Sub(preItem.DataTime).Hours() / float64(24))
+		//utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
+		//fmt.Println("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime:" + currentItem.DataTime.Format(utils.FormatDate))
+
+		for k := 1; k < day; k++ {
+			needDay := preDate.AddDate(0, 0, k)
+			dateDataMap[needDay] = preItem.Value
 		}
 		dateDataMap[currentItem.DataTime] = currentItem.Value
+
+		// 如果是最后一期的数据
+		if i == lastI {
+			day = int(nowDay.Sub(currentDate).Hours() / float64(24))
+			//utils.FileLog.Info("preItem.DataTime:" + currentDate.Format(utils.FormatDate) + ";currentItem.DataTime" + nowDay.Format(utils.FormatDate))
+
+			for k := 1; k <= day; k++ {
+				needDay := currentDate.AddDate(0, 0, k)
+				dateDataMap[needDay] = currentItem.Value
+			}
+		}
+
 	}
 
 	return
@@ -788,6 +814,7 @@ func (obj BaseCalculate) DownFrequency() (dateDataMap map[time.Time]float64, err
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -954,6 +981,7 @@ func (obj BaseCalculate) TimeShift() (dateDataMap map[time.Time]float64, err err
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1017,6 +1045,7 @@ func (obj BaseCalculate) Cjjx() (dateDataMap map[time.Time]float64, err error, e
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1260,6 +1289,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMap := make(map[int]float64)
 		yearList := make([]int, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			year := item.DataTime.Year()
 			yearVal, ok := yearMap[year]
 			if ok {
@@ -1278,6 +1308,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1309,6 +1340,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1344,6 +1376,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1371,6 +1404,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			dayInt := itemDate.Year()*100 + int(itemDate.Month())
 			var currTime time.Time
@@ -1414,6 +1448,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1504,6 +1539,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1534,6 +1570,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		}
 	default:
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			yearVal, ok := yearMap[year]
@@ -1624,6 +1661,7 @@ func calculateExponentialSmoothingData(dataList []*EdbInfoData, alpha float64) (
 	alphaDecimal := decimal.NewFromFloat(alpha)
 	subAlpha := decimal.NewFromFloat(1).Sub(alphaDecimal)
 	for k, d := range dataList {
+		d.DataTime = time.Date(d.DataTime.Year(), d.DataTime.Month(), d.DataTime.Day(), 0, 0, 0, 0, d.DataTime.Location())
 		// 首期的值以原始值作为指数修匀的计算值
 		if k == 0 {
 			newDataList = append(newDataList, EdbInfoData{
@@ -1684,6 +1722,7 @@ func (obj BaseCalculate) Rjz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}

models/base_from_smm.go

@@ -37,6 +37,33 @@ func GetBaseFromSmmDataByCondition(condition string, pars []interface{}) (list [
 	return
 }
 
+func UpdateBaseFromSmmDataListByIndexCode(items []*BaseFromSmmData) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_smm_data SET value=? WHERE index_code=? AND data_time=? `
+	stmt, err := o.Raw(sql).Prepare()
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+	for _, item := range items {
+		_, err = stmt.Exec(item.Value, item.IndexCode, item.DataTime)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func GetBaseFromSmmDataByIds(smmDataIds []int) (list []*BaseFromSmmData, err error) {
+	if len(smmDataIds) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_smm_data WHERE 1=1 AND base_from_smm_index_id in (` + utils.GetOrmInReplace(len(smmDataIds)) + `)`
+	_, err = o.Raw(sql, smmDataIds).QueryRows(&list)
+	return
+}
+
 // 新增有色指标数据
 func AddEdbDataFromSmm(edbCode string, smmBaseDataAll []*BaseFromSmmData) (err error) {
 	o := orm.NewOrm()

models/edb_data_calculate_bp.go

@@ -348,177 +348,107 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	if err != nil {
 		return err
 	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	fromDataMap := make(map[string]float64)
-	//来源指指标数据
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-		fromDataMap[v.DataTime] = v.Value
+
+	// 来源指标没有数据，那么需要删除所有的计算指标数据
+	if len(dataList) <= 0 {
+		// todo 删除所有的计算指标数据
+		return
+	}
+	// 来源指标的第一个日期
+	fromFirstDate, err := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
+	if err != nil {
+		return
+	}
+	fromFirstDate = time.Date(fromFirstDate.Year(), fromFirstDate.Month(), fromFirstDate.Day(), 0, 0, 0, 0, time.Local)
+
+	// 变频计算
+	newDataList, err := EdbInfoSearchDataToData(dataList)
+	if err != nil {
+		return
+	}
+
+	baseCalculate := BaseCalculate{
+		DataList:      newDataList,
+		Frequency:     "",
+		Formula:       nil,
+		Calendar:      "",
+		MoveType:      0,
+		MoveFrequency: "",
+		FromFrequency: "",
+		Source:        source,
+	}
+	dateDataMap, err, _ := baseCalculate.UpFrequency()
+	if err != nil {
+		return
 	}
-	fmt.Println("source:", source)
 
-	//获取升频指标所有数据
+	// 获取升频指所有已经存在的计算指标数据
 	existDataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
 	if err != nil {
 		return
 	}
 	//计算指标的map
 	existDataMap := make(map[string]*EdbData, 0)
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v
+	}
 
 	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 
-	var lastValue float64   //最后数据的值(float64)
-	var lastValueStr string //最后数据的值(string)
-	//待删除的日期
-	removeDateList := make([]string, 0)
-	if len(existDataList) > 0 {
-		//第一个已经入库的日期
-		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
-
-			lastValue = fromDataMap[firstFromDataTimeStr]
-			lastValueStr = decimal.NewFromFloat(lastValue).String()
-			//第一步： 判断来源指标的开始时间与计算指标的开始时间是否相等，相等的话，那么就不需要对两个时间之间的数据做处理
-			if firstExistDataTimeStr != firstFromDataTimeStr {
-				if firstExistDataTime.Before(firstFromDataTime) { //如果计算指标第一条数据的开始时间 早于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 删除处理
-					for _, v := range existDataList {
-						if v.DataTime == firstFromDataTimeStr {
-							if tmpLastValue, ok := fromDataMap[firstFromDataTimeStr]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-						removeDateList = append(removeDateList, v.DataTime)
-					}
-				} else {
-					for _, v := range dateArr { //如果计算指标第一条数据的开始时间 晚于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 新增处理
-						vDataTime, _ := time.ParseInLocation(utils.FormatDate, v, time.Local) //当前日期（time类型）
-						if firstExistDataTime.Equal(vDataTime) || firstExistDataTime.Before(vDataTime) {
-							if tmpLastValue, ok := fromDataMap[v]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
+	now := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
 
-						currentDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						addSql += GetAddSql(edbInfoIdStr, edbCode, v, timestampStr, lastValueStr)
+	for currDate := fromFirstDate; !currDate.After(now); currDate = currDate.AddDate(0, 0, 1) {
+		currDateStr := currDate.Format(utils.FormatDate)
+		timestamp := currDate.UnixNano() / 1e6
+		timestampStr := fmt.Sprintf("%d", timestamp)
 
-						isAdd = true
-					}
-				}
-			}
-
-			//第二步 剩余数据每天修改
-
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
-
-			//第三步： 已经入库的数据处理
-			for _, v := range existDataList {
-				existDataMap[v.DataTime] = v
-			}
+		// 当前计算的值
+		currValue, ok := dateDataMap[currDate]
+		if !ok {
+			// 没有计算成功就过滤
+			continue
+		}
+		lastValueStr := decimal.NewFromFloat(currValue).Round(4).String()
+
+		// 已经入库的值
+		existData, ok := existDataMap[currDateStr]
+		if !ok {
+			// 没有入库那么就插入添加
+			isAdd = true
+			addSql += GetAddSql(edbInfoIdStr, edbCode, currDateStr, timestampStr, lastValueStr)
+			continue
+		}
 
-			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				tmpExistData, ok := existDataMap[needDayStr]
-				if ok {
-					if tmpLastValue, ok := fromDataMap[tmpExistData.DataTime]; ok { //来源指标当天的数据
-						lastValue = tmpLastValue
-						//lastValueStr = decimal.NewFromFloat(lastValue).String()
-						lastValueStr = fmt.Sprintf("%.4f", lastValue)
-					}
-					//如果对应的值不匹配
-					if tmpExistData.Value != lastValueStr {
-						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, lastValueStr)
-						if err != nil {
-							return err
-						}
-					}
-				} else {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, lastValueStr)
+		// 将已经入库的值转换为decimal类型，然后再保留4位小数，目的是为了做匹配，要不然取出来的数据与计算的数据不一致
+		existDataValueDec, tmpErr := decimal.NewFromString(existData.Value)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		existDataValueStr := existDataValueDec.Round(4).String()
 
-					isAdd = true
-				}
-			}
-		} else {
-			//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
-			tableName := GetEdbDataTableName(source, subSource)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
-			_, err = to.Raw(sql, edbInfoId).Exec()
+		// 如果该日期已经入库了，且两个值不匹配，那么就更新
+		if lastValueStr != existDataValueStr {
+			err = ModifyEdbDataById(source, subSource, existData.EdbDataId, lastValueStr)
 			if err != nil {
-				err = fmt.Errorf("删除所有的升频指标数据失败,Err:" + err.Error())
-				return
+				return err
 			}
-
-			//for _, v := range existDataList {
-			//	removeDateList = append(removeDateList, v.DataTime)
-			//}
 		}
-	} else {
-		existMap := make(map[string]string)
-		dataLen := len(dataList)
 
-		for i := 0; i < dataLen; i++ {
-			//当期
-			currentItem := dataList[i]
-			currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-			var day int
-			var preItem *EdbInfoSearchData
-			var preDate time.Time
-			if i == 0 {
-				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-				preDate = time.Now()
-			} else {
-				j := i - 1
-				if j < dataLen {
-					preItem = dataList[j]
-					preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
-					day = int(preDate.Sub(currentDate).Hours() / float64(24))
-					utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
-				}
-			}
-			for k := 0; k <= day; k++ {
-				needDay := preDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				existKey := edbCode + needDayStr
-				if _, ok := existMap[existKey]; !ok {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					valStr := decimal.NewFromFloat(currentItem.Value).String()
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
-					isAdd = true
-				}
-				existMap[existKey] = needDayStr
-			}
-			existKey := edbCode + currentItem.DataTime
-			if _, ok := existMap[existKey]; !ok {
-				currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				valStr := decimal.NewFromFloat(currentItem.Value).String()
-				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
-				isAdd = true
-			}
-			existMap[existKey] = currentItem.DataTime
-		}
+		// 该日期已经处理过了，所以需要移除，如果后面该map还有数据，那么需要删除该map里面的日期数据
+		delete(existDataMap, currDateStr)
+
 	}
 
 	// 删除不需要的指标数据
-	if len(removeDateList) > 0 {
+	if len(existDataMap) > 0 {
+		//待删除的日期
+		removeDateList := make([]string, 0)
+		for date := range existDataMap {
+			removeDateList = append(removeDateList, date)
+		}
+
 		removeDateStr := strings.Join(removeDateList, `","`)
 		removeDateStr = `"` + removeDateStr + `"`
 		//如果拼接指标变更了，那么需要删除所有的指标数据
@@ -532,9 +462,11 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 		}
 	}
 
+	// 新增的数据值
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = to.Raw(addSql).Exec()
 	}
+
 	return
 }

models/edb_info.go

@@ -1384,6 +1384,16 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 			err = errors.New("指标信息不全")
 			return
 		}
+
+		// 兼容数据
+		{
+			if req.Frequency == `` {
+				req.Frequency = tmpItem.Frequency
+			}
+			if req.Unit == `` {
+				req.Unit = tmpItem.Unit
+			}
+		}
 	}
 	//获取该层级下最大的排序数
 	maxSort, err := GetEdbAndClassifyMaxSort(req.ClassifyId, 0)

services/base_from_baiinfo.go

@@ -91,12 +91,13 @@ func HandleBaiinfoIndex(baseFilePath, terminalCode, renameFilePath, indexName, i
 
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
 	for date, value := range excelDataMap {
+		dateT, tmpErr := utils.DealExcelDate(date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			return
+		}
+		date = dateT.Format(utils.FormatDate)
 		if findData, ok := exitDataMap[date]; !ok {
-			_, err := time.ParseInLocation(utils.FormatDate, date, time.Local)
-			if err != nil {
-				fmt.Println("time.ParseInLocation Err:" + err.Error())
-				return
-			}
 			if !strings.Contains(value, "#N/A") {
 				var saveDataTime time.Time
 				if strings.Contains(date, "00:00:00") {