新增有色基础数据更新

models/data_manage/base_from_smm.go

@@ -43,7 +43,8 @@ func AddBaseFromSmm(item *BaseFromSmm) (lastId int64, err error) {
 func GetBaseFromSmmList() (list []*BaseFromSmm, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_smm `
+	//sql := `SELECT * FROM base_from_smm WHERE interface='cu_annual_copper_cathode_demand_supply_balance_in_china' `
+	sql := `SELECT * FROM base_from_smm WHERE end_date IS NOT NULL `
 	_, err = o.Raw(sql).QueryRows(&list)
 	return
 }
@@ -132,6 +133,15 @@ func GetBaseFromSmmMaxOrMinDate(indexCode string) (min_date, max_date string, er
 	return
 }
 
+
+func ModifyBaseFromSmmSmmType(baseFromSmmId int, smmType string) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := ` UPDATE base_from_smm SET smm_type=? WHERE base_from_smm_id=? `
+	_, err = o.Raw(sql, smmType, baseFromSmmId).Exec()
+	return
+}
+
 func ModifyBaseFromSmmMinDateAndMaxDate(baseFromSmmIndexId, baseFromSmmId int, minDate, maxDate string) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
@@ -142,10 +152,21 @@ func ModifyBaseFromSmmMinDateAndMaxDate(baseFromSmmIndexId, baseFromSmmId int, m
 	return
 }
 
-func ModifyBaseFromSmmSmmType(baseFromSmmId int, smmType string) (err error) {
+type BaseFromSmmDataSimple struct {
+	SmmDataId          int `orm:"column(smm_data_id);pk"`
+	BaseFromSmmIndexId int
+	IndexCode          string
+	DataTime           string
+	Value              float64
+}
+
+func GetBaseFromSmmDataByCondition(condition string, pars []interface{}) (list []*BaseFromSmmDataSimple, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := ` UPDATE base_from_smm SET smm_type=? WHERE base_from_smm_id=? `
-	_, err = o.Raw(sql, smmType, baseFromSmmId).Exec()
+	sql := `SELECT * FROM base_from_smm_data WHERE 1=1 `
+	if condition!="" {
+		sql+=condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
 	return
-}
+}

models/data_manage/edb_data_ys.go

@@ -17,112 +17,345 @@ import (
 	"net/url"
 	"rdluck_tools/orm"
 	"strconv"
-	"strings"
 	"time"
 )
 
-//刷新有色指标数据
-func RefreshEdbDataByYs(edbInfoId int, edbCode, startDate, endDate string) (err error) {
+type EdbDataYs struct {
+	EdbDataId     int `orm:"column(edb_data_id);pk"`
+	EdbInfoId     int
+	EdbCode       string
+	DataTime      string
+	Value         float64
+	Status        int
+	CreateTime    time.Time
+	ModifyTime    time.Time
+	DataTimestamp int64
+}
+
+func AddEdbDataSmm(items []*EdbDataYs) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	o.Begin()
-	defer func() {
-		if err != nil {
-			o.Rollback()
+	_, err = o.InsertMulti(1, items)
+	return
+}
+
+func ModifyEdbDataSmm(edbCode, dataTime string, value float64) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := ` UPDATE edb_data_ys SET value=?,status=1,modify_time=NOW() WHERE edb_code=? AND data_time=? `
+	_, err = o.Raw(sql, value, edbCode, dataTime).Exec()
+	return
+}
+
+//刷新有色指标数据
+func RefreshEdbDataByYs(edbInfoId int, edbCode, startDate, endDate string) (err error) {
+	var condition string
+	var pars []interface{}
+	condition += " AND base_from_smm_data=? "
+	pars = append(pars, edbCode)
+
+	condition += " AND data_time>=? "
+	pars = append(pars, startDate)
+	condition += " AND data_time<=? "
+	pars = append(pars, endDate)
+
+	ysBaseDataAll, err := GetBaseFromSmmDataByCondition(condition, pars)
+	if err != nil {
+		err = errors.New("GetBaseFromSmmDataByCondition Err:" + err.Error())
+		return
+	}
+	var existCondition string
+	var existPars []interface{}
+	existCondition += " AND edb_code=? "
+	existPars = append(pars, edbCode)
+	existDataList, err := GetEdbDataListAll(existCondition, existPars, utils.DATA_SOURCE_YS, 1)
+	if err != nil {
+		err = errors.New("GetEdbDataListAll Err:" + err.Error())
+		return err
+	}
+	existMap := make(map[string]float64)
+	for _, ev := range existDataList {
+		existMap[ev.DataTime] = ev.Value
+	}
+	ysList := make([]*EdbDataYs, 0)
+	for _, dv := range ysBaseDataAll {
+		if existItem, ok := existMap[dv.DataTime]; !ok { //新增
+			dataDate, _ := time.Parse(utils.FormatDate, dv.DataTime)
+			timestamp := dataDate.UnixNano() / 1e6
+			item := new(EdbDataYs)
+			item.EdbCode = edbCode
+			item.EdbInfoId = edbInfoId
+			item.CreateTime = time.Now()
+			item.ModifyTime = time.Now()
+			item.DataTime = dv.DataTime
+			item.Value = dv.Value
+			item.Status = 1
+			item.DataTimestamp = timestamp
+			ysList = append(ysList, item)
 		} else {
-			o.Commit()
+			if dv.Value != existItem { //修改
+				ModifyEdbDataSmm(edbCode, dv.DataTime, dv.Value)
+			}
+		}
+	}
+	if len(ysList) > 0 {
+		err = AddEdbDataSmm(ysList)
+		if err != nil {
+			return
 		}
-	}()
+	}
+	return
+}
+
+/*
 
+//有色基础指标同步
+func SyncSmmIndexDataBase() {
+	utils.FileLog.Info("start:" + time.Now().Format(utils.FormatDateTime))
+	list, err := data_manage.GetBaseFromSmmList()
+	if err != nil {
+		fmt.Println("GetBaseFromSmmList Err:" + err.Error())
+		return
+	}
 	token, err := getToken("pqian@hzinsights.com", "hz123456")
 	if err != nil {
 		fmt.Println(err)
 		return
 	}
-
-	ysResult, err := getApiData(token, edbCode, startDate, endDate)
+	endDate := time.Now().Format(utils.FormatDate)
+	smmIndexAll, err := data_manage.GetBaseFromSmmIndex()
 	if err != nil {
-		fmt.Println(err)
+		fmt.Println("GetBaseFromSmmIndex Err:" + err.Error())
 		return
 	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	if ysResult != nil && len(ysResult.Data.Content) > 0 {
-		var isAdd bool
-		addSql := ` INSERT INTO edb_data_ys(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-		existMap := make(map[string]string)
-		frequency := ysResult.Data.Frequency
-		for _, rv := range ysResult.Data.Content {
-			var eDate, sValue string
-			for ck, cv := range rv {
-				if ck == 0 {
-					eDate = cv
-				}
-				if ck == 1 {
-					sValue = strings.Replace(cv, ",", "", -1)
+	existIndexMap := make(map[string]*data_manage.BaseFromSmmIndex)
+	for _, item := range smmIndexAll {
+		existIndexMap[item.IndexCode] = item
+	}
+
+	for k, v := range list {
+		edbCode := v.Interface
+		var startDate string
+		if v.EndDate == "" || v.EndDate == "0000-00-00" {
+			startDate = v.ApiStartTime
+		} else {
+			if v.EndDate != "" {
+				dateFormate, _ := time.Parse(utils.FormatDate, v.EndDate)
+				var syncDay int
+				if v.ApiUpdateType == "day" {
+					syncDay = utils.DATA_REFRESH
+				} else if v.ApiUpdateType == "month" {
+					syncDay = utils.DATA_REFRESH * 30
+				} else if v.ApiUpdateType == "week" {
+					syncDay = utils.DATA_REFRESH * 7
+				} else if v.ApiUpdateType == "year" {
+					syncDay = utils.DATA_REFRESH * 365
 				}
+				startDate = dateFormate.AddDate(0, 0, -syncDay).Format(utils.FormatDate)
 			}
-			if sValue=="" {
-				continue
+		}
+
+		ysItem, err := getApiData(token, edbCode, startDate, endDate)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		if ysItem != nil && ysItem.Code == 200 {
+
+			frequency := ysItem.Data.Frequency
+
+			indexMap := make(map[string]int)
+			smmIndexIdMap := make(map[int]int)
+			indexCodeMap := make(map[int]string)
+
+			indexKey := 0
+			var isDateIndex int
+			for fk, fv := range ysItem.Data.Field {
+				if fv.IsDate == "1" {
+					isDateIndex = fk
+				} else {
+					if !strings.Contains(fv.Name, "产品名称") &&
+						!strings.Contains(fv.Name, "单位") &&
+						!strings.Contains(fv.Name, "时间") &&
+						!strings.Contains(fv.Name, "备注") {
+
+						indexMap[fv.DBColName] = fk
+
+						indexKey += 1
+						indexCode := v.Interface + "#" + strconv.Itoa(indexKey)
+
+						if findItem, ok := existIndexMap[indexCode]; !ok {
+							ssmIndex := new(data_manage.BaseFromSmmIndex)
+							ssmIndex.Interface = v.Interface
+							ssmIndex.Name = v.Name
+							ssmIndex.IndexCode = indexCode
+							ssmIndex.IndexName = v.Name + "_" + fv.Name
+							ssmIndex.Type1 = v.Type1
+							ssmIndex.Type2 = v.Type2
+							ssmIndex.Type3 = v.Type3
+							ssmIndex.Frequency = frequency
+							ssmIndex.Unit = fv.Unit
+							ssmIndex.ApiStartTime = v.ApiStartTime
+							ssmIndex.ApiUpdateTime = v.ApiUpdateTime
+							ssmIndex.StartTime = v.StartTime
+							ssmIndex.FinishTime = v.FinishTime
+							ssmIndex.CreateTime = time.Now()
+							ssmIndex.ModifyTime = time.Now()
+							lastIndexId, err := data_manage.AddBaseFromSmmIndex(ssmIndex)
+							if err != nil {
+								return
+							}
+							smmIndexIdMap[fk] = int(lastIndexId)
+							indexCodeMap[fk] = indexCode
+						} else {
+							smmIndexIdMap[fk] = findItem.BaseFromSmmIndexId
+							indexCodeMap[fk] = findItem.IndexCode
+						}
+					}
+				}
 			}
-			if frequency == "月" {
-				monthDate, err := time.Parse("2006-01", eDate)
+
+			existDataMap := make(map[string]*data_manage.BaseFromSmmData)
+			for _, mv := range indexCodeMap {
+				indexCode := mv
+				dataAllList, err := data_manage.GetBaseFromSmmDataAll(indexCode)
 				if err != nil {
-					fmt.Println("time.Parse:" + err.Error())
+					fmt.Println("GetBaseFromSmmData Err:" + err.Error())
+					return
 				}
-				lastTime := monthDate.AddDate(0, 1, -1)
-				lastYear, lastMonth, lastDay := lastTime.Date()
-				var lastDate string
-				if int(lastMonth) < 10 {
-					lastDate = strconv.Itoa(lastYear) + "-" + "0" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
-				} else {
-					lastDate = strconv.Itoa(lastYear) + "-" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+				for _, item := range dataAllList {
+					key := item.IndexCode + item.DataTime
+					existDataMap[key] = item
 				}
-				eDate = lastDate
-			}
-			count, err := GetEdbDataBaseByCodeAndDate(utils.DATA_SOURCE_YS, edbCode, eDate)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				return err
 			}
-			if count <= 0 {
-				dataTime, err := time.Parse(utils.FormatDate, eDate)
+
+			addExistDataMap := make(map[string]string)
+
+			for _, dv := range ysItem.Data.Content {
+				var dataTime string
+				dataTime = dv[isDateIndex]
+				if strings.Contains(dataTime, "Q1") {
+					dataTime = strings.Replace(dataTime, "Q1", "-01", -1)
+					dataTime += "-31"
+				}
+				if strings.Contains(dataTime, "Q2") {
+					dataTime = strings.Replace(dataTime, "Q2", "-06", -1)
+					dataTime += "-30"
+				}
+				if strings.Contains(dataTime, "Q3") {
+					dataTime = strings.Replace(dataTime, "Q3", "-09", -1)
+					dataTime += "-30"
+				}
+				if strings.Contains(dataTime, "Q4") {
+					dataTime = strings.Replace(dataTime, "Q4", "-12", -1)
+					dataTime += "-31"
+				}
+
+				if strings.Contains(dataTime, "H1") {
+					dataTime = strings.Replace(dataTime, "H1", "-06", -1)
+					dataTime += "-30"
+				}
+
+				if strings.Contains(dataTime, "H2") {
+					dataTime = strings.Replace(dataTime, "H2", "-12", -1)
+					dataTime += "-31"
+				}
+
+				if frequency == "月" {
+					monthDate, err := time.Parse("2006-01", dataTime)
+					if err != nil {
+						fmt.Println("time.Parse:" + err.Error())
+					}
+					lastTime := monthDate.AddDate(0, 1, -1)
+					lastYear, lastMonth, lastDay := lastTime.Date()
+					var lastDate string
+					if int(lastMonth) < 10 {
+						lastDate = strconv.Itoa(lastYear) + "-" + "0" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+					} else {
+						lastDate = strconv.Itoa(lastYear) + "-" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+					}
+					dataTime = lastDate
+				} else if frequency == "年" {
+					dataTime = dataTime + "-12-31"
+				}
+				saveDataTime, err := time.Parse(utils.FormatDate, dataTime)
 				if err != nil {
-					return err
+					fmt.Println("saveDataTime parse err:" + err.Error() + " dataTime:" + dataTime)
+					panic(err)
 				}
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				if _, ok := existMap[eDate]; !ok {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
-					isAdd = true
+				timestamp := saveDataTime.UnixNano() / 1e6
+				//循环指标
+				for _, v := range indexMap {
+
+					indexCode := indexCodeMap[v]
+					smmIndexId := smmIndexIdMap[v]
+					dataVal := dv[v]
+
+					if indexCode != "" {
+
+						key := indexCode + dataTime
+						val := strings.Replace(dataVal, ",", "", -1)
+						if findData, dataOk := existDataMap[key]; !dataOk {
+							if _, addOK := addExistDataMap[key]; !addOK {
+								if val != "" && val != "-" {
+									dataItem := new(data_manage.BaseFromSmmData)
+									dataItem.BaseFromSmmIndexId = int(smmIndexId)
+									dataItem.IndexCode = indexCode
+									dataItem.DataTime = dataTime
+									dataItem.Value = val
+									dataItem.CreateTime = time.Now()
+									dataItem.ModifyTime = time.Now()
+									dataItem.DataTimestamp = timestamp
+									_, err = data_manage.AddBaseFromSmmData(dataItem)
+									if err != nil && !strings.Contains(err.Error(), "idx_index_code_date") {
+										fmt.Println("AddBaseFromSmmData Err:" + err.Error())
+										return
+									}
+								}
+							}
+						} else {
+							if findData != nil && findData.Value != val { //修改
+								if _, addOK := addExistDataMap[key]; !addOK {
+									if val != "" && val != "-" {
+										err = data_manage.ModifyBaseFromSmmData(findData.SmmDataId, val)
+										if err != nil {
+											fmt.Println("ModifyBaseFromSmmData Err:" + err.Error())
+											return
+										}
+									}
+								}
+							}
+						}
+						addExistDataMap[key] = key
+					}
 				}
-			} else {
-				sql := ` UPDATE edb_data_ys SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-				_, err = o.Raw(sql, sValue, edbInfoId, eDate).Exec()
 			}
-			existMap[eDate] = sValue
-		}
-		if isAdd {
-			addSql = strings.TrimRight(addSql, ",")
-			_, err = o.Raw(addSql).Exec()
-			if err != nil {
-				return err
-			}
-		}
-	}
 
-	maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(utils.DATA_SOURCE_LZ, edbCode)
-	if err != nil {
-		return err
-	}
-	if maxAndMinItem != nil {
-		err = ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem)
-		if err != nil {
-			return err
+			//修改数据开始，结束日期
+			{
+				indexList, err := data_manage.GetBaseFromSmmIndexByCode(v.Interface)
+				if err != nil {
+					fmt.Println("GetBaseFromSmmIndexByCode Err:" + err.Error())
+				}
+				for _, sv := range indexList {
+					minDate, maxDate, err := data_manage.GetBaseFromSmmMaxOrMinDate(sv.IndexCode)
+					if err != nil {
+						fmt.Println("GetEdbDataSmmMaxOrMinDate Err:" + err.Error())
+					} else {
+						err = data_manage.ModifyBaseFromSmmMinDateAndMaxDate(sv.BaseFromSmmIndexId, v.BaseFromSmmId, minDate, maxDate)
+						if err != nil {
+							fmt.Println("ModifyBaseFromSmmMinDateAndMaxDate Err:" + err.Error())
+						}
+					}
+				}
+			}
 		}
+		fmt.Println(v.Interface, v.Name, k)
+		time.Sleep(1 * time.Second)
 	}
-	return
+	utils.FileLog.Info("end:" + time.Now().Format(utils.FormatDateTime))
 }
-
-
+*/
 
 const (
 	dataUrl = "https://dataapi.smm.cn/GetData/" //data url (中文)

models/data_manage/edb_info_calculate_nszydpjjs.go

@@ -47,7 +47,7 @@ func RefreshCalculateNszydpjjs(edbInfoId, source, formulaInt int, fromEdbInfo *E
 	var condition string
 	var pars []interface{}
 	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
+	pars = append(pars, edbInfoId)
 
 	dataList, err := GetEdbDataListAll(condition, pars, source, 0)
 	if err != nil {
@@ -56,18 +56,21 @@ func RefreshCalculateNszydpjjs(edbInfoId, source, formulaInt int, fromEdbInfo *E
 
 	existDataMap := make(map[string]*EdbInfoSearchData)
 	for _, v := range dataList {
+		utils.FileLog.Info("exist:",v.DataTime,v.Value)
 		existDataMap[v.DataTime] = v
 	}
 
+	fmt.Println("existDataMap:",existDataMap)
+
 	//计算来源数据
-	startDate := time.Now().AddDate(-2, 0, 0).Format(utils.FormatDate)
+	//startDate := time.Now().AddDate(-2, 0, 0).Format(utils.FormatDate)
 
 	var fromCondition string
 	var fromPars []interface{}
 	fromCondition += " AND edb_info_id=? "
 	fromPars = append(fromPars, fromEdbInfo.EdbInfoId)
-	fromCondition += " AND data_time>=? "
-	fromPars = append(fromPars, startDate)
+	//fromCondition += " AND data_time>=? "
+	//fromPars = append(fromPars, startDate)
 
 	fromDataList, err := GetEdbDataListAll(fromCondition, fromPars, fromEdbInfo.Source, 0)
 	if err != nil {
@@ -86,58 +89,67 @@ func RefreshCalculateNszydpjjs(edbInfoId, source, formulaInt int, fromEdbInfo *E
 
 	arrLen := len(fromDateArr)
 	for ak, av := range fromDateArr {
-		if _, existOk := existDataMap[av]; !existOk {
-			//处理第一个值
-			var valArr []float64
-			if findItem, ok := fromDataMap[av]; ok {
-				valArr = append(valArr, findItem.Value)
-			} else {
-				continue
-			}
-			if ak+1 != arrLen {
-				//处理除第一个值之外的N-1个值
-				for i := 1; i < formulaInt; i++ {
-					arrIndex := ak + i
-					if arrIndex >= arrLen {
-						break
-					}
-					arrVal := fromDateArr[arrIndex]
-					if findItem, ok := fromDataMap[arrVal]; ok {
-						valArr = append(valArr, findItem.Value)
-					} else {
-						continue
-					}
+		//处理第一个值
+		var valArr []float64
+		if findItem, ok := fromDataMap[av]; ok {
+			valArr = append(valArr, findItem.Value)
+		} else {
+			continue
+		}
+		if ak+1 != arrLen {
+			//处理除第一个值之外的N-1个值
+			for i := 1; i < formulaInt; i++ {
+				arrIndex := ak + i
+				if arrIndex >= arrLen {
+					break
+				}
+				arrVal := fromDateArr[arrIndex]
+				if findItem, ok := fromDataMap[arrVal]; ok {
+					valArr = append(valArr, findItem.Value)
+				} else {
+					continue
 				}
 			}
-			valArrLen := len(valArr)
-			totalVal := decimal.NewFromFloat(0.00)
-			for _, v := range valArr {
-				//totalVal = totalVal + v
-				newDecimal := decimal.NewFromFloat(v)
-				totalVal = totalVal.Add(newDecimal)
-			}
-			//af := decimal.NewFromFloat(totalVal)
-			af := totalVal
-
-			bf := decimal.NewFromFloat(float64(valArrLen))
-			val, _ := af.Div(bf).Float64()
-			currentDate, err := time.Parse(utils.FormatDate, av)
-			if err != nil {
-				return err
-			}
+		}
+		valArrLen := len(valArr)
+		totalVal := decimal.NewFromFloat(0.00)
+		for _, v := range valArr {
+			//totalVal = totalVal + v
+			newDecimal := decimal.NewFromFloat(v)
+			totalVal = totalVal.Add(newDecimal)
+		}
+		//af := decimal.NewFromFloat(totalVal)
+		af := totalVal
 
-			timestamp := currentDate.UnixNano() / 1e6
-			timestampStr := fmt.Sprintf("%d", timestamp)
-			valStr := utils.SubFloatToString(val, 4)
+		bf := decimal.NewFromFloat(float64(valArrLen))
+		val, _ := af.Div(bf).Float64()
+		currentDate, err := time.Parse(utils.FormatDate, av)
+		if err != nil {
+			return err
+		}
 
-			count, err := GetEdbDataCalculateNszydpjjsByCodeAndDate(edbCode, av)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				return err
-			}
-			if count <= 0 {
-				addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, valStr)
-				isAdd = true
-			} else {
+		timestamp := currentDate.UnixNano() / 1e6
+		timestampStr := fmt.Sprintf("%d", timestamp)
+		valStr := utils.SubFloatToString(val, 4)
+
+		if existItem, existOk := existDataMap[av]; !existOk {
+			//count, err := GetEdbDataCalculateNszydpjjsByCodeAndDate(edbCode, av)
+			//if err != nil && err.Error() != utils.ErrNoRow() {
+			//	return err
+			//}
+			//if count <= 0 {
+			//	addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, valStr)
+			//	isAdd = true
+			//} else {
+			//	err = ModifyEdbDataCalculateNszydpjjs(int64(edbInfoId), av, valStr)
+			//	if err != nil {
+			//		return err
+			//	}
+			//}
+			addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, valStr)
+			isAdd = true
+		} else {
+			if existItem.Value != val {
 				err = ModifyEdbDataCalculateNszydpjjs(int64(edbInfoId), av, valStr)
 				if err != nil {
 					return err

models/db.go

@@ -79,5 +79,6 @@ func init() {
 		new(data_manage.BaseFromSmm),
 		new(data_manage.BaseFromSmmIndex),
 		new(data_manage.BaseFromSmmData),
+		new(data_manage.EdbDataYs),
 	)
 }

services/data/base_from_smm.go

@@ -334,20 +334,19 @@ func GetSmmIndexData() {
 	}
 }
 
-func SyncSmmIndexData() {
+//有色基础指标同步
+func SyncSmmIndexDataBase() {
 	utils.FileLog.Info("start:" + time.Now().Format(utils.FormatDateTime))
 	list, err := data_manage.GetBaseFromSmmList()
 	if err != nil {
 		fmt.Println("GetBaseFromSmmList Err:" + err.Error())
 		return
 	}
-
 	token, err := getToken("pqian@hzinsights.com", "hz123456")
 	if err != nil {
 		fmt.Println(err)
 		return
 	}
-
 	endDate := time.Now().Format(utils.FormatDate)
 	smmIndexAll, err := data_manage.GetBaseFromSmmIndex()
 	if err != nil {
@@ -359,10 +358,10 @@ func SyncSmmIndexData() {
 		existIndexMap[item.IndexCode] = item
 	}
 
-	for _, v := range list {
+	for k, v := range list {
 		edbCode := v.Interface
 		var startDate string
-		if v.EndDate == "" {
+		if v.EndDate == "" || v.EndDate == "0000-00-00" {
 			startDate = v.ApiStartTime
 		} else {
 			if v.EndDate != "" {
@@ -381,197 +380,203 @@ func SyncSmmIndexData() {
 			}
 		}
 
-		if startDate == "" {
-			startDate = endDate
-		}
-
-		var syncDay int
-		if v.ApiUpdateType == "day" {
-			syncDay = utils.DATA_REFRESH
-		} else if v.ApiUpdateType == "month" {
-			syncDay = utils.DATA_REFRESH * 30
-		} else if v.ApiUpdateType == "week" {
-			syncDay = utils.DATA_REFRESH * 7
-		} else if v.ApiUpdateType == "year" {
-			syncDay = utils.DATA_REFRESH * 365
-		}
-		startDate = time.Now().AddDate(0, 0, -syncDay).Format(utils.FormatDate)
-
-		fmt.Println("getApiData:", edbCode, startDate, endDate)
 		ysItem, err := getApiData(token, edbCode, startDate, endDate)
 		if err != nil {
 			fmt.Println(err)
 			return
 		}
 		if ysItem != nil && ysItem.Code == 200 {
-			colKey := ""
-			for _, fv := range ysItem.Data.Field {
-				colKey += fv.DBColName + "#"
+
+			frequency := ysItem.Data.Frequency
+
+			indexMap := make(map[string]int)
+			smmIndexIdMap := make(map[int]int)
+			indexCodeMap := make(map[int]string)
+
+			indexKey := 0
+			var isDateIndex int
+			for fk, fv := range ysItem.Data.Field {
+				if fv.IsDate == "1" {
+					isDateIndex = fk
+				} else {
+					if !strings.Contains(fv.Name, "产品名称") &&
+						!strings.Contains(fv.Name, "单位") &&
+						!strings.Contains(fv.Name, "时间") &&
+						!strings.Contains(fv.Name, "备注") {
+
+						indexMap[fv.DBColName] = fk
+
+						indexKey += 1
+						indexCode := v.Interface + "#" + strconv.Itoa(indexKey)
+
+						if findItem, ok := existIndexMap[indexCode]; !ok {
+							ssmIndex := new(data_manage.BaseFromSmmIndex)
+							ssmIndex.Interface = v.Interface
+							ssmIndex.Name = v.Name
+							ssmIndex.IndexCode = indexCode
+							ssmIndex.IndexName = v.Name + "_" + fv.Name
+							ssmIndex.Type1 = v.Type1
+							ssmIndex.Type2 = v.Type2
+							ssmIndex.Type3 = v.Type3
+							ssmIndex.Frequency = frequency
+							ssmIndex.Unit = fv.Unit
+							ssmIndex.ApiStartTime = v.ApiStartTime
+							ssmIndex.ApiUpdateTime = v.ApiUpdateTime
+							ssmIndex.StartTime = v.StartTime
+							ssmIndex.FinishTime = v.FinishTime
+							ssmIndex.CreateTime = time.Now()
+							ssmIndex.ModifyTime = time.Now()
+							lastIndexId, err := data_manage.AddBaseFromSmmIndex(ssmIndex)
+							if err != nil {
+								return
+							}
+							smmIndexIdMap[fk] = int(lastIndexId)
+							indexCodeMap[fk] = indexCode
+						} else {
+							smmIndexIdMap[fk] = findItem.BaseFromSmmIndexId
+							indexCodeMap[fk] = findItem.IndexCode
+						}
+					}
+				}
+			}
+
+			existDataMap := make(map[string]*data_manage.BaseFromSmmData)
+			for _, mv := range indexCodeMap {
+				indexCode := mv
+				dataAllList, err := data_manage.GetBaseFromSmmDataAll(indexCode)
+				if err != nil {
+					fmt.Println("GetBaseFromSmmData Err:" + err.Error())
+					return
+				}
+				for _, item := range dataAllList {
+					key := item.IndexCode + item.DataTime
+					existDataMap[key] = item
+				}
+			}
+
+			addExistDataMap := make(map[string]string)
+
+			for _, dv := range ysItem.Data.Content {
+				var dataTime string
+				dataTime = dv[isDateIndex]
+				if strings.Contains(dataTime, "Q1") {
+					dataTime = strings.Replace(dataTime, "Q1", "-01", -1)
+					dataTime += "-31"
+				}
+				if strings.Contains(dataTime, "Q2") {
+					dataTime = strings.Replace(dataTime, "Q2", "-06", -1)
+					dataTime += "-30"
+				}
+				if strings.Contains(dataTime, "Q3") {
+					dataTime = strings.Replace(dataTime, "Q3", "-09", -1)
+					dataTime += "-30"
+				}
+				if strings.Contains(dataTime, "Q4") {
+					dataTime = strings.Replace(dataTime, "Q4", "-12", -1)
+					dataTime += "-31"
+				}
+
+				if strings.Contains(dataTime, "H1") {
+					dataTime = strings.Replace(dataTime, "H1", "-06", -1)
+					dataTime += "-30"
+				}
+
+				if strings.Contains(dataTime, "H2") {
+					dataTime = strings.Replace(dataTime, "H2", "-12", -1)
+					dataTime += "-31"
+				}
+
+				if frequency == "月" {
+					monthDate, err := time.Parse("2006-01", dataTime)
+					if err != nil {
+						fmt.Println("time.Parse:" + err.Error())
+					}
+					lastTime := monthDate.AddDate(0, 1, -1)
+					lastYear, lastMonth, lastDay := lastTime.Date()
+					var lastDate string
+					if int(lastMonth) < 10 {
+						lastDate = strconv.Itoa(lastYear) + "-" + "0" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+					} else {
+						lastDate = strconv.Itoa(lastYear) + "-" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+					}
+					dataTime = lastDate
+				} else if frequency == "年" {
+					dataTime = dataTime + "-12-31"
+				}
+				saveDataTime, err := time.Parse(utils.FormatDate, dataTime)
+				if err != nil {
+					fmt.Println("saveDataTime parse err:" + err.Error() + " dataTime:" + dataTime)
+					panic(err)
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+				//循环指标
+				for _, v := range indexMap {
+
+					indexCode := indexCodeMap[v]
+					smmIndexId := smmIndexIdMap[v]
+					dataVal := dv[v]
+
+					if indexCode != "" {
+
+						key := indexCode + dataTime
+						val := strings.Replace(dataVal, ",", "", -1)
+						if findData, dataOk := existDataMap[key]; !dataOk {
+							if _, addOK := addExistDataMap[key]; !addOK {
+								if val != "" && val != "-" {
+									dataItem := new(data_manage.BaseFromSmmData)
+									dataItem.BaseFromSmmIndexId = int(smmIndexId)
+									dataItem.IndexCode = indexCode
+									dataItem.DataTime = dataTime
+									dataItem.Value = val
+									dataItem.CreateTime = time.Now()
+									dataItem.ModifyTime = time.Now()
+									dataItem.DataTimestamp = timestamp
+									_, err = data_manage.AddBaseFromSmmData(dataItem)
+									if err != nil && !strings.Contains(err.Error(), "idx_index_code_date") {
+										fmt.Println("AddBaseFromSmmData Err:" + err.Error())
+										return
+									}
+								}
+							}
+						} else {
+							if findData != nil && findData.Value != val { //修改
+								if _, addOK := addExistDataMap[key]; !addOK {
+									if val != "" && val != "-" {
+										err = data_manage.ModifyBaseFromSmmData(findData.SmmDataId, val)
+										if err != nil {
+											fmt.Println("ModifyBaseFromSmmData Err:" + err.Error())
+											return
+										}
+									}
+								}
+							}
+						}
+						addExistDataMap[key] = key
+					}
+				}
+			}
+
+			//修改数据开始，结束日期
+			{
+				indexList, err := data_manage.GetBaseFromSmmIndexByCode(v.Interface)
+				if err != nil {
+					fmt.Println("GetBaseFromSmmIndexByCode Err:" + err.Error())
+				}
+				for _, sv := range indexList {
+					minDate, maxDate, err := data_manage.GetBaseFromSmmMaxOrMinDate(sv.IndexCode)
+					if err != nil {
+						fmt.Println("GetEdbDataSmmMaxOrMinDate Err:" + err.Error())
+					} else {
+						err = data_manage.ModifyBaseFromSmmMinDateAndMaxDate(sv.BaseFromSmmIndexId, v.BaseFromSmmId, minDate, maxDate)
+						if err != nil {
+							fmt.Println("ModifyBaseFromSmmMinDateAndMaxDate Err:" + err.Error())
+						}
+					}
+				}
 			}
-			fmt.Println("start colKey:", colKey)
-			colKey = strings.TrimRight(colKey, "#")
-			fmt.Println("end colKey:", colKey)
-			data_manage.ModifyBaseFromSmmSmmType(v.BaseFromSmmId, colKey)
 		}
-		//if ysItem != nil && ysItem.Code == 200 {
-		//
-		//	frequency := ysItem.Data.Frequency
-		//	var smmIndexIdArr []int64
-		//	var indexCodeArr []string
-		//	indexKey := 0
-		//	containsCount := 0
-		//	existDataMap := make(map[string]*data_manage.BaseFromSmmData)
-		//	for fk, fv := range ysItem.Data.Field {
-		//		if fk == 0 {
-		//			continue
-		//		}
-		//		if strings.Contains(fv.Name, "产品名称") ||
-		//			strings.Contains(fv.Name, "单位") ||
-		//			strings.Contains(fv.Name, "时间") ||
-		//			strings.Contains(fv.Name, "备注") {
-		//			containsCount++
-		//		}
-		//		if !strings.Contains(fv.Name, "产品名称") &&
-		//			!strings.Contains(fv.Name, "单位") &&
-		//			!strings.Contains(fv.Name, "时间") &&
-		//			!strings.Contains(fv.Name, "备注") {
-		//
-		//			indexKey += 1
-		//			indexCode := v.Interface + "_" + strconv.Itoa(indexKey)
-		//
-		//			dataAllList, err := data_manage.GetBaseFromSmmDataAll(indexCode)
-		//			if err != nil {
-		//				fmt.Println("GetBaseFromSmmData Err:" + err.Error())
-		//				return
-		//			}
-		//
-		//			for _, item := range dataAllList {
-		//				key := item.IndexCode + item.DataTime
-		//				existDataMap[key] = item
-		//			}
-		//
-		//			if findItem, ok := existIndexMap[indexCode]; !ok {
-		//				ssmIndex := new(data_manage.BaseFromSmmIndex)
-		//				ssmIndex.Interface = v.Interface
-		//				ssmIndex.Name = v.Name
-		//				ssmIndex.IndexCode = indexCode
-		//				ssmIndex.IndexName = v.Name + "_" + fv.Name
-		//				ssmIndex.Type1 = v.Type1
-		//				ssmIndex.Type2 = v.Type2
-		//				ssmIndex.Type3 = v.Type3
-		//				ssmIndex.Frequency = frequency
-		//				ssmIndex.Unit = fv.Unit
-		//				ssmIndex.ApiStartTime = v.ApiStartTime
-		//				ssmIndex.ApiUpdateTime = v.ApiUpdateTime
-		//				ssmIndex.StartTime = v.StartTime
-		//				ssmIndex.FinishTime = v.FinishTime
-		//				ssmIndex.CreateTime = time.Now()
-		//				ssmIndex.ModifyTime = time.Now()
-		//				lastIndexId, err := data_manage.AddBaseFromSmmIndex(ssmIndex)
-		//				if err != nil {
-		//					return
-		//				}
-		//				smmIndexIdArr = append(smmIndexIdArr, lastIndexId)
-		//				indexCodeArr = append(indexCodeArr, indexCode)
-		//			} else {
-		//				smmIndexIdArr = append(smmIndexIdArr, int64(findItem.BaseFromSmmIndexId))
-		//				indexCodeArr = append(indexCodeArr, findItem.IndexCode)
-		//			}
-		//		}
-		//	}
-		//
-		//	for _, dv := range ysItem.Data.Content {
-		//		var dataTime string
-		//		for dataK, dataV := range dv {
-		//			fmt.Println(dataK, dataV)
-		//			if dataK == 0 {
-		//				dataTime = dataV
-		//
-		//				if frequency == "月" {
-		//					monthDate, err := time.Parse("2006-01", dataTime)
-		//					if err != nil {
-		//						fmt.Println("time.Parse:" + err.Error())
-		//					}
-		//					lastTime := monthDate.AddDate(0, 1, -1)
-		//					lastYear, lastMonth, lastDay := lastTime.Date()
-		//					var lastDate string
-		//					if int(lastMonth) < 10 {
-		//						lastDate = strconv.Itoa(lastYear) + "-" + "0" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
-		//					} else {
-		//						lastDate = strconv.Itoa(lastYear) + "-" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
-		//					}
-		//					dataTime = lastDate
-		//				}
-		//
-		//			} else {
-		//				if !strings.Contains(ysItem.Data.Field[dataK].Name, "产品名称") &&
-		//					!strings.Contains(ysItem.Data.Field[dataK].Name, "单位") &&
-		//					!strings.Contains(ysItem.Data.Field[dataK].Name, "时间") &&
-		//					!strings.Contains(ysItem.Data.Field[dataK].Name, "备注") {
-		//
-		//					saveDataTime, err := time.Parse(utils.FormatDate, dataTime)
-		//					if err != nil {
-		//
-		//					}
-		//					timestamp := saveDataTime.UnixNano() / 1e6
-		//					var smmIndexId int64
-		//					var indexCode string
-		//					smmIndexId = smmIndexIdArr[dataK-(containsCount+1)]
-		//					indexCode = indexCodeArr[dataK-(containsCount+1)]
-		//
-		//					key := indexCode + dataTime
-		//					val := strings.Replace(dataV, ",", "", -1)
-		//
-		//					if findData, dataOk := existDataMap[key]; !dataOk {
-		//						dataItem := new(data_manage.BaseFromSmmData)
-		//						dataItem.BaseFromSmmIndexId = int(smmIndexId)
-		//						dataItem.IndexCode = indexCode
-		//						dataItem.DataTime = dataTime
-		//						dataItem.Value = val
-		//						dataItem.CreateTime = time.Now()
-		//						dataItem.ModifyTime = time.Now()
-		//						dataItem.DataTimestamp = timestamp
-		//						_, err = data_manage.AddBaseFromSmmData(dataItem)
-		//						if err != nil && !strings.Contains(err.Error(), "idx_index_code_date") {
-		//							fmt.Println("AddBaseFromSmmData Err:" + err.Error())
-		//							return
-		//						}
-		//					} else {
-		//						fmt.Println("modify:", findData.SmmDataId, findData.Value, val)
-		//						if findData != nil && findData.Value != val { //修改
-		//							err = data_manage.ModifyBaseFromSmmData(findData.SmmDataId, val)
-		//							if err != nil {
-		//								fmt.Println("ModifyBaseFromSmmData Err:" + err.Error())
-		//								return
-		//							}
-		//						}
-		//					}
-		//				}
-		//			}
-		//		}
-		//	}
-		//}
-		//
-		////修改数据开始，结束日期
-		//{
-		//	indexList, err := data_manage.GetBaseFromSmmIndexByCode(v.Interface)
-		//	if err != nil {
-		//		fmt.Println("GetBaseFromSmmIndexByCode Err:" + err.Error())
-		//	}
-		//	for _, sv := range indexList {
-		//		minDate, maxDate, err := data_manage.GetBaseFromSmmMaxOrMinDate(sv.IndexCode)
-		//		if err != nil {
-		//			fmt.Println("GetEdbDataSmmMaxOrMinDate Err:" + err.Error())
-		//		} else {
-		//			err = data_manage.ModifyBaseFromSmmMinDateAndMaxDate(sv.BaseFromSmmIndexId, v.BaseFromSmmId, minDate, maxDate)
-		//			if err != nil {
-		//				fmt.Println("ModifyBaseFromSmmMinDateAndMaxDate Err:" + err.Error())
-		//			}
-		//		}
-		//	}
-		//}
-		//fmt.Println(v.Interface, v.Name)
-		//time.Sleep(3 * time.Second)
+		fmt.Println(v.Interface, v.Name, k)
+		time.Sleep(1 * time.Second)
 	}
 	utils.FileLog.Info("end:" + time.Now().Format(utils.FormatDateTime))
 }

services/data/edb_info.go

@@ -244,7 +244,7 @@ func RefreshDataFromCalculateAll() (err error) {
 	condition += " AND edb_type=? "
 	pars = append(pars, 2)
 	//condition += " AND edb_info_id=? "
-	//pars = append(pars, 100341)
+	//pars = append(pars, 100691)
 
 	items, err := data_manage.GetEdbInfoByCondition(condition, pars, 0)
 	if err != nil {

services/task.go

@@ -59,12 +59,6 @@ func Task() {
 	fmt.Println("task end")
 }
 
-func Task123() {
-	fmt.Println("ys start")
-	data.SyncSmmIndexData()
-	fmt.Println("ys end")
-}
-
 //生产环境需要走的任务
 func releaseTask() {
 
@@ -108,6 +102,10 @@ func releaseTask() {
 	refreshData := task.NewTask("refreshData", "0 1 0,19 * * *", RefreshData)
 	task.AddTask("refreshData", refreshData)
 
+	//刷新指标基础数据
+	//refreshBaseData := task.NewTask("refreshBaseData", "0 1 18,23 * * *", RefreshBaseData)
+	//task.AddTask("refreshBaseData", refreshBaseData)
+
 	// 定时往同花顺同步微信群的截止日期（凌晨4点）
 	syncThsWxGroupEveryDay := task.NewTask("syncThsWxGroupEveryDay", "0 1 4 * * * ", SyncWxGroupEveryDay)
 	task.AddTask("定时往同花顺同步微信群的截止日期", syncThsWxGroupEveryDay)
@@ -176,6 +174,13 @@ func RefreshData(cont context.Context) (err error) {
 	return
 }
 
+//刷新基础数据
+func RefreshBaseData(cont context.Context) (err error) {
+	//同步有色基础指标数据
+	go data.SyncSmmIndexDataBase()
+	return
+}
+
 //func RefreshCalculateData(cont context.Context) (err error) {
 //	//计算指标
 //	go data.RefreshDataFromCalculateAll()