Merge remote-tracking branch 'origin/fix/sci_end_date'

controllers/base_from_mtjh.go

@@ -254,6 +254,23 @@ func (this *MtjhDataController) Mtjh() {
 			fmt.Println("添加指标名称成功", newId)
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromMtjhMapping)
+			for _, indexCode := range codeMap {
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	fmt.Println("数据操作完成")
 
 	br.Ret = 200

controllers/commodity_coal.go

@@ -224,6 +224,23 @@ func (this *CoalMineDataController) JsmHistory() {
 		}
 	}
 
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+			for _, v := range indexItems {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -449,6 +466,24 @@ func (this *CoalMineDataController) CoastalHistory() {
 			}
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+			for _, v := range indexItems {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -653,6 +688,24 @@ func (this *CoalMineDataController) InlandHistory() {
 			}
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+			for _, v := range indexItems {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -1001,6 +1054,56 @@ func (this *CoalMineDataController) Jsm() {
 
 	}
 
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+
+			for _, v := range items {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+
+			for _, v := range itemsIndex {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+
+			for _, v := range itemsCompany {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+			for _, v := range itemsCompanyIndex {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -1210,6 +1313,24 @@ func (this *CoalMineDataController) Coastal() {
 			}
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+			for _, v := range indexItems {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -1421,6 +1542,24 @@ func (this *CoalMineDataController) Inland() {
 			}
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+			for _, v := range indexItems {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -1447,7 +1586,6 @@ func (this *CoalMineDataController) Firm() {
 		return
 	}
 
-
 	var year, month, day string
 	var yearMap = make(map[int]string)
 	var monthMap = make(map[int]string)
@@ -1655,6 +1793,27 @@ func (this *CoalMineDataController) Firm() {
 			}
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			obj := new(models.BaseFromCoalmineMapping)
+			for _, v := range indexItems {
+				indexCode := v.IndexCode
+				var dateItem *models.EdbInfoMaxAndMinInfo
+				dateItem, err = obj.GetMaxAndMinDateByIndexCode(indexCode)
+				if err != nil {
+					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+					return
+				}
+				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			}
+		}()
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
 	return
 }
 

controllers/sci99_crawler.go

@@ -113,6 +113,12 @@ func (this *Sci99Crawler) RefreshSci99Classify() {
 
 	existDataMap := make(map[string]*models.BaseFromSci99Data)
 
+	sic99Info, ok := IndexMap[indexName]
+	if !ok {
+		utils.FileLog.Info("指标异常，通过名称找不到指标")
+		return
+	}
+
 	//获取所有指标信息
 	allData, e := models.GetBaseFromTradeSci99DataAll(indexCode)
 	if e != nil {
@@ -143,7 +149,7 @@ func (this *Sci99Crawler) RefreshSci99Classify() {
 		} else {
 			// 新增
 			dataItem := models.BaseFromSci99Data{
-				BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
+				BaseFromSciIndexId: sic99Info.BaseFromSciIndexId,
 				IndexCode:          indexCode,
 				DataTime:           v.RealDate,
 				Value:              v.MDataValue,
@@ -163,6 +169,20 @@ func (this *Sci99Crawler) RefreshSci99Classify() {
 			return
 		}
 	}
+
+	// 更新指标最新日期
+	{
+		go func() {
+			var dateItem *models.EdbInfoMaxAndMinInfo
+			dateItem, err = sic99Info.GetMaxAndMinDateByIndexCode(indexCode)
+			if err != nil {
+				err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+				return
+			}
+			sic99Info.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+		}()
+	}
+
 	time.Sleep(1 * time.Second)
 
 	br.Ret = 200

models/base_from_coalmine.go

@@ -44,15 +44,14 @@ type BaseFromCoalmineCompanyIndex struct {
 	ModifyTime                     time.Time `description:"修改时间"`
 }
 
-
-//添加指标
+// 添加指标
 func AddBaseFromCoalmineMapping(item *BaseFromCoalmineMapping) (lastId int64, err error) {
 	o := orm.NewOrm()
 	lastId, err = o.Insert(item)
 	return
 }
 
-//查询指标
+// 查询指标
 func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_coalmine_mapping`
@@ -60,7 +59,7 @@ func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error)
 	return
 }
 
-//查询数据
+// 查询数据
 func GetBaseFromCoalmineIndex() (items []*BaseFromCoalmineJsmIndex, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_coalmine_jsm_index`
@@ -75,21 +74,21 @@ func UpdateBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (err error) {
 	return
 }
 
-//添加数据
+// 添加数据
 func AddBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (lastId int64, err error) {
 	o := orm.NewOrm()
 	lastId, err = o.Insert(item)
 	return
 }
 
-//添加公司指标
+// 添加公司指标
 func AddBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (lastId int64, err error) {
 	o := orm.NewOrm()
 	lastId, err = o.Insert(item)
 	return
 }
 
-//查询公司指标
+// 查询公司指标
 func GetBaseFromCoalmineCompanyIndex() (items []*BaseFromCoalmineCompanyIndex, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_coalmine_company_index`
@@ -104,22 +103,47 @@ func UpdateBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (err
 	return
 }
 
+func (m *BaseFromCoalmineMapping) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_sci99_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var latest_value float64
+	sql = ` SELECT value AS latest_value FROM base_from_sci99_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latest_value)
+	if err != nil {
+		return
+	}
+	item.LatestValue = latest_value
+
+	return
+}
+
+func (m *BaseFromCoalmineMapping) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_sci99_index SET start_date=?,end_date=?,latest_value=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
+	return
+}
+
 type CoalMineDataReq struct {
 	SheetData []SheetData
 }
 
-
 type SheetData struct {
-	Name        string
-	Rows        []Row
-	Cols        []*Col
-	MaxRow      int
-	MaxCol      int
-	Hidden      bool
-	Selected    bool
+	Name     string
+	Rows     []Row
+	Cols     []*Col
+	MaxRow   int
+	MaxCol   int
+	Hidden   bool
+	Selected bool
 }
 
-
 type Row struct {
 	Cells        []Cell
 	Hidden       bool
@@ -129,16 +153,15 @@ type Row struct {
 }
 
 type Col struct {
-	Min             int
-	Max             int
-	Hidden          bool
-	Width           float64
-	Collapsed       bool
-	OutlineLevel    uint8
-	numFmt          string
+	Min          int
+	Max          int
+	Hidden       bool
+	Width        float64
+	Collapsed    bool
+	OutlineLevel uint8
+	numFmt       string
 }
 
-
 type Cell struct {
-	Value          string
+	Value string
 }

models/base_from_mtjh.go

@@ -281,3 +281,45 @@ func GetMtjhDataByTradeCode(condition string, pars []interface{}) (item []*BaseF
 	_, err = o.Raw(sql, pars).QueryRows(&item)
 	return
 }
+
+func (m *BaseFromMtjhMapping) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	var suffix string
+	if strings.Contains(indexCode, "jsm") {
+		suffix = "jsm_index"
+	} else if strings.Contains(indexCode, "company") {
+		suffix = "company_index"
+	} else if strings.Contains(indexCode, "firm") {
+		suffix = "firm_index"
+	} else if strings.Contains(indexCode, "coastal") {
+		suffix = "coastal_index"
+	} else if strings.Contains(indexCode, "inland") {
+		suffix = "inland_index"
+	}
+
+	tableName := fmt.Sprintf("base_from_coalmine_%s", suffix)
+
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(deal_value) AS min_value,MAX(deal_value) AS max_value FROM %s WHERE index_code=? `, tableName)
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var latest_value float64
+	sql = fmt.Sprintf(` SELECT value AS deal_value FROM %s WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `, tableName)
+	err = o.Raw(sql, indexCode).QueryRow(&latest_value)
+	if err != nil {
+		return
+	}
+	item.LatestValue = latest_value
+
+	return
+}
+
+func (m *BaseFromMtjhMapping) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_mtjh_mapping SET start_date=?,end_date=?,latest_value=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
+	return
+}

models/base_from_sci.go

@@ -235,6 +235,7 @@ type BaseFromSciIndex struct {
 	ModifyTime         time.Time `gorm:"autoUpdateTime:milli;column:modify_time" json:"modify_time"` //最后更新时间
 	FilePath           string    `gorm:"column:file_path" json:"file_path"`                          // 文件路径
 	TerminalCode       string    `gorm:"column:terminal_code" json:"terminal_code"`                  // 指标编码
+	LatestValue        float64   `gorm:"column:latest_value" json:"latest_value"`                    // 数据最新值
 }
 
 // TableName get sql table name.获取数据库表名

models/base_from_sci99.go

@@ -20,6 +20,9 @@ type BaseFromSci99Index struct {
 	Describe           string    // 指标描述
 	CreateTime         time.Time // 创建时间
 	ModifyTime         time.Time // 修改时间
+	StartDate          time.Time `gorm:"column:start_date"`
+	EndDate            time.Time `gorm:"column:end_date"`
+	LatestValue        float64   `gorm:"column:latest_value"` // 数据最新值
 }
 
 // BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
@@ -39,7 +42,7 @@ type BaseFromSci99Data struct {
 	BaseFromSciIndexId int       // 指标id
 	IndexCode          string    // 指标编码
 	DataTime           string    // 数据日期
-	Value              float64    // 数据值
+	Value              float64   // 数据值
 	CreateTime         time.Time // 创建时间
 	ModifyTime         time.Time // 修改时间
 }
@@ -278,4 +281,31 @@ func UpdateBaseFromSci99Data(value float64, indexCode, dataTime string) (err err
 	sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
 	_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
 	return
-}
+}
+
+func (m *BaseFromSci99Index) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_sci99_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var latest_value float64
+	sql = ` SELECT value AS latest_value FROM base_from_sci99_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latest_value)
+	if err != nil {
+		return
+	}
+	item.LatestValue = latest_value
+
+	return
+}
+
+func (m *BaseFromSci99Index) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_sci99_index SET start_date=?,end_date=?,latest_value=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
+	return
+}

services/base_from_baiinfo.go

@@ -70,10 +70,18 @@ func HandleBaiinfoIndex(baseFilePath, terminalCode, renameFilePath, indexName, i
 		//修改排序
 		go models.ModifyBaiinfoIndexSort(indexId)
 	} else if isAdd == 2 {
-		if item.TerminalCode == `` && terminalCode != `` {
+		if terminalCode != `` && item.TerminalCode != terminalCode {
 			item.TerminalCode = terminalCode
 			item.Update([]string{"TerminalCode"})
 		}
+		if baseFilePath != `` && item.BaseFileName != baseFilePath {
+			item.BaseFileName = baseFilePath
+			item.Update([]string{"BaseFileName"})
+		}
+		if renameFilePath != `` && item.RenameFileName != renameFilePath {
+			item.RenameFileName = renameFilePath
+			item.Update([]string{"RenameFileName"})
+		}
 
 		//获取已存在的所有数据
 		exitDataList, err := models.GetBaseFromBaiinfoDataByCode(indexCode)

services/base_from_pb.go

@@ -45,6 +45,10 @@ func GetEdbDataFromPb(edbCode, startDate, endDate, edbTerminalCode string) (item
 		utils.FileLog.Info("GetEdbDataByPb Unmarshal Err:" + err.Error())
 		return
 	}
+	if item == nil {
+		err = fmt.Errorf("彭博数据转结构体失败，body：" + string(body))
+		return
+	}
 	edbCode, _ = url.QueryUnescape(edbCode)
 	return
 }
@@ -86,6 +90,11 @@ func GetEdbDataFromPbFinance(companyCode, edbCode, startDate, endDate, edbTermin
 		utils.FileLog.Info("GetEdbDataByPb Unmarshal Err:" + err.Error())
 		return
 	}
+	if item == nil {
+		err = fmt.Errorf("彭博财务数据转结构体失败，body：" + string(body))
+		return
+	}
+
 	edbCode, _ = url.QueryUnescape(edbCode)
 	return
 }

services/base_from_sci.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/shopspring/decimal"
 	"strings"
 	"time"
 )
@@ -125,12 +126,25 @@ func HandleSciData(dataMap map[string]map[string]string) {
 		}
 		addSciDataList := make([]*models.BaseFromSciData, 0)
 
+		var tmpStartDate, tmpEndDate time.Time
+		var latestValue string
+
 		for currDate, currVal := range data {
 			currDataTime, tmpErr := time.ParseInLocation(utils.FormatDate, currDate, time.Local)
 			if tmpErr != nil {
 				errMsgList = append(errMsgList, fmt.Sprint("时间格式化失败,指标编码：", currDate, ";错误原因：", tmpErr.Error()))
 				continue
 			}
+			// 如果开始日期为空、或者当前日期早于开始日期，那么给开始日期赋值为当前日期
+			if tmpStartDate.IsZero() || currDataTime.Before(tmpStartDate) {
+				tmpStartDate = currDataTime
+			}
+			// 如果结束日期为空、或者当前日期晚于结束日期，那么给结束日期赋值为当前日期，且赋值最新值
+			if tmpEndDate.IsZero() || currDataTime.After(tmpEndDate) {
+				tmpEndDate = currDataTime
+				latestValue = currVal
+			}
+
 			timestamp := currDataTime.UnixNano() / 1e6
 
 			sciData, ok := indexDataExistMap[currDate]
@@ -174,8 +188,24 @@ func HandleSciData(dataMap map[string]map[string]string) {
 				continue
 			}
 		}
+
+		updateCols := []string{"ModifyTime"}
 		indexInfo.ModifyTime = time.Now()
-		indexInfo.Update([]string{"ModifyTime"})
+		if !tmpStartDate.IsZero() {
+			indexInfo.StartDate = tmpStartDate
+			updateCols = append(updateCols, "StartDate")
+		}
+		if !tmpEndDate.IsZero() {
+			indexInfo.EndDate = tmpEndDate
+			updateCols = append(updateCols, "EndDate")
+		}
+
+		latestValueDeci, tmpErr := decimal.NewFromString(latestValue)
+		if tmpErr == nil {
+			indexInfo.LatestValue, _ = latestValueDeci.Float64()
+			updateCols = append(updateCols, "LatestValue")
+		}
+		indexInfo.Update(updateCols)
 
 		// 同步刷新ETA图库红桃3的指标
 		{

services/base_from_smm.go

@@ -104,6 +104,9 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		if renameFilePath != `` && item.RenameFileName != renameFilePath {
 			updateParams["rename_file_name"] = renameFilePath
 		}
+		if terminalCode != `` && item.TerminalCode != terminalCode {
+			updateParams["terminal_code"] = terminalCode
+		}
 
 		whereParam := make(map[string]interface{})
 		whereParam["index_code"] = indexCode