Merge branch 'ETA_1.6.4'

# Conflicts:
#	models/db.go
#	routers/router.go
#	utils/constants.go

controllers/base_from_mtjh.go

@@ -0,0 +1,379 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"strconv"
+	"time"
+)
+
+type MtjhDataController struct {
+	BaseAuthController
+}
+
+
+// @Title 煤炭江湖数据
+// @Description 刷新煤炭江湖数据接口
+// @Param	request	body models.CoalMineDataReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /data [post]
+func (this *MtjhDataController) Mtjh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.CoalMineDataReq
+
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	var area string
+	var port string
+	var description string
+
+	var dateMap = make(map[int]string)
+	var varietyMap = make(map[int]string)
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var codeCompanyMap = make(map[string]string)
+	var indexCompanyMap = make(map[string]string)
+	var items []*models.BaseFromMtjhMapping
+	var itemsIndex []*models.BaseFromMtjhIndex
+	codeList, err := models.GetBaseFromMtjhMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return
+	}
+
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+
+	codeCompanyList, err := models.GetBaseFromMtjhMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(codeCompanyList) > 0 {
+		for _, v := range codeCompanyList {
+			codeCompanyMap[v.IndexName] = v.IndexCode
+		}
+	}
+
+	indexList, err := models.GetBaseFromMtjhIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(indexList) > 0 {
+		for _, v := range indexList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	indexCompanyList, err := models.GetBaseFromCoalmineCompanyIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexCompanyMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for _, sheet := range req.SheetData {
+		//遍历行读取
+		maxRow := sheet.MaxRow
+		for i := 1; i < maxRow; i++ {
+			//获取样本情况
+			row := sheet.Rows[i]
+			cells := row.Cells
+
+			// 获取日期
+			if i == 1 {
+				var date string
+				for k, cell := range cells {
+					if k > 2 {
+						text := cell.Value
+						if text != "" {
+							stamp, _ := time.ParseInLocation("01-02-06", text, time.Now().Location())
+							date = stamp.Format(utils.FormatDate)
+						}
+						dateMap[k] = date
+					}
+				}
+			}
+
+			// 获取品种
+			if i == 2 {
+				for k, cell := range cells {
+					if k > 2 {
+						text := cell.Value
+						if text != "" {
+							varietyMap[k] = text
+						}
+					}
+				}
+			}
+
+			//for i := 0 ;i < len(cells);i ++ {
+			//	fmt.Println("k:",i)
+			//	fmt.Println("dateMap:",dateMap[i])
+			//	fmt.Println("varietyMap:",varietyMap[i])
+			//}
+			if i > 2 {
+				for k, cell := range cells {
+					text := cell.Value
+					if text != "" {
+						if k == 1 {
+							area = text
+						}
+						if k == 2 {
+							port = text
+
+							for j := 0; j < 3; j++ {
+								item := new(models.BaseFromMtjhMapping)
+								switch j {
+								case 0:
+									description = "动力煤"
+								case 1:
+									description = "焦煤"
+								case 2:
+									description = "合计"
+								}
+								itemName := area + port + description
+
+								//取处理后公司名首字母缩写
+								var strResult string
+
+								a := pinyin.NewArgs()
+								pys := [][]string{}
+								for _, r := range itemName {
+									py := pinyin.SinglePinyin(r, a)
+									if len(py) > 0 {
+										pys = append(pys, py)
+									} else {
+										if r != '-' && r != '（'  && r != '）'{
+											pys = append(pys, []string{string(r)})
+										}
+									}
+								}
+								for i := 0; i < len(pys); i++ {
+									if len(pys[i]) != 0 {
+										str := pys[i][0]
+										pi := str[0:1]
+										strResult += pi
+									}
+								}
+
+								strResult = "mtjh" + strResult
+
+								item.IndexCode = strResult
+								item.Area = area
+								item.Port = port
+								item.Variety = description
+								item.Unit = "万吨"
+								item.Frequency = "周度"
+								item.IndexName = itemName
+
+								item.CreateTime = time.Now()
+								items = append(items, item)
+							}
+						}
+						if k > 2 {
+							item := new(models.BaseFromMtjhIndex)
+							item.IndexName = area + port + varietyMap[k]
+							item.IndexCode = codeMap[item.IndexName]
+							item.DealValue = text
+							item.DataTime = dateMap[k]
+							item.Area = area
+							item.Port = port
+							item.Variety = varietyMap[k]
+							item.Unit = "万吨"
+							item.Frequency = "周度"
+							item.ModifyTime = time.Now()
+							item.CreateTime = time.Now()
+							itemsIndex = append(itemsIndex, item)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	//添加数据到数据库
+	mappingAddList := make([]*models.BaseFromMtjhMapping, 0)
+	for _, v := range items {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			mappingAddList = append(mappingAddList, v)
+		}
+	}
+	if len(mappingAddList) > 0 {
+		newId, err := models.AddBaseFromMtjhMappingMuti(mappingAddList)
+		if err != nil {
+			fmt.Println("添加指标名称错误",err.Error())
+		} else {
+			fmt.Println("添加指标名称成功", newId)
+		}
+	}
+
+	fmt.Println("指标操作完成")
+
+	indexAddList := make([]*models.BaseFromMtjhIndex, 0)
+	for _, v := range itemsIndex {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			indexAddList = append(indexAddList, v)
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromMtjhIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	if len(indexAddList) > 0 {
+		newId, err := models.AddBaseFromMtjhIndexMuti(indexAddList)
+		if err != nil {
+			fmt.Println("添加指标名称错误",err.Error())
+		} else {
+			fmt.Println("添加指标名称成功", newId)
+		}
+	}
+	fmt.Println("数据操作完成")
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+	return
+}
+
+
+// @Title 新增煤炭网指标接口
+// @Description 新增煤炭网指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *MtjhDataController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_MTJH
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		err = models.AddEdbDataFromMtjh(req.EdbCode)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromMtjh，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// @Title 刷新中国煤炭网指标接口
+// @Description 刷新中国煤炭网指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *MtjhDataController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_MTJH
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+
+	err = models.RefreshEdbDataFromMtjh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromCoal，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

controllers/commodity_coal.go

@@ -659,8 +659,8 @@ func (this *CoalMineDataController) InlandHistory() {
 	return
 }
 
-// @Title 刷新数据
-// @Description 刷新数据接口
+// @Title 晋陕蒙数据
+// @Description 刷新晋陕蒙数据接口
 // @Param	request	body models.CoalMineDataReq true "type json string"
 // @Success 200 {object} models.EdbClassify
 // @router /jsm [post]
@@ -1007,8 +1007,8 @@ func (this *CoalMineDataController) Jsm() {
 	return
 }
 
-// @Title 刷新数据
-// @Description 刷新数据接口
+// @Title 沿海数据
+// @Description 刷新沿海数据接口
 // @Param	request	body models.CoalMineDataReq true "type json string"
 // @Success 200 {object} models.EdbClassify
 // @router /coastal [post]
@@ -1209,8 +1209,8 @@ func (this *CoalMineDataController) Coastal() {
 	return
 }
 
-// @Title 刷新数据
-// @Description 刷新数据接口
+// @Title 内陆数据
+// @Description 刷新内陆数据接口
 // @Param	request	body models.CoalMineDataReq true "type json string"
 // @Success 200 {object} models.EdbClassify
 // @router /inland [post]
@@ -1414,6 +1414,238 @@ func (this *CoalMineDataController) Inland() {
 	br.Msg = "获取成功"
 	return
 }
+
+// @Title 内陆数据
+// @Description 刷新内陆数据接口
+// @Param	request	body models.CoalMineDataReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /firm [post]
+func (this *CoalMineDataController) Firm() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.CoalMineDataReq
+
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+
+	var year, month, day string
+	var yearMap = make(map[int]string)
+	var monthMap = make(map[int]string)
+	var dateMap = make(map[int]string)
+	var mappingItems []*models.BaseFromCoalmineMapping
+	var indexItems []*models.BaseFromCoalmineFirmIndex
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var groupName string
+	codeList, err := models.GetBaseFromCoalmineMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return
+	}
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+	indexCompanyList, err := models.GetBaseFromCoalmineFirmIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for _, sheet := range req.SheetData {
+		//遍历行读取
+		maxRow := sheet.MaxRow
+		for i := 0; i < maxRow; i++ {
+			//获取年份
+			//if i == 2 {
+			//	row := sheet.Row(i)
+			//	cells := row.Cells
+			//	for k, cell := range cells {
+			//		text := cell.String()
+			//		if k > 0 && text != "" {
+			//			year = text
+			//			fmt.Println("year:",year)
+			//			fmt.Println("k:",k)
+			//			yearMap[k] = year
+			//		}
+			//	}
+			//}
+			//获取月份和年度
+			if i == 3 {
+				row := sheet.Rows[i]
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.Value
+					if k > 0 && text != "" {
+						month = text
+						month = utils.ConvertToFormatDay(month)
+						time, _ := time.Parse(utils.FormatDate, month)
+						yearInt := time.Year()
+						monthInt := int(time.Month())
+						year = strconv.Itoa(yearInt)
+						yearMap[k] = year
+						month = strconv.Itoa(monthInt)
+						monthMap[k] = month
+					}
+				}
+			}
+			//获取旬度
+			if i == 4 {
+				row := sheet.Rows[i]
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.Value
+					if k > 0 && text != "" {
+						if yearMap[k] != "" {
+							year = yearMap[k]
+						}
+						if monthMap[k] != "" {
+							month = monthMap[k]
+						}
+						day = text
+						dateMap[k] = year + "年" + month + "月" + day
+					}
+				}
+			}
+			//获取企业或地区名及信息
+			if i > 4 {
+				row := sheet.Rows[i]
+				cells := row.Cells
+				var companyName string
+				for k, cell := range cells {
+					if k == 0 {
+						companyName = cell.Value
+
+						//省名
+						if !strings.Contains(companyName, " ") {
+							groupName = companyName
+						}
+						if groupName == "全国合计" {
+							groupName = "全国"
+						}
+						//若有下面四个名称要拼上省名不然会重复
+						if companyName == "  地方合计" || companyName == "    地方国有" ||
+							companyName == "    地方乡镇" || companyName == "  国有重点" {
+							companyName = groupName + companyName
+						}
+						companyName = strings.Replace(companyName, " ", "", -1)
+						var item models.BaseFromCoalmineMapping
+						//生成code
+						exists := ContainsSpecialName(companyName)
+						var code string
+						if exists {
+							abbr := trimProvinceName(companyName)
+							//取处理后公司名首字母缩写
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(companyName[9:], a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = abbr[:2] + code + "firm"
+
+						} else {
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(companyName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = code + "firm"
+						}
+						item.IndexName = companyName
+						item.CreateTime = time.Now()
+						mappingItems = append(mappingItems, &item)
+					} else {
+						dealValue := cell.Value
+
+						item := models.BaseFromCoalmineFirmIndex{
+							IndexName:  companyName,
+							IndexCode:  codeMap[companyName],
+							DataTime:   dateMap[k],
+							DealValue:  dealValue,
+							GroupName:  groupName,
+							Source:     "全国分企业",
+							Unit:       "万吨",
+							Frequency:  "旬度",
+							CreateTime: time.Now(),
+							ModifyTime: time.Now(),
+						}
+
+						indexItems = append(indexItems, &item)
+					}
+				}
+			}
+		}
+	}
+	//添加数据到数据库
+	for _, v := range mappingItems {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			newId, err := models.AddBaseFromCoalmineMapping(v)
+			if err != nil {
+				for i := 0; i < 10; i++ {
+					v.IndexCode = v.IndexCode + strconv.Itoa(i)
+					codeMap[v.IndexName] = v.IndexCode
+					newId, err := models.AddBaseFromCoalmineMapping(v)
+					if err != nil {
+						fmt.Println("再次添加公司指标名称错误", err)
+						continue
+					} else {
+						fmt.Println("新增公司成功", newId)
+						break
+					}
+				}
+			} else {
+				fmt.Println("新增公司成功", newId)
+			}
+		}
+	}
+	fmt.Println("指标操作完成")
+
+	//给indexItem中的code赋值并插入index表
+	for _, v := range indexItems {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			newId, err := models.AddBaseFromCoalFirmIndex(v)
+			if err != nil {
+				fmt.Println("添加数据错误", err)
+			} else {
+				fmt.Println("新增成功", newId)
+			}
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromCoalFirmIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
 func ttrimProvinceName(name string) string {
 	name = strings.Replace(name, "陕西省", "sn", -1)
 	name = strings.Replace(name, "陕西", "sn", -1)

models/base_from_mtjh.go

@@ -0,0 +1,283 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type BaseFromMtjhMapping struct {
+	BaseFromMtjhMappingId int       `orm:"column(base_from_Mtjh_mapping_id);pk"`
+	IndexName             string    `description:"持买单量指标名称"`
+	IndexCode             string    `description:"持买单量指标编码"`
+	CreateTime            time.Time `description:"时间"`
+	Area                  string    `description:"区域"`
+	Port                  string    `description:"港口或码头"`
+	Variety               string    `description:"品种"`
+	Unit                  string    `description:"单位"`
+	Frequency             string    `description:"频率"`
+}
+
+type BaseFromMtjhIndex struct {
+	BaseFromMtjhIndexId int       `orm:"column(base_from_mtjh_index_id);pk"`
+	IndexName           string    `description:"持买单量指标名称"`
+	IndexCode           string    `description:"持买单量指标编码"`
+	DealValue           string    `description:"成交量"`
+	DataTime            string    `description:"数据日期"`
+	Area                string    `description:"区域"`
+	Port                string    `description:"港口或码头"`
+	Variety             string    `description:"品种"`
+	Unit                string    `description:"单位"`
+	Frequency           string    `description:"频率"`
+	CreateTime          time.Time `description:"插入时间"`
+	ModifyTime          time.Time `description:"修改时间"`
+}
+
+// 查询指标
+func GetBaseFromMtjhMapping() (items []*BaseFromMtjhMapping, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mtjh_mapping`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+// 查询指标
+func GetBaseFromMtjhIndex() (items []*BaseFromMtjhIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mtjh_index`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+// 添加数据
+func AddBaseFromMtjhIndex(item *BaseFromMtjhIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromMtjhIndexMuti(items []*BaseFromMtjhIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertMulti(500, items)
+	return
+}
+
+// 添加指标
+func AddBaseFromMtjhMapping(item *BaseFromMtjhMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromMtjhMappingMuti(items []*BaseFromMtjhMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertMulti(500, items)
+	return
+}
+
+func UpdateBaseFromMtjhIndex(item *BaseFromMtjhIndex) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_mtjh_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
+	return
+}
+
+func AddEdbDataFromMtjh(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	coalBaseDataAll, err := GetMtjhindexByCode(edbCode)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_mtjh(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range coalBaseDataAll {
+		eDate := sv.DataTime
+		var timeStr string
+		var dataTime time.Time
+		var sDataTime string
+		var timestamp int64
+
+		sDataTime = eDate
+		dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			fmt.Println("time.Parse Err:" + eDate)
+			return err
+		}
+		timestamp = dataTime.UnixNano() / 1e6
+		timeStr = fmt.Sprintf("%d", timestamp)
+
+		value := strings.Replace(sv.DealValue, "%", "", -1)
+		if _, ok := existMap[sDataTime]; !ok {
+			addSql += GetAddSql("0", edbCode, sDataTime, timeStr, value)
+			fmt.Println("edbCode：", edbCode)
+			fmt.Println("sDataTime：", sDataTime)
+			fmt.Println("timeStr：", timeStr)
+			fmt.Println("value：", value)
+			isAdd = true
+		}
+		existMap[eDate] = value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+func RefreshEdbDataFromMtjh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := utils.DATA_SOURCE_MTJH
+	subSource := utils.DATA_SUB_SOURCE_EDB
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	glDataList, err := GetMtjhDataByTradeCode(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	addSql := ` INSERT INTO edb_data_mtjh(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range glDataList {
+		var value string
+		value = strings.Replace(v.DealValue, "%", "", -1)
+		item := v
+		itemValue := value
+		if _, ok := existMap[v.DataTime]; !ok {
+			eDate := item.DataTime
+			var timeStr string
+			var dataTime time.Time
+			var sDataTime string
+			var timestamp int64
+
+			sDataTime = eDate
+			dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+			if err != nil {
+				fmt.Println("time.Parse Err:" + eDate)
+				return err
+			}
+			timestamp = dataTime.UnixNano() / 1e6
+			timeStr = fmt.Sprintf("%d", timestamp)
+
+			sValue := itemValue
+			if sValue != "" {
+				saveValue := sValue
+
+				if findItem, ok := existMap[eDate]; !ok {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, sDataTime, timeStr, saveValue)
+					isAdd = true
+				} else {
+					if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
+						err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+
+			// 下面代码主要目的是处理掉手动插入的数据判断
+			{
+				if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+					realDataMaxDate = dataTime
+				}
+				if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+					isFindConfigDateRealData = true
+				}
+			}
+
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+
+	return
+}
+
+// GetMtjhindexByCode
+func GetMtjhindexByCode(indexCode string) (items []*BaseFromMtjhIndex, err error) {
+	o := orm.NewOrm()
+	sql := "SELECT * FROM base_from_mtjh_index WHERE index_code=? "
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+	return
+}
+
+func GetMtjhDataByTradeCode(condition string, pars []interface{}) (item []*BaseFromMtjhIndex, err error) {
+	sql := ` SELECT * FROM base_from_mtjh_index WHERE 1=1 `
+	o := orm.NewOrm()
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY data_time DESC `
+	_, err = o.Raw(sql, pars).QueryRows(&item)
+	return
+}

models/db.go

@@ -119,6 +119,9 @@ func initBaseIndex() {
 		new(BaseFromCoalmineCoastalIndex),
 		new(BaseFromCoalmineInlandIndex),
 		new(BaseFromCoalmineCompanyIndex),
+		new(BaseFromCoalmineFirmIndex),
+		new(BaseFromMtjhMapping),
+		new(BaseFromMtjhIndex),
 		new(BaseFromFenweiIndex),
 		new(BaseFromFenweiData),
 	)

routers/commentsRouter.go

@@ -214,6 +214,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"],
+        beego.ControllerComments{
+            Method: "Firm",
+            Router: `/firm`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"],
         beego.ControllerComments{
             Method: "Inland",
@@ -592,6 +601,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"],
+        beego.ControllerComments{
+            Method: "Mtjh",
+            Router: `/data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -229,6 +229,11 @@ func init() {
 				&controllers.GzController{},
 			),
 		),
+		beego.NSNamespace("/mtjh",
+			beego.NSInclude(
+				&controllers.MtjhDataController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

utils/constants.go

@@ -103,6 +103,7 @@ const (
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74
 	DATA_SOURCE_CALCULATE_RJZ                                   // 日均值计算->75
 	DATA_SOURCE_GFEX                                 = 78       // 广州期货交易所->78
+	DATA_SOURCE_MTJH                                 = 80       // 煤炭江湖->80
 )
 
 // 指标来源的中文展示
@@ -183,6 +184,7 @@ const (
 	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`             //自定义分析->74
 	DATA_SOURCE_NAME_YONYI                                = `涌益咨询`              // 涌益咨询
 	DATA_SOURCE_NAME_FENWEI                               = `汾渭数据`              // 汾渭煤炭
+	DATA_SOURCE_NAME_MTJH                                 = `煤炭江湖`              // 煤炭江湖->80
 )
 
 // 基础数据初始化日期