修改隆众数据同步

main.go

@@ -1,9 +1,16 @@
 package main
 
 import (
+	"fmt"
 	"github.com/beego/beego/v2/server/web"
 	_ "hongze/hongze_task/routers"
 	"hongze/hongze_task/services"
+	"hongze/hongze_task/utils"
+	"runtime"
+	"time"
+
+	"github.com/beego/beego/v2/server/web/context"
+	"github.com/beego/beego/v2/adapter/logs"
 )
 
 func main() {
@@ -13,5 +20,37 @@ func main() {
 	}
 	go services.Task()
 	//services.TaskTest()
+	web.BConfig.RecoverFunc = Recover
 	web.Run()
 }
+
+func Recover(ctx *context.Context, conf *web.Config) {
+	if err := recover(); err != nil {
+		if err == web.ErrAbort {
+			return
+		}
+		if !web.BConfig.RecoverPanic {
+			panic(err)
+		}
+		stack := ""
+		msg := fmt.Sprintf("The request url is  %v", ctx.Input.URL())
+		stack += msg + "</br>"
+		logs.Critical(msg)
+		msg = fmt.Sprintf("The request data is %v", string(ctx.Input.RequestBody))
+		stack += msg + "</br>"
+		logs.Critical(msg)
+		msg = fmt.Sprintf("Handler crashed with error %v", err)
+		stack += msg + "</br>"
+		logs.Critical(msg)
+		for i := 1; ; i++ {
+			_, file, line, ok := runtime.Caller(i)
+			if !ok {
+				break
+			}
+			logs.Critical(fmt.Sprintf("%s:%d", file, line))
+			stack = stack + fmt.Sprintln(fmt.Sprintf("%s:%d</br>", file, line))
+		}
+		go utils.SendEmail(utils.APPNAME+"崩了"+time.Now().Format("2006-01-02 15:04:05"), stack, utils.EmailSendToUsers)
+	}
+	return
+}

models/data_manage/edb_data_gl.go

@@ -16,7 +16,6 @@ func AddEdbDataGlBySql(sqlStr string) (err error) {
 	return
 }
 
-
 type GlData struct {
 	InputValue string `orm:"column(DATA_VALUE)" description:"日期"`
 	DataTime   string `orm:"column(DATA_DATE)" description:"值"`
@@ -85,20 +84,29 @@ func RefreshEdbDataByGl(edbInfoId int, edbCode, startDate, endDate string) (err
 		condition += " AND DATA_DATE<=? "
 		pars = append(pars, endDate)
 	}
+	//获取已存在指标所有数据
+	existDataList := make([]*EdbDataBase, 0)
+	dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_GL)
+	sql := `SELECT * FROM %s WHERE edb_info_id=? `
+	sql = fmt.Sprintf(sql, dataTableName)
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)
+	if err != nil {
+		return err
+	}
+	existDataMap := make(map[string]string)
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v.Value
+	}
 
 	glDataList, err := GetGlDataByTradeCode(condition, pars)
 
 	addSql := ` INSERT INTO edb_data_gl(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
 	var isAdd bool
-	existMap := make(map[string]string)
+	addExistMap := make(map[string]string)
 	for _, v := range glDataList {
 		item := v
-		if _, ok := existMap[v.DataTime]; !ok {
-			count, err := GetEdbDataGlByCodeAndDate(edbCode, v.DataTime)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				return err
-			}
-			if count <= 0 {
+		if _, ok := addExistMap[v.DataTime]; !ok {
+			if existVal, existOk := existDataMap[v.DataTime]; !existOk {
 				eDate := item.DataTime
 				sValue := item.InputValue
 				if sValue != "" {
@@ -112,13 +120,17 @@ func RefreshEdbDataByGl(edbInfoId int, edbCode, startDate, endDate string) (err
 					isAdd = true
 				}
 			} else {
-				err = ModifyEdbDataGl(int64(edbInfoId), v.DataTime, v.InputValue)
-				if err != nil {
-					return err
+				if existVal != item.InputValue {
+					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
+					sql = fmt.Sprintf(sql, dataTableName)
+					_, err = o.Raw(sql, item.InputValue, edbInfoId, v.DataTime).Exec()
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
-		existMap[v.DataTime] = v.InputValue
+		addExistMap[v.DataTime] = v.InputValue
 	}
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
@@ -129,4 +141,4 @@ func RefreshEdbDataByGl(edbInfoId int, edbCode, startDate, endDate string) (err
 	}
 
 	return
-}
+}

models/data_manage/edb_data_lz.go

@@ -124,35 +124,51 @@ func RefreshEdbDataByLz(edbInfoId int, edbCode, startDate, endDate string) (err
 
 	lzDataList, err := GetLzSurveyDataByTradeCode(condition, pars)
 
+	//获取已存在指标所有数据
+	existDataList := make([]*EdbDataBase, 0)
+	dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_LZ)
+	sql := `SELECT * FROM %s WHERE edb_info_id=? `
+	sql = fmt.Sprintf(sql, dataTableName)
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)
+	if err != nil {
+		return err
+	}
+	existDataMap := make(map[string]string)
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v.Value
+	}
+
 	addSql := ` INSERT INTO edb_data_lz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
 	var isAdd bool
+	addExistMap := make(map[string]string)
 	for _, v := range lzDataList {
 		item := v
-		count, err := GetEdbDataLzByCodeAndDate(edbCode, v.DataTime)
-		if err != nil && err.Error() != utils.ErrNoRow() {
-			return err
-		}
-		if count <= 0 {
-			nowStr := time.Now().Format(utils.FormatDateTime)
-			eDate := item.DataTime
-			sValue := item.InputValue
-			dataTime, err := time.Parse(utils.FormatDate, eDate)
-			if err != nil {
-				return err
-			}
-			timestamp := dataTime.UnixNano() / 1e6
-			timeStr := fmt.Sprintf("%d", timestamp)
-			addSql += "("
-			addSql += edbInfoIdStr + "," + "'" + edbCode + "'" + "," + "'" + eDate + "'" + "," + sValue + "," + "'" + nowStr + "'" +
-				"," + "'" + nowStr + "'" + "," + "1" + "," + "'" + timeStr + "'"
-			addSql += "),"
-			isAdd = true
-		} else {
-			err = ModifyEdbDataLz(int64(edbInfoId), v.DataTime, v.InputValue)
-			if err != nil {
-				return err
+		if _, ok := addExistMap[v.DataTime]; !ok {
+			if existVal, existOk := existDataMap[v.DataTime]; !existOk {
+				eDate := item.DataTime
+				sValue := item.InputValue
+				if sValue != "" {
+					dataTime, err := time.Parse(utils.FormatDate, eDate)
+					if err != nil {
+						return err
+					}
+					timestamp := dataTime.UnixNano() / 1e6
+					timeStr := fmt.Sprintf("%d", timestamp)
+					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+					isAdd = true
+				}
+			} else {
+				if existVal != item.InputValue {
+					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
+					sql = fmt.Sprintf(sql, dataTableName)
+					_, err = o.Raw(sql, item.InputValue, edbInfoId, v.DataTime).Exec()
+					if err != nil {
+						return err
+					}
+				}
 			}
 		}
+		addExistMap[v.DataTime] = v.InputValue
 	}
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
@@ -161,16 +177,5 @@ func RefreshEdbDataByLz(edbInfoId int, edbCode, startDate, endDate string) (err
 			return err
 		}
 	}
-
-	maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(utils.DATA_SOURCE_LZ, edbCode)
-	if err != nil {
-		return err
-	}
-	if maxAndMinItem != nil {
-		err = ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem)
-		if err != nil {
-			return err
-		}
-	}
 	return
 }

models/data_manage/edb_data_ys.go

@@ -54,10 +54,15 @@ func RefreshEdbDataByYs(edbInfoId int, edbCode, startDate, endDate string) (err
 	condition += " AND base_from_smm_data=? "
 	pars = append(pars, edbCode)
 
-	condition += " AND data_time>=? "
-	pars = append(pars, startDate)
-	condition += " AND data_time<=? "
-	pars = append(pars, endDate)
+	if startDate!="" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	if endDate!="" {
+		condition += " AND data_time<=? "
+		pars = append(pars, endDate)
+	}
 
 	ysBaseDataAll, err := GetBaseFromSmmDataByCondition(condition, pars)
 	if err != nil {

services/data/edb_info_sync.go

@@ -0,0 +1,109 @@
+package data
+
+import (
+	"hongze/hongze_task/models/data_manage"
+	"hongze/hongze_task/utils"
+)
+
+//同步钢炼数据
+func SyncGlDataBase() {
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info("SyncGlDataBase Err:" + err.Error())
+		}
+	}()
+	var condition string
+	var pars []interface{}
+	condition += " AND source=? "
+	pars = append(pars, utils.DATA_SOURCE_GL)
+	items, err := data_manage.GetEdbInfoByCondition(condition, pars, 0)
+	if err != nil {
+		return
+	}
+	for _, v := range items {
+		err = data_manage.RefreshEdbDataByGl(v.EdbInfoId, v.EdbCode, v.StartDate.Format(utils.FormatDate), "")
+		if err != nil {
+			return
+		}
+
+		item, err := data_manage.GetEdbInfoMaxAndMinInfo(v.Source, v.EdbCode)
+		if err != nil {
+			return
+		}
+		if item != nil {
+			err = data_manage.ModifyEdbInfoMaxAndMinInfo(v.EdbInfoId, item)
+			if err != nil {
+				return
+			}
+		}
+	}
+}
+
+//同步隆重数据
+func SyncLzDataBase() {
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info("SyncLzDataBase Err:" + err.Error())
+		}
+	}()
+	var condition string
+	var pars []interface{}
+	condition += " AND source=? "
+	pars = append(pars, utils.DATA_SOURCE_LZ)
+	items, err := data_manage.GetEdbInfoByCondition(condition, pars, 0)
+	if err != nil {
+		return
+	}
+	for _, v := range items {
+		err = data_manage.RefreshEdbDataByLz(v.EdbInfoId, v.EdbCode, v.StartDate.Format(utils.FormatDate), "")
+		if err != nil {
+			return
+		}
+		item, err := data_manage.GetEdbInfoMaxAndMinInfo(v.Source, v.EdbCode)
+		if err != nil {
+			return
+		}
+		if item != nil {
+			err = data_manage.ModifyEdbInfoMaxAndMinInfo(v.EdbInfoId, item)
+			if err != nil {
+				return
+			}
+		}
+	}
+}
+
+//同步有色数据
+func SyncSmmDataBase() {
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info("SyncSmmDataBase Err:" + err.Error())
+		}
+	}()
+	var condition string
+	var pars []interface{}
+	condition += " AND source=? "
+	pars = append(pars, utils.DATA_SOURCE_YS)
+	items, err := data_manage.GetEdbInfoByCondition(condition, pars, 0)
+	if err != nil {
+		return
+	}
+	for _, v := range items {
+		err = data_manage.RefreshEdbDataByYs(v.EdbInfoId, v.EdbCode, v.StartDate.Format(utils.FormatDate), "")
+		if err != nil {
+			return
+		}
+		item, err := data_manage.GetEdbInfoMaxAndMinInfo(v.Source, v.EdbCode)
+		if err != nil {
+			return
+		}
+		if item != nil {
+			err = data_manage.ModifyEdbInfoMaxAndMinInfo(v.EdbInfoId, item)
+			if err != nil {
+				return
+			}
+		}
+	}
+}

services/data_source_longzhong.go

@@ -1185,8 +1185,8 @@ func GetLzSurveyProductData(cont context.Context) (err error) {
 							return err
 						}
 					} else {
-						if findItem.InputValue != n.InputValue && n.InputValue != "0" {
-							models.ModifyLzSurveyData(n.InputValue,findItem.SurveyDataId)
+						if findItem != nil && findItem.InputValue != n.InputValue && n.InputValue != "0" {
+							models.ModifyLzSurveyData(n.InputValue, findItem.SurveyDataId)
 						}
 						fmt.Println("exist:", shouldDateTimeStr)
 					}

services/task.go

@@ -77,10 +77,10 @@ func releaseTask() {
 	//task.AddTask("getLzProductPriceData", getLzProductPriceData)
 
 	//隆众调研指标获取
-	getLzSurveyProduct := task.NewTask("getLzSurveyProduct", "0 5 11-19/1 * * * ", GetLzSurveyProduct)
+	getLzSurveyProduct := task.NewTask("getLzSurveyProduct", "0 5 08-19/1 * * * ", GetLzSurveyProduct)
 	task.AddTask("getLzSurveyProduct", getLzSurveyProduct)
 	//隆众调研指标数据获取
-	getLzSurveyProductData := task.NewTask("getLzSurveyProductData", "0 10 11-19/1 * * * ", GetLzSurveyProductData)
+	getLzSurveyProductData := task.NewTask("getLzSurveyProductData", "0 10 08-19/1 * * * ", GetLzSurveyProductData)
 	task.AddTask("getLzSurveyProductData", getLzSurveyProductData)
 
 	//发送邮件
@@ -106,6 +106,10 @@ func releaseTask() {
 	//refreshBaseData := task.NewTask("refreshBaseData", "0 1 18,23 * * *", RefreshBaseData)
 	//task.AddTask("refreshBaseData", refreshBaseData)
 
+	//刷新弘则数据库中来自，钢联，隆众，有色等基础数据
+	//refreshHzBaseData := task.NewTask("refreshHzBaseData", "0 */5 * * * * ", RefreshHzBaseData)
+	//task.AddTask("refreshHzBaseData", refreshHzBaseData)
+
 	// 定时往同花顺同步微信群的截止日期（凌晨4点）
 	syncThsWxGroupEveryDay := task.NewTask("syncThsWxGroupEveryDay", "0 1 4 * * * ", SyncWxGroupEveryDay)
 	task.AddTask("定时往同花顺同步微信群的截止日期", syncThsWxGroupEveryDay)
@@ -181,6 +185,15 @@ func RefreshBaseData(cont context.Context) (err error) {
 	return
 }
 
+//刷新基础数据
+func RefreshHzBaseData(cont context.Context) (err error) {
+	//同步钢联基础数据
+	go data.SyncGlDataBase()
+	//同步隆众基础数据
+	//同步有色基础数据
+	return
+}
+
 //func RefreshCalculateData(cont context.Context) (err error) {
 //	//计算指标
 //	go data.RefreshDataFromCalculateAll()

utils/constants.go

@@ -22,7 +22,7 @@ const (
 const (
 	APPNAME                        = "弘则-task"
 	EmailSendToUsers               = "317699326@qq.com;984198890@qq.com;512188925@qq.com"
-	RefreshEdbInfoEmailSendToUsers = "317699326@qq.com;jhwang@hzinsights.com;lnyan@hzinsights.com"
+	RefreshEdbInfoEmailSendToUsers = "317699326@qq.com;jhwang@hzinsights.com;lnyan@hzinsights.com;vwang@hzinsights.com"
 )
 
 //手机号，电子邮箱正则