报告同步任务

controllers/article.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/PuerkitoBio/goquery"
 	"hongze/hongze_cygx/models"
 	"hongze/hongze_cygx/services"
 	"hongze/hongze_cygx/utils"
@@ -398,7 +399,7 @@ func (this *ArticleCommonController) Tongbu() {
 	}()
 
 	fmt.Println("同步数据")
-	//indexName := utils.IndexName
+	indexName := utils.IndexName
 	//endDate := time.Now().AddDate(0, 0, -7).Format(utils.FormatDate)
 	//list, err := models.GetTacticsList(endDate)
 	list, err := models.GetTacticsListAll2()
@@ -406,48 +407,65 @@ func (this *ArticleCommonController) Tongbu() {
 		fmt.Println("GetTacticsList Err:", err.Error())
 		return
 	}
-
 	fmt.Println("list len:", len(list))
 
+	listSummary, errsu := models.GetReportMappingCategoryID()
+	if errsu != nil {
+		fmt.Println("GetTacticsList Err:", errsu.Error())
+		return
+	}
+
+	listPermission, errper := models.GetPermissionMappingCategoryID()
+	if errper != nil {
+		fmt.Println("GetTacticsList Err:", errper.Error())
+		return
+	}
 	for k, v := range list {
 
 		//是否属于纪要库
-		countSummary, err := models.GetPermissionMappingById(v.CategoryId)
-		if err != nil && err.Error() != utils.ErrNoRow() {
-			br.Msg = "参数解析异常！"
-			br.ErrMsg = "参数解析失败，Err:" + err.Error()
-			return
-		}
-		if countSummary > 0 {
-			v.IsSummary = 1
+		//countSummary, err := models.GetPermissionMappingById(v.CategoryId)
+		//if err != nil && err.Error() != utils.ErrNoRow() {
+		//	br.Msg = "参数解析异常！"
+		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		//	return
+		//}
+		//if countSummary > 0 {
+		//	v.IsSummary = 1
+		//}
+		for _, vSum := range listSummary {
+			if v.CategoryId == vSum.CategoryId {
+				v.IsSummary = 1
+			}
 		}
 		//是否属于报告
-		countReport, err := models.GetReportMappingById(v.CategoryId)
-		if err != nil && err.Error() != utils.ErrNoRow() {
-			br.Msg = "参数解析异常！"
-			br.ErrMsg = "参数解析失败，Err:" + err.Error()
-			return
+		//countReport, err := models.GetReportMappingById(v.CategoryId)
+		//if err != nil && err.Error() != utils.ErrNoRow() {
+		//	br.Msg = "参数解析异常！"
+		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		//	return
+		//}
+		for _, vPer := range listPermission {
+			if v.CategoryId == vPer.CategoryId {
+				v.IsReport = 1
+			}
 		}
-		if countReport > 0 {
-			v.IsReport = 1
+		if v.IsReport > 0 {
 			//是否属于策略 策略自动归类
-			if v.CategoryId == 7 || v.CategoryId == 11 || v.CategoryId == 51 || v.CategoryId == 52 {
+			//是否属于行业报告 行业报告自动归类
+			if v.CategoryId == 7 || v.CategoryId == 11 || v.CategoryId == 51 || v.CategoryId == 52 || v.CategoryId == 64 || v.CategoryId == 80 || v.CategoryId == 87 {
 				v.IsClass = 1
 			}
-
 			if v.CategoryId == 64 || v.CategoryId == 87 || v.CategoryId == 80 {
 				v.ReportType = 2 //是否属于行业报告
 			} else {
 				v.ReportType = 1 //是否属于产业报告
 			}
 		}
-
 		v.Department = "弘则权益研究"
 		fmt.Println(k, v.ArticleId)
 		hh, _ := time.ParseDuration("8h")
 		//pDate := publishDate.Add(hh)
 		v.PublishDate = v.PublishDate.Add(hh)
-
 		//判断是否已经存在
 		if v.ArticleId < 0 {
 			if err != nil {
@@ -462,11 +480,20 @@ func (this *ArticleCommonController) Tongbu() {
 			br.ErrMsg = "参数解析失败，Err:" + err.Error()
 			return
 		}
-
+		fmt.Println(v.IsClass)
 		v.Body = strings.Replace(v.Body, "http://vmp.hzinsights.com", "https://vmp.hzinsights.com", -1)
 		expertNumStr, expertContentStr, interviewDateStr := services.BodyAnalysis(v.Body)
 		if count > 0 {
 			fmt.Println(k, v.ArticleId, "edit")
+			//articleInfo, err := models.GetArticleDetailById(v.ArticleId)
+			//if err != nil {
+			//	br.Msg = "同步失败！文章ID：" + strconv.Itoa(v.ArticleId)
+			//	br.ErrMsg = "同步失败，Err:" + err.Error()
+			//	return
+			//}
+			//if articleInfo.IsClass == 1 {
+			//	v.IsClass = 1
+			//}
 			bodyText, _ := services.GetReportContentTextSub(v.Body)
 			updateParams := make(map[string]interface{})
 			updateParams["Title"] = v.Title
@@ -484,7 +511,7 @@ func (this *ArticleCommonController) Tongbu() {
 			updateParams["ExpertBackground"] = expertContentStr
 			updateParams["ExpertNumber"] = expertNumStr
 			updateParams["InterviewDate"] = interviewDateStr
-			updateParams["IsClass"] = v.IsClass
+			//updateParams["IsClass"] = v.IsClass
 			updateParams["IsSummary"] = v.IsSummary
 			updateParams["IsReport"] = v.IsReport
 			updateParams["ReportType"] = v.ReportType
@@ -530,25 +557,27 @@ func (this *ArticleCommonController) Tongbu() {
 				return
 			}
 		}
-
-		//content := html.UnescapeString(v.Body)
-		//doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
-		//if err != nil {
-		//	fmt.Println("create doc err:", err.Error())
-		//	br.Msg = "参数解析异常！"
-		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
-		//	return
-		//}
-		//doc.Find("a").Each(func(i int, a *goquery.Selection) {
-		//	a.Remove()
-		//})
-		//bodyText := doc.Text()
-		//item := new(ElasticTestArticleDetail)
-		//item.ArticleId = v.ArticleId
-		//item.Title = v.Title
-		//item.BodyText = bodyText
-		//item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
-		//EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
+		//纪要库的数据同步到Es
+		if v.IsSummary == 1 {
+			content := html.UnescapeString(v.Body)
+			doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+			if err != nil {
+				fmt.Println("create doc err:", err.Error())
+				br.Msg = "参数解析异常！"
+				br.ErrMsg = "参数解析失败，Err:" + err.Error()
+				return
+			}
+			doc.Find("a").Each(func(i int, a *goquery.Selection) {
+				a.Remove()
+			})
+			bodyText := doc.Text()
+			item := new(services.ElasticTestArticleDetail)
+			item.ArticleId = v.ArticleId
+			item.Title = v.Title
+			item.BodyText = bodyText
+			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
+			services.EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
+		}
 	}
 	br.Ret = 200
 	br.Success = true

models/article.go

@@ -73,6 +73,7 @@ type ArticleDetail struct {
 	SellerMobile         string `description:"销售手机号"`
 	SellerName           string `description:"销售名称"`
 	ArticleIdMd5         string `description:"纪要id"`
+	IsClass              int    `description:"是否归类，1是，0否"`
 	SellerList           []*SellerRep
 }
 
@@ -266,3 +267,25 @@ func GetReportPermission(categoryName string) (item []*ChartPermission, err erro
 	_, err = o.Raw(sql, categoryName).QueryRows(&item)
 	return
 }
+
+type ReportMappingCategoryRep struct {
+	CategoryId int `description:"文章分类id"`
+}
+
+func GetReportMappingCategoryID() (item []*ReportMappingCategoryRep, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT category_id FROM cygx_permission_mapping  GROUP BY category_id `
+	_, err = o.Raw(sql).QueryRows(&item)
+	return
+}
+
+type PermissionMappingCategoryRep struct {
+	CategoryId int `description:"文章分类id"`
+}
+
+func GetPermissionMappingCategoryID() (item []*PermissionMappingCategoryRep, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT category_id FROM cygx_report_mapping  GROUP BY category_id `
+	_, err = o.Raw(sql).QueryRows(&item)
+	return
+}

models/tactics.go

@@ -88,7 +88,7 @@ func GetTacticsListAll2() (list []*Tactics2, err error) {
 			INNER JOIN article_content AS b ON a.id=b.article_id
 			INNER JOIN article_category AS c ON a.id=c.article_id
 			INNER JOIN article_categoryinfo AS d ON c.category_id=d.id
-			WHERE a.publish_status=1 AND d.id IN (7,11,28,32,45,50,51,52,57,62,64,65,66,67,69,71,74,79,80,84,85,86,87,88,89,90,95) 
+			WHERE a.publish_status=1 AND d.id IN (7,11,28,32,45,47,50,51,52,57,62,64,65,66,67,69,71,74,79,80,84,85,86,87,88,89,90,95) 
             AND a.id NOT IN (3454,3456,3457,3459,2449,2450,2453,2454,2459,2530,2583,2663,2670,2699,2715,2732,2748,2759,2399,2356,2870,3173,2978,2826,3470) `
 	//IN(85,71)
 	_, err = o.Raw(sql).QueryRows(&list)

services/tactics.go

@@ -281,3 +281,190 @@ func BodyAnalysis(body string) (expertNumStr, expertContentStr, interviewDateStr
 	}
 	return
 }
+
+//同步策略文章
+func SyncTacticsListAddreport() (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("同步失败，Err:", err.Error())
+		}
+	}()
+	fmt.Println("同步数据")
+	indexName := utils.IndexName
+	//endDate := time.Now().AddDate(0, 0, -7).Format(utils.FormatDate)
+	//list, err := models.GetTacticsList(endDate)
+	list, err := models.GetTacticsListAll2()
+	if err != nil {
+		fmt.Println("GetTacticsList Err:", err.Error())
+		return
+	}
+	fmt.Println("list len:", len(list))
+
+	listSummary, errsu := models.GetReportMappingCategoryID()
+	if errsu != nil {
+		fmt.Println("GetTacticsList Err:", errsu.Error())
+		return
+	}
+
+	listPermission, errper := models.GetPermissionMappingCategoryID()
+	if errper != nil {
+		fmt.Println("GetTacticsList Err:", errper.Error())
+		return
+	}
+	for k, v := range list {
+
+		//是否属于纪要库
+		//countSummary, err := models.GetPermissionMappingById(v.CategoryId)
+		//if err != nil && err.Error() != utils.ErrNoRow() {
+		//	br.Msg = "参数解析异常！"
+		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		//	return
+		//}
+		//if countSummary > 0 {
+		//	v.IsSummary = 1
+		//}
+		for _, vSum := range listSummary {
+			if v.CategoryId == vSum.CategoryId {
+				v.IsSummary = 1
+			}
+		}
+		//是否属于报告
+		//countReport, err := models.GetReportMappingById(v.CategoryId)
+		//if err != nil && err.Error() != utils.ErrNoRow() {
+		//	br.Msg = "参数解析异常！"
+		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		//	return
+		//}
+		for _, vPer := range listPermission {
+			if v.CategoryId == vPer.CategoryId {
+				v.IsReport = 1
+			}
+		}
+		if v.IsReport > 0 {
+			//是否属于策略 策略自动归类
+			//是否属于行业报告 行业报告自动归类
+			if v.CategoryId == 7 || v.CategoryId == 11 || v.CategoryId == 51 || v.CategoryId == 52 || v.CategoryId == 64 || v.CategoryId == 80 || v.CategoryId == 87 {
+				v.IsClass = 1
+			}
+			if v.CategoryId == 64 || v.CategoryId == 87 || v.CategoryId == 80 {
+				v.ReportType = 2 //是否属于行业报告
+			} else {
+				v.ReportType = 1 //是否属于产业报告
+			}
+		}
+		v.Department = "弘则权益研究"
+		fmt.Println(k, v.ArticleId)
+		hh, _ := time.ParseDuration("8h")
+		//pDate := publishDate.Add(hh)
+		v.PublishDate = v.PublishDate.Add(hh)
+		//判断是否已经存在
+		if v.ArticleId < 0 {
+			if err != nil {
+				fmt.Println("AddCygxArticle Err:", err.Error())
+				return err
+			}
+		}
+		count, err := models.GetArticleCountById(v.ArticleId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			if err != nil {
+				fmt.Println("AddCygxArticle Err:", err.Error())
+				return err
+			}
+		}
+		fmt.Println(v.IsClass)
+		v.Body = strings.Replace(v.Body, "http://vmp.hzinsights.com", "https://vmp.hzinsights.com", -1)
+		expertNumStr, expertContentStr, interviewDateStr := BodyAnalysis(v.Body)
+		if count > 0 {
+			fmt.Println(k, v.ArticleId, "edit")
+			//articleInfo, err := models.GetArticleDetailById(v.ArticleId)
+			//if err != nil {
+			//	br.Msg = "同步失败！文章ID：" + strconv.Itoa(v.ArticleId)
+			//	br.ErrMsg = "同步失败，Err:" + err.Error()
+			//	return
+			//}
+			//if articleInfo.IsClass == 1 {
+			//	v.IsClass = 1
+			//}
+			bodyText, _ := GetReportContentTextSub(v.Body)
+			updateParams := make(map[string]interface{})
+			updateParams["Title"] = v.Title
+			updateParams["TitleEn"] = v.TitleEn
+			updateParams["UpdateFrequency"] = v.UpdateFrequency
+			updateParams["CreateDate"] = v.CreateDate
+			updateParams["PublishDate"] = v.PublishDate
+			updateParams["Body"] = html.EscapeString(v.Body)
+			updateParams["BodyText"] = bodyText
+			updateParams["Abstract"] = html.EscapeString(v.Abstract)
+			updateParams["CategoryName"] = v.CategoryName
+			updateParams["SubCategoryName"] = v.SubCategoryName
+			updateParams["CategoryId"] = v.CategoryId
+			updateParams["PublishStatus"] = v.PublishStatus
+			updateParams["ExpertBackground"] = expertContentStr
+			updateParams["ExpertNumber"] = expertNumStr
+			updateParams["InterviewDate"] = interviewDateStr
+			//updateParams["IsClass"] = v.IsClass
+			updateParams["IsSummary"] = v.IsSummary
+			updateParams["IsReport"] = v.IsReport
+			updateParams["ReportType"] = v.ReportType
+			if v.Department != "弘则权益研究" {
+				v.Department = "弘则权益研究"
+			}
+			updateParams["Department"] = v.Department
+			whereParam := map[string]interface{}{"article_id": v.ArticleId}
+			err = models.UpdateByExpr(models.CygxArticle{}, whereParam, updateParams)
+			if err != nil {
+				fmt.Println("UpdateByExpr Err:" + err.Error())
+			}
+		} else {
+			fmt.Println(k, v.ArticleId, "add")
+			item := new(models.CygxArticle)
+			articleIdInt := v.ArticleId
+			item.ArticleId = articleIdInt
+			item.Title = v.Title
+			item.TitleEn = v.TitleEn
+			item.UpdateFrequency = v.UpdateFrequency
+			item.CreateDate = v.CreateDate
+			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
+			item.Body = html.EscapeString(v.Body)
+			item.Abstract = html.EscapeString(v.Abstract)
+			item.CategoryName = v.CategoryName
+			item.SubCategoryName = v.SubCategoryName
+			item.CategoryId = v.CategoryId
+			item.PublishStatus = v.PublishStatus
+			item.ExpertBackground = expertContentStr
+			item.ExpertNumber = expertNumStr
+			item.InterviewDate = interviewDateStr
+			item.Department = v.Department
+			item.ArticleIdMd5 = utils.MD5(strconv.Itoa(articleIdInt))
+			item.IsClass = v.IsClass
+			item.IsSummary = v.IsSummary
+			item.IsReport = v.IsReport
+			item.ReportType = v.ReportType
+			_, err = models.AddCygxArticles(item)
+			if err != nil {
+				fmt.Println("AddCygxArticle Err:", err.Error())
+				return err
+			}
+		}
+		//纪要库的数据同步到Es
+		if v.IsSummary == 1 {
+			content := html.UnescapeString(v.Body)
+			doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+			if err != nil {
+				fmt.Println("AddCygxArticle Err:", err.Error())
+				return err
+			}
+			doc.Find("a").Each(func(i int, a *goquery.Selection) {
+				a.Remove()
+			})
+			bodyText := doc.Text()
+			item := new(ElasticTestArticleDetail)
+			item.ArticleId = v.ArticleId
+			item.Title = v.Title
+			item.BodyText = bodyText
+			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
+			EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
+		}
+	}
+	return
+}

services/task.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"github.com/astaxie/beego/toolbox"
 	"hongze/hongze_cygx/utils"
-
 	//"github.com/astaxie/beego/cache"
 )
 
@@ -12,8 +11,12 @@ func Task() {
 	fmt.Println("start")
 	//同步数据
 	if utils.RunMode == "release" {
-		syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList)
-		toolbox.AddTask("syncTacticsList", syncTacticsList)
+		//syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList)
+		//toolbox.AddTask("syncTacticsList", syncTacticsList)
+		//toolbox.StartTask()
+
+		syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 */5 * * * *", SyncTacticsListAddreport)
+		toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
 		toolbox.StartTask()
 	}
 	//SyncCygxArticleList()