策略，报告同步

controllers/article.go

@@ -3,7 +3,6 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/PuerkitoBio/goquery"
 	"hongze/hongze_cygx/models"
 	"hongze/hongze_cygx/services"
 	"hongze/hongze_cygx/utils"
@@ -408,196 +407,196 @@ func (this *ArticleCommonController) Detail() {
 // @Param   ArticleIdMd5   query   int  true       "报告ID"
 // @Success 200 {object} models.ArticleDetailResp
 // @router /tongbu [get]
-func (this *ArticleCommonController) Tongbu() {
-	br := new(models.BaseResponse).Init()
-	defer func() {
-		this.Data["json"] = br
-		this.ServeJSON()
-	}()
-
-	fmt.Println("同步数据")
-	indexName := utils.IndexName
-	//endDate := time.Now().AddDate(0, 0, -7).Format(utils.FormatDate)
-	//list, err := models.GetTacticsList(endDate)
-	list, err := models.GetTacticsListAll2()
-	if err != nil {
-		fmt.Println("GetTacticsList Err:", err.Error())
-		return
-	}
-	fmt.Println("list len:", len(list))
-
-	listSummary, errsu := models.GetReportMappingCategoryID()
-	if errsu != nil {
-		fmt.Println("GetTacticsList Err:", errsu.Error())
-		return
-	}
-
-	listPermission, errper := models.GetPermissionMappingCategoryID()
-	if errper != nil {
-		fmt.Println("GetTacticsList Err:", errper.Error())
-		return
-	}
-	for k, v := range list {
-
-		//是否属于纪要库
-		//countSummary, err := models.GetPermissionMappingById(v.CategoryId)
-		//if err != nil && err.Error() != utils.ErrNoRow() {
-		//	br.Msg = "参数解析异常！"
-		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
-		//	return
-		//}
-		//if countSummary > 0 {
-		//	v.IsSummary = 1
-		//}
-		for _, vSum := range listSummary {
-			if v.CategoryId == vSum.CategoryId {
-				v.IsSummary = 1
-			}
-		}
-		//是否属于报告
-		//countReport, err := models.GetReportMappingById(v.CategoryId)
-		//if err != nil && err.Error() != utils.ErrNoRow() {
-		//	br.Msg = "参数解析异常！"
-		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
-		//	return
-		//}
-		for _, vPer := range listPermission {
-			if v.CategoryId == vPer.CategoryId {
-				v.IsReport = 1
-			}
-		}
-		if v.IsReport > 0 {
-			//是否属于策略 策略自动归类
-			//是否属于行业报告 行业报告自动归类
-			if v.CategoryId == 7 || v.CategoryId == 11 || v.CategoryId == 51 || v.CategoryId == 52 || v.CategoryId == 64 || v.CategoryId == 80 || v.CategoryId == 87 {
-				v.IsClass = 1
-			}
-			if v.CategoryId == 64 || v.CategoryId == 87 || v.CategoryId == 80 {
-				v.ReportType = 2 //是否属于行业报告
-			} else {
-				v.ReportType = 1 //是否属于产业报告
-			}
-		}
-		v.Department = "弘则权益研究"
-		fmt.Println(k, v.ArticleId)
-		hh, _ := time.ParseDuration("8h")
-		//pDate := publishDate.Add(hh)
-		v.PublishDate = v.PublishDate.Add(hh)
-		//判断是否已经存在
-		if v.ArticleId < 0 {
-			if err != nil {
-				br.Msg = "参数解析异常！"
-				br.ErrMsg = "参数解析失败，Err:" + err.Error()
-				return
-			}
-		}
-		count, err := models.GetArticleCountById(v.ArticleId)
-		if err != nil && err.Error() != utils.ErrNoRow() {
-			br.Msg = "参数解析异常！"
-			br.ErrMsg = "参数解析失败，Err:" + err.Error()
-			return
-		}
-		fmt.Println(v.IsClass)
-		v.Body = strings.Replace(v.Body, "http://vmp.hzinsights.com", "https://vmp.hzinsights.com", -1)
-		expertNumStr, expertContentStr, interviewDateStr := services.BodyAnalysis(v.Body)
-		if count > 0 {
-			fmt.Println(k, v.ArticleId, "edit")
-			//articleInfo, err := models.GetArticleDetailById(v.ArticleId)
-			//if err != nil {
-			//	br.Msg = "同步失败！文章ID：" + strconv.Itoa(v.ArticleId)
-			//	br.ErrMsg = "同步失败，Err:" + err.Error()
-			//	return
-			//}
-			//if articleInfo.IsClass == 1 {
-			//	v.IsClass = 1
-			//}
-			bodyText, _ := services.GetReportContentTextSub(v.Body)
-			updateParams := make(map[string]interface{})
-			updateParams["Title"] = v.Title
-			updateParams["TitleEn"] = v.TitleEn
-			updateParams["UpdateFrequency"] = v.UpdateFrequency
-			updateParams["CreateDate"] = v.CreateDate
-			updateParams["PublishDate"] = v.PublishDate
-			updateParams["Body"] = html.EscapeString(v.Body)
-			updateParams["BodyText"] = bodyText
-			updateParams["Abstract"] = html.EscapeString(v.Abstract)
-			updateParams["CategoryName"] = v.CategoryName
-			updateParams["SubCategoryName"] = v.SubCategoryName
-			updateParams["CategoryId"] = v.CategoryId
-			updateParams["PublishStatus"] = v.PublishStatus
-			updateParams["ExpertBackground"] = expertContentStr
-			updateParams["ExpertNumber"] = expertNumStr
-			updateParams["InterviewDate"] = interviewDateStr
-			//updateParams["IsClass"] = v.IsClass
-			updateParams["IsSummary"] = v.IsSummary
-			updateParams["IsReport"] = v.IsReport
-			updateParams["ReportType"] = v.ReportType
-			if v.Department != "弘则权益研究" {
-				v.Department = "弘则权益研究"
-			}
-			updateParams["Department"] = v.Department
-			whereParam := map[string]interface{}{"article_id": v.ArticleId}
-			err = models.UpdateByExpr(models.CygxArticle{}, whereParam, updateParams)
-			if err != nil {
-				fmt.Println("UpdateByExpr Err:" + err.Error())
-			}
-		} else {
-			fmt.Println(k, v.ArticleId, "add")
-			item := new(models.CygxArticle)
-			articleIdInt := v.ArticleId
-			item.ArticleId = articleIdInt
-			item.Title = v.Title
-			item.TitleEn = v.TitleEn
-			item.UpdateFrequency = v.UpdateFrequency
-			item.CreateDate = v.CreateDate
-			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
-			item.Body = html.EscapeString(v.Body)
-			item.Abstract = html.EscapeString(v.Abstract)
-			item.CategoryName = v.CategoryName
-			item.SubCategoryName = v.SubCategoryName
-			item.CategoryId = v.CategoryId
-			item.PublishStatus = v.PublishStatus
-			item.ExpertBackground = expertContentStr
-			item.ExpertNumber = expertNumStr
-			item.InterviewDate = interviewDateStr
-			item.Department = v.Department
-			item.ArticleIdMd5 = utils.MD5(strconv.Itoa(articleIdInt))
-			item.IsClass = v.IsClass
-			item.IsSummary = v.IsSummary
-			item.IsReport = v.IsReport
-			item.ReportType = v.ReportType
-			_, err = models.AddCygxArticles(item)
-			if err != nil {
-				fmt.Println("AddCygxArticle Err:", err.Error())
-				br.Msg = "参数解析异常！"
-				br.ErrMsg = "参数解析失败，Err:" + err.Error()
-				return
-			}
-		}
-		//纪要库的数据同步到Es
-		if v.IsSummary == 1 {
-			content := html.UnescapeString(v.Body)
-			doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
-			if err != nil {
-				fmt.Println("create doc err:", err.Error())
-				br.Msg = "参数解析异常！"
-				br.ErrMsg = "参数解析失败，Err:" + err.Error()
-				return
-			}
-			doc.Find("a").Each(func(i int, a *goquery.Selection) {
-				a.Remove()
-			})
-			bodyText := doc.Text()
-			item := new(services.ElasticTestArticleDetail)
-			item.ArticleId = v.ArticleId
-			item.Title = v.Title
-			item.BodyText = bodyText
-			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
-			services.EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
-		}
-	}
-	br.Ret = 200
-	br.Success = true
-	br.Msg = "同步成功"
-
-}
+//func (this *ArticleCommonController) Tongbu() {
+//	br := new(models.BaseResponse).Init()
+//	defer func() {
+//		this.Data["json"] = br
+//		this.ServeJSON()
+//	}()
+//
+//	fmt.Println("同步数据")
+//	indexName := utils.IndexName
+//	//endDate := time.Now().AddDate(0, 0, -7).Format(utils.FormatDate)
+//	//list, err := models.GetTacticsList(endDate)
+//	list, err := models.GetTacticsListAll2()
+//	if err != nil {
+//		fmt.Println("GetTacticsList Err:", err.Error())
+//		return
+//	}
+//	fmt.Println("list len:", len(list))
+//
+//	listSummary, errsu := models.GetReportMappingCategoryID()
+//	if errsu != nil {
+//		fmt.Println("GetTacticsList Err:", errsu.Error())
+//		return
+//	}
+//
+//	listPermission, errper := models.GetPermissionMappingCategoryID()
+//	if errper != nil {
+//		fmt.Println("GetTacticsList Err:", errper.Error())
+//		return
+//	}
+//	for k, v := range list {
+//
+//		//是否属于纪要库
+//		//countSummary, err := models.GetPermissionMappingById(v.CategoryId)
+//		//if err != nil && err.Error() != utils.ErrNoRow() {
+//		//	br.Msg = "参数解析异常！"
+//		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
+//		//	return
+//		//}
+//		//if countSummary > 0 {
+//		//	v.IsSummary = 1
+//		//}
+//		for _, vSum := range listSummary {
+//			if v.CategoryId == vSum.CategoryId {
+//				v.IsSummary = 1
+//			}
+//		}
+//		//是否属于报告
+//		//countReport, err := models.GetReportMappingById(v.CategoryId)
+//		//if err != nil && err.Error() != utils.ErrNoRow() {
+//		//	br.Msg = "参数解析异常！"
+//		//	br.ErrMsg = "参数解析失败，Err:" + err.Error()
+//		//	return
+//		//}
+//		for _, vPer := range listPermission {
+//			if v.CategoryId == vPer.CategoryId {
+//				v.IsReport = 1
+//			}
+//		}
+//		if v.IsReport > 0 {
+//			//是否属于策略 策略自动归类
+//			//是否属于行业报告 行业报告自动归类
+//			if v.CategoryId == 7 || v.CategoryId == 11 || v.CategoryId == 51 || v.CategoryId == 52 || v.CategoryId == 64 || v.CategoryId == 80 || v.CategoryId == 87 {
+//				v.IsClass = 1
+//			}
+//			if v.CategoryId == 64 || v.CategoryId == 87 || v.CategoryId == 80 {
+//				v.ReportType = 2 //是否属于行业报告
+//			} else {
+//				v.ReportType = 1 //是否属于产业报告
+//			}
+//		}
+//		v.Department = "弘则权益研究"
+//		fmt.Println(k, v.ArticleId)
+//		hh, _ := time.ParseDuration("8h")
+//		//pDate := publishDate.Add(hh)
+//		v.PublishDate = v.PublishDate.Add(hh)
+//		//判断是否已经存在
+//		if v.ArticleId < 0 {
+//			if err != nil {
+//				br.Msg = "参数解析异常！"
+//				br.ErrMsg = "参数解析失败，Err:" + err.Error()
+//				return
+//			}
+//		}
+//		count, err := models.GetArticleCountById(v.ArticleId)
+//		if err != nil && err.Error() != utils.ErrNoRow() {
+//			br.Msg = "参数解析异常！"
+//			br.ErrMsg = "参数解析失败，Err:" + err.Error()
+//			return
+//		}
+//		fmt.Println(v.IsClass)
+//		v.Body = strings.Replace(v.Body, "http://vmp.hzinsights.com", "https://vmp.hzinsights.com", -1)
+//		expertNumStr, expertContentStr, interviewDateStr := services.BodyAnalysis(v.Body)
+//		if count > 0 {
+//			fmt.Println(k, v.ArticleId, "edit")
+//			//articleInfo, err := models.GetArticleDetailById(v.ArticleId)
+//			//if err != nil {
+//			//	br.Msg = "同步失败！文章ID：" + strconv.Itoa(v.ArticleId)
+//			//	br.ErrMsg = "同步失败，Err:" + err.Error()
+//			//	return
+//			//}
+//			//if articleInfo.IsClass == 1 {
+//			//	v.IsClass = 1
+//			//}
+//			bodyText, _ := services.GetReportContentTextSub(v.Body)
+//			updateParams := make(map[string]interface{})
+//			updateParams["Title"] = v.Title
+//			updateParams["TitleEn"] = v.TitleEn
+//			updateParams["UpdateFrequency"] = v.UpdateFrequency
+//			updateParams["CreateDate"] = v.CreateDate
+//			updateParams["PublishDate"] = v.PublishDate
+//			updateParams["Body"] = html.EscapeString(v.Body)
+//			updateParams["BodyText"] = bodyText
+//			updateParams["Abstract"] = html.EscapeString(v.Abstract)
+//			updateParams["CategoryName"] = v.CategoryName
+//			updateParams["SubCategoryName"] = v.SubCategoryName
+//			updateParams["CategoryId"] = v.CategoryId
+//			updateParams["PublishStatus"] = v.PublishStatus
+//			updateParams["ExpertBackground"] = expertContentStr
+//			updateParams["ExpertNumber"] = expertNumStr
+//			updateParams["InterviewDate"] = interviewDateStr
+//			updateParams["IsClass"] = v.IsClass
+//			updateParams["IsSummary"] = v.IsSummary
+//			updateParams["IsReport"] = v.IsReport
+//			updateParams["ReportType"] = v.ReportType
+//			if v.Department != "弘则权益研究" {
+//				v.Department = "弘则权益研究"
+//			}
+//			updateParams["Department"] = v.Department
+//			whereParam := map[string]interface{}{"article_id": v.ArticleId}
+//			err = models.UpdateByExpr(models.CygxArticle{}, whereParam, updateParams)
+//			if err != nil {
+//				fmt.Println("UpdateByExpr Err:" + err.Error())
+//			}
+//		} else {
+//			fmt.Println(k, v.ArticleId, "add")
+//			item := new(models.CygxArticle)
+//			articleIdInt := v.ArticleId
+//			item.ArticleId = articleIdInt
+//			item.Title = v.Title
+//			item.TitleEn = v.TitleEn
+//			item.UpdateFrequency = v.UpdateFrequency
+//			item.CreateDate = v.CreateDate
+//			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
+//			item.Body = html.EscapeString(v.Body)
+//			item.Abstract = html.EscapeString(v.Abstract)
+//			item.CategoryName = v.CategoryName
+//			item.SubCategoryName = v.SubCategoryName
+//			item.CategoryId = v.CategoryId
+//			item.PublishStatus = v.PublishStatus
+//			item.ExpertBackground = expertContentStr
+//			item.ExpertNumber = expertNumStr
+//			item.InterviewDate = interviewDateStr
+//			item.Department = v.Department
+//			item.ArticleIdMd5 = utils.MD5(strconv.Itoa(articleIdInt))
+//			item.IsClass = v.IsClass
+//			item.IsSummary = v.IsSummary
+//			item.IsReport = v.IsReport
+//			item.ReportType = v.ReportType
+//			_, err = models.AddCygxArticles(item)
+//			if err != nil {
+//				fmt.Println("AddCygxArticle Err:", err.Error())
+//				br.Msg = "参数解析异常！"
+//				br.ErrMsg = "参数解析失败，Err:" + err.Error()
+//				return
+//			}
+//		}
+//		//纪要库的数据同步到Es
+//		if v.IsSummary == 1 {
+//			content := html.UnescapeString(v.Body)
+//			doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+//			if err != nil {
+//				fmt.Println("create doc err:", err.Error())
+//				br.Msg = "参数解析异常！"
+//				br.ErrMsg = "参数解析失败，Err:" + err.Error()
+//				return
+//			}
+//			doc.Find("a").Each(func(i int, a *goquery.Selection) {
+//				a.Remove()
+//			})
+//			bodyText := doc.Text()
+//			item := new(services.ElasticTestArticleDetail)
+//			item.ArticleId = v.ArticleId
+//			item.Title = v.Title
+//			item.BodyText = bodyText
+//			item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
+//			services.EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
+//		}
+//	}
+//	br.Ret = 200
+//	br.Success = true
+//	br.Msg = "同步成功"
+//
+//}

models/tactics.go

@@ -80,6 +80,21 @@ type Tactics2 struct {
 	ReportType      int       `description:"报告类型，1行业报告，2产业报告,0无"`
 }
 
+func GetTacticsList2(endDate string) (list []*Tactics2, err error) {
+	o := orm.NewOrm()
+	o.Using("tactics")
+	sql := `SELECT a.*,b.body,b.abstract,b.annotation,b.article_id,d.category_name,d.sub_category_name,c.category_id
+			FROM article_articleinfo AS a
+			INNER JOIN article_content AS b ON a.id=b.article_id
+			INNER JOIN article_category AS c ON a.id=c.article_id
+			INNER JOIN article_categoryinfo AS d ON c.category_id=d.id
+			WHERE a.publish_status=1  AND a.publish_date>=? AND d.id IN (7,11,28,32,45,47,50,51,52,57,62,64,65,66,67,69,71,74,79,80,84,85,86,87,88,89,90,95) 
+            AND a.id NOT IN (3454,3456,3457,3459,2449,2450,2453,2454,2459,2530,2583,2663,2670,2699,2715,2732,2748,2759,2399,2356,2870,3173,2978,2826,3470) `
+	//IN(85,71)
+	_, err = o.Raw(sql, endDate).QueryRows(&list)
+	return
+}
+
 func GetTacticsListAll2() (list []*Tactics2, err error) {
 	o := orm.NewOrm()
 	o.Using("tactics")
@@ -88,7 +103,7 @@ func GetTacticsListAll2() (list []*Tactics2, err error) {
 			INNER JOIN article_content AS b ON a.id=b.article_id
 			INNER JOIN article_category AS c ON a.id=c.article_id
 			INNER JOIN article_categoryinfo AS d ON c.category_id=d.id
-			WHERE a.publish_status=1 AND d.id IN (7,11,28,32,45,47,50,51,52,57,62,64,65,66,67,69,71,74,79,80,84,85,86,87,88,89,90,95) 
+			WHERE a.publish_status=1  AND d.id IN (7,11,28,32,45,47,50,51,52,57,62,64,65,66,67,69,71,74,79,80,84,85,86,87,88,89,90,95) 
             AND a.id NOT IN (3454,3456,3457,3459,2449,2450,2453,2454,2459,2530,2583,2663,2670,2699,2715,2732,2748,2759,2399,2356,2870,3173,2978,2826,3470) `
 	//IN(85,71)
 	_, err = o.Raw(sql).QueryRows(&list)

services/tactics.go

@@ -21,7 +21,7 @@ func SyncTacticsList() (err error) {
 	fmt.Println("同步数据")
 	indexName := utils.IndexName
 	endDate := time.Now().AddDate(0, 0, -30).Format(utils.FormatDate)
-	list, err := models.GetTacticsList(endDate)
+	list, err := models.GetTacticsList2(endDate)
 	//list, err := models.GetTacticsListAll()
 	if err != nil {
 		fmt.Println("GetTacticsList Err:", err.Error())
@@ -282,7 +282,7 @@ func BodyAnalysis(body string) (expertNumStr, expertContentStr, interviewDateStr
 	return
 }
 
-//同步策略文章
+//同步策略,报告文章
 func SyncTacticsListAddreport() (err error) {
 	defer func() {
 		if err != nil {
@@ -291,9 +291,9 @@ func SyncTacticsListAddreport() (err error) {
 	}()
 	fmt.Println("同步数据")
 	indexName := utils.IndexName
-	//endDate := time.Now().AddDate(0, 0, -7).Format(utils.FormatDate)
-	//list, err := models.GetTacticsList(endDate)
-	list, err := models.GetTacticsListAll2()
+	endDate := time.Now().AddDate(0, 0, -30).Format(utils.FormatDate)
+	list, err := models.GetTacticsList2(endDate)
+	//list, err := models.GetTacticsListAll2()
 	if err != nil {
 		fmt.Println("GetTacticsList Err:", err.Error())
 		return
@@ -312,7 +312,6 @@ func SyncTacticsListAddreport() (err error) {
 		return
 	}
 	for k, v := range list {
-
 		//是否属于纪要库
 		//countSummary, err := models.GetPermissionMappingById(v.CategoryId)
 		//if err != nil && err.Error() != utils.ErrNoRow() {
@@ -402,7 +401,7 @@ func SyncTacticsListAddreport() (err error) {
 			updateParams["ExpertBackground"] = expertContentStr
 			updateParams["ExpertNumber"] = expertNumStr
 			updateParams["InterviewDate"] = interviewDateStr
-			//updateParams["IsClass"] = v.IsClass
+			updateParams["IsClass"] = v.IsClass
 			updateParams["IsSummary"] = v.IsSummary
 			updateParams["IsReport"] = v.IsReport
 			updateParams["ReportType"] = v.ReportType

services/task.go

@@ -15,11 +15,10 @@ func Task() {
 		//toolbox.AddTask("syncTacticsList", syncTacticsList)
 		//toolbox.StartTask()
 
-		syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 */5 * * * *", SyncTacticsListAddreport)
+		syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 16 16 * * *", SyncTacticsListAddreport)
 		toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
 		toolbox.StartTask()
 	}
-	//SyncCygxArticleList()
 	fmt.Println("end")
 }