hongze_cygx/services/task.go

package services

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/astaxie/beego/toolbox"
	"hongze/hongze_cygx/models"
	"hongze/hongze_cygx/utils"
	"html"
	"strconv"
	"strings"
	//"github.com/astaxie/beego/cache"
)

func Task() {
	fmt.Println("start")
	//同步数据
	if utils.RunMode == "release" {
		//syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList)
		//toolbox.AddTask("syncTacticsList", syncTacticsList)
		//toolbox.StartTask()
		syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 */5 * * * *", SyncTacticsListAddreport)
		toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)

		//同步纪要库中的Es
		syncTacticsListToEs := toolbox.NewTask("syncTacticsListToEs", "0 */30 * * * *", SyncTacticsListToEs)
		toolbox.AddTask("syncTacticsListToEs", syncTacticsListToEs)
		toolbox.StartTask()
	}

	//if utils.RunMode == "debug" {
	//	//syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList)
	//	//toolbox.AddTask("syncTacticsList", syncTacticsList)
	//	//toolbox.StartTask()
	//	syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 01 01 * * *", SyncTacticsListAddreport)
	//	toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
	//
	//	//同步纪要库中的Es
	//	syncTacticsListToEs := toolbox.NewTask("syncTacticsListToEs", "0 01 01 * * *", SyncTacticsListToEs)
	//	toolbox.AddTask("syncTacticsListToEs", syncTacticsListToEs)
	//	toolbox.StartTask()
	//}
	//syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 42 09 * * *", SyncTacticsListAddreport)
	//toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
	//toolbox.StartTask()

	fmt.Println("end")
}

func ElasticOption() {
	//SyncTacticsList()
	//toolbox.StartTask()
	//keyWord := "专家"
	//pageSize := 20
	//SearchByKeyWord(keyWord, pageSize)
	//SaveData()
	//GetIndustryMap()
	//解析报告
	//GetArticleExpert()
	//SearchByKeyWordQuery("")
	//AddMap()

	//同步数据
	//ImportData()
	//AddMap()
	//Search("光伏行业")

	//修复报告内容
	//GetArticleAll()
	//indexName := "article_list"
	//		mappingJson := `{
	//"mappings": {
	// "dynamic": true,
	// "properties": {
	//   "ArticleId": {
	//     "type": "integer"
	//   },
	//   "Title": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "TitleEn": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "UpdateFrequency": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "CreateDate": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "PublishDate": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "Abstract": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "CategoryName": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "SubCategoryName": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "InterviewDate": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "ExpertBackground": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "ExpertNumber": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "Department": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "BodyText": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   },
	//   "ArticleIdMd5": {
	//     "type": "text",
	//     "analyzer": "ik_smart"
	//   }
	// }
	//}
	//}`
	//EsCreateIndex(indexName,mappingJson)

	//插入数据
	//allList, err := models.GetArticleAll()
	//if err != nil {
	//	fmt.Println("GetArticleAll Err:", err.Error())
	//	return
	//}
	//
	//indexName := "article_list"
	//
	//for _, v := range allList {
	//	content := html.UnescapeString(v.Body)
	//	doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
	//	if err != nil {
	//		fmt.Println("create doc err:", err.Error())
	//		return
	//	}
	//	bodyText := doc.Text()
	//
	//	item := new(ElasticArticleDetail)
	//	item.ArticleId = v.ArticleId
	//	item.Title = v.Title
	//	item.TitleEn = v.TitleEn
	//	item.UpdateFrequency = v.UpdateFrequency
	//	item.CreateDate = v.CreateDate
	//	item.PublishDate = v.PublishDate
	//	item.Abstract = v.Abstract
	//	item.CategoryName = v.CategoryName
	//	item.SubCategoryName = v.SubCategoryName
	//	item.InterviewDate = v.InterviewDate
	//	item.ExpertBackground = v.ExpertBackground
	//	item.ExpertNumber = v.ExpertNumber
	//	item.Department = v.Department
	//	item.ArticleIdMd5 = v.ArticleIdMd5
	//	item.BodyText = bodyText
	//	EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
	//}

	//fmt.Println("bm")
	//bm, err := cache.NewCache("file", `{"CachePath":"./cache","FileSuffix":".cache", "EmbedExpiry": "120"}`)
	////
	//if err != nil {
	//	fmt.Println("false",err)
	//}
	////fmt.Println(bm)
	//key:="put_key_one"
	////bm.Put(key,1,1*time.Second)
	//b:=bm.IsExist(key)
	//fmt.Println(b)

	//同步策略文章
	//SyncTacticsList()

	//indexName := "article_list"
	//EsDeleteData(indexName,)
}

/*
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
searchItem.Body = v.Highlight["BodyText"]
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
*/
func CreateIndex() {
	indexName := "cygx_article_v4" //utils.IndexName
	mappingJson := `{
 "mappings": {
   "dynamic": true,
   "properties": {
     "ArticleId": {
       "type": "integer"
     },
     "BodyText": {
       "type": "text",
       "term_vector": "with_positions_offsets",
		"analyzer": "ik_smart"
     },
     "PublishDate": {
       "type": "keyword"
     },
     "SubCategoryName": {
       "type": "text",
       "term_vector": "with_positions_offsets",
		"analyzer": "ik_smart"
     },
     "Title": {
       "type": "text",
       "term_vector": "with_positions_offsets",
		"analyzer": "ik_smart"
     },
     "TitleEn": {
       "type": "text",
       "term_vector": "with_positions_offsets"
     },
     "UpdateFrequency": {
       "type": "text",
       "term_vector": "with_positions_offsets"
     }
   }
 }
}`
	EsCreateIndex(indexName, mappingJson)
}

func AddAllArticle() {
	allList, err := models.GetArticleAll()
	if err != nil {
		fmt.Println("GetArticleAll Err:", err.Error())
		return
	}

	indexName := "cygx_article_v3"

	for _, v := range allList {
		content := html.UnescapeString(v.Body)
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
		if err != nil {
			fmt.Println("create doc err:", err.Error())
			return
		}
		bodyText := doc.Text()

		item := new(ElasticTestArticleDetail)
		item.ArticleId = v.ArticleId
		item.Title = v.Title
		item.PublishDate = v.PublishDate
		item.BodyText = bodyText
		EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
		fmt.Println(v.ArticleId)
	}
}

// "term_vector": "with_positions_offsets"

type ElasticArticleDetail struct {
	ArticleId        int    `description:"报告id"`
	Title            string `description:"标题"`
	TitleEn          string `description:"英文标题 "`
	UpdateFrequency  string `description:"更新周期"`
	CreateDate       string `description:"创建时间"`
	PublishDate      string `description:"发布时间"`
	Abstract         string `description:"摘要"`
	CategoryName     string `description:"一级分类"`
	SubCategoryName  string `description:"二级分类"`
	InterviewDate    string `description:"访谈时间"`
	ExpertBackground string `description:"专家背景"`
	ExpertNumber     string `description:"专家编号"`
	Department       string `description:"作者"`
	ArticleIdMd5     string `description:"纪要id"`
	BodyText         string `description:"内容"`
}

type ElasticTestArticleDetail struct {
	ArticleId   int    `description:"报告id"`
	Title       string `description:"标题"`
	BodyText    string `description:"内容"`
	PublishDate string `description:"发布时间"`
}