package services import ( "fmt" "github.com/PuerkitoBio/goquery" //"github.com/beego/beego/v2/adapter/toolbox" "github.com/beego/beego/v2/task" "hongze/hongze_cygx/models" "hongze/hongze_cygx/utils" "html" "strconv" "strings" "time" //"github.com/beego/beego/v2/adapter/cache" ) func Task() { fmt.Println("start") //同步数据 if utils.RunMode == "release" { //syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList) //toolbox.AddTask("syncTacticsList", syncTacticsList) //toolbox.StartTask() syncTacticsListAddreport := task.NewTask("syncTacticsListAddreport", "0 */5 * * * *", SyncTacticsListAddreport) task.AddTask("syncTacticsListAddreport", syncTacticsListAddreport) //同步纪要库中的Es //syncTacticsListToEs := toolbox.NewTask("syncTacticsListToEs", "0 */30 * * * *", SyncTacticsListToEs) //toolbox.AddTask("syncTacticsListToEs", syncTacticsListToEs) getSummarytoEs := task.NewTask("getSummarytoEs", "0 */30 * * * *", GetSummarytoEs) //同步纪要库内容到Es task.AddTask("getSummarytoEs", getSummarytoEs) //toolbox.StartTask() } //UpdateActivitySattus() //修改任务状态 updateActivitySattus := task.NewTask("syncTacticsListAddreport", "0 */1 8-22 * * *", UpdateActivitySattus) task.AddTask("updateActivitySattus", updateActivitySattus) //更新深 和新的标签 updateIndustrialManagementLabel := task.NewTask("updateIndustrialManagementLabel", "0 */30 * * * *", UpdateIndustrialManagementLabel) task.AddTask("updateIndustrialManagementLabel", updateIndustrialManagementLabel) //更新报告的搜索栏内容 //UpdateIndustrialManagementSubjectNnames() //会议提醒模板消息推送 sendActivityBeginMsg := task.NewTask("sendActivityBeginMsg", "0 */10 8-22 * * *", SendActivityBeginMsg) //会议前60分钟的提醒 task.AddTask("sendActivityBeginMsg", sendActivityBeginMsg) sendActivityBeginMsgMeeting := task.NewTask("sendActivityBeginMsgMeeting", "0 */5 8-22 * * *", SendActivityBeginMsgMeeting) //会议前15分钟的提醒 task.AddTask("sendActivityBeginMsgMeeting", sendActivityBeginMsgMeeting) //预约外呼名单,会前1小时自动发送邮件给专家组 sendEmailFileToExpert := task.NewTask("sendEmailFileToExpert", "0 */5 8-22 * * *", SendEmailFileToExpert) //预约外呼名单,会前1小时自动发送邮件给专家组 task.AddTask("sendEmailFileToExpert", sendEmailFileToExpert) //editOutboundMobile := task.NewTask("editOutboundMobile", "0 */1 8-22 * * *", EditOutboundMobile) //同步外呼号码与手机号 //task.AddTask("sendEmailFileToExpert", editOutboundMobile) //editUserOutboundMobile := task.NewTask("editUserOutboundMobile", "0 27 14 * * * ", EditUserOutboundMobile) //同步外呼号码与手机号 //task.AddTask("editUserOutboundMobile", editUserOutboundMobile) task.StartTask() fmt.Println("end") } func ElasticOption() { //SyncTacticsList() //toolbox.StartTask() //keyWord := "专家" //pageSize := 20 //SearchByKeyWord(keyWord, pageSize) //SaveData() //GetIndustryMap() //解析报告 //GetArticleExpert() //SearchByKeyWordQuery("") //AddMap() //同步数据 //ImportData() //AddMap() //Search("光伏行业") //修复报告内容 //GetArticleAll() //indexName := "article_list" // mappingJson := `{ //"mappings": { // "dynamic": true, // "properties": { // "ArticleId": { // "type": "integer" // }, // "Title": { // "type": "text", // "analyzer": "ik_smart" // }, // "TitleEn": { // "type": "text", // "analyzer": "ik_smart" // }, // "UpdateFrequency": { // "type": "text", // "analyzer": "ik_smart" // }, // "CreateDate": { // "type": "text", // "analyzer": "ik_smart" // }, // "PublishDate": { // "type": "text", // "analyzer": "ik_smart" // }, // "Abstract": { // "type": "text", // "analyzer": "ik_smart" // }, // "CategoryName": { // "type": "text", // "analyzer": "ik_smart" // }, // "SubCategoryName": { // "type": "text", // "analyzer": "ik_smart" // }, // "InterviewDate": { // "type": "text", // "analyzer": "ik_smart" // }, // "ExpertBackground": { // "type": "text", // "analyzer": "ik_smart" // }, // "ExpertNumber": { // "type": "text", // "analyzer": "ik_smart" // }, // "Department": { // "type": "text", // "analyzer": "ik_smart" // }, // "BodyText": { // "type": "text", // "analyzer": "ik_smart" // }, // "ArticleIdMd5": { // "type": "text", // "analyzer": "ik_smart" // } // } //} //}` //EsCreateIndex(indexName,mappingJson) //插入数据 //allList, err := models.GetArticleAll() //if err != nil { // fmt.Println("GetArticleAll Err:", err.Error()) // return //} // //indexName := "article_list" // //for _, v := range allList { // content := html.UnescapeString(v.Body) // doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) // if err != nil { // fmt.Println("create doc err:", err.Error()) // return // } // bodyText := doc.Text() // // item := new(ElasticArticleDetail) // item.ArticleId = v.ArticleId // item.Title = v.Title // item.TitleEn = v.TitleEn // item.UpdateFrequency = v.UpdateFrequency // item.CreateDate = v.CreateDate // item.PublishDate = v.PublishDate // item.Abstract = v.Abstract // item.CategoryName = v.CategoryName // item.SubCategoryName = v.SubCategoryName // item.InterviewDate = v.InterviewDate // item.ExpertBackground = v.ExpertBackground // item.ExpertNumber = v.ExpertNumber // item.Department = v.Department // item.ArticleIdMd5 = v.ArticleIdMd5 // item.BodyText = bodyText // EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item) //} //fmt.Println("bm") //bm, err := cache.NewCache("file", `{"CachePath":"./cache","FileSuffix":".cache", "EmbedExpiry": "120"}`) //// //if err != nil { // fmt.Println("false",err) //} ////fmt.Println(bm) //key:="put_key_one" ////bm.Put(key,1,1*time.Second) //b:=bm.IsExist(key) //fmt.Println(b) //同步策略文章 //SyncTacticsList() //indexName := "article_list" //EsDeleteData(indexName,) } /* searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) searchItem.Body = v.Highlight["BodyText"] searchItem.Title = title searchItem.PublishDate = article.PublishDate */ func CreateIndex() { indexName := "cygx_article_v4" //utils.IndexName mappingJson := `{ "mappings": { "dynamic": true, "properties": { "ArticleId": { "type": "integer" }, "BodyText": { "type": "text", "term_vector": "with_positions_offsets", "analyzer": "ik_smart" }, "PublishDate": { "type": "keyword" }, "SubCategoryName": { "type": "text", "term_vector": "with_positions_offsets", "analyzer": "ik_smart" }, "Title": { "type": "text", "term_vector": "with_positions_offsets", "analyzer": "ik_smart" }, "TitleEn": { "type": "text", "term_vector": "with_positions_offsets" }, "UpdateFrequency": { "type": "text", "term_vector": "with_positions_offsets" } } } }` EsCreateIndex(indexName, mappingJson) } func AddAllArticle() { endDate := time.Now().AddDate(0, 0, -30).Format(utils.FormatDate) //allList, err := models.GetArticleAll() allList, err := models.GetArticleAllDate(endDate) if err != nil { fmt.Println("GetArticleAll Err:", err.Error()) return } indexName := utils.IndexName for _, v := range allList { content := html.UnescapeString(v.Body) doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) if err != nil { fmt.Println("create doc err:", err.Error()) return } bodyText := doc.Text() item := new(ElasticTestArticleDetail) item.ArticleId = v.ArticleId item.Title = v.Title item.PublishDate = v.PublishDate item.BodyText = bodyText EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item) fmt.Println(v.ArticleId) } } func AddAllArticleV4() { allList, err := models.GetArticleAll2() if err != nil { fmt.Println("GetArticleAll Err:", err.Error()) return } indexName := "cygx_article_v4" for _, v := range allList { content := html.UnescapeString(v.Body) doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) if err != nil { fmt.Println("create doc err:", err.Error()) return } bodyText := doc.Text() item := new(ElasticTestArticleDetailV4) item.ArticleId = v.ArticleId item.Title = v.Title item.PublishDate = v.PublishDate item.BodyText = bodyText item.IsSummary = v.IsSummary item.IsReport = v.IsReport //if v.IsReport == 1 { // item.IsReport = true //} //if v.IsSummary == 1 { // item.IsSummary = true //} EsAddOrEditDataV4(indexName, strconv.Itoa(v.ArticleId), item) fmt.Println(v.ArticleId) } } // "term_vector": "with_positions_offsets" type ElasticArticleDetail struct { ArticleId int `description:"报告id"` Title string `description:"标题"` TitleEn string `description:"英文标题 "` UpdateFrequency string `description:"更新周期"` CreateDate string `description:"创建时间"` PublishDate string `description:"发布时间"` Abstract string `description:"摘要"` CategoryName string `description:"一级分类"` SubCategoryName string `description:"二级分类"` InterviewDate string `description:"访谈时间"` ExpertBackground string `description:"专家背景"` ExpertNumber string `description:"专家编号"` Department string `description:"作者"` ArticleIdMd5 string `description:"纪要id"` BodyText string `description:"内容"` } type ElasticTestArticleDetail struct { ArticleId int `description:"报告id"` Title string `description:"标题"` BodyText string `description:"内容"` PublishDate string `description:"发布时间"` } type ElasticTestArticleDetailV4 struct { ArticleId int `description:"报告id"` Title string `description:"标题"` BodyText string `description:"内容"` PublishDate string `description:"发布时间"` IsSummary int `description:"是否属于纪要库"` IsReport int `description:"是否属于报告"` }