task.go 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. package services
  2. import (
  3. "fmt"
  4. "github.com/PuerkitoBio/goquery"
  5. "github.com/astaxie/beego/toolbox"
  6. "hongze/hongze_cygx/models"
  7. "hongze/hongze_cygx/utils"
  8. "html"
  9. "strconv"
  10. "strings"
  11. //"github.com/astaxie/beego/cache"
  12. )
  13. func Task() {
  14. fmt.Println("start")
  15. //同步数据
  16. if utils.RunMode == "release" {
  17. //syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList)
  18. //toolbox.AddTask("syncTacticsList", syncTacticsList)
  19. //toolbox.StartTask()
  20. syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 */5 * * * *", SyncTacticsListAddreport)
  21. toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
  22. //同步纪要库中的Es
  23. syncTacticsListToEs := toolbox.NewTask("syncTacticsListToEs", "0 */30 * * * *", SyncTacticsListToEs)
  24. toolbox.AddTask("syncTacticsListToEs", syncTacticsListToEs)
  25. toolbox.StartTask()
  26. }
  27. //updateActivitySattus := toolbox.NewTask("updateActivitySattus", "0 */5 * * * *", UpdateActivitySattus)
  28. //toolbox.AddTask("updateActivitySattus", updateActivitySattus)
  29. //SyncTacticsListAddreport()
  30. //UpdateActivitySattus()
  31. updateActivitySattus := toolbox.NewTask("syncTacticsListAddreport", "0 */10 * * * *", UpdateActivitySattus)
  32. toolbox.AddTask("updateActivitySattus", updateActivitySattus)
  33. //
  34. //updateIndustrialManagementLabel := toolbox.NewTask("updateIndustrialManagementLabel", "0 */30 * * * *", UpdateIndustrialManagementLabel)
  35. //toolbox.AddTask("updateIndustrialManagementLabel", updateIndustrialManagementLabel)
  36. //UpdateIndustrialManagementSubjectNnames()
  37. SendActivityBeginMsg()
  38. toolbox.StartTask()
  39. //if utils.RunMode == "debug" {
  40. // //syncTacticsList := toolbox.NewTask("syncTacticsList", "0 */5 * * * *", SyncTacticsList)
  41. // //toolbox.AddTask("syncTacticsList", syncTacticsList)
  42. // //toolbox.StartTask()
  43. // syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 01 01 * * *", SyncTacticsListAddreport)
  44. // toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
  45. //
  46. // //同步纪要库中的Es
  47. // syncTacticsListToEs := toolbox.NewTask("syncTacticsListToEs", "0 01 01 * * *", SyncTacticsListToEs)
  48. // toolbox.AddTask("syncTacticsListToEs", syncTacticsListToEs)
  49. // toolbox.StartTask()
  50. //}
  51. //syncTacticsListAddreport := toolbox.NewTask("syncTacticsListAddreport", "0 42 09 * * *", SyncTacticsListAddreport)
  52. //toolbox.AddTask("syncTacticsListAddreport", syncTacticsListAddreport)
  53. //toolbox.StartTask()
  54. //AddAllArticle()
  55. //SyncTacticsListToEs()
  56. fmt.Println("end")
  57. }
  58. func ElasticOption() {
  59. //SyncTacticsList()
  60. //toolbox.StartTask()
  61. //keyWord := "专家"
  62. //pageSize := 20
  63. //SearchByKeyWord(keyWord, pageSize)
  64. //SaveData()
  65. //GetIndustryMap()
  66. //解析报告
  67. //GetArticleExpert()
  68. //SearchByKeyWordQuery("")
  69. //AddMap()
  70. //同步数据
  71. //ImportData()
  72. //AddMap()
  73. //Search("光伏行业")
  74. //修复报告内容
  75. //GetArticleAll()
  76. //indexName := "article_list"
  77. // mappingJson := `{
  78. //"mappings": {
  79. // "dynamic": true,
  80. // "properties": {
  81. // "ArticleId": {
  82. // "type": "integer"
  83. // },
  84. // "Title": {
  85. // "type": "text",
  86. // "analyzer": "ik_smart"
  87. // },
  88. // "TitleEn": {
  89. // "type": "text",
  90. // "analyzer": "ik_smart"
  91. // },
  92. // "UpdateFrequency": {
  93. // "type": "text",
  94. // "analyzer": "ik_smart"
  95. // },
  96. // "CreateDate": {
  97. // "type": "text",
  98. // "analyzer": "ik_smart"
  99. // },
  100. // "PublishDate": {
  101. // "type": "text",
  102. // "analyzer": "ik_smart"
  103. // },
  104. // "Abstract": {
  105. // "type": "text",
  106. // "analyzer": "ik_smart"
  107. // },
  108. // "CategoryName": {
  109. // "type": "text",
  110. // "analyzer": "ik_smart"
  111. // },
  112. // "SubCategoryName": {
  113. // "type": "text",
  114. // "analyzer": "ik_smart"
  115. // },
  116. // "InterviewDate": {
  117. // "type": "text",
  118. // "analyzer": "ik_smart"
  119. // },
  120. // "ExpertBackground": {
  121. // "type": "text",
  122. // "analyzer": "ik_smart"
  123. // },
  124. // "ExpertNumber": {
  125. // "type": "text",
  126. // "analyzer": "ik_smart"
  127. // },
  128. // "Department": {
  129. // "type": "text",
  130. // "analyzer": "ik_smart"
  131. // },
  132. // "BodyText": {
  133. // "type": "text",
  134. // "analyzer": "ik_smart"
  135. // },
  136. // "ArticleIdMd5": {
  137. // "type": "text",
  138. // "analyzer": "ik_smart"
  139. // }
  140. // }
  141. //}
  142. //}`
  143. //EsCreateIndex(indexName,mappingJson)
  144. //插入数据
  145. //allList, err := models.GetArticleAll()
  146. //if err != nil {
  147. // fmt.Println("GetArticleAll Err:", err.Error())
  148. // return
  149. //}
  150. //
  151. //indexName := "article_list"
  152. //
  153. //for _, v := range allList {
  154. // content := html.UnescapeString(v.Body)
  155. // doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
  156. // if err != nil {
  157. // fmt.Println("create doc err:", err.Error())
  158. // return
  159. // }
  160. // bodyText := doc.Text()
  161. //
  162. // item := new(ElasticArticleDetail)
  163. // item.ArticleId = v.ArticleId
  164. // item.Title = v.Title
  165. // item.TitleEn = v.TitleEn
  166. // item.UpdateFrequency = v.UpdateFrequency
  167. // item.CreateDate = v.CreateDate
  168. // item.PublishDate = v.PublishDate
  169. // item.Abstract = v.Abstract
  170. // item.CategoryName = v.CategoryName
  171. // item.SubCategoryName = v.SubCategoryName
  172. // item.InterviewDate = v.InterviewDate
  173. // item.ExpertBackground = v.ExpertBackground
  174. // item.ExpertNumber = v.ExpertNumber
  175. // item.Department = v.Department
  176. // item.ArticleIdMd5 = v.ArticleIdMd5
  177. // item.BodyText = bodyText
  178. // EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
  179. //}
  180. //fmt.Println("bm")
  181. //bm, err := cache.NewCache("file", `{"CachePath":"./cache","FileSuffix":".cache", "EmbedExpiry": "120"}`)
  182. ////
  183. //if err != nil {
  184. // fmt.Println("false",err)
  185. //}
  186. ////fmt.Println(bm)
  187. //key:="put_key_one"
  188. ////bm.Put(key,1,1*time.Second)
  189. //b:=bm.IsExist(key)
  190. //fmt.Println(b)
  191. //同步策略文章
  192. //SyncTacticsList()
  193. //indexName := "article_list"
  194. //EsDeleteData(indexName,)
  195. }
  196. /*
  197. searchItem := new(models.SearchItem)
  198. searchItem.ArticleId, _ = strconv.Atoi(v.Id)
  199. searchItem.Body = v.Highlight["BodyText"]
  200. searchItem.Title = title
  201. searchItem.PublishDate = article.PublishDate
  202. */
  203. func CreateIndex() {
  204. indexName := "cygx_article_v4" //utils.IndexName
  205. mappingJson := `{
  206. "mappings": {
  207. "dynamic": true,
  208. "properties": {
  209. "ArticleId": {
  210. "type": "integer"
  211. },
  212. "BodyText": {
  213. "type": "text",
  214. "term_vector": "with_positions_offsets",
  215. "analyzer": "ik_smart"
  216. },
  217. "PublishDate": {
  218. "type": "keyword"
  219. },
  220. "SubCategoryName": {
  221. "type": "text",
  222. "term_vector": "with_positions_offsets",
  223. "analyzer": "ik_smart"
  224. },
  225. "Title": {
  226. "type": "text",
  227. "term_vector": "with_positions_offsets",
  228. "analyzer": "ik_smart"
  229. },
  230. "TitleEn": {
  231. "type": "text",
  232. "term_vector": "with_positions_offsets"
  233. },
  234. "UpdateFrequency": {
  235. "type": "text",
  236. "term_vector": "with_positions_offsets"
  237. }
  238. }
  239. }
  240. }`
  241. EsCreateIndex(indexName, mappingJson)
  242. }
  243. func AddAllArticle() {
  244. allList, err := models.GetArticleAll()
  245. if err != nil {
  246. fmt.Println("GetArticleAll Err:", err.Error())
  247. return
  248. }
  249. indexName := "cygx_article_v3"
  250. for _, v := range allList {
  251. content := html.UnescapeString(v.Body)
  252. doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
  253. if err != nil {
  254. fmt.Println("create doc err:", err.Error())
  255. return
  256. }
  257. bodyText := doc.Text()
  258. item := new(ElasticTestArticleDetail)
  259. item.ArticleId = v.ArticleId
  260. item.Title = v.Title
  261. item.PublishDate = v.PublishDate
  262. item.BodyText = bodyText
  263. EsAddOrEditData(indexName, strconv.Itoa(v.ArticleId), item)
  264. fmt.Println(v.ArticleId)
  265. }
  266. }
  267. func AddAllArticleV4() {
  268. allList, err := models.GetArticleAll2()
  269. if err != nil {
  270. fmt.Println("GetArticleAll Err:", err.Error())
  271. return
  272. }
  273. indexName := "cygx_article_v4"
  274. for _, v := range allList {
  275. content := html.UnescapeString(v.Body)
  276. doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
  277. if err != nil {
  278. fmt.Println("create doc err:", err.Error())
  279. return
  280. }
  281. bodyText := doc.Text()
  282. item := new(ElasticTestArticleDetailV4)
  283. item.ArticleId = v.ArticleId
  284. item.Title = v.Title
  285. item.PublishDate = v.PublishDate
  286. item.BodyText = bodyText
  287. item.IsSummary = v.IsSummary
  288. item.IsReport = v.IsReport
  289. //if v.IsReport == 1 {
  290. // item.IsReport = true
  291. //}
  292. //if v.IsSummary == 1 {
  293. // item.IsSummary = true
  294. //}
  295. EsAddOrEditDataV4(indexName, strconv.Itoa(v.ArticleId), item)
  296. fmt.Println(v.ArticleId)
  297. }
  298. }
  299. // "term_vector": "with_positions_offsets"
  300. type ElasticArticleDetail struct {
  301. ArticleId int `description:"报告id"`
  302. Title string `description:"标题"`
  303. TitleEn string `description:"英文标题 "`
  304. UpdateFrequency string `description:"更新周期"`
  305. CreateDate string `description:"创建时间"`
  306. PublishDate string `description:"发布时间"`
  307. Abstract string `description:"摘要"`
  308. CategoryName string `description:"一级分类"`
  309. SubCategoryName string `description:"二级分类"`
  310. InterviewDate string `description:"访谈时间"`
  311. ExpertBackground string `description:"专家背景"`
  312. ExpertNumber string `description:"专家编号"`
  313. Department string `description:"作者"`
  314. ArticleIdMd5 string `description:"纪要id"`
  315. BodyText string `description:"内容"`
  316. }
  317. type ElasticTestArticleDetail struct {
  318. ArticleId int `description:"报告id"`
  319. Title string `description:"标题"`
  320. BodyText string `description:"内容"`
  321. PublishDate string `description:"发布时间"`
  322. }
  323. type ElasticTestArticleDetailV4 struct {
  324. ArticleId int `description:"报告id"`
  325. Title string `description:"标题"`
  326. BodyText string `description:"内容"`
  327. PublishDate string `description:"发布时间"`
  328. IsSummary int `description:"是否属于纪要库"`
  329. IsReport int `description:"是否属于报告"`
  330. }