article.go 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. package services
  2. import (
  3. "context"
  4. "encoding/json"
  5. "fmt"
  6. "github.com/PuerkitoBio/goquery"
  7. "github.com/rdlucklib/rdluck_tools/orm"
  8. "hongze/hongze_cygx/models"
  9. "hongze/hongze_cygx/utils"
  10. "html"
  11. "io/ioutil"
  12. nhttp "net/http"
  13. "strconv"
  14. "strings"
  15. "time"
  16. )
  17. func GetReportContentSub(content string) (contentSub string, err error) {
  18. content = html.UnescapeString(content)
  19. doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
  20. if err != nil {
  21. fmt.Println("create doc err:", err.Error())
  22. return
  23. }
  24. n := 0
  25. doc.Find("p").Each(func(i int, s *goquery.Selection) {
  26. if n > 3 {
  27. return
  28. }
  29. n++
  30. phtml, err := s.Html()
  31. if err != nil {
  32. fmt.Println("get html err", err.Error())
  33. return
  34. }
  35. if s.Text() != "" || strings.Contains(phtml, "src") {
  36. contentSub = contentSub + "<p>" + phtml + "</p>"
  37. }
  38. })
  39. return
  40. }
  41. func GetReportContentTextSub(content string) (contentSub string, err error) {
  42. content = html.UnescapeString(content)
  43. doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
  44. docText := doc.Text()
  45. bodyRune := []rune(docText)
  46. bodyRuneLen := len(bodyRune)
  47. if bodyRuneLen > 200 {
  48. bodyRuneLen = 200
  49. }
  50. body := string(bodyRune[:bodyRuneLen])
  51. contentSub = body
  52. return
  53. }
  54. //解析文章内容
  55. func GetArticleAll() {
  56. var err error
  57. defer func() {
  58. if err != nil {
  59. fmt.Println("err:", err.Error())
  60. return
  61. }
  62. }()
  63. list, err := models.GetArticleAll()
  64. if err != nil {
  65. return
  66. }
  67. for _, v := range list {
  68. fmt.Println(v.ArticleId, v.Title)
  69. FixArticleContent(v.ArticleId)
  70. }
  71. }
  72. //解析报告
  73. func FixArticleContent(articleId int) {
  74. item, err := models.GetArticleDetailById(articleId)
  75. if err != nil {
  76. fmt.Println("GetArticleDetailById Err:" + err.Error())
  77. return
  78. }
  79. content := item.Body
  80. bodyText, _ := GetReportContentTextSub(content)
  81. content = html.UnescapeString(content)
  82. content = strings.Replace(content, "http", "https", -1)
  83. doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
  84. if err != nil {
  85. fmt.Println("create doc err:", err.Error())
  86. return
  87. }
  88. var expertNumArr []string
  89. var expertContentArr []string
  90. var interviewDateArr []string
  91. doc.Find("p").Each(func(i int, s *goquery.Selection) {
  92. contentTxt := s.Text()
  93. if strings.Contains(contentTxt, "#访谈时间:") || strings.Contains(contentTxt, "访谈时间:") {
  94. interviewDate := s.Next().Text()
  95. interviewDateArr = append(interviewDateArr, interviewDate)
  96. }
  97. if strings.Contains(contentTxt, "#专家评价") || strings.Contains(contentTxt, "专家评价") {
  98. expertContent := s.Next().Text()
  99. if expertContent == "" {
  100. expertContent = contentTxt
  101. }
  102. if expertContent != "" {
  103. rightIndex := strings.Index(expertContent, ")")
  104. if rightIndex == 0 {
  105. rightIndex = strings.Index(expertContent, ")")
  106. }
  107. if rightIndex > 0 {
  108. expertNum := expertContent[:rightIndex]
  109. expertNum = strings.Replace(expertNum, "(", "", -1)
  110. expertNum = strings.Replace(expertNum, "(", "", -1)
  111. expertNum = strings.Replace(expertNum, "专家评价", "", -1)
  112. if expertNum != "" {
  113. expertNumArr = append(expertNumArr, expertNum)
  114. rightIndex = rightIndex
  115. expertContentStr := expertContent[rightIndex:]
  116. expertContentStr = strings.Replace(expertContentStr, ")", "", -1)
  117. expertContentStr = strings.TrimLeft(expertContentStr, ":")
  118. expertContentStr = strings.TrimRight(expertContentStr, "(推荐")
  119. expertContentArr = append(expertContentArr, expertContentStr)
  120. }
  121. }
  122. }
  123. }
  124. })
  125. if len(expertContentArr) <= 0 {
  126. doc.Find("pre").Each(func(i int, pre *goquery.Selection) {
  127. pre.Find("span").Each(func(n int, span *goquery.Selection) {
  128. contentTxt := span.Text()
  129. if strings.Contains(contentTxt, "#专家评价") || strings.Contains(contentTxt, "专家评价") {
  130. span.Find("span").Each(func(m int, subspan *goquery.Selection) {
  131. subspanText := subspan.Text()
  132. if strings.Contains(subspanText, "专家评价") {
  133. expertContent := subspan.Next().Text()
  134. if expertContent != "" {
  135. rightIndex := strings.Index(expertContent, ")")
  136. if rightIndex == 0 {
  137. rightIndex = strings.Index(expertContent, ")")
  138. }
  139. if rightIndex > 0 {
  140. expertNum := expertContent[:rightIndex]
  141. expertNum = strings.Replace(expertNum, "(", "", -1)
  142. expertNum = strings.Replace(expertNum, "(", "", -1)
  143. expertNum = strings.Replace(expertNum, "专家评价", "", -1)
  144. if expertNum != "" {
  145. expertNumArr = append(expertNumArr, expertNum)
  146. rightIndex = rightIndex
  147. expertContentStr := expertContent[rightIndex:]
  148. expertContentStr = strings.Replace(expertContentStr, ")", "", -1)
  149. expertContentStr = strings.TrimLeft(expertContentStr, ":")
  150. expertContentStr = strings.TrimRight(expertContentStr, "(推荐")
  151. expertContentArr = append(expertContentArr, expertContentStr)
  152. }
  153. }
  154. }
  155. }
  156. })
  157. }
  158. span.Find("span").Each(func(k int, sspan *goquery.Selection) {
  159. sspanText := sspan.Text()
  160. if strings.Contains(sspanText, "访谈时间") {
  161. sspanText = strings.Replace(sspanText, "#访谈时间:", "", -1)
  162. sspanText = strings.Replace(sspanText, "访谈时间:", "", -1)
  163. sspanText = strings.Replace(sspanText, "\n", "", -1)
  164. sspanText = strings.Replace(sspanText, " ", "", -1)
  165. sspanText = strings.Trim(sspanText, " ")
  166. sspanText = sspanText[:10]
  167. interviewDate := sspanText
  168. if interviewDate != "" {
  169. interviewDateArr = append(interviewDateArr, interviewDate)
  170. }
  171. }
  172. })
  173. })
  174. })
  175. }
  176. if len(expertContentArr) <= 0 {
  177. doc.Find("span").Each(func(i int, span *goquery.Selection) {
  178. span.Find("strong").Each(func(n int, strong *goquery.Selection) {
  179. spanText := span.Text()
  180. strongText := strong.Text()
  181. if strings.Contains(strongText, "#专家评价") || strings.Contains(strongText, "专家评价") {
  182. expertContent := strong.Parents().Text()
  183. if expertContent != "" {
  184. rightIndex := strings.Index(expertContent, ")")
  185. if rightIndex == 0 {
  186. rightIndex = strings.Index(expertContent, ")")
  187. }
  188. if rightIndex > 0 {
  189. expertNum := expertContent[:rightIndex]
  190. expertNum = strings.Replace(expertNum, "(", "", -1)
  191. expertNum = strings.Replace(expertNum, "(", "", -1)
  192. expertNum = strings.Replace(expertNum, "专家评价", "", -1)
  193. expertNum = strings.Replace(expertNum, "#", "", -1)
  194. expertNum = strings.Replace(expertNum, ":", "", -1)
  195. expertNum = strings.Replace(expertNum, "\n", "", -1)
  196. if expertNum != "" {
  197. expertNumArr = append(expertNumArr, expertNum)
  198. rightIndex = rightIndex
  199. expertContentStr := expertContent[rightIndex:]
  200. expertContentStr = strings.Replace(expertContentStr, ")", "", -1)
  201. expertContentStr = strings.TrimLeft(expertContentStr, ":")
  202. expertContentStr = strings.TrimRight(expertContentStr, "(推荐")
  203. expertContentArr = append(expertContentArr, expertContentStr)
  204. return
  205. }
  206. }
  207. }
  208. }
  209. if strings.Contains(spanText, "访谈时间") {
  210. spanText = strings.Replace(spanText, "#访谈时间:", "", -1)
  211. spanText = strings.Replace(spanText, "访谈时间:", "", -1)
  212. spanText = strings.Replace(spanText, "\n", "", -1)
  213. spanText = strings.Replace(spanText, " ", "", -1)
  214. spanText = strings.Trim(spanText, " ")
  215. spanText = spanText[:10]
  216. interviewDate := spanText
  217. if interviewDate != "" {
  218. interviewDateArr = append(interviewDateArr, interviewDate)
  219. }
  220. }
  221. })
  222. })
  223. }
  224. var expertNumStr, expertContentStr, interviewDateStr string
  225. if len(expertNumArr) > 0 {
  226. expertNumStr = expertNumArr[0]
  227. }
  228. if len(expertContentArr) > 0 {
  229. expertContentStr = expertContentArr[0]
  230. }
  231. if len(interviewDateArr) > 0 {
  232. interviewDateStr = interviewDateArr[0]
  233. }
  234. expertNumStr = strings.Replace(expertNumStr, "#:", "", -1)
  235. err = models.ModifyArticleExpert(articleId, expertNumStr, expertContentStr, interviewDateStr, bodyText)
  236. if err != nil {
  237. fmt.Println("ModifyArticleExpert Err:" + err.Error())
  238. return
  239. }
  240. }
  241. func FixArticleImgUrl(body string) (contentSub string, err error) {
  242. r := strings.NewReader(string(body))
  243. doc, err := goquery.NewDocumentFromReader(r)
  244. if err != nil {
  245. fmt.Println(err)
  246. }
  247. doc.Find("img").Each(func(i int, s *goquery.Selection) {
  248. src, _ := s.Attr("src")
  249. if i == 0 && src != "" {
  250. contentSub = src
  251. }
  252. })
  253. return
  254. }
  255. //获取标签里的第一个内容
  256. func FixArticleFirstCount(body string) (contentSub string, err error) {
  257. doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
  258. if err != nil {
  259. fmt.Println("create doc err:", err.Error())
  260. return
  261. }
  262. doc.Find("p").Each(func(i int, s *goquery.Selection) {
  263. contentTxt := s.Text()
  264. fmt.Println(contentTxt)
  265. })
  266. return
  267. }
  268. func GetArticleListByApi(cont context.Context) (err error) {
  269. defer func() {
  270. if err != nil {
  271. fmt.Println("GetArticleListByApi Err:" + err.Error())
  272. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "GetArticleListByApi ErrMsg:"+err.Error(), utils.EmailSendToUsers)
  273. }
  274. }()
  275. url := "https://vmp.hzinsights.com/v2api/articles/mp?take=100&skip=0&publish_status=1"
  276. method := "GET"
  277. client := &nhttp.Client{}
  278. req, err := nhttp.NewRequest(method, url, nil)
  279. if err != nil {
  280. fmt.Println("GetListApi Err:", err.Error())
  281. return err
  282. }
  283. req.Header.Add("Authorization", "bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkiLCJwaG9uZV9udW1iZXIiOiIxMjM0NTY3ODkiLCJuYW1lIjoi5YW25LuWIiwiZW50cmFuY2UiOiJwYXNzd3dvcmQiLCJpYXQiOjE2MzQ4NzA1OTQsImV4cCI6MTYzNDg3NDE5NH0.tho2L9jsbDPn8ltEGUVDve_nHsh0Kzf6ZrSz0RcZ0ag")
  284. res, err := client.Do(req)
  285. if err != nil {
  286. fmt.Println(err)
  287. return err
  288. }
  289. defer res.Body.Close()
  290. body, err := ioutil.ReadAll(res.Body)
  291. if err != nil {
  292. fmt.Println("Getres.Body Err:", err.Error())
  293. return err
  294. }
  295. var pdfResult models.ArticleResultApi
  296. err = json.Unmarshal(body, &pdfResult)
  297. if err != nil {
  298. fmt.Println("Getres.pdfResult Err:", err.Error())
  299. return err
  300. }
  301. exitMap := make(map[int]int)
  302. classMap := make(map[int]int)
  303. reportMap := make(map[int]int)
  304. summaryMap := make(map[int]int)
  305. listMap, err := models.GetArticleApiMap()
  306. if err != nil {
  307. fmt.Println("GetlistMap Err:", err.Error())
  308. return err
  309. }
  310. //新旧分类 反向隐射,是否归类,是否是报告,是否是纪要库
  311. for _, v := range listMap {
  312. exitMap[v.Id] = v.OldId
  313. if v.IsClass == 1 {
  314. classMap[v.OldId] = 1
  315. }
  316. if v.IsReport == 1 {
  317. reportMap[v.OldId] = 1
  318. }
  319. if v.IsSummary == 1 {
  320. summaryMap[v.OldId] = 1
  321. }
  322. }
  323. listData := pdfResult.Data
  324. var list []*models.Tactics2
  325. var listAuthor []*models.CygxArticleAuthor
  326. for _, v := range listData {
  327. if exitMap[v.SeriesId] > 0 {
  328. v.PublishDate = time.Date(v.PublishDate.Year(), v.PublishDate.Month(), v.PublishDate.Day(), v.PublishDate.Hour(), v.PublishDate.Minute(), v.PublishDate.Second(), v.PublishDate.Nanosecond(), time.Local)
  329. item := new(models.Tactics2)
  330. itemAuthor := new(models.CygxArticleAuthor)
  331. item.ArticleId = v.ArticleId
  332. item.Title = v.Title
  333. item.TitleEn = v.TitleEn
  334. if v.Frequency == "日度" {
  335. item.UpdateFrequency = "daily"
  336. } else if v.Frequency == "周度" {
  337. item.UpdateFrequency = "weekly"
  338. } else if v.Frequency == "月度" {
  339. item.UpdateFrequency = "monthly"
  340. } else if v.Frequency == "季度" {
  341. item.UpdateFrequency = "quarterly"
  342. } else if v.Frequency == "年度" {
  343. item.UpdateFrequency = "yearly"
  344. } else {
  345. item.UpdateFrequency = "unknow"
  346. }
  347. item.CreateDate = v.CreateDate
  348. item.PublishDate = v.PublishDate
  349. item.PublishStatus = v.PublishStatus
  350. item.Body = v.Content.Body
  351. item.Abstract = v.Content.Abstract
  352. item.CategoryName = v.Industry.Name
  353. item.CategoryId = exitMap[v.SeriesId]
  354. item.SubCategoryName = v.Series.Name
  355. list = append(list, item)
  356. itemAuthor.ArticleId = v.ArticleId
  357. itemAuthor.Name = v.Author.Name
  358. itemAuthor.Mobile = v.Author.PhoneNumber
  359. listAuthor = append(listAuthor, itemAuthor)
  360. }
  361. }
  362. //同步作者
  363. for _, v := range listAuthor {
  364. var count int
  365. count, err = models.GetActivityAuthorCount(v.ArticleId, v.Mobile)
  366. if err != nil {
  367. fmt.Println("GetCount Err:", err.Error())
  368. return err
  369. }
  370. if count == 0 {
  371. _, err = models.AddCygxActivityAuthor(v)
  372. if err != nil {
  373. fmt.Println("AddCygxActivityAuthor Err:", err.Error())
  374. return err
  375. }
  376. }
  377. }
  378. fmt.Println("同步文章条数:", len(list))
  379. listCustomArticle, err := models.GetCustomArticleId() //手动归类的文章,不替换文章类型
  380. if err != nil {
  381. fmt.Println("GetTacticsList Err:", err.Error())
  382. return err
  383. }
  384. listGetMatchTypeName, errMatch := models.GetMatchTypeNamenNotNull() //手动归类的文章,不替换文章类型
  385. if errMatch != nil {
  386. fmt.Println("GetTacticsList Err:", errMatch.Error())
  387. return err
  388. }
  389. fmt.Println("list len:", len(list))
  390. noSummaryArticleIds := "3454,3456,3457,3459,2449,2450,2453,2454,2459,2530,2583,2663,2670,2699,2715,2732,2748,2759,2399,2356,2870,3173,2978,2826,3470" //非纪要库类型的文章ID
  391. listNoSummaryArticleIds := strings.Split(noSummaryArticleIds, ",")
  392. for k, v := range list {
  393. //同步匹配类型
  394. matchTypeName := ""
  395. for _, vMatch := range listGetMatchTypeName {
  396. if v.CategoryId == vMatch.CategoryId {
  397. matchTypeName = vMatch.MatchTypeName
  398. }
  399. }
  400. //是否属于纪要库的数据
  401. if _, has := summaryMap[v.CategoryId]; has {
  402. v.IsSummary = 1
  403. }
  404. //排除不属于纪要库类型的文章
  405. for _, vArt := range listNoSummaryArticleIds {
  406. vArtInt, _ := strconv.Atoi(vArt)
  407. if v.ArticleId == vArtInt {
  408. v.IsSummary = 0
  409. }
  410. }
  411. if _, has := reportMap[v.CategoryId]; has {
  412. v.IsReport = 1
  413. if _, ok := classMap[v.CategoryId]; ok {
  414. v.IsClass = 1
  415. v.ReportType = 1 //是否属于行业报告
  416. } else {
  417. v.ReportType = 2 //是否属于产业报告
  418. }
  419. }
  420. v.Department = "弘则权益研究"
  421. //判断是否已经存在
  422. if v.ArticleId < 0 {
  423. fmt.Println("AddCygxArticle Err:")
  424. return err
  425. }
  426. var count int
  427. count, err = models.GetArticleCountById(v.ArticleId)
  428. if err != nil && err.Error() != utils.ErrNoRow() {
  429. fmt.Println("AddCygxArticle Err:", err.Error())
  430. return err
  431. }
  432. v.Body = strings.Replace(v.Body, "http://vmp.hzinsights.com", "https://vmp.hzinsights.com", -1)
  433. expertNumStr, expertContentStr, interviewDateStr, fileLink, bodyReturn := BodyAnalysis2(v.Body)
  434. if strings.Index(v.Body, "报告全文(") > 0 && strings.Index(v.Body, "PDF格式报告下载.pdf") > 0 {
  435. v.Body = strings.Replace(v.Body, "报告全文(", "", -1)
  436. v.Body = strings.Replace(v.Body, "PDF格式报告下载.pdf", "", -1)
  437. v.Body = strings.Replace(v.Body, "):", "", -1)
  438. }
  439. var titleNew string
  440. titleNew = v.Title
  441. // 7资金流向 、11大类资产 、51每日复盘 、80医药周报、9估值研究
  442. if v.CategoryId == 7 || v.CategoryId == 11 || v.CategoryId == 51 || v.CategoryId == 9 {
  443. if v.UpdateFrequency == "daily" {
  444. var daystr string
  445. daystr = strconv.Itoa(v.PublishDate.Day())
  446. if len(daystr) == 1 {
  447. daystr = "0" + daystr
  448. }
  449. titleNew = v.Title + "(" + strconv.Itoa(v.PublishDate.Year())[2:len(strconv.Itoa(v.PublishDate.Year()))-0] + v.PublishDate.Format("01") + daystr + ")"
  450. } else if v.UpdateFrequency == "weekly" {
  451. titleNew = v.Title + utils.WeekByDate(v.PublishDate)
  452. }
  453. }
  454. if v.CategoryId == 80 {
  455. titleNew = v.Title + utils.WeekByDate(v.PublishDate)
  456. }
  457. if count > 0 {
  458. fmt.Println(k, v.ArticleId, "edit")
  459. var isCustom bool
  460. bodyText, _ := GetReportContentTextSub(v.Body)
  461. updateParams := make(map[string]interface{})
  462. //updateParams["Title"] = v.Title
  463. updateParams["Title"] = titleNew
  464. updateParams["TitleEn"] = v.TitleEn
  465. updateParams["UpdateFrequency"] = v.UpdateFrequency
  466. updateParams["CreateDate"] = v.CreateDate
  467. updateParams["PublishDate"] = v.PublishDate
  468. //updateParams["Body"] = html.EscapeString(v.Body)
  469. updateParams["Body"] = html.EscapeString(bodyReturn)
  470. updateParams["BodyText"] = bodyText
  471. updateParams["Abstract"] = html.EscapeString(v.Abstract)
  472. updateParams["CategoryName"] = v.CategoryName
  473. for _, vCustom := range listCustomArticle {
  474. if v.ArticleId == vCustom.ArticleId {
  475. fmt.Println("手动归类的文章:" + strconv.Itoa(v.ArticleId))
  476. isCustom = true
  477. }
  478. }
  479. if isCustom == false {
  480. updateParams["CategoryId"] = v.CategoryId
  481. updateParams["MatchTypeName"] = matchTypeName
  482. updateParams["IsSummary"] = v.IsSummary
  483. updateParams["IsReport"] = v.IsReport
  484. updateParams["ReportType"] = v.ReportType
  485. updateParams["SubCategoryName"] = v.SubCategoryName
  486. }
  487. //updateParams["CategoryId"] = v.CategoryId
  488. updateParams["PublishStatus"] = v.PublishStatus
  489. updateParams["ExpertBackground"] = expertContentStr
  490. updateParams["ExpertNumber"] = expertNumStr
  491. updateParams["InterviewDate"] = interviewDateStr
  492. //updateParams["IsClass"] = v.IsClass
  493. v.Department = "弘则权益研究"
  494. updateParams["Department"] = v.Department
  495. updateParams["FileLink"] = fileLink
  496. whereParam := map[string]interface{}{"article_id": v.ArticleId}
  497. err = models.UpdateByExpr(models.CygxArticle{}, whereParam, updateParams)
  498. if err != nil {
  499. fmt.Println("UpdateByExpr Err:" + err.Error())
  500. return err
  501. }
  502. } else {
  503. fmt.Println(k, v.ArticleId, "add")
  504. item := new(models.CygxArticle)
  505. articleIdInt := v.ArticleId
  506. item.ArticleId = articleIdInt
  507. //item.Title = v.Title
  508. item.Title = titleNew
  509. item.TitleEn = v.TitleEn
  510. item.UpdateFrequency = v.UpdateFrequency
  511. item.CreateDate = v.CreateDate
  512. item.PublishDate = v.PublishDate.Format(utils.FormatDateTime)
  513. //item.Body = html.EscapeString(v.Body)
  514. item.Body = html.EscapeString(bodyReturn)
  515. item.Abstract = html.EscapeString(v.Abstract)
  516. item.CategoryName = v.CategoryName
  517. item.SubCategoryName = v.SubCategoryName
  518. item.CategoryId = v.CategoryId
  519. item.PublishStatus = v.PublishStatus
  520. item.ExpertBackground = expertContentStr
  521. item.ExpertNumber = expertNumStr
  522. item.InterviewDate = interviewDateStr
  523. item.Department = v.Department
  524. item.ArticleIdMd5 = utils.MD5(strconv.Itoa(articleIdInt))
  525. item.IsClass = v.IsClass
  526. item.IsSummary = v.IsSummary
  527. item.IsReport = v.IsReport
  528. item.ReportType = v.ReportType
  529. item.FileLink = fileLink
  530. item.MatchTypeName = matchTypeName
  531. _, err = models.AddCygxArticles(item)
  532. if err != nil {
  533. fmt.Println("AddCygxArticle Err:", err.Error())
  534. return err
  535. }
  536. }
  537. }
  538. return
  539. }
  540. func SynchronizationArtclehistory() {
  541. fmt.Println("同步开始")
  542. list, err := models.GetArticleHistoryList()
  543. if err != nil {
  544. fmt.Println("获取列表失败", err)
  545. }
  546. fmt.Println(len(list))
  547. for _, v := range list {
  548. //endDate := v.ModifyTime.Add(+time.Minute * 10).Format(utils.FormatDateTime)
  549. //detail, err := models.GetNewArticleHistoryRecordNewpv(v.UserId, v.ArticleId, endDate)
  550. //if err != nil && err.Error() != utils.ErrNoRow() {
  551. // fmt.Println("获取信息失败", err)
  552. //}
  553. v.OutType = 1
  554. //fmt.Println(v.Id)
  555. //if detail == nil {
  556. // _, err = models.AddCygxArticleViewRecordNewpv(v)
  557. // if err != nil {
  558. // fmt.Println("新增失败", err)
  559. // }
  560. //} else {
  561. // err = models.UpdateCygxArticleViewRecordNewpvList(v, v.StopTime)
  562. // if err != nil {
  563. // fmt.Println("修改失败", err)
  564. // }
  565. //}
  566. newId, err := models.AddCygxArticleViewRecordNewpv(v)
  567. fmt.Println("新增", newId)
  568. if err != nil {
  569. fmt.Println("新增失败", err)
  570. }
  571. }
  572. fmt.Println("同步结束")
  573. }
  574. //统计报表
  575. func StatisticalReport() {
  576. var isSummaryNumAll, isClassNum, pvNumAll, uvNumAll int
  577. list, err := models.GetChartPermissionActivity()
  578. if err != nil {
  579. fmt.Println("获取列表失败", err)
  580. }
  581. for _, v := range list {
  582. var listPv []*models.ReportMappingStatistical
  583. if v.PermissionName == "研选" {
  584. listPv, err = models.GetStatisticalReportArtilceExpert()
  585. if err != nil {
  586. fmt.Println("获取Pv列表失败", err)
  587. }
  588. } else {
  589. listPv, err = models.GetStatisticalReportArtilce(v.ChartPermissionId)
  590. if err != nil {
  591. fmt.Println("获取Pv列表失败", err)
  592. }
  593. }
  594. var pvNum, uvNum, isSummaryNum int
  595. for _, v2 := range listPv {
  596. pvNum += v2.Pv
  597. uvNum += v2.Uv
  598. if v2.IsSummary == "1" {
  599. isSummaryNum += 1
  600. }
  601. if v2.IsClass == "1" && v.ChartPermissionId <= 22 {
  602. isClassNum += 1
  603. }
  604. if v2.IsSummary == "1" && v.ChartPermissionId <= 22 {
  605. isSummaryNumAll += 1
  606. }
  607. }
  608. if v.ChartPermissionId <= 22 {
  609. pvNumAll += pvNum
  610. uvNumAll += uvNum
  611. }
  612. fmt.Println(v.PermissionName+"行业", len(listPv), "篇,其中主观类报告", isSummaryNum, "篇,客观类报告", len(listPv)-isSummaryNum, "篇。共产生阅读量pv-,", pvNum, ",uv-", uvNum)
  613. }
  614. fmt.Println("目前同步四大行业的总报告(已归类)数量", isClassNum, "篇,其中主观类报告", isSummaryNumAll, "篇,客观类报告", isClassNum-isSummaryNumAll, "篇。共产生阅读量pv-", pvNumAll, ",uv-", uvNumAll)
  615. var totalOnline int //线上
  616. var totalOffline int //线下
  617. var totalPeople int //共累计预约外呼人数
  618. var totalSignUpOff int //线下报名人数
  619. var totalSignUpOffTime int //线下报名人数
  620. var totalPeopleMeet int //线下参会人数
  621. o := orm.NewOrm()
  622. sql := `SELECT COUNT(1) FROM cygx_activity WHERE activity_type_id IN (1,2,3) AND publish_status = 1 AND is_submit_meeting = 1 AND activity_time <= NOW();`
  623. err = o.Raw(sql).QueryRow(&totalOnline)
  624. if err != nil {
  625. fmt.Println("获取线上", err)
  626. }
  627. sql = `SELECT COUNT(1) FROM cygx_activity WHERE activity_type_id IN (4,5,6) AND publish_status = 1 AND is_submit_meeting = 1 AND activity_time <= NOW();`
  628. err = o.Raw(sql).QueryRow(&totalOffline)
  629. if err != nil {
  630. fmt.Println("获取线下", err)
  631. }
  632. sql = `SELECT COUNT( 1 ) FROM
  633. cygx_activity_signup as s
  634. INNER JOIN cygx_activity as a ON a.activity_id = s.activity_id
  635. WHERE
  636. s.do_fail_type = 0
  637. AND a.is_submit_meeting = 1
  638. AND a.activity_time <= NOW()
  639. AND a.publish_status = 1`
  640. err = o.Raw(sql).QueryRow(&totalPeople)
  641. if err != nil {
  642. fmt.Println("共累计预约外呼人数", err)
  643. }
  644. sql = `SELECT COUNT( 1 ) FROM
  645. cygx_activity_signup as s
  646. INNER JOIN cygx_activity as a ON a.activity_id = s.activity_id
  647. WHERE
  648. s.do_fail_type = 0
  649. AND a.is_submit_meeting = 1
  650. AND a.activity_time <= NOW()
  651. AND a.activity_type_id IN (4,5,6)
  652. AND a.publish_status = 1`
  653. err = o.Raw(sql).QueryRow(&totalSignUpOff)
  654. if err != nil {
  655. fmt.Println("共累计预约外呼人数", err)
  656. }
  657. sql = `SELECT COUNT( 1 ) FROM
  658. cygx_activity_signup as s
  659. INNER JOIN cygx_activity as a ON a.activity_id = s.activity_id
  660. WHERE
  661. s.do_fail_type = 0
  662. AND a.publish_status = 1
  663. AND a.is_submit_meeting = 1
  664. AND a.activity_time <= NOW()
  665. AND a.is_submit_meeting = 1
  666. AND a.activity_type_id IN (4,5,6);`
  667. err = o.Raw(sql).QueryRow(&totalSignUpOffTime)
  668. if err != nil {
  669. fmt.Println("线下报名参会人数", err)
  670. }
  671. sql = `SELECT COUNT( 1 ) FROM
  672. cygx_activity_signup as s
  673. INNER JOIN cygx_activity as a ON a.activity_id = s.activity_id
  674. WHERE
  675. s.do_fail_type = 0
  676. AND a.is_submit_meeting = 1
  677. AND a.activity_time <= NOW()
  678. AND a.publish_status = 1
  679. AND s.is_meeting = 1
  680. AND a.activity_type_id IN (4,5,6);`
  681. err = o.Raw(sql).QueryRow(&totalPeopleMeet)
  682. if err != nil {
  683. fmt.Println("线下参会人数", err)
  684. }
  685. fmt.Println("共上线活动", totalOnline+totalOffline, "个,其中线上", totalOnline, "个,线下", totalOffline, "个")
  686. fmt.Println("共累计预约外呼人数", totalPeople, "人")
  687. fmt.Println("报名线下参会", totalSignUpOff, "人,实际到会人数", totalPeopleMeet, "人,线下到会率约", totalPeopleMeet*100/totalSignUpOff, "%")
  688. num := totalPeopleMeet / totalSignUpOffTime
  689. fmt.Println(num)
  690. fmt.Println(totalOnline)
  691. fmt.Println(totalOffline)
  692. fmt.Println(totalPeople)
  693. fmt.Println(totalSignUpOff)
  694. fmt.Println(totalPeopleMeet)
  695. fmt.Println(totalSignUpOffTime)
  696. fmt.Println(totalPeopleMeet / totalSignUpOffTime)
  697. return
  698. }