Merge branch 'feature/eta_2.3.5' into debug

controllers/data_source/data_source.go

@@ -78,23 +78,22 @@ func (c *DataSourceController) SearchByEs() {
 		// 由于start_date、end_date和latest_value字段不全的历史遗留问题，这里查出来并补充进ES里面去
 		if v.StartDate == "" || v.EndDate == "" {
 			minMax, e := dataSourceModel.GetBaseIndexDataMinMax(v.Source, v.SubSource, v.IndexCode)
-			if e != nil {
-				if e.Error() == utils.ErrNoRow() {
-					continue
-				}
+			if e != nil && e.Error() != utils.ErrNoRow() {
 				br.Msg = "获取失败"
 				br.ErrMsg = fmt.Sprintf("获取指标开始结束时间失败, %v", e)
 				return
 			}
-			v.StartDate = minMax.MinDate
-			v.EndDate = minMax.MaxDate
-			v.LatestValue = minMax.LatestValue
+			if minMax != nil {
+				v.StartDate = minMax.MinDate
+				v.EndDate = minMax.MaxDate
+				v.LatestValue = minMax.LatestValue
 
-			// 写入ES更新队列
-			if e := utils.Rc.LPush(utils.CACHE_DATA_SOURCE_ES_HANDLE, v); e != nil {
-				br.Msg = "获取失败"
-				br.ErrMsg = fmt.Sprintf("写入ES更新队列失败, Source: %d, IndexCode: %s, err: %v", v.Source, v.IndexCode, e)
-				return
+				// 写入ES更新队列
+				if e := utils.Rc.LPush(utils.CACHE_DATA_SOURCE_ES_HANDLE, v); e != nil {
+					br.Msg = "获取失败"
+					br.ErrMsg = fmt.Sprintf("写入ES更新队列失败, Source: %d, IndexCode: %s, err: %v", v.Source, v.IndexCode, e)
+					return
+				}
 			}
 		}
 		// 转换成map返回

models/data_source/data_source.go

@@ -1950,14 +1950,12 @@ func GetBaseIndexDataMinMax(source, subSource int, indexCode string) (item *Base
 	// 煤炭江湖
 	if source == utils.DATA_SOURCE_MTJH {
 		sql = `SELECT MIN(data_time) AS min_date, MAX(data_time) AS max_date FROM base_from_mtjh_index WHERE index_code = ?`
-		if e := o.Raw(sql, indexCode).QueryRow(&item); e != nil {
-			err = fmt.Errorf("获取数据源开始结束时间失败, %v", e)
+		if err = o.Raw(sql, indexCode).QueryRow(&item); err != nil {
 			return
 		}
 
 		sql = `SELECT deal_value AS latest_value FROM base_from_mtjh_index WHERE index_code = ? ORDER BY data_time DESC LIMIT 1`
-		if e := o.Raw(sql, indexCode).QueryRow(&latestVal); e != nil {
-			err = fmt.Errorf("获取数据源最新值失败, %v", e)
+		if err = o.Raw(sql, indexCode).QueryRow(&latestVal); err != nil {
 			return
 		}
 		item.LatestValue = latestVal
@@ -1977,15 +1975,13 @@ func GetBaseIndexDataMinMax(source, subSource int, indexCode string) (item *Base
 		}
 		sql = `SELECT MIN(%s) AS min_date, MAX(%s) AS max_date FROM %s WHERE index_code = ?`
 		sql = fmt.Sprintf(sql, fieldDataTime, fieldDataTime, dataTable)
-		if e := o.Raw(sql, indexCode).QueryRow(&item); e != nil {
-			err = fmt.Errorf("获取数据源开始结束时间失败, %v", e)
+		if err = o.Raw(sql, indexCode).QueryRow(&item); err != nil {
 			return
 		}
 
 		sql = `SELECT deal_value AS latest_value FROM %s WHERE index_code = ? ORDER BY %s DESC LIMIT 1`
 		sql = fmt.Sprintf(sql, dataTable, fieldDataTime)
-		if e := o.Raw(sql, indexCode).QueryRow(&latestVal); e != nil {
-			err = fmt.Errorf("获取数据源最新值失败, %v", e)
+		if err = o.Raw(sql, indexCode).QueryRow(&latestVal); err != nil {
 			return
 		}
 		item.LatestValue = latestVal
@@ -2001,15 +1997,13 @@ func GetBaseIndexDataMinMax(source, subSource int, indexCode string) (item *Base
 
 	sql = `SELECT MIN(data_time) AS min_date, MAX(data_time) AS max_date FROM %s WHERE index_code = ?`
 	sql = fmt.Sprintf(sql, dataTable)
-	if e := o.Raw(sql, indexCode).QueryRow(&item); e != nil {
-		err = fmt.Errorf("获取数据源开始结束时间失败, %v", e)
+	if err = o.Raw(sql, indexCode).QueryRow(&item); err != nil {
 		return
 	}
 
 	sql = `SELECT value AS latest_value FROM %s WHERE index_code = ? ORDER BY data_time DESC LIMIT 1`
 	sql = fmt.Sprintf(sql, dataTable)
-	if e := o.Raw(sql, indexCode).QueryRow(&latestVal); e != nil {
-		err = fmt.Errorf("获取数据源最新值失败, %v", e)
+	if err = o.Raw(sql, indexCode).QueryRow(&latestVal); err != nil {
 		return
 	}
 	item.LatestValue = latestVal

services/elastic/elastic.go

@@ -1964,27 +1964,27 @@ func SearchDataSourceIndex(indexName, keyword string, source, subSource int, cla
 	})
 
 	// 指标编码/名称
-	//shouldMap := make(map[string]interface{}, 0)
+	shouldMap := make(map[string]interface{}, 0)
 	if keyword != "" {
-		//shouldMap["should"] = []interface{}{
-		//	map[string]interface{}{
-		//		"match": map[string]interface{}{
-		//			"IndexCode": keyword,
-		//		},
-		//	},
-		//	map[string]interface{}{
-		//		"match": map[string]interface{}{
-		//			"IndexName": keyword,
-		//		},
-		//	},
-		//}
-		queryFields := []string{"IndexName.keyword", "IndexName.ik", "IndexName.ngram", "IndexCode.keyword", "IndexCode.ngram"}
-		mustMap = append(mustMap, map[string]interface{}{
-			"multi_match": map[string]interface{}{
-				"query":  keyword,
-				"fields": queryFields,
+		shouldMap["should"] = []interface{}{
+			map[string]interface{}{
+				"match": map[string]interface{}{
+					"IndexCode": keyword,
+				},
 			},
-		})
+			map[string]interface{}{
+				"match": map[string]interface{}{
+					"IndexName": keyword,
+				},
+			},
+		}
+		//queryFields := []string{"IndexName.keyword", "IndexName.ik", "IndexName.ngram", "IndexCode.keyword", "IndexCode.ngram"}
+		//mustMap = append(mustMap, map[string]interface{}{
+		//	"multi_match": map[string]interface{}{
+		//		"query":  keyword,
+		//		"fields": queryFields,
+		//	},
+		//})
 	}
 
 	// 来源/子来源
@@ -2031,9 +2031,9 @@ func SearchDataSourceIndex(indexName, keyword string, source, subSource int, cla
 	}
 
 	// 关键字匹配
-	//mustMap = append(mustMap, map[string]interface{}{
-	//	"bool": shouldMap,
-	//})
+	mustMap = append(mustMap, map[string]interface{}{
+		"bool": shouldMap,
+	})
 	queryMap := map[string]interface{}{
 		"query": map[string]interface{}{
 			"bool": map[string]interface{}{
@@ -2057,14 +2057,14 @@ func SearchDataSourceIndex(indexName, keyword string, source, subSource int, cla
 	total = t
 
 	// 表格名称高亮，分页
-	//highlightKeyName := "IndexName"
-	highlightIk := "IndexName.ik"
-	highlightKeyword := "IndexName.keyword"
+	highlightKeyName := "IndexName"
+	//highlightIk := "IndexName.ik"
+	//highlightKeyword := "IndexName.keyword"
 	queryMap["highlight"] = map[string]interface{}{
 		"fields": map[string]interface{}{
-			//highlightKeyName:      map[string]interface{}{},
-			highlightIk:      map[string]interface{}{},
-			highlightKeyword: map[string]interface{}{},
+			highlightKeyName: map[string]interface{}{},
+			//highlightIk:      map[string]interface{}{},
+			//highlightKeyword: map[string]interface{}{},
 		},
 		"pre_tags":  "<span style=\"color:#0052D9\">",
 		"post_tags": "</span>",
@@ -2106,19 +2106,19 @@ func SearchDataSourceIndex(indexName, keyword string, source, subSource int, cla
 			err = fmt.Errorf("hits json unmarshal err: %v", e)
 			return
 		}
-		//if len(v.Highlight[highlightKeyName]) > 0 {
-		//	item.SearchText = v.Highlight[highlightKeyName][0]
-		//}
+		if len(v.Highlight[highlightKeyName]) > 0 {
+			item.SearchText = v.Highlight[highlightKeyName][0]
+		}
 
 		// 优先取keyword的高亮，其次是ik分词之后的高亮
-		if len(v.Highlight) > 0 {
-			if len(v.Highlight[highlightKeyword]) > 0 {
-				item.SearchText = v.Highlight[highlightKeyword][0]
-			}
-			if len(v.Highlight[highlightKeyword]) == 0 && len(v.Highlight[highlightIk]) > 0 {
-				item.SearchText = v.Highlight[highlightIk][0]
-			}
-		}
+		//if len(v.Highlight) > 0 {
+		//	if len(v.Highlight[highlightKeyword]) > 0 {
+		//		item.SearchText = v.Highlight[highlightKeyword][0]
+		//	}
+		//	if len(v.Highlight[highlightKeyword]) == 0 && len(v.Highlight[highlightIk]) > 0 {
+		//		item.SearchText = v.Highlight[highlightIk][0]
+		//	}
+		//}
 		if item.SearchText == "" {
 			item.SearchText = item.IndexName
 		}

services/temp_fix.go

@@ -54,8 +54,8 @@ func FixNewEs() {
 		return
 	}
 	fmt.Println("开始创建数据源索引：", utils.EsDataSourceIndexName)
-	//indexMappingB := `{"mappings":{"properties":{"ClassifyId":{"type":"long"},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"EndDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Frequency":{"type":"text","analyzer":"ik_smart"},"IndexCode":{"type":"text","analyzer":"ik_smart"},"IndexName":{"type":"text","analyzer":"ik_smart"},"IsDeleted":{"type":"long"},"LatestValue":{"type":"double"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"PrimaryId":{"type":"long"},"Source":{"type":"long"},"SourceName":{"type":"text","analyzer":"ik_smart"},"StartDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Unit":{"type":"text","analyzer":"ik_smart"}}}}`
-	indexMappingB := `{"settings":{"analysis":{"analyzer":{"ik_analyzer":{"type":"custom","tokenizer":"ik_max_word"},"ngram_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","ngram_filter"]}},"filter":{"ngram_filter":{"type":"ngram","min_gram":1,"max_gram":2,"token_chars":["letter","digit"]}}}},"mappings":{"properties":{"ClassifyId":{"type":"long"},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"EndDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Frequency":{"type":"text","analyzer":"ik_smart"},"IndexCode":{"type":"text","fields":{"ngram":{"type":"text","analyzer":"ngram_analyzer","search_analyzer":"standard"},"keyword":{"type":"keyword","ignore_above":256}}},"IndexName":{"type":"text","fields":{"ik":{"type":"text","analyzer":"ik_analyzer","search_analyzer":"ik_analyzer"},"ngram":{"type":"text","analyzer":"ngram_analyzer","search_analyzer":"standard"},"keyword":{"type":"keyword","ignore_above":256}}},"IsDeleted":{"type":"long"},"LatestValue":{"type":"double"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"PrimaryId":{"type":"long"},"Source":{"type":"long"},"SourceName":{"type":"text","analyzer":"ik_smart"},"SubSource":{"type":"long"},"StartDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Unit":{"type":"text","analyzer":"ik_smart"}}}}`
+	indexMappingB := `{"mappings":{"properties":{"ClassifyId":{"type":"long"},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"EndDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Frequency":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IndexCode":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IndexName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IsDeleted":{"type":"long"},"LatestValue":{"type":"double"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"PrimaryId":{"type":"long"},"Source":{"type":"long"},"SourceName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"SubSource":{"type":"long"},"StartDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Unit":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}`
+	//indexMappingB := `{"settings":{"analysis":{"analyzer":{"ik_analyzer":{"type":"custom","tokenizer":"ik_max_word"},"ngram_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","ngram_filter"]}},"filter":{"ngram_filter":{"type":"ngram","min_gram":1,"max_gram":2,"token_chars":["letter","digit"]}}}},"mappings":{"properties":{"ClassifyId":{"type":"long"},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"EndDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Frequency":{"type":"text","analyzer":"ik_smart"},"IndexCode":{"type":"text","fields":{"ngram":{"type":"text","analyzer":"ngram_analyzer","search_analyzer":"standard"},"keyword":{"type":"keyword","ignore_above":256}}},"IndexName":{"type":"text","fields":{"ik":{"type":"text","analyzer":"ik_analyzer","search_analyzer":"ik_analyzer"},"ngram":{"type":"text","analyzer":"ngram_analyzer","search_analyzer":"standard"},"keyword":{"type":"keyword","ignore_above":256}}},"IsDeleted":{"type":"long"},"LatestValue":{"type":"double"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"PrimaryId":{"type":"long"},"Source":{"type":"long"},"SourceName":{"type":"text","analyzer":"ik_smart"},"SubSource":{"type":"long"},"StartDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Unit":{"type":"text","analyzer":"ik_smart"}}}}`
 	if e := elastic.CreateEsIndex(utils.EsDataSourceIndexName, indexMappingB); e != nil {
 		err = fmt.Errorf("创建数据源索引失败, %v", e)
 		return