fix

controllers/data_manage/ai_predict_model/index.go

@@ -15,12 +15,13 @@ import (
 	"eta/eta_api/services/elastic"
 	"eta/eta_api/utils"
 	"fmt"
-	"github.com/rdlucklib/rdluck_tools/paging"
-	"github.com/tealeg/xlsx"
 	"os"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/rdlucklib/rdluck_tools/paging"
+	"github.com/tealeg/xlsx"
 )
 
 // AiPredictModelIndexController AI预测模型标的
@@ -167,6 +168,135 @@ func (this *AiPredictModelIndexController) List() {
 	br.Msg = "获取成功"
 }
 
+// GetAll
+// @Title 获取标的全量数据
+// @Description 获取标的全量数据
+// @Param   ClassifyId   query   int   false   "分类id"
+// @Param   IndexId   query   int   false   "模型标的ID"
+// @Param   Keyword   query   string   false   "搜索关键词"
+// @Success 200 {object} data_manage.ChartListResp
+// @router /index/all [get]
+func (this *AiPredictModelIndexController) GetAll() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	sysUser := this.SysUser
+	if sysUser == nil {
+		br.Msg = "请登录"
+		br.ErrMsg = "请登录，SysUser Is Empty"
+		br.Ret = 408
+		return
+	}
+	classifyId, _ := this.GetInt("ClassifyId")
+	indexId, _ := this.GetInt("IndexId")
+	keyword := this.GetString("KeyWord")
+	if keyword == "" {
+		keyword = this.GetString("Keyword")
+	}
+	keyword = strings.TrimSpace(keyword)
+	resp := new(aiPredictModel.AiPredictModelIndexPageListResp)
+
+	// 分类
+	classifyIdName := make(map[int]string)
+	{
+		classifyOb := new(aiPredictModel.AiPredictModelClassify)
+		list, e := classifyOb.GetItemsByCondition("", make([]interface{}, 0), []string{}, "")
+		if e != nil {
+			br.Msg = "获取失败"
+			br.ErrMsg = fmt.Sprintf("获取分类失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			classifyIdName[v.AiPredictModelClassifyId] = v.ClassifyName
+		}
+	}
+
+	// 筛选条件
+	highlightMap := make(map[int]string)
+	indexOb := new(aiPredictModel.AiPredictModelIndex)
+	var cond string
+	var pars []interface{}
+	{
+		if indexId > 0 {
+			cond += fmt.Sprintf(" AND %s = ?", indexOb.Cols().PrimaryId)
+			pars = append(pars, indexId)
+		}
+		if classifyId > 0 {
+			cond += fmt.Sprintf(" AND %s = ?", indexOb.Cols().ClassifyId)
+			pars = append(pars, classifyId)
+		}
+
+		// 有关键词从es中搜索
+		if keyword != "" {
+			// 使用scroll API获取所有匹配的数据
+			scrollId, list, e := elastic.SearchDataSourceIndexWithScroll(utils.EsDataSourceIndexName, keyword, utils.DATA_SOURCE_AI_PREDICT_MODEL, 0, []int{}, []int{}, []string{}, 1000)
+			if e != nil {
+				br.Msg = "获取失败"
+				br.ErrMsg = fmt.Sprintf("ES-搜索AI预测模型列表失败, %v", e)
+				return
+			}
+
+			// 如果scrollId不为空，说明还有更多数据，继续获取
+			for scrollId != "" {
+				nextScrollId, nextList, e := elastic.ScrollDataSourceIndex(utils.EsDataSourceIndexName, scrollId)
+				if e != nil {
+					br.Msg = "获取失败"
+					br.ErrMsg = fmt.Sprintf("ES-获取更多数据失败, %v", e)
+					return
+				}
+				if len(nextList) > 0 {
+					list = append(list, nextList...)
+				}
+				scrollId = nextScrollId
+			}
+
+			if len(list) == 0 {
+				resp.List = make([]*aiPredictModel.AiPredictModelIndexItem, 0)
+				br.Ret = 200
+				br.Success = true
+				br.Msg = "获取成功"
+				br.Data = resp
+				return
+			}
+			var ids []int
+			for _, v := range list {
+				ids = append(ids, v.PrimaryId)
+				highlightMap[v.PrimaryId] = v.SearchText
+			}
+			cond += fmt.Sprintf(` AND %s IN (%s)`, indexOb.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+			pars = append(pars, ids)
+		}
+	}
+
+	// 获取列表
+	list, e := indexOb.GetItemsByCondition(cond, pars, []string{}, "")
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = fmt.Sprintf("获取列表失败, %v", e)
+		return
+	}
+	pageList := make([]*aiPredictModel.AiPredictModelIndexItem, 0)
+	for _, v := range list {
+		t := v.Format2Item()
+		t.ClassifyName = classifyIdName[v.ClassifyId]
+		// 搜索高亮
+		t.SearchText = v.IndexName
+		s := highlightMap[v.AiPredictModelIndexId]
+		if s != "" {
+			t.SearchText = s
+		}
+		pageList = append(pageList, t)
+	}
+
+	resp.List = pageList
+	br.Data = resp
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
 // Import
 // @Title 导入标的和数据
 // @Description 导入标的和数据

controllers/data_manage/ai_predict_model/index_config.go

@@ -24,7 +24,7 @@ type AiPredictModelIndexConfigController struct {
 // @Description 列表
 // @Param   PageSize   query   int  true       "每页数据条数"
 // @Param   CurrentIndex   query   int  true       "当前页页码，从1开始"
-// @Param   AiPredictModelIndexId   query   int  true       "标的id"
+// @Param   IndexId   query   int  true       "标的id"
 // @Success 200 {object} []*data_manage.AiPredictModelIndexConfigView
 // @router /index_config/list [get]
 func (c *AiPredictModelIndexConfigController) List() {
@@ -42,7 +42,7 @@ func (c *AiPredictModelIndexConfigController) List() {
 	}
 	pageSize, _ := c.GetInt("PageSize")
 	currentIndex, _ := c.GetInt("CurrentIndex")
-	indexId, _ := c.GetInt("AiPredictModelIndexId")
+	indexId, _ := c.GetInt("IndexId")
 
 	if indexId <= 0 {
 		br.Msg = "标的id不能为空"
@@ -399,7 +399,7 @@ func (c *AiPredictModelIndexConfigController) Train() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	if req.AiPredictModelIndexId <= 0 {
+	if req.IndexId <= 0 {
 		br.Msg = "标的id不能为空"
 		br.IsSendEmail = false
 		return
@@ -414,7 +414,7 @@ func (c *AiPredictModelIndexConfigController) Train() {
 
 	// 查询标的情况
 	indexOb := new(data_manage.AiPredictModelIndex)
-	indexItem, err := indexOb.GetItemById(req.AiPredictModelIndexId)
+	indexItem, err := indexOb.GetItemById(req.IndexId)
 	if err != nil {
 		br.Msg = "训练失败，查找标的失败"
 		br.ErrMsg = fmt.Sprintf("训练失败，查找标的失败, %v", err)

models/ai_predict_model/request/index_config.go

@@ -7,7 +7,7 @@ type DelConfigReq struct {
 // TrainReq
 // @Description: 训练模型请求参数
 type TrainReq struct {
-	AiPredictModelIndexId       int          `description:"模型id"`
+	IndexId                     int          `description:"模型id"`
 	AiPredictModelIndexConfigId int          `description:"配置id"`
 	Params                      ConfigParams `description:"训练参数"`
 }

services/elastic/elastic.go

@@ -10,9 +10,10 @@ import (
 	dataSourceModel "eta/eta_api/models/data_source"
 	"eta/eta_api/utils"
 	"fmt"
-	"github.com/olivere/elastic/v7"
 	"strconv"
 	"strings"
+
+	"github.com/olivere/elastic/v7"
 )
 
 // indexName:索引名称
@@ -2205,3 +2206,203 @@ func ClearEsIndex(indexName string) (err error) {
 	fmt.Printf("Cleared index: %s, Deleted documents count: %d\n", indexName, res.Deleted)
 	return
 }
+
+// SearchDataSourceIndexWithScroll 使用scroll API查询es中的数据源
+func SearchDataSourceIndexWithScroll(indexName, keyword string, source, subSource int, classifyIds, adminIds []int, frequency []string, size int) (scrollId string, list []*dataSourceModel.SearchDataSourceItem, err error) {
+	list = make([]*dataSourceModel.SearchDataSourceItem, 0)
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("SearchDataSourceIndexWithScroll err: %v", err)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	client := utils.EsClient
+
+	mustMap := make([]interface{}, 0)
+	mustNotMap := make([]interface{}, 0)
+	mustMap = append(mustMap, map[string]interface{}{
+		"term": map[string]interface{}{
+			"IsDeleted": 0,
+		},
+	})
+
+	// 指标编码/名称
+	shouldMap := make(map[string]interface{}, 0)
+	if keyword != "" {
+		shouldMap["should"] = []interface{}{
+			map[string]interface{}{
+				"match": map[string]interface{}{
+					"IndexCode": keyword,
+				},
+			},
+			map[string]interface{}{
+				"match": map[string]interface{}{
+					"IndexName": keyword,
+				},
+			},
+		}
+	}
+
+	// 来源/子来源
+	if source > 0 {
+		mustMap = append(mustMap, map[string]interface{}{
+			"term": map[string]interface{}{
+				"Source": source,
+			},
+		})
+	}
+	if subSource > 0 {
+		mustMap = append(mustMap, map[string]interface{}{
+			"term": map[string]interface{}{
+				"SubSource": subSource,
+			},
+		})
+	}
+
+	// 分类
+	if len(classifyIds) > 0 {
+		mustMap = append(mustMap, map[string]interface{}{
+			"term": map[string]interface{}{
+				"ClassifyId": classifyIds,
+			},
+		})
+	}
+
+	// 创建人
+	if len(adminIds) > 0 {
+		mustMap = append(mustMap, map[string]interface{}{
+			"term": map[string]interface{}{
+				"SysUserId": adminIds,
+			},
+		})
+	}
+
+	// 频度
+	if len(frequency) > 0 {
+		mustMap = append(mustMap, map[string]interface{}{
+			"term": map[string]interface{}{
+				"Frequency": frequency,
+			},
+		})
+	}
+
+	// 关键字匹配
+	mustMap = append(mustMap, map[string]interface{}{
+		"bool": shouldMap,
+	})
+	queryMap := map[string]interface{}{
+		"query": map[string]interface{}{
+			"bool": map[string]interface{}{
+				"must":     mustMap,
+				"must_not": mustNotMap,
+			},
+		},
+	}
+
+	// 表格名称高亮
+	highlightKeyName := "IndexName"
+	queryMap["highlight"] = map[string]interface{}{
+		"fields": map[string]interface{}{
+			highlightKeyName: map[string]interface{}{},
+		},
+		"pre_tags":  "<span style=\"color:#0052D9\">",
+		"post_tags": "</span>",
+	}
+	queryMap["size"] = size
+
+	// 使用scroll API
+	request := client.Scroll(indexName).Body(queryMap).Scroll("5m")
+	searchResp, e := request.Do(context.Background())
+	if e != nil {
+		err = fmt.Errorf("search do err: %v", e)
+		return
+	}
+
+	if searchResp.Status != 0 {
+		return
+	}
+	if searchResp.Hits == nil {
+		return
+	}
+
+	scrollId = searchResp.ScrollId
+	searchMap := make(map[string]string)
+	for _, v := range searchResp.Hits.Hits {
+		if _, ok := searchMap[v.Id]; ok {
+			continue
+		}
+		j, e := v.Source.MarshalJSON()
+		if e != nil {
+			err = fmt.Errorf("hits json err: %v", e)
+			return
+		}
+		item := new(dataSourceModel.SearchDataSourceItem)
+		if e = json.Unmarshal(j, &item); e != nil {
+			err = fmt.Errorf("hits json unmarshal err: %v", e)
+			return
+		}
+		if len(v.Highlight[highlightKeyName]) > 0 {
+			item.SearchText = v.Highlight[highlightKeyName][0]
+		}
+		if item.SearchText == "" {
+			item.SearchText = item.IndexName
+		}
+		list = append(list, item)
+		searchMap[v.Id] = v.Id
+	}
+	return
+}
+
+// ScrollDataSourceIndex 使用scroll API获取下一页数据
+func ScrollDataSourceIndex(indexName, scrollId string) (nextScrollId string, list []*dataSourceModel.SearchDataSourceItem, err error) {
+	list = make([]*dataSourceModel.SearchDataSourceItem, 0)
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("ScrollDataSourceIndex err: %v", err)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	client := utils.EsClient
+
+	// 使用scroll API获取下一页
+	request := client.Scroll().ScrollId(scrollId).Scroll("5m")
+	searchResp, e := request.Do(context.Background())
+	if e != nil {
+		err = fmt.Errorf("scroll do err: %v", e)
+		return
+	}
+
+	if searchResp.Status != 0 {
+		return
+	}
+	if searchResp.Hits == nil {
+		return
+	}
+
+	nextScrollId = searchResp.ScrollId
+	searchMap := make(map[string]string)
+	for _, v := range searchResp.Hits.Hits {
+		if _, ok := searchMap[v.Id]; ok {
+			continue
+		}
+		j, e := v.Source.MarshalJSON()
+		if e != nil {
+			err = fmt.Errorf("hits json err: %v", e)
+			return
+		}
+		item := new(dataSourceModel.SearchDataSourceItem)
+		if e = json.Unmarshal(j, &item); e != nil {
+			err = fmt.Errorf("hits json unmarshal err: %v", e)
+			return
+		}
+		if len(v.Highlight["IndexName"]) > 0 {
+			item.SearchText = v.Highlight["IndexName"][0]
+		}
+		if item.SearchText == "" {
+			item.SearchText = item.IndexName
+		}
+		list = append(list, item)
+		searchMap[v.Id] = v.Id
+	}
+	return
+}