package data_manage import ( "errors" "fmt" "github.com/beego/beego/v2/client/orm" "hongze/hz_crm_api/utils" "strings" "time" ) // AddCalculateZjpj 新增直接拼接数据 func AddCalculateZjpj(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, edbCode, uniqueCode string, sysUserId int, sysUserRealName string) (edbInfoId int, err error) { fmt.Println("AddCalculateZjpj") o := orm.NewOrmUsingDB("data") to, err := o.Begin() if err != nil { return } defer func() { if err != nil { fmt.Println("AddCalculateZjpj,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() var edbInfo *EdbInfo if req.EdbInfoId <= 0 { edbInfo = &EdbInfo{ SourceName: "直接拼接", Source: utils.DATA_SOURCE_CALCULATE_ZJPJ, EdbCode: edbCode, EdbName: req.EdbName, EdbNameSource: req.EdbName, Frequency: req.Frequency, Unit: req.Unit, StartDate: firstEdbInfo.StartDate, EndDate: firstEdbInfo.EndDate, ClassifyId: req.ClassifyId, SysUserId: sysUserId, SysUserRealName: sysUserRealName, UniqueCode: uniqueCode, CreateTime: time.Now(), ModifyTime: time.Now(), CalculateFormula: req.Formula, EdbType: 2, } newEdbInfoId, err := to.Insert(edbInfo) if err != nil { return edbInfoId, err } edbInfoId = int(newEdbInfoId) } else { edbInfoId = req.EdbInfoId //查询 tmpEdbInfo, tmpErr := GetEdbInfoById(edbInfoId) if tmpErr != nil { err = tmpErr return } tmpEdbInfo.EdbName = req.EdbName tmpEdbInfo.ClassifyId = req.ClassifyId tmpEdbInfo.Frequency = req.Frequency tmpEdbInfo.Unit = req.Unit tmpEdbInfo.CalculateFormula = req.Formula edbInfo = tmpEdbInfo //删除指标数据 dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_CALCULATE_ZJPJ) fmt.Println("dataTableName:" + dataTableName) deleteSql := ` DELETE FROM %s WHERE edb_info_id=? ` deleteSql = fmt.Sprintf(deleteSql, dataTableName) _, err = to.Raw(deleteSql, req.EdbInfoId).Exec() if err != nil { return 0, err } //删除指标关系 sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id=? ` _, err = to.Raw(sql, edbInfoId).Exec() if err != nil { return } } //关联关系 //第一个指标 { calculateMappingItem := new(EdbInfoCalculateMapping) calculateMappingItem.CreateTime = time.Now() calculateMappingItem.ModifyTime = time.Now() calculateMappingItem.Sort = 1 calculateMappingItem.EdbCode = edbCode calculateMappingItem.EdbInfoId = edbInfoId calculateMappingItem.FromEdbInfoId = firstEdbInfo.EdbInfoId calculateMappingItem.FromEdbCode = firstEdbInfo.EdbCode calculateMappingItem.FromEdbName = firstEdbInfo.EdbName calculateMappingItem.FromSource = firstEdbInfo.Source calculateMappingItem.FromSourceName = firstEdbInfo.SourceName calculateMappingItem.FromTag = "A" calculateMappingItem.Source = edbInfo.Source calculateMappingItem.SourceName = edbInfo.SourceName _, err = to.Insert(calculateMappingItem) if err != nil { return } } //第二个指标 { calculateMappingItem := new(EdbInfoCalculateMapping) calculateMappingItem.CreateTime = time.Now() calculateMappingItem.ModifyTime = time.Now() calculateMappingItem.Sort = 1 calculateMappingItem.EdbCode = edbCode calculateMappingItem.EdbInfoId = edbInfoId calculateMappingItem.FromEdbInfoId = secondEdbInfo.EdbInfoId calculateMappingItem.FromEdbCode = secondEdbInfo.EdbCode calculateMappingItem.FromEdbName = secondEdbInfo.EdbName calculateMappingItem.FromSource = secondEdbInfo.Source calculateMappingItem.FromSourceName = secondEdbInfo.SourceName calculateMappingItem.FromTag = "B" calculateMappingItem.Source = edbInfo.Source calculateMappingItem.SourceName = edbInfo.SourceName _, err = to.Insert(calculateMappingItem) if err != nil { return } } addDataList := make([]*EdbDataCalculateZjpj, 0) //拼接数据 //第一个指标 { var condition string var pars []interface{} condition += " AND data_time < ? AND edb_info_id=? " pars = append(pars, req.Formula, firstEdbInfo.EdbInfoId) //第一个指标的数据列表 firstDataList, tmpErr := GetEdbDataListAll(condition, pars, firstEdbInfo.Source, 0) if tmpErr != nil { return edbInfoId, tmpErr } for _, v := range firstDataList { //时间戳 currentDate, _ := time.Parse(utils.FormatDate, v.DataTime) timestamp := currentDate.UnixNano() / 1e6 edbDataZjpj := &EdbDataCalculateZjpj{ EdbInfoId: edbInfoId, EdbCode: edbInfo.EdbCode, DataTime: v.DataTime, Value: v.Value, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataZjpj) if len(addDataList) >= 200 { _, tmpErr := to.InsertMulti(len(addDataList), addDataList) if tmpErr != nil { err = tmpErr return } //重新初始化需要加入的数据切片 addDataList = make([]*EdbDataCalculateZjpj, 0) } } } //第二个指标 { var condition string var pars []interface{} condition += " AND data_time >= ? AND edb_info_id = ? " pars = append(pars, req.Formula, secondEdbInfo.EdbInfoId) //第二个指标的数据列表 secondDataList, tmpErr := GetEdbDataListAll(condition, pars, secondEdbInfo.Source, 0) if tmpErr != nil { return edbInfoId, tmpErr } for _, v := range secondDataList { //时间戳 currentDate, _ := time.Parse(utils.FormatDate, v.DataTime) timestamp := currentDate.UnixNano() / 1e6 edbDataZjpj := &EdbDataCalculateZjpj{ EdbInfoId: edbInfoId, EdbCode: edbInfo.EdbCode, DataTime: v.DataTime, Value: v.Value, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataZjpj) if len(addDataList) >= 200 { _, tmpErr := to.InsertMulti(len(addDataList), addDataList) if tmpErr != nil { err = tmpErr return } //重新初始化需要加入的数据切片 addDataList = make([]*EdbDataCalculateZjpj, 0) } } } //将剩余的数据入库 if len(addDataList) > 0 { _, tmpErr := to.InsertMulti(len(addDataList), addDataList) if tmpErr != nil { err = tmpErr return } } return } // EditCalculateZjpj 编辑直接拼接数据 func EditCalculateZjpj(req *EdbInfoCalculateBatchEditReq, nowEdbInfo, firstEdbInfo, secondEdbInfo *EdbInfo) (edbInfoId int, err error) { edbInfoId = req.EdbInfoId o := orm.NewOrmUsingDB("data") to, err := o.Begin() if err != nil { return } defer func() { if err != nil { _ = to.Rollback() } else { _ = to.Commit() } }() //修改指标信息 sql := ` UPDATE edb_info SET edb_name =?, edb_name_source=?, frequency = ?, unit = ?, classify_id = ?, calculate_formula=?, modify_time = NOW() WHERE edb_info_id = ? ` _, err = to.Raw(sql, req.EdbName, req.EdbName, req.Frequency, req.Unit, req.ClassifyId, req.Formula, edbInfoId).Exec() if err != nil { return } var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfoId) //查询出所有的关联指标 existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars) if err != nil { err = errors.New("判断指标是否改变失败,Err:" + err.Error()) return } var existItemA, existItemB *EdbInfoCalculateMapping for _, existItem := range existList { if existItem.FromTag == "A" { existItemA = existItem } else if existItem.FromTag == "B" { existItemB = existItem } } addDataList := make([]*EdbDataCalculateZjpj, 0) firstDataList := make([]*EdbInfoSearchData, 0) secondDataList := make([]*EdbInfoSearchData, 0) //如果 之前的拼接日期 与 现在的拼接日期 不一致的话,需要做以下处理 nowFormulaDate, _ := time.ParseInLocation(utils.FormatDate, nowEdbInfo.CalculateFormula, time.Local) reqFormulaDate, _ := time.ParseInLocation(utils.FormatDate, req.Formula, time.Local) //如果前后选择的日期不一致,那么需要删除一部分数据 if nowEdbInfo.CalculateFormula != req.Formula { var startDate, endDate time.Time //如果当前选择的日期 小于 之前选择的日期 if reqFormulaDate.Before(nowFormulaDate) { startDate = reqFormulaDate endDate = nowFormulaDate } else { //如果当前选择的日期 大于 之前选择的日期 startDate = nowFormulaDate endDate = reqFormulaDate } //删除 之前日期 与 当前日期 之间的指标数据 sql = ` DELETE FROM edb_data_calculate_zjpj WHERE edb_info_id = ? and data_time >= ? and data_time < ?` _, err = to.Raw(sql, edbInfoId, startDate, endDate).Exec() if err != nil { err = errors.New("删除 之前日期 与 当前日期 之间的指标数据失败,Err:" + err.Error()) return } } //第一个指标数据 { var condition string var pars []interface{} if existItemA.FromEdbInfoId != firstEdbInfo.EdbInfoId { //删除之前的A指标关联关系 sql = ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?` _, err = to.Raw(sql, edbInfoId, existItemA.FromEdbInfoId).Exec() if err != nil { err = errors.New("删除拼接日期之前的指标关联关系失败,Err:" + err.Error()) return } //删除之前所有的A指标数据 sql = ` DELETE FROM edb_data_calculate_zjpj WHERE edb_info_id = ? and data_time < ?` _, err = to.Raw(sql, edbInfoId, req.Formula).Exec() if err != nil { err = errors.New("删除拼接日期之前的数据失败,Err:" + err.Error()) return } //获取第一个指标的数据列表 condition += " AND data_time < ? AND edb_info_id=? " pars = append(pars, req.Formula, firstEdbInfo.EdbInfoId) tmpFirstDataList, tmpErr := GetEdbDataListAll(condition, pars, firstEdbInfo.Source, 0) if tmpErr != nil { return edbInfoId, tmpErr } firstDataList = tmpFirstDataList //添加新的指标关系 { calculateMappingItem := new(EdbInfoCalculateMapping) calculateMappingItem.CreateTime = time.Now() calculateMappingItem.ModifyTime = time.Now() calculateMappingItem.Sort = 1 calculateMappingItem.EdbCode = nowEdbInfo.EdbCode calculateMappingItem.EdbInfoId = edbInfoId calculateMappingItem.FromEdbInfoId = firstEdbInfo.EdbInfoId calculateMappingItem.FromEdbCode = firstEdbInfo.EdbCode calculateMappingItem.FromEdbName = firstEdbInfo.EdbName calculateMappingItem.FromSource = firstEdbInfo.Source calculateMappingItem.FromSourceName = firstEdbInfo.SourceName calculateMappingItem.FromTag = "A" calculateMappingItem.Source = nowEdbInfo.Source calculateMappingItem.SourceName = nowEdbInfo.SourceName _, err = to.Insert(calculateMappingItem) if err != nil { return } } } else { if req.Formula != nowEdbInfo.CalculateFormula { //获取第一个指标的数据列表 condition += " AND data_time >= ? AND data_time < ? AND edb_info_id=? " pars = append(pars, nowFormulaDate, reqFormulaDate, firstEdbInfo.EdbInfoId) tmpFirstDataList, tmpErr := GetEdbDataListAll(condition, pars, firstEdbInfo.Source, 0) if tmpErr != nil { return edbInfoId, tmpErr } firstDataList = tmpFirstDataList } } //待插入数据 for _, v := range firstDataList { //时间戳 currentDate, _ := time.Parse(utils.FormatDate, v.DataTime) timestamp := currentDate.UnixNano() / 1e6 edbDataZjpj := &EdbDataCalculateZjpj{ EdbInfoId: edbInfoId, EdbCode: nowEdbInfo.EdbCode, DataTime: v.DataTime, Value: v.Value, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataZjpj) } } //第二个指标数据 { var condition string var pars []interface{} if existItemB.FromEdbInfoId != secondEdbInfo.EdbInfoId { //删除之前的B指标关联关系 sql = ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?` _, err = to.Raw(sql, edbInfoId, existItemB.FromEdbInfoId).Exec() if err != nil { err = errors.New("删除拼接日期之后的指标关联关系失败,Err:" + err.Error()) return } //删除历史拼接日期之前所有的B指标数据 sql = ` DELETE FROM edb_data_calculate_zjpj WHERE edb_info_id = ? and data_time >= ?` _, err = to.Raw(sql, edbInfoId, nowEdbInfo.CalculateFormula).Exec() if err != nil { err = errors.New("删除历史拼接日期之后的数据失败,Err:" + err.Error()) return } //第二个指标的数据列表 condition = " AND data_time >= ? AND edb_info_id=? " pars = append(pars, reqFormulaDate, secondEdbInfo.EdbInfoId) tmpSecondDataList, tmpErr := GetEdbDataListAll(condition, pars, secondEdbInfo.Source, 0) if tmpErr != nil { return edbInfoId, tmpErr } secondDataList = tmpSecondDataList //添加新的指标关系 { calculateMappingItem := new(EdbInfoCalculateMapping) calculateMappingItem.CreateTime = time.Now() calculateMappingItem.ModifyTime = time.Now() calculateMappingItem.Sort = 1 calculateMappingItem.EdbCode = nowEdbInfo.EdbCode calculateMappingItem.EdbInfoId = edbInfoId calculateMappingItem.FromEdbInfoId = secondEdbInfo.EdbInfoId calculateMappingItem.FromEdbCode = secondEdbInfo.EdbCode calculateMappingItem.FromEdbName = secondEdbInfo.EdbName calculateMappingItem.FromSource = secondEdbInfo.Source calculateMappingItem.FromSourceName = secondEdbInfo.SourceName calculateMappingItem.FromTag = "B" calculateMappingItem.Source = nowEdbInfo.Source calculateMappingItem.SourceName = nowEdbInfo.SourceName _, err = to.Insert(calculateMappingItem) if err != nil { return } } } else { if req.Formula != nowEdbInfo.CalculateFormula { //获取第二个指标的数据列表 condition += " AND data_time >= ? AND data_time < ? AND edb_info_id=? " pars = append(pars, reqFormulaDate, nowFormulaDate, secondEdbInfo.EdbInfoId) tmpSecondDataList, tmpErr := GetEdbDataListAll(condition, pars, secondEdbInfo.Source, 0) if tmpErr != nil { return edbInfoId, tmpErr } secondDataList = tmpSecondDataList } } //待插入数据 for _, v := range secondDataList { //时间戳 currentDate, _ := time.Parse(utils.FormatDate, v.DataTime) timestamp := currentDate.UnixNano() / 1e6 edbDataZjpj := &EdbDataCalculateZjpj{ EdbInfoId: edbInfoId, EdbCode: nowEdbInfo.EdbCode, DataTime: v.DataTime, Value: v.Value, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataZjpj) } } //数据入库 tmpAddDataList := make([]*EdbDataCalculateZjpj, 0) for _, v := range addDataList { tmpAddDataList = append(tmpAddDataList, v) if len(tmpAddDataList) >= 200 { _, tmpErr := to.InsertMulti(len(tmpAddDataList), tmpAddDataList) if tmpErr != nil { err = tmpErr return } //重新初始化需要加入的数据切片 tmpAddDataList = make([]*EdbDataCalculateZjpj, 0) } } //最后如果还有需要新增的数据,那么就统一入库 if len(tmpAddDataList) > 0 { _, tmpErr := to.InsertMulti(len(tmpAddDataList), tmpAddDataList) if tmpErr != nil { err = tmpErr return } } return } // RefreshAllCalculateZjpj 刷新所有 直接拼接 数据 func RefreshAllCalculateZjpj(edbInfo *EdbInfo) (err error) { o := orm.NewOrmUsingDB("data") to, err := o.Begin() if err != nil { return } defer func() { if err != nil { _ = to.Rollback() } else { _ = to.Commit() } }() //查询当前指标现有的数据 var condition string var pars []interface{} condition += " AND edb_info_id=? " pars = append(pars, edbInfo.EdbInfoId) dataList, err := GetAllEdbDataCalculateZjpjByEdbInfoId(edbInfo.EdbInfoId) if err != nil { return err } var dateArr []string dataMap := make(map[string]*EdbDataCalculateZjpj) removeDataTimeMap := make(map[string]int) //需要移除的日期数据 for _, v := range dataList { dateArr = append(dateArr, v.DataTime) dataMap[v.DataTime] = v removeDataTimeMap[v.DataTime] = 1 } //查询关联指标信息 var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfo.EdbInfoId) existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars) if err != nil { err = errors.New("判断指标是否改变失败,Err:" + err.Error()) return } var existItemA, existItemB *EdbInfoCalculateMapping for _, existItem := range existList { if existItem.FromTag == "A" { existItemA = existItem } else if existItem.FromTag == "B" { existItemB = existItem } } addDataList := make([]*EdbDataCalculateZjpj, 0) //第一个指标 { var condition string var pars []interface{} condition += " AND data_time < ? AND edb_info_id=? " pars = append(pars, edbInfo.CalculateFormula, existItemA.FromEdbInfoId) //第一个指标的数据列表 firstDataList, tmpErr := GetEdbDataListAll(condition, pars, existItemA.FromSource, 0) if tmpErr != nil { return tmpErr } for _, v := range firstDataList { //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素 if _, ok := removeDataTimeMap[v.DataTime]; ok { delete(removeDataTimeMap, v.DataTime) } //时间戳 if edbData, ok := dataMap[v.DataTime]; ok { if edbData.Value != v.Value { //更新指标数据 edbData.Value = v.Value _, err = to.Update(edbData, "Value") if err != nil { return } } } else { //时间戳 currentDate, _ := time.Parse(utils.FormatDate, v.DataTime) timestamp := currentDate.UnixNano() / 1e6 edbDataZjpj := &EdbDataCalculateZjpj{ EdbInfoId: edbInfo.EdbInfoId, EdbCode: edbInfo.EdbCode, DataTime: v.DataTime, Value: v.Value, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataZjpj) } } } //第二个指标 { condition = `` pars = make([]interface{}, 0) condition += " AND data_time >= ? AND edb_info_id = ? " pars = append(pars, edbInfo.CalculateFormula, existItemB.FromEdbInfoId) //第二个指标的数据列表 secondDataList, tmpErr := GetEdbDataListAll(condition, pars, existItemB.FromSource, 0) if tmpErr != nil { return tmpErr } for _, v := range secondDataList { //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素 if _, ok := removeDataTimeMap[v.DataTime]; ok { delete(removeDataTimeMap, v.DataTime) } if edbData, ok := dataMap[v.DataTime]; ok { if edbData.Value != v.Value { //更新指标数据 edbData.Value = v.Value edbData.ModifyTime = time.Now() _, tmpErr := to.Update(edbData, "Value", "ModifyTime") if tmpErr != nil { fmt.Println("tmpErr:", tmpErr) } } } else { //时间戳 currentDate, _ := time.Parse(utils.FormatDate, v.DataTime) timestamp := currentDate.UnixNano() / 1e6 edbDataZjpj := &EdbDataCalculateZjpj{ EdbInfoId: edbInfo.EdbInfoId, EdbCode: edbInfo.EdbCode, DataTime: v.DataTime, Value: v.Value, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataZjpj) } } } //删除已经不存在的累计同比拼接指标数据(由于同比值当日的数据删除了) { removeDateList := make([]string, 0) for dateTime := range removeDataTimeMap { removeDateList = append(removeDateList, dateTime) } if len(removeDateList) > 0 { removeDateStr := strings.Join(removeDateList, `","`) removeDateStr = `"` + removeDateStr + `"` //如果拼接指标变更了,那么需要删除所有的指标数据 tableName := GetEdbDataTableName(edbInfo.Source) sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr) _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec() if err != nil { err = errors.New("删除不存在的直接拼接指标数据失败,Err:" + err.Error()) return } } } //数据入库 tmpAddDataList := make([]*EdbDataCalculateZjpj, 0) for _, v := range addDataList { tmpAddDataList = append(tmpAddDataList, v) if len(tmpAddDataList) >= 200 { _, tmpErr := to.InsertMulti(len(tmpAddDataList), tmpAddDataList) if tmpErr != nil { err = tmpErr return } //重新初始化需要加入的数据切片 tmpAddDataList = make([]*EdbDataCalculateZjpj, 0) } } //最后如果还有需要新增的数据,那么就统一入库 if len(tmpAddDataList) > 0 { _, tmpErr := to.InsertMulti(len(tmpAddDataList), tmpAddDataList) if tmpErr != nil { err = tmpErr return } } return }