package services import ( "eta/eta_crawler/models" "eta/eta_crawler/services/alarm_msg" "eta/eta_crawler/utils" "fmt" "github.com/mozillazg/go-pinyin" "github.com/tealeg/xlsx" "strconv" "strings" "time" ) func FileCoalJsm() { var err error defer func() { if err != nil { fmt.Println("RefreshDataFromCoalJsm Err:" + err.Error()) msg := "失败提醒" + "RefreshDataFromCoalJsm ErrMsg:" + err.Error() go alarm_msg.SendAlarmMsg(msg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalJsm ErrMsg:"+err.Error(), utils.EmailSendToUsers) } }() //path := "static/file/442家晋陕蒙煤矿周度产量数据-20220328.xlsx" var exist bool path := "/home/code/python/coal_mail/emailFile/442家晋陕蒙煤矿周度产量数据-%s.xlsx" for i := 0; i < 10; i++ { npath := fmt.Sprintf(path, time.Now().AddDate(0, 0, -i).Format(utils.FormatDateUnSpace)) fmt.Println(npath) exist, err = PathExists(npath) if err != nil { fmt.Println(err) return } if exist { path = npath break } } var xlFile *xlsx.File if exist { xlFile, err = xlsx.OpenFile(path) if err != nil { return } } else { return } var province string var description string var exchange string var city string var companyName string var group string var dateMap = make(map[int]string) var codeMap = make(map[string]string) var indexMap = make(map[string]string) var codeCompanyMap = make(map[string]string) var indexCompanyMap = make(map[string]string) var items []*models.BaseFromCoalmineMapping var itemsCompany []*models.BaseFromCoalmineMapping var itemsIndex []*models.BaseFromCoalmineJsmIndex var itemsCompanyIndex []*models.BaseFromCoalmineCompanyIndex codeList, err := models.GetBaseFromCoalmineMapping() if err != nil && err.Error() != utils.ErrNoRow() { utils.FileLog.Info("获取煤炭指标失败:", err) return } if len(codeList) > 0 { for _, v := range codeList { codeMap[v.IndexName] = v.IndexCode } } codeCompanyList, err := models.GetBaseFromCoalmineMapping() if err != nil && err.Error() != utils.ErrNoRow() { utils.FileLog.Info("获取煤炭公司指标失败:", err) return } if len(codeCompanyList) > 0 { for _, v := range codeCompanyList { codeCompanyMap[v.IndexName] = v.IndexCode } } indexList, err := models.GetBaseFromCoalmineIndex() if err != nil && err.Error() != utils.ErrNoRow() { utils.FileLog.Info("获取煤炭公司指标失败:", err) return } if len(indexList) > 0 { for _, v := range indexList { indexMap[v.IndexName+v.DataTime] = v.DealValue } } indexCompanyList, err := models.GetBaseFromCoalmineCompanyIndex() if err != nil && err.Error() != utils.ErrNoRow() { utils.FileLog.Info("获取煤炭公司指标失败:", err) return } if len(indexCompanyList) > 0 { for _, v := range indexCompanyList { indexCompanyMap[v.IndexName+v.DataTime] = v.DealValue } } for _, sheet := range xlFile.Sheets { //遍历行读取 maxRow := sheet.MaxRow for i := 0; i < maxRow; i++ { //获取样本情况 if i > 3 && i < 16 { row := sheet.Row(i) cells := row.Cells Loop2: for k, cell := range cells { text := cell.String() if text != "" { if k == 1 { province = text } if k == 3 { description = text } if k == 4 { item := new(models.BaseFromCoalmineMapping) exchange = text exists := ContainsSpecialName(province) var strResult string if exists { abbr := TrimProvinceName(province) //取处理后公司名首字母缩写 a := pinyin.NewArgs() rows := pinyin.Pinyin(exchange, a) for i := 0; i < len(rows); i++ { if len(rows[i]) != 0 { str := rows[i][0] pi := str[0:1] strResult += pi } } item.IndexCode = abbr + strResult + "jsm" } else { //取处理后公司名首字母缩写 a := pinyin.NewArgs() rows := pinyin.Pinyin(province+exchange, a) for i := 0; i < len(rows); i++ { if len(rows[i]) != 0 { str := rows[i][0] pi := str[0:1] strResult += pi } } item.IndexCode = strResult + "jsm" } item.IndexName = province + exchange item.CreateTime = time.Now() items = append(items, item) break Loop2 } } } } //获取日期 if i == 3 { row := sheet.Row(i) cells := row.Cells for k, cell := range cells { text := cell.String() stamp, _ := time.ParseInLocation("01-02-06", text, time.Now().Location()) if k > 4 { dateMap[k] = stamp.Format(utils.FormatDate) } } } //获取具体产量 if i > 3 && i < 16 { row := sheet.Row(i) cells := row.Cells for k, cell := range cells { text := cell.String() if k > 4 { item := new(models.BaseFromCoalmineJsmIndex) item.IndexName = province + exchange item.IndexCode = codeMap[item.IndexName] item.Exchange = exchange item.DealValue = text item.DataTime = dateMap[k] item.Exchange = exchange item.Description = description item.Province = province item.Source = "三省周度" if exchange == "产量" { item.Unit = "万吨" } else { item.Unit = "百分比" } item.Frequency = "周度" item.ModifyTime = time.Now() item.CreateTime = time.Now() itemsIndex = append(itemsIndex, item) } } } //获取公司指标名称 if i > 17 { row := sheet.Row(i) cells := row.Cells Loop3: for k, cell := range cells { text := cell.String() if text != "" { if k == 1 { province = text } if k == 2 { city = text } if k == 3 { companyName = text } if k == 4 { item := new(models.BaseFromCoalmineMapping) group = text trimName := TrimCompanyName(companyName) item.IndexName = trimName //取处理后公司名首字母缩写 strResult := "" a := pinyin.NewArgs() rows := pinyin.Pinyin(trimName, a) for i := 0; i < len(rows); i++ { if len(rows[i]) != 0 { str := rows[i][0] pi := str[0:1] strResult += pi } } item.IndexCode = strResult + "company" item.CreateTime = time.Now() itemsCompany = append(itemsCompany, item) break Loop3 } } } } //获取公司具体产量 if i > 18 { row := sheet.Row(i) cells := row.Cells for k, cell := range cells { text := cell.String() if k > 4 { item := new(models.BaseFromCoalmineCompanyIndex) companyName = TrimCompanyName(companyName) item.IndexName = companyName item.IndexCode = codeCompanyMap[item.IndexName] item.DealValue = text item.DataTime = dateMap[k] item.Province = province item.City = city //处理无类别名时的情况 if group == "无" { item.GroupName = companyName } else { item.GroupName = group } item.Source = "三省企业" item.Unit = "万吨" item.Frequency = "周度" item.ModifyTime = time.Now() item.CreateTime = time.Now() //fmt.Println(item) itemsCompanyIndex = append(itemsCompanyIndex, item) } } } } } //添加数据到数据库 for _, v := range items { if codeMap[v.IndexName] == "" { codeMap[v.IndexName] = v.IndexCode newId, err := models.AddBaseFromCoalmineMapping(v) if err != nil { fmt.Println("添加指标名称错误") } else { fmt.Println("添加指标名称成功", newId) } } } fmt.Println("指标操作完成") for _, v := range itemsIndex { v.IndexCode = codeMap[v.IndexName] if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" { newId, err := models.AddBaseFromCoalmineIndex(v) if err != nil { fmt.Println("添加数据错误", err) } else { fmt.Println("新增成功", newId) } } else { if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" { err = models.UpdateBaseFromCoalmineIndex(v) if err != nil { fmt.Println("修改数据错误错误", err) return } } } } fmt.Println("数据操作完成") for _, v := range itemsCompany { if codeCompanyMap[v.IndexName] == "" { codeCompanyMap[v.IndexName] = v.IndexCode newId, err := models.AddBaseFromCoalmineMapping(v) if err != nil { for i := 0; i < 10; i++ { v.IndexCode = v.IndexCode + strconv.Itoa(i) codeCompanyMap[v.IndexName] = v.IndexCode newId, err := models.AddBaseFromCoalmineMapping(v) if err != nil { fmt.Println("再次添加公司指标名称错误", err) fmt.Println(v.IndexName, v.IndexCode) continue } else { fmt.Println("新增公司成功", newId) break } } } else { fmt.Println("新增公司成功", newId) } } } fmt.Println("公司指标操作完成") for _, v := range itemsCompanyIndex { v.IndexCode = codeCompanyMap[v.IndexName] if indexCompanyMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" { newId, err := models.AddBaseFromCoalmineCompanyIndex(v) if err != nil { fmt.Println("添加公司数据错误", err) } else { fmt.Println("新增公司数据成功", newId) } } else { if indexCompanyMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" { err = models.UpdateBaseFromCoalmineCompanyIndex(v) if err != nil { fmt.Println("修改数据错误错误", err) } } } } return } func TrimCompanyName(name string) string { name = strings.Replace(name, "有限", "", -1) name = strings.Replace(name, "股份", "", -1) name = strings.Replace(name, "责任", "", -1) name = strings.Replace(name, "公司", "", -1) name = strings.Replace(name, "(", "", -1) name = strings.Replace(name, ")", "", -1) return name } func ContainsSpecialName(name string) bool { return strings.Contains(name, "陕西") || strings.Contains(name, "海南") || strings.Contains(name, "河南") || strings.Contains(name, "河北") || strings.Contains(name, "澳门") || strings.Contains(name, "内蒙古") || strings.Contains(name, "黑龙江") } func TrimProvinceName(name string) string { name = strings.Replace(name, "陕西省", "sns", -1) name = strings.Replace(name, "陕西", "sn", -1) name = strings.Replace(name, "海南市", "hi", -1) name = strings.Replace(name, "海南", "hi", -1) name = strings.Replace(name, "河南省", "has", -1) name = strings.Replace(name, "河南", "ha", -1) name = strings.Replace(name, "河北省", "hes", -1) name = strings.Replace(name, "河北", "he", -1) name = strings.Replace(name, "澳门", "mo", -1) name = strings.Replace(name, "内蒙古自治区", "nm", -1) name = strings.Replace(name, "内蒙古", "nm", -1) name = strings.Replace(name, "黑龙江", "hl", -1) name = strings.Replace(name, "(", "", -1) name = strings.Replace(name, ")", "", -1) name = strings.Replace(name, "+", "", -1) return name }