Merge branch 'ETA_1.8.7'

models/db.go

@@ -11,19 +11,21 @@ import (
 
 func init() {
 
-	_ = orm.RegisterDataBase("default", "mysql", utils.MYSQL_URL)
-	orm.SetMaxIdleConns("default", 50)
-	orm.SetMaxOpenConns("default", 100)
+	if utils.BusinessCode == utils.BusinessCodeRelease {
+		_ = orm.RegisterDataBase("default", "mysql", utils.MYSQL_URL)
+		orm.SetMaxIdleConns("default", 50)
+		orm.SetMaxOpenConns("default", 100)
 
-	db, _ := orm.GetDB("default")
-	db.SetConnMaxLifetime(10 * time.Minute)
+		db, _ := orm.GetDB("default")
+		db.SetConnMaxLifetime(10 * time.Minute)
 
-	_ = orm.RegisterDataBase("data", "mysql", utils.MYSQL_URL_DATA)
-	orm.SetMaxIdleConns("data", 50)
-	orm.SetMaxOpenConns("data", 100)
+		_ = orm.RegisterDataBase("data", "mysql", utils.MYSQL_URL_DATA)
+		orm.SetMaxIdleConns("data", 50)
+		orm.SetMaxOpenConns("data", 100)
 
-	data_db, _ := orm.GetDB("data")
-	data_db.SetConnMaxLifetime(10 * time.Minute)
+		data_db, _ := orm.GetDB("data")
+		data_db.SetConnMaxLifetime(10 * time.Minute)
+	}
 
 	orm.Debug = true
 	orm.DebugLog = orm.NewLog(utils.Binlog)

services/sci99/sci99.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"eta/eta_crawler/models"
 	"eta/eta_crawler/utils"
 	"fmt"
 	"github.com/mozillazg/go-pinyin"
@@ -16,8 +15,6 @@ import (
 )
 
 var IndexCodeMap = make(map[string]string)
-var IndexMap = make(map[string]*models.BaseFromSci99Index)
-var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
 
 var PriceTypeIDs = []int{34320, 34318}
 var requestList = []ListRequestBody{
@@ -76,39 +73,8 @@ func Scii99(cont context.Context) (err error) {
 	listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
 	detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
 
-	allClassify, err := models.GetBaseFromSci99Classify()
-	if err != nil {
-		fmt.Println("select Code err:", err)
-		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
-		return
-	}
-	for _, item := range allClassify {
-		ClassifyMap[item.ClassifyName] = item
-	}
-
 	for _, request := range requestList {
-		classifyId := 0
-		if classify, ok := ClassifyMap[request.Ppname]; !ok {
-			classifyItem := models.BaseFromSci99Classify{
-				ClassifyName: request.Ppname,
-				CreateTime:   time.Now(),
-				ModifyTime:   time.Now(),
-			}
-
-			id, e := models.AddBaseFromSci99Classify(&classifyItem)
-			if e != nil {
-				err = e
-				fmt.Println("Error inserting into database:", err)
-				utils.FileLog.Info("Error inserting into database:", err)
-				return
-			}
-			fmt.Println("新增分类：", classifyItem.ClassifyName)
-			classifyId = int(id)
-			classifyItem.BaseFromSciClassifyId = int(id)
-			ClassifyMap[request.Ppname] = &classifyItem
-		} else {
-			classifyId = classify.BaseFromSciClassifyId
-		}
+		classifyName := request.Ppname
 
 		// 分别获取国内价格和国际价格
 		for _, priceTypeID := range PriceTypeIDs {
@@ -171,17 +137,6 @@ func Scii99(cont context.Context) (err error) {
 				utils.FileLog.Info("Error unmarshalling JSON:", err)
 				return
 			}
-			allCode, e := models.GetBaseFromSci99Index()
-			if e != nil {
-				err = e
-				fmt.Println("select Code err:", err)
-				utils.FileLog.Info("GetBaseFromSci99Index err:", err)
-				return
-			}
-			for _, item := range allCode {
-				IndexCodeMap[item.IndexName] = item.IndexCode
-				IndexMap[item.IndexName] = item
-			}
 
 			for _, v := range listResponse.FirstData.SecondData.Items {
 				if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 {
@@ -194,31 +149,6 @@ func Scii99(cont context.Context) (err error) {
 				} else {
 					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.FactorySampleName + "：" + v.Model
 				}
-				fmt.Println("indexName:", indexName)
-				indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType
-				indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model)
-				if needAdd {
-					index := models.BaseFromSci99Index{
-						IndexCode:  indexCode,
-						IndexName:  indexName,
-						ClassifyId: classifyId,
-						Unit:       v.Unit,
-						Frequency:  "日度",
-						Describe:   v.DataItemName,
-						CreateTime: time.Now(),
-						ModifyTime: time.Now(),
-					}
-					id, e := models.AddBaseFromSci99Index(&index)
-					if e != nil {
-						err = e
-						fmt.Println("Error inserting into database:", err)
-						utils.FileLog.Info("Error inserting into database:", err)
-						return
-					}
-					fmt.Println("新增指标：", index.IndexName)
-					index.BaseFromSciIndexId = int(id)
-					IndexMap[indexName] = &index
-				}
 
 				ppid := strconv.Itoa(v.ProductID)
 				dataTypeID := strconv.Itoa(v.DataTypeID)
@@ -232,16 +162,9 @@ func Scii99(cont context.Context) (err error) {
 					CycleType:  "day",
 				}
 
-				// 获取最新日期
-				dateTime, e := models.GetBaseFromTradeSci99LatestData(indexCode)
-				if e != nil && e.Error() != utils.ErrNoRow() {
-					err = e
-					utils.FileLog.Info("GetBaseFromTradeSci99LatestData err:", err)
-					return
-				}
-				if !dateTime.IsZero() {
-					fmt.Println("最新日期:", dateTime)
-					deailReq.Start = dateTime.AddDate(0,0,-3).Format(utils.FormatDate2)
+				// 初始化是不加日期获取全部，初始化后只拿前三天的
+				if utils.IS_INIT_SCI99 != "true" {
+					deailReq.Start = time.Now().AddDate(0,0,-3).Format(utils.FormatDate2)
 					deailReq.End = time.Now().Format(utils.FormatDate2)
 				}
 
@@ -294,59 +217,33 @@ func Scii99(cont context.Context) (err error) {
 					return
 				}
 
-
-				existDataMap := make(map[string]*models.BaseFromSci99Data)
-
-				//获取所有指标信息
-				allIndex, e := models.GetBaseFromTradeSci99IndexAll(indexCode)
+				param := make(map[string]interface{})
+				param["ListData"] = v
+				param["DetailData"] = detailResponse
+				param["IndexName"] = indexName
+				param["ClassifyName"] = classifyName
+				urlStr := `sci99_crawler/refresh/list`
+				postUrl := utils.EDB_LIB_URL + urlStr
+				postData, e := json.Marshal(param)
 				if e != nil {
 					err = e
-					fmt.Println("select err:", err)
-					utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
-				}
-				for _, v := range allIndex {
-					indexKey := v.IndexCode + v.DataTime
-					existDataMap[indexKey] = v
+					utils.FileLog.Info("Marshal Err:" + err.Error())
+					fmt.Println(err)
+					return
 				}
-
-
-				// 新增data数据
-				addList := make([]*models.BaseFromSci99Data, 0)
-				for _, v := range detailResponse.Data.List {
-					if dataItem, ok := existDataMap[indexCode + v.RealDate]; ok {
-						if dataItem.Value != v.MDataValue {
-							// 更新
-							fmt.Println("更新指标：", indexCode + v.RealDate)
-							e = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate)
-							if e != nil {
-								err = e
-								fmt.Println("Error update into database:", err)
-								utils.FileLog.Info("Error update into database:", err)
-								return
-							}
-						}
-					} else {
-						// 新增
-						dataItem := models.BaseFromSci99Data{
-							BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
-							IndexCode:          indexCode,
-							DataTime:           v.RealDate,
-							Value:              v.MDataValue,
-							CreateTime:         time.Now(),
-							ModifyTime:         time.Now(),
-						}
-						addList = append(addList, &dataItem)
-						fmt.Println("新增数据：", indexCode + v.RealDate)
-					}
+				result, e := HttpPost(postUrl, string(postData), "application/json")
+				if e != nil {
+					err = e
+					fmt.Println(err)
+					utils.FileLog.Info("HttpPost Err:" + err.Error())
+					return
 				}
-				if len(addList) > 0 {
-					e = models.AddBaseFromSci99DataMulti(addList)
-					if e != nil {
-						err = e
-						fmt.Println("Error inserting into database:", err)
-						utils.FileLog.Info("Error inserting into database:", err)
-						return
-					}
+				utils.FileLog.Info("postRefreshEdbData:" + postUrl + ";" + string(postData) + ";result:" + string(result))
+				err = json.Unmarshal(result, &resp)
+				if err != nil {
+					fmt.Println(err)
+					utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
+					return
 				}
 				time.Sleep(1 * time.Second)
 			}
@@ -447,3 +344,23 @@ var ProvinceMap = map[string]string{
 	"香港":  "hongkong",
 	"黑龙江": "heilongjiang",
 }
+
+func HttpPost(url, postData string, params ...string) ([]byte, error) {
+	body := ioutil.NopCloser(strings.NewReader(postData))
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	contentType := "application/x-www-form-urlencoded;charset=utf-8"
+	if len(params) > 0 && params[0] != "" {
+		contentType = params[0]
+	}
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
+	resp, err := client.Do(req)
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	fmt.Println("HttpPost:" + string(b))
+	return b, err
+}

services/task.go

@@ -54,7 +54,6 @@ func Task() {
 
 	if utils.BusinessCode == utils.BusinessCodeFuBang {
 		// 仅富邦
-		//sci99.Scii99(nil)
 		sci99 := task.NewTask("refreshSci99", "0 0,30 16-20 * * *", sci99.Scii99) //卓创资讯
 		task.AddTask("卓创资讯", sci99)
 	}

utils/config.go

@@ -24,10 +24,18 @@ var (
 	LogMaxDays int //日志最大保留天数
 )
 
-var BusinessCode  string //商户号
+var BusinessCode string //商户号
 
+// 卓创资讯
 var (
 	SCI99_COOKIE_PATH string
+	IS_INIT_SCI99     string
+)
+
+var (
+	EDB_LIB_URL         string
+	APP_EDB_LIB_NAME_EN string
+	EDB_LIB_Md5_KEY     string
 )
 
 func init() {
@@ -56,11 +64,23 @@ func init() {
 		panic("配置文件读取错误 " + err.Error())
 	}
 	beeLogger.Log.Info(RunMode + " 模式")
-	MYSQL_URL = config["mysql_url"]
-	MYSQL_URL_DATA = config["mysql_url_data"]
-	SCI99_COOKIE_PATH = config["sci99_cookie_path"]
+
 	// 商家编码
 	BusinessCode = config["business_code"]
+
+	if BusinessCode == BusinessCodeRelease {
+		MYSQL_URL = config["mysql_url"]
+		MYSQL_URL_DATA = config["mysql_url_data"]
+	}
+
+	if BusinessCode == BusinessCodeFuBang {
+		SCI99_COOKIE_PATH = config["sci99_cookie_path"]
+		IS_INIT_SCI99 = config["is_init_sci99"]
+		EDB_LIB_URL = config["edb_lib_url"]
+		APP_EDB_LIB_NAME_EN = config["app_edb_lib_name_en"]
+		EDB_LIB_Md5_KEY = config["edb_lib_md5_key"]
+	}
+
 	if RunMode == "release" {
 
 	} else {