base_from_python.go 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. package services
  2. import (
  3. "bytes"
  4. "encoding/json"
  5. "eta/eta_index_lib/models"
  6. "eta/eta_index_lib/utils"
  7. "fmt"
  8. "github.com/shopspring/decimal"
  9. "os"
  10. "os/exec"
  11. "reflect"
  12. "strings"
  13. )
  14. func Test() (err error) {
  15. defer func() {
  16. if err != nil {
  17. fmt.Println("err:", err)
  18. }
  19. }()
  20. ///usr/local/bin/python3.9
  21. //exec.Command("bin/bash", "-c")
  22. //cmd := exec.Command("/usr/local/bin/python3.9", "-c", "/Users/roc/go/src/eta/eta_index_lib/test2.py")
  23. //cmd := exec.Command("python3", "-c", "./test2.py")
  24. cmd := exec.Command("python3", "/Users/roc/go/src/eta/eta_index_lib/test2.py")
  25. outputByte, err := cmd.Output()
  26. //fmt.Println(err)
  27. fmt.Println("start")
  28. if err != nil {
  29. return
  30. }
  31. fmt.Println(string(outputByte))
  32. arr := strings.Split(string(outputByte), "result=")
  33. arrLen := len(arr)
  34. //fmt.Println(arr)
  35. if arrLen <= 1 {
  36. err = fmt.Errorf("python运算结果异常")
  37. return
  38. }
  39. resultStr := arr[arrLen-1]
  40. fmt.Println(resultStr)
  41. var dataMap map[string]float64
  42. json.Unmarshal([]byte(resultStr), &dataMap)
  43. fmt.Println(dataMap)
  44. //i, err := python3.Py_Main(os.Args)
  45. //if err != nil {
  46. // fmt.Printf("error launching the python interpreter: %s\n", err)
  47. // os.Exit(1)
  48. //}
  49. ////commStr := "#!/usr/bin/python\n# -*- coding: UTF-8 -*-\n\nimport pymysql\nimport pandas as pd\nfrom test_bak import sql_config\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n index_str: str = \"data_time\",\n value_str: str = \"value\"\n ) -> dict:\n \"\"\"\n Parameters\n ----------\n data : pandas的DataFrame数据结构.\n index_str : 对象下标字符串,在pandas的DataFrame中的列名.\n value_str : 对象值字符串,在pandas的DataFrame中的列名\n\n Returns\n -------\n DataFrame or Iterator[DataFrame]\n 例子:{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n \"\"\"\n tmp_result = {}\n for num in range(1, data.index.size): # 迭代 所有的指标\n tmp_result[data[index_str][num]] = data[value_str][num]\n return tmp_result\n\n\ndef query():\n edb_code = 's0033227'\n data_time = '2002-03-17'\n # field_name = '平均温度'\n sql1 = f\"\"\"SELECT data_time,`value` FROM edb_data_wind WHERE edb_code = '{edb_code}' and data_time > '{data_time}' ;\"\"\"\n raw = pandas_fetch_all(sql1, db)\n raw['data_time_str'] = raw['data_time'].apply(lambda x: x.strftime(\"%Y-%m-%d\"))\n format_result = format_data(raw, \"data_time_str\", \"value\")\n print(format_result)\n return format_result\n\n\nresult = query()\ndb.close()\n"
  50. ////i := python3.PyRun_SimpleString(commStr)
  51. //fmt.Println(i)
  52. return
  53. }
  54. // EdbDataStrFromPython 通过python代码获取到的指标数据(interface数据)
  55. type EdbDataStrFromPython struct {
  56. Date map[int]string `json:"date"`
  57. Value map[int]interface{} `json:"value"`
  58. }
  59. // ExecPythonCode 执行Python代码
  60. func ExecPythonCode(edbCode, reqCode string) (dataMap models.EdbDataFromPython, err error, errMsg string) {
  61. defer func() {
  62. if err != nil {
  63. fmt.Println("err:", err)
  64. }
  65. }()
  66. //获取python文件的绝对地址
  67. pythonFile, err := getPythonFileAbsolutePath(edbCode)
  68. if err != nil {
  69. return
  70. }
  71. pthonCodeStr := getPythonFrontStr() + reqCode + getPythonLaterStr()
  72. fileHandle, err := os.OpenFile(pythonFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0766)
  73. if err != nil {
  74. return
  75. }
  76. defer func() {
  77. os.Remove(pythonFile)
  78. }()
  79. _, err = fileHandle.Write([]byte(pthonCodeStr))
  80. if err != nil {
  81. return
  82. }
  83. fileHandle.Close()
  84. cmd := exec.Command(utils.PYTHON_PATH, pythonFile)
  85. var out, errMsgOut bytes.Buffer
  86. cmd.Stdout = &out
  87. cmd.Stderr = &errMsgOut
  88. err = cmd.Run()
  89. if err != nil {
  90. errMsg = errMsgOut.String()
  91. return
  92. }
  93. outputByte := out.String()
  94. if err != nil {
  95. return
  96. }
  97. //fmt.Println(string(outputByte))
  98. arr := strings.Split(string(outputByte), "result=")
  99. arrLen := len(arr)
  100. //fmt.Println(arr)
  101. if arrLen <= 1 {
  102. err = fmt.Errorf("python运算结果异常")
  103. return
  104. }
  105. resultStr := arr[arrLen-1]
  106. //fmt.Println(resultStr)
  107. var dataMapStr EdbDataStrFromPython
  108. //先将value转为interface
  109. err = json.Unmarshal([]byte(resultStr), &dataMapStr)
  110. if err != nil {
  111. return
  112. }
  113. dataMap.Date = make(map[int]string)
  114. dataMap.Value = make(map[int]float64)
  115. //将value为nil的给过滤掉
  116. i := 0
  117. lenData := len(dataMapStr.Date)
  118. for k := 0; k < lenData; k++ {
  119. date := dataMapStr.Date[k]
  120. tmpValue := dataMapStr.Value[k]
  121. if reflect.TypeOf(tmpValue) != nil {
  122. if reflect.TypeOf(tmpValue).Kind() == reflect.Float64 {
  123. dataMap.Date[i] = date
  124. dataMap.Value[i], _ = decimal.NewFromFloat(reflect.ValueOf(tmpValue).Float()).Truncate(4).Float64() //保留4位小数
  125. i++
  126. }
  127. }
  128. }
  129. //err = json.Unmarshal([]byte(resultStr), &dataMap)
  130. //fmt.Println(dataMap)
  131. return
  132. }
  133. // getPythonFileAbsolutePath 获取python文件的绝对地址
  134. func getPythonFileAbsolutePath(edbCode string) (pythonFile string, err error) {
  135. uploadDir := utils.STATIC_DIR + "python/"
  136. err = os.MkdirAll(uploadDir, 0766)
  137. if err != nil {
  138. return
  139. }
  140. pythonFile = uploadDir + fmt.Sprint(edbCode, "_", utils.GetRandDigit(16), ".py")
  141. if utils.RunMode != "release" {
  142. dir, tmpErr := os.Getwd()
  143. if tmpErr != nil {
  144. err = tmpErr
  145. return
  146. }
  147. pythonFile = dir + "/" + pythonFile
  148. }
  149. return
  150. }
  151. // getPythonFrontStr 获取python前面的代码
  152. func getPythonFrontStr() string {
  153. //return "#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n 'host': 'rm-uf67kg347rhjfep5c1o.mysql.rds.aliyuncs.com',\n 'port': 3306,#主机号\n 'user': 'hz_technology',#账户名\n 'passwd': 'hongze@2021',#密码\n 'db': 'test_hz_data',\n 'charset': 'utf8mb4',\n 'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n index_str: str = \"data_time\",\n value_str: str = \"value\"\n ) -> pd.DataFrame:\n \"\"\"\n Parameters\n ----------\n data : pandas的DataFrame数据结构.\n index_str : 对象下标字符串,在pandas的DataFrame中的列名.\n value_str : 对象值字符串,在pandas的DataFrame中的列名\n\n Returns\n -------\n DataFrame or Iterator[DataFrame]\n 例子:{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n \"\"\"\n index_list = [] # 空列表\n value_list = [] # 空列表\n\n for num in range(1, data.index.size): # 迭代 所有的指标\n index_list.append(data[index_str][num])\n value_list.append(data[value_str][num])\n\n tmp_data = {\n \"date\": index_list,\n \"value\": value_list\n }\n pd_data = pd.DataFrame(tmp_data)\n # print(pd_data)\n return pd_data\n"
  154. str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n 'host': '%s',\n 'port': 3306,#主机号\n 'user': '%s',#账户名\n 'passwd': '%s',#密码\n 'db': '%s',\n 'charset': 'utf8mb4',\n 'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n index_str: str = \"data_time\",\n value_str: str = \"value\"\n ) -> pd.DataFrame:\n \"\"\"\n Parameters\n ----------\n data : pandas的DataFrame数据结构.\n index_str : 对象下标字符串,在pandas的DataFrame中的列名.\n value_str : 对象值字符串,在pandas的DataFrame中的列名\n\n Returns\n -------\n DataFrame or Iterator[DataFrame]\n 例子:{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n \"\"\"\n index_list = [] # 空列表\n value_list = [] # 空列表\n\n for num in range(0, data.index.size): # 迭代 所有的指标\n index_list.append(data[index_str][num])\n value_list.append(data[value_str][num])\n\n tmp_data = {\n \"date\": index_list,\n \"value\": value_list\n }\n pd_data = pd.DataFrame(tmp_data)\n # print(pd_data)\n return pd_data\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
  155. return str
  156. }
  157. // getPythonLaterStr 获取python结尾的代码
  158. func getPythonLaterStr() string {
  159. return "\n\nprint(\"result=\", result.to_json())\ndb.close()"
  160. }