diff --git a/docker/dockerfile b/docker/dockerfile index b082f9e3..9571d1e1 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -9,18 +9,22 @@ RUN apt-get clean && apt-get update && apt-get upgrade -y # Set the locale RUN apt-get install -y locales && locale-gen en_US.UTF-8 -#Install required python packages +#Install required python packages RUN pip install paho-mqtt -RUN pip install requests +RUN pip install requests RUN pip install influxdb -RUN pip install influxdb-client +RUN pip install influxdb-client +RUN pip install psutil COPY grott.py /app/grott.py COPY grottconf.py /app/grottconf.py COPY grottdata.py /app/grottdata.py COPY grottproxy.py /app/grottproxy.py COPY grottsniffer.py /app/grottsniffer.py -COPY grott.ini /app/grott.ini +COPY grottserver.py /app/grottserver.py +COPY examples/grott.ini /app/grott.ini +COPY examples/Extensions/grottext.py /app/grottext.py +COPY examples/Extensions/grotcsv.py /app/grotcsv.py WORKDIR /app -CMD ["python", "-u", "grott.py", "-v"] \ No newline at end of file +CMD ["python", "-u", "grott.py", "-v"] diff --git a/grottdata.py b/grottdata.py index 35ca2bf5..df57924f 100644 --- a/grottdata.py +++ b/grottdata.py @@ -700,8 +700,20 @@ def procdata(conf,data): "d" : pvodate, "t" : pvotime, #2.7.1 "v1" : definedkey["pvenergytoday"]*100, - "v2" : definedkey["pvpowerout"]/10, - "v6" : definedkey["pvgridvoltage"]/10 + "v2" : definedkey["pvpowerin"]/10, # Power generated by solar + "v3" : definedkey["elocalloadtod"]*100 - definedkey["edischarge1tod"]*100 + definedkey["eharge1tod"]*100, # Energy used by house today, excluding from battery, including battery charge + "v4" : definedkey["plocaloadr"]/10 - definedkey["pdischarge1"]/10 + definedkey["pcharge1"]/10, # Power used by house, excluding battery discharge, including battery charge + "v6" : definedkey["pvgridvoltage"]/10, + + "v7" : definedkey["soc"], # Battery state of charge + "v8" : definedkey["pcharge1"]/10 if definedkey["pcharge1"] != 0 else -definedkey["pdischarge1"]/10, # Battery chage power + # "v9" : definedkey["pdischarge1"]/10, # Battery discharge power + + "v9" : definedkey["plocaloadr"]/10, # Power used by house + "v10" : definedkey["elocalloadtod"]*100, # Energy used by house today + + "v11" : definedkey["pactogridtot"]/10 if definedkey["pactogridtot"] != 0 else -definedkey["pactouserr"]/10, # Power export/imort to/from grid. Negative = import + "v12" : definedkey["etogridtod"]*100 - definedkey["etousertod"]*100 # Energy exported/imported to/from grid. Negative = import } if not conf.pvdisv1 : pvdata["v1"] = definedkey["pvenergytoday"]*100 @@ -859,4 +871,3 @@ def procdata(conf,data): ##print("\t -", ext_result) else: if conf.verbose : print("\t - " + "Grott extension processing disabled ") - diff --git a/grottserver.py b/grottserver.py index f3267c0a..c4326fd2 100644 --- a/grottserver.py +++ b/grottserver.py @@ -14,6 +14,7 @@ from collections import defaultdict import logging import os,psutil +import traceback from grottdata import procdata #set logging definities @@ -45,6 +46,11 @@ #dataloggerrespwait = 5 #ConnectionTimeout = 300 is now configurable in grott.ini +inverterReadRegisters = "05" +dataLoggerSend = "19" +inverterWriteSingleRegister = "06" +inverterWriteMultipleRegisters = "10" + def addLoggingLevel(levelName, levelNum, methodName=None): if not methodName: methodName = levelName.lower() @@ -132,7 +138,7 @@ def calc_crc(data): return crc def validate_record(xdata): - # validata data record on length and CRC (for "05" and "06" records) + # validata data record on length and CRC (for inverterReadRegisters and inverterWriteSingleRegister records) logger.debug("validate data record") data = bytes.fromhex(xdata) ldata = len(data) @@ -140,7 +146,7 @@ def validate_record(xdata): header = "".join("{:02x}".format(n) for n in data[0:8]) protocol = header[6:8] - if protocol in ("05","06"): + if protocol in (inverterReadRegisters,inverterWriteSingleRegister): lcrc = 4 crc = int.from_bytes(data[ldata-2:ldata],"big") else: @@ -176,7 +182,7 @@ def createtimecommand(self, protocol,deviceid,loggerid,sequenceno) : sequenceno = sequenceno bodybytes = loggerid.encode('ISO-8859-1') body = bodybytes.hex() - if protocol == "06" : + if protocol == inverterWriteSingleRegister : body = body + "0000000000000000000000000000000000000000" register = 31 body = body + "{:04x}".format(int(register)) @@ -286,10 +292,10 @@ def do_GET(self): elif self.path.startswith("datalogger") or self.path.startswith("inverter") : if self.path.startswith("datalogger"): if verbose: print("\t - " + "Grotthttpserver - datalogger get received : ", urlquery) - sendcommand = "19" + sendcommand = dataLoggerSend else: if verbose: print("\t - " + "Grotthttpserver - inverter get received : ", urlquery) - sendcommand = "05" + sendcommand = inverterReadRegisters #validcommand = False if urlquery == {} : @@ -324,7 +330,7 @@ def do_GET(self): # test if datalogger and / or inverter id is specified. try: - if sendcommand == "05" : + if sendcommand == inverterReadRegisters : inverterid_found = False try: #test if inverter id is specified and get loggerid @@ -358,7 +364,7 @@ def do_GET(self): # no set default format op dec. formatval = "dec" - if sendcommand == "19" : + if sendcommand == dataLoggerSend : # if read datalogger info. dataloggerid = urlquery["datalogger"][0] @@ -410,7 +416,7 @@ def do_GET(self): bodybytes = dataloggerid.encode('ISO-8859-1') body = bodybytes.hex() - if loggerreg[dataloggerid]["protocol"] == "06" : + if loggerreg[dataloggerid]["protocol"] == inverterWriteSingleRegister : body = body + "0000000000000000000000000000000000000000" body = body + "{:04x}".format(int(register)) #assumption now only 1 reg query; other put below end register @@ -421,7 +427,7 @@ def do_GET(self): #device id for datalogger is by default "01" for inverter deviceid is inverterid! deviceid = "01" # test if it is inverter command and set - if sendcommand == "05": + if sendcommand == inverterReadRegisters: deviceid = (loggerreg[dataloggerid][inverterid]["inverterno"]) print("\t - Grotthttpserver: selected deviceid :", deviceid) @@ -457,7 +463,7 @@ def do_GET(self): #wait for response #Set #retry waiting loop for datalogger or inverter - if sendcommand == "05" : + if sendcommand == inverterReadRegisters : wait = round(self.conf.inverterrespwait/self.conf.apirespwait) #if verbose: print("\t - Grotthttpserver - wait Cycles:", wait ) else : @@ -469,7 +475,7 @@ def do_GET(self): try: comresp = commandresponse[sendcommand][regkey] - if sendcommand == "05" : + if sendcommand == inverterReadRegisters : if formatval == "dec" : comresp["value"] = int(comresp["value"],16) elif formatval == "text" : @@ -520,6 +526,266 @@ def do_GET(self): except Exception as e: print("\t - Grottserver - exception in httpserver thread - get occured : ", e) + print(traceback.format_exc()) + + def do_POST(self): + try: + if(verbose): print("\t - Grotthttpserver - POST received") + + # Load request + contentLength = int(self.headers.get("Content-Length")) + if(contentLength == 0): + responsetxt = b'no content' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + rawContent = self.rfile.read(contentLength).decode("UTF-8") + try: + request = json.loads(rawContent) + except Exception as ex: + responsetxt = b'invalid content' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + +# EXAMPLE INVERTER JSON +#{ +# "inverterId": "EGM3N7K06U", +# "startRegister": 1100, +# "values": +# [ +# { +# "value": "0105", +# "type": "time" +# }, +# { +# "value": "0655", +# "type": "time" +# }, +# { +# "value": 1, +# "type": "dec" +# } +# ] +#} + + if(self.path.startswith("/inverter")): + if(verbose): print("\t - Grotthttpserver - datalogger json POST received") + # Get inverter ID + if("inverterId" not in request): + responsetxt = b'inverterId not in json' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + foundInverter = False + for key in loggerreg.keys() : + for key2 in loggerreg[key].keys() : + if(key2 == request["inverterId"]): + dataloggerId = key + foundInverter = True + break + if(not foundInverter): + responsetxt = b'specified inverterId not found in system' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + + # Get start register + if("startRegister" not in request): + responsetxt = b'startRegister not in json' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + startRegister = request["startRegister"] + + # Check register values + if("values" not in request): + responsetxt = b'values not in json' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + + self.sendCommand( + dataloggerId = dataloggerId, + commandType = inverterWriteMultipleRegisters, + commandValue = request["values"], + inverterId = request["inverterId"], + startRegister = startRegister + ) + else: + responsetxt = b'path not valid' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + + except Exception as e: + print("\t - Grottserver - exception in httpserver thread - put occured : ", e) + print(traceback.format_exc()) + + def validateTime(self, timeValue: str) -> hex: + if(len(timeValue) != 4): raise ValueError("Specified time not of required length (4)") + hour = int(timeValue[0:2]) + minute = int(timeValue[2:]) + if(hour > 23 or hour < 0): raise ValueError("Specified hour not in 24 hour format (between 0-23)") + if(minute > 59 or minute < 0): raise ValueError("Specified minute not in range (0-59)") + return "{:02x}{:02x}".format(hour, minute) + + def sendCommand(self, dataloggerId, commandType, commandValue, inverterId, startRegister): + bodybytes = dataloggerId.encode('ISO-8859-1') + body = bodybytes.hex() + + if loggerreg[dataloggerId]["protocol"] == inverterWriteSingleRegister : + body = body + "0000000000000000000000000000000000000000" + + # Convert values to hex + if commandType == inverterWriteSingleRegister : + value = "{:04x}".format(commandValue) + valueLength = "" + elif(commandType == inverterWriteMultipleRegisters): + # Value is already in hex format + pass + else: + value = commandValue.encode('ISO-8859-1').hex() + valueLength = int(len(value)/2) + valueLength = "{:04x}".format(valueLength) + + # Build body + if(commandType == inverterWriteMultipleRegisters): + registerValues = "" + for valueInfo in commandValue: + if("value" not in valueInfo): + responsetxt = b'value value missing!' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + if("type" not in valueInfo): + responsetxt = b'value type not specified' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + if(valueInfo["type"] not in ["dec", "hex", "time"]): + responsetxt = b'specified value type not valid' + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + if(valueInfo["type"] == "hex"): registerValues += "{:04x}".format(int(valueInfo["value"], 16)) + elif(valueInfo["type"] == "dec"): registerValues += "{:04x}".format(int(valueInfo["value"])) + elif(valueInfo["type"] == "time"): + try: + registerValues += self.validateTime(valueInfo["value"]) + except Exception as ex: + responsetxt = str(ex).encode() + responserc = 400 + responseheader = "text/html" + htmlsendresp(self, responserc, responseheader, responsetxt) + return + + body += "{:04x}".format(int(startRegister)) + endRegister = int(startRegister) + len(commandValue) + body += "{:04x}".format(endRegister) + body += registerValues + else: + body += "{:04x}".format(int(startRegister)) + valueLength+ value + + bodylen = int(len(body)/2+2) + + #device id for datalogger is by default "01" for inverter deviceid is inverterId! + deviceid = "01" + # test if it is inverter command and set deviceid + if commandType in (inverterWriteSingleRegister,inverterWriteMultipleRegisters) : + deviceid = (loggerreg[dataloggerId][inverterId]["inverterno"]) + print("\t - Grotthttpserver: selected deviceid :", deviceid) + + #create header + header = "{:04x}".format(sendseq) + "00" + loggerreg[dataloggerId]["protocol"] + "{:04x}".format(bodylen) + deviceid + commandType + body = header + body + print("XXXXXX") + print(body) + body = bytes.fromhex(body) + + if verbose: + print("\t - Grotthttpserver - unencrypted put command:") + print(format_multi_line("\t\t ",body)) + + if loggerreg[dataloggerId]["protocol"] != "02" : + #encrypt message + body = decrypt(body) + crc16 = calc_crc(bytes.fromhex(body)) + body = bytes.fromhex(body) + crc16.to_bytes(2, "big") + + # queue command + qname = loggerreg[dataloggerId]["ip"] + "_" + str(loggerreg[dataloggerId]["port"]) + self.send_queuereg[qname].put(body) + responseno = "{:04x}".format(sendseq) + if commandType == inverterWriteMultipleRegisters: + regkey = "{:04x}".format(int(startRegister)) + "{:04x}".format(int(endRegister)) + else : + regkey = "{:04x}".format(int(startRegister)) + + try: + #delete response: be aware a 18 command give 19 response, 06 send command gives 06 response in different format! + if commandType == "18" : + del commandresponse[commandType][regkey] + else: + del commandresponse[commandType][regkey] + except: + pass + + #wait for response + #Set #retry waiting loop for datalogger or inverter + if commandType == inverterWriteSingleRegister : + wait = round(self.conf.inverterrespwait/self.conf.apirespwait) + #if verbose: print("\t - Grotthttpserver - wait Cycles:", wait ) + else : + wait = round(self.conf.dataloggerrespwait/self.conf.apirespwait) + #if verbose: print("\t - Grotthttpserver - wait Cycles:", wait ) + + for x in range(wait): + if verbose: print("\t - Grotthttpserver - wait for PUT response") + try: + #read response: be aware a 18 command give 19 response, 06 send command gives 06 response in differnt format! + if commandType == "18" : + comresp = commandresponse["18"][regkey] + else: + comresp = commandresponse[commandType][regkey] + if verbose: print("\t - " + "Grotthttperver - Commandresponse ", responseno, startRegister, commandresponse[commandType][regkey]) + break + except: + #wait for second and try again + #Set retry waiting cycle time loop for datalogger or inverter + time.sleep(self.conf.apirespwait) + try: + if comresp != "" : + responsetxt = b'OK' + responserc = 200 + responseheader = "text/body" + htmlsendresp(self,responserc,responseheader,responsetxt) + return + + except : + responsetxt = b'no or invalid response received' + responserc = 400 + responseheader = "text/body" + htmlsendresp(self,responserc,responseheader,responsetxt) + return + + + responsetxt = b'OK' + responserc = 200 + responseheader = "text/body" + if verbose: print("\t - " + "Grott: datalogger command response :", responserc, responsetxt, responseheader) + htmlsendresp(self,responserc,responseheader,responsetxt) + return def do_PUT(self): try: @@ -539,7 +805,7 @@ def do_PUT(self): else: if verbose: print("\t - Grotthttpserver - inverter PUT received : ", urlquery) # Must be an inverter. Use 06 for now. May change to 10 later. - sendcommand = "06" + sendcommand = inverterWriteSingleRegister if urlquery == "" : #no command entered return loggerreg info: @@ -571,7 +837,7 @@ def do_PUT(self): # test if datalogger and / or inverter id is specified. try: - if sendcommand == "06" : + if sendcommand == inverterWriteSingleRegister : inverterid_found = False try: #test if inverter id is specified and get loggerid @@ -646,7 +912,7 @@ def do_PUT(self): elif command == "multiregister" : # Switch to multiregister command - sendcommand = "10" + sendcommand = inverterWriteMultipleRegisters # TODO: Too much copy/paste here. Refactor into methods. @@ -690,7 +956,7 @@ def do_PUT(self): elif command == "datetime" : #process set datetime, only allowed for datalogger!!! - if sendcommand == "06" : + if sendcommand == inverterWriteSingleRegister : responsetxt = b'datetime command not allowed for inverter' responserc = 400 responseheader = "text/body" @@ -709,7 +975,7 @@ def do_PUT(self): return #test value: - if sendcommand == "06" : + if sendcommand == inverterWriteSingleRegister : try: # is format keyword specified? (dec, text, hex) formatval = urlquery["format"][0] @@ -747,14 +1013,14 @@ def do_PUT(self): bodybytes = dataloggerid.encode('ISO-8859-1') body = bodybytes.hex() - if loggerreg[dataloggerid]["protocol"] == "06" : + if loggerreg[dataloggerid]["protocol"] == inverterWriteSingleRegister : body = body + "0000000000000000000000000000000000000000" - if sendcommand == "06" : + if sendcommand == inverterWriteSingleRegister : value = "{:04x}".format(value) valuelen = "" - elif sendcommand == "10" : + elif sendcommand == inverterWriteMultipleRegisters : # Value is already in hex format pass @@ -763,7 +1029,7 @@ def do_PUT(self): valuelen = int(len(value)/2) valuelen = "{:04x}".format(valuelen) - if sendcommand == "10" : + if sendcommand == inverterWriteMultipleRegisters : body = body + "{:04x}".format(int(startregister)) + "{:04x}".format(int(endregister)) + value else : @@ -774,7 +1040,7 @@ def do_PUT(self): #device id for datalogger is by default "01" for inverter deviceid is inverterid! deviceid = "01" # test if it is inverter command and set deviceid - if sendcommand in ("06","10") : + if sendcommand in (inverterWriteSingleRegister,inverterWriteMultipleRegisters) : deviceid = (loggerreg[dataloggerid][inverterid]["inverterno"]) print("\t - Grotthttpserver: selected deviceid :", deviceid) @@ -797,7 +1063,7 @@ def do_PUT(self): qname = loggerreg[dataloggerid]["ip"] + "_" + str(loggerreg[dataloggerid]["port"]) self.send_queuereg[qname].put(body) responseno = "{:04x}".format(sendseq) - if sendcommand == "10": + if sendcommand == inverterWriteMultipleRegisters: regkey = "{:04x}".format(int(startregister)) + "{:04x}".format(int(endregister)) else : regkey = "{:04x}".format(int(register)) @@ -813,7 +1079,7 @@ def do_PUT(self): #wait for response #Set #retry waiting loop for datalogger or inverter - if sendcommand == "06" : + if sendcommand == inverterWriteSingleRegister : wait = round(self.conf.inverterrespwait/self.conf.apirespwait) #if verbose: print("\t - Grotthttpserver - wait Cycles:", wait ) else : @@ -859,6 +1125,7 @@ def do_PUT(self): except Exception as e: print("\t - Grottserver - exception in httpserver thread - put occured : ", e) + print(traceback.format_exc()) class GrottHttpServer: @@ -982,7 +1249,7 @@ def handle_readable_socket(self, conf, s, trname): header = "".join("{:02x}".format(n) for n in msgbuffer[0:8]) protocol = header[6:8] #set recordlength correction header + crc (if included). - if protocol in ("05","06"): + if protocol in (inverterReadRegisters,inverterWriteSingleRegister): lcrc = 8 else: lcrc = 6 @@ -1011,7 +1278,7 @@ def handle_readable_socket(self, conf, s, trname): else: logger.debug("handle_readble_socket, process data to sent to growatt server") - if rectype in ("03", "04", "16","50", "1b", "19","20","29"): + if rectype in ("03", "04", "16","50", "1b", dataLoggerSend,"20","29"): #forward only specific recordtypes #get qname for growatt server based on growatt address and client addres gLaddr = self.channel[s].getsockname() @@ -1290,7 +1557,7 @@ def process_data(self, conf, s, data): protocol = header[6:8] #command = header[14:16] rectype = header[14:16] - if protocol in ("05","06") : + if protocol in (inverterReadRegisters,inverterWriteSingleRegister) : result_string = decrypt(data) else : result_string = "".join("{:02x}".format(n) for n in data) @@ -1342,7 +1609,7 @@ def process_data(self, conf, s, data): if rectype in ("03") : # init record register logger/inverter id (including sessionid?) # decrypt body. - if header[6:8] in ("05","06") : + if header[6:8] in (inverterReadRegisters,inverterWriteSingleRegister) : #print("header1 : ", header[6:8]) result_string = decrypt(data) else : @@ -1350,7 +1617,7 @@ def process_data(self, conf, s, data): loggerid = result_string[16:36] loggerid = codecs.decode(loggerid, "hex").decode('ISO-8859-1') - if header[6:8] in ("02","05") : + if header[6:8] in ("02",inverterReadRegisters) : inverterid = result_string[36:56] else : inverterid = result_string[76:96] @@ -1374,15 +1641,15 @@ def process_data(self, conf, s, data): response = createtimecommand(self,protocol,deviceid,loggerid,"0001") if verbose: print("\t - Grottserver 03 announce data record processed") - elif rectype in ("19","05","06","18"): + elif rectype in (dataLoggerSend,inverterReadRegisters,inverterWriteSingleRegister,"18"): if verbose: print("\t - Grottserver - " + header[12:16] + " Command Response record received, no response needed") offset = 0 - if protocol in ("06") : + if protocol in (inverterWriteSingleRegister) : offset = 40 register = int(result_string[36+offset:40+offset],16) - if rectype == "05" : + if rectype == inverterReadRegisters : #value = result_string[40+offset:44+offset] #v0.0.14: test if empty response is sent (this will give CRC code as values) #print("length resultstring:", len(result_string)) @@ -1391,24 +1658,24 @@ def process_data(self, conf, s, data): if verbose: print("\t - Grottserver - empty register get response recieved, response ignored") else: value = result_string[44+offset:48+offset] - elif rectype == "06" : + elif rectype == inverterWriteSingleRegister : result = result_string[40+offset:42+offset] #print("06 response result :", result) value = result_string[42+offset:46+offset] elif rectype == "18" : result = result_string[40+offset:42+offset] else : - # "19" response take length into account + # dataLoggerSend response take length into account valuelen = int(result_string[40+offset:44+offset],16) #value = codecs.decode(result_string[44+offset:44+offset+valuelen*2], "hex").decode('ISO-8859-1') value = codecs.decode(result_string[44+offset:44+offset+valuelen*2], "hex").decode('ISO-8859-1') regkey = "{:04x}".format(register) - if rectype == "06" : + if rectype == inverterWriteSingleRegister : # command 06 response has ack (result) + value. We will create a 06 response and a 05 response (for reg administration) - commandresponse["06"][regkey] = {"value" : value , "result" : result} - commandresponse["05"][regkey] = {"value" : value} + commandresponse[inverterWriteSingleRegister][regkey] = {"value" : value , "result" : result} + commandresponse[inverterReadRegisters][regkey] = {"value" : value} if rectype == "18" : commandresponse["18"][regkey] = {"result" : result} else : @@ -1417,7 +1684,7 @@ def process_data(self, conf, s, data): response = None - elif rectype in ("10") : + elif rectype in (inverterWriteMultipleRegisters) : if verbose: print("\t - Grottserver - " + header[12:16] + " record received, no response needed") startregister = int(result_string[76:80],16)