Update Pybytes to version 1.6.1

Author: Xykon

esp32/frozen/Pybytes/_OTA.py

@@ -137,7 +137,6 @@ def update(self, customManifest=None, fwtype=None, token=None):
 
     def get_file(self, f):
         new_path = "{}.new".format(f['dst_path'])
-
         # If a .new file exists from a previously failed update delete it
         try:
             os.remove(new_path)
@@ -184,6 +183,15 @@ def delete_file(self, f):
 
     def write_firmware(self, f):
         # hash =
+        url = f['URL'].split("//")[1].split("/")[0]
+
+        if url.find(":") > -1:
+            self.ip = url.split(":")[0]
+            self.port = int(url.split(":")[1])
+        else:
+            self.ip = url
+            self.port = 443
+
         self.get_data(
             f['URL'].split("/", 3)[-1],
             hash=True,
@@ -222,7 +230,6 @@ def _http_get(self, path, host):
 
     def get_data(self, req, dest_path=None, hash=False, firmware=False):
         h = None
-
         useSSL = int(self.port) == 443
 
         # Connect to server
@@ -232,11 +239,9 @@ def get_data(self, req, dest_path=None, hash=False, firmware=False):
         if (int(self.port) == 443):
             print("Wrapping socket")
             s = ssl.wrap_socket(s)
-
         print("Sending request")
         # Request File
         s.sendall(self._http_get(req, "{}:{}".format(self.ip, self.port)))
-
         try:
             content = bytearray()
             fp = None
@@ -247,6 +252,7 @@ def get_data(self, req, dest_path=None, hash=False, firmware=False):
                 fp = open(dest_path, 'wb')
 
             if firmware:
+                print_debug(4, "Starting OTA...")
                 pycom.ota_start()
 
             h = uhashlib.sha1()

esp32/frozen/Pybytes/_pybytes_constants.py

@@ -69,11 +69,12 @@ class constants:
     __TYPE_OTA = 0x05
     __TYPE_FCOTA = 0x06
     __TYPE_PONG = 0x07
-    __TYPE_PYMESH = 0x0D
-    __TYPE_PYBYTES = 0x0E
-    __TYPE_RELEASE_INFO = 0x0B
     __TYPE_RELEASE_DEPLOY = 0x0A
+    __TYPE_RELEASE_INFO = 0x0B
     __TYPE_DEVICE_NETWORK_DEPLOY = 0x0C
+    __TYPE_PYMESH = 0x0D
+    __TYPE_PYBYTES = 0x0E
+    __TYPE_ML = 0x0F
     __PYBYTES_PROTOCOL = ">B%ds"
     __PYBYTES_PROTOCOL_PING = ">B"
     __PYBYTES_INTERNAL_PROTOCOL = ">BBH"
@@ -90,6 +91,8 @@ class constants:
     __COMMAND_ANALOG_WRITE = 4
     __COMMAND_CUSTOM_METHOD = 5
     __COMMAND_CUSTOM_LOCATION = 6
+    __COMMAND_START_SAMPLE = 7
+    __COMMAND_DEPLOY_MODEL = 8
 
     __FCOTA_COMMAND_HIERARCHY_ACQUISITION = 0x00
     __FCOTA_COMMAND_FILE_ACQUISITION = 0x01

esp32/frozen/Pybytes/_pybytes_machine_learning.py

@@ -0,0 +1,176 @@
+'''
+Copyright (c) 2020, Pycom Limited.
+This software is licensed under the GNU GPL version 3 or any
+later version, with permitted additional terms. For more information
+see the Pycom Licence v1.0 document supplied with this file, or
+available at https://www.pycom.io/opensource/licensing
+'''
+
+import math
+import json
+
+try:
+    from pybytes_debug import print_debug
+except:
+    from _pybytes_debug import print_debug
+
+try:
+    import urequest
+except:
+    import _urequest as urequest
+
+try:
+    from pybytes_constants import constants
+except:
+    from _pybytes_constants import constants
+
+import pycom
+
+try:
+    from LIS2HH12 import *
+except:
+    print_debug(5, "LIS2HH12 not imported")
+
+# 20 seconds, max window in time for recording
+MAX_LEN_MSEC = const(20000)
+
+# 350Hz, max frequency
+MAX_FREQ_HZ = const(350)
+
+
+class MlFeatures():
+    def __init__(self, pybytes_protocol=None, parameters=None):
+        if parameters is not None:
+            self.__length = parameters["length"]
+            self.__label = parameters["label"]
+            self.__sampleName = parameters["sampleName"]
+            self.__type = parameters["type"]
+            self.__device = parameters["device"]
+            self.__model = parameters["model"]
+            self.__mlSample = parameters["mlSample"]
+            self.__frequency = parameters["frequency"]
+            self.__pybytes_protocol = pybytes_protocol
+            self.__data = []
+
+    def _debug_hack(self, pybytes):
+        self.__pybytes = pybytes
+
+    def start_sampling(self, pin):
+        # here define the required libraries
+        try:
+            from pysense import Pysense
+        except:
+            print_debug(5, "pysense not imported")
+
+        try:
+            from pytrack import Pytrack
+        except:
+            print_debug(5, "pytrack not imported")
+
+        lib = False
+        try:
+            py = Pysense()
+            lib = True
+        except NameError:
+            print_debug(5, "Pysense not defined")
+
+        if not lib:
+            try:
+                py = Pytrack()
+            except NameError:
+                print_debug(5, "Check if Pysense/Pytrack libraries are loaded")
+                return
+
+        try:
+            li = LIS2HH12(py)
+        except NameError:
+            print_debug(5, "LIS2HH12 library are not loaded")
+            return
+        li.set_odr(ODR_400_HZ)
+
+        # make the max record length to 20 seconds
+        self.__length = min(MAX_LEN_MSEC, self.__length)
+
+        # make the max frequency to 350Hz
+        self.__frequency = min(MAX_FREQ_HZ, self.__frequency)
+
+        # compute time interval between 2 consecutive samples
+        delta_t_us = int(1000000.0 / self.__frequency)
+        # compute the number of samples to be acquisition
+        samples_num = math.ceil(self.__length * self.__frequency / 1000) + 1
+
+        try:
+            pycom.heartbeat(False)
+            pycom.rgbled(0x7f7f00)
+        except:
+            pass
+        time.sleep(0.5)
+
+        self.__data = []
+        index = 0
+        print("Start acquisition data for %d msec, freq %d Hz" % (self.__length, self.__frequency))
+
+        next_ts = time.ticks_us()
+        ts_orig = next_ts
+        while True:
+            while time.ticks_diff(next_ts, time.ticks_us()) > 0:
+                pass
+            acc = li.acceleration()
+            ts = next_ts
+            self.__data.append((ts - ts_orig, acc))
+            next_ts = ts + delta_t_us
+            index += 1
+            if index >= samples_num:
+                break  # done
+
+        print("Done acquisition %d samples, real freq %.1f Hz" % (index, index / (self.__length / 1000)))
+        self._parse_data(pin)
+
+    def _send_data(self, data, pin, acc, ts):
+        if self.__pybytes_protocol is not None:
+            if self.__type == 2:
+                self.__label = self.__sampleName
+            self.__pybytes_protocol.send_pybytes_custom_method_values(pin, [
+                data],
+                'sample/{}/{}/{}/{}/{}'.format(self.__label, self.__type, self.__model, self.__device, self.__mlSample))
+        else:
+            self.__pybytes.send_signal(pin & 0xFF, str((int(ts / 1000), acc)))
+
+    def _parse_data(self, pin):
+        print("_parse_data, %d samples" % len(self.__data))
+        try:
+            pycom.rgbled(0x8d05f5)
+        except:
+            pass
+        data = ['{"data": "ml"}']
+        for (ts, acc) in self.__data:
+            data.append('{' + '"data": [{},{},{}], "ms": {}'.format(acc[0], acc[1], acc[2], int(ts / 1000)) + '}')
+            if len(data) > 25:
+                self._send_data(data, pin, acc, ts)
+                data = ['{"data": "ml"}']
+        self._send_data(data, pin, acc, ts)
+        try:
+            pycom.heartbeat(True)
+        except:
+            pass
+
+    def deploy_model(self, modelId, silent=False):
+        try:
+            file = '/flash/model_definition.json'
+            modelDefinition = {}
+            url = '{}://{}/ml/{}'.format(
+                constants.__DEFAULT_PYCONFIG_PROTOCOL,
+                constants.__DEFAULT_PYCONFIG_DOMAIN,
+                modelId
+            )
+            print_debug(2, '{}'.format(url))
+            result = urequest.get(url, headers={'content-type': 'application/json'})
+            modelDefinition = json.loads(result.content.decode())
+            print_debug(2, 'modelDefinition: {}'.format(modelDefinition))
+            f = open(file, 'w')
+            f.write(json.dumps(modelDefinition).encode('utf-8'))
+            f.close()
+            print_debug(2, "Model definition written to {}".format(file))
+        except Exception as e:
+            if not silent:
+                print_debug(2, "Exception: {}".format(e))

esp32/frozen/Pybytes/_pybytes_protocol.py

@@ -31,6 +31,11 @@
 except:
     from _pybytes_pymesh_config import PybytesPymeshConfig
 
+try:
+    from pybytes_machine_learning import MlFeatures
+except:
+    from _pybytes_machine_learning import MlFeatures
+
 try:
     from pybytes_config_reader import PybytesConfigReader
 except:
@@ -281,10 +286,10 @@ def __process_recv_message(self, message):
                     splittedBody = bodyString.split(',')
                     if (len(splittedBody) >= 2):
                         path = splittedBody[0]
-                        print_debug(2, path[len(path)-7:len(path)])
-                        if (path[len(path)-7:len(path)] != '.pymakr'):
+                        print_debug(2, path[len(path) - 7:len(path)])
+                        if (path[len(path) - 7:len(path)] != '.pymakr'):
                             self.send_fcota_ping('updating file...')
-                        newContent = bodyString[len(path)+1:len(body)]
+                        newContent = bodyString[len(path) + 1:len(body)]
                         if (self.__FCOTA.update_file_content(path, newContent) is True): # noqa
                             size = self.__FCOTA.get_file_size(path)
                             self.send_fcota_file(newContent, path, size)
@@ -319,7 +324,18 @@ def __process_recv_message(self, message):
                 if (len(body) > 3):
                     value = body[2] << 8 | body[3]
 
-                if (command == constants.__COMMAND_PIN_MODE):
+                if (command == constants.__COMMAND_START_SAMPLE):
+                    parameters = ujson.loads(body[2: len(body)].decode("utf-8"))
+                    sampling = MlFeatures(self, parameters=parameters)
+                    sampling.start_sampling(pin=parameters["pin"])
+                    self.send_ota_response(result=2, topic='sample')
+                elif (command == constants.__COMMAND_DEPLOY_MODEL):
+                    parameters = ujson.loads(body[2: len(body)].decode("utf-8"))
+                    sampling = MlFeatures()
+                    sampling.deploy_model(modelId=parameters["modelId"])
+                    self.send_ota_response(result=2, topic='deploymlmodel')
+
+                elif (command == constants.__COMMAND_PIN_MODE):
                     pass
 
                 elif (command == constants.__COMMAND_DIGITAL_READ):
@@ -633,16 +649,11 @@ def write_firmware(self, customManifest=None):
     def get_application_details(self, body):
         application = self.__conf.get('application')
         if application is not None:
-            if 'id' in application and application['id']:
-                applicationID = application['id']
-            else:
-                applicationID = body['applicationId']
             if 'release' in application and 'codeFilename' in application['release']:
                 currentReleaseID = application['release']['codeFilename']
             else:
                 currentReleaseID = None
         else:
-            applicationID = body['applicationId']
             currentReleaseID = None
             self.__conf['application'] = {
                 "id": "",
@@ -652,6 +663,7 @@ def get_application_details(self, body):
                       "version": 0
                 }
             }
+        applicationID = body['applicationId']
         return (applicationID, currentReleaseID)
 
     def get_update_manifest(self, applicationID, newReleaseID, currentReleaseID):
@@ -755,21 +767,46 @@ def update_network_config(self, letResp):
         except Exception as e:
             print_debug(1, "error while updating network config pybytes_config.json! {}".format(e))
 
-    def update_firmware(self, body):
+    def update_firmware(self, body, applicationID, fw_type='pybytes'):
         if "firmware" not in body:
             print_debug(0, "no firmware to update")
             return
-        version = body['firmware']["version"]
-        print_debug(0, "updating firmware to {}".format(version))
-        customManifest = {
-            "firmware": {
-                "URL": "https://{}/downloads/appimg/firmware_{}_{}.bin".format(
-                    constants.__DEFAULT_SW_HOST,
-                    os.uname().sysname,
-                    version),
+
+        if "version" in body['firmware']:
+            version = body['firmware']["version"]
+            print_debug(0, "updating firmware to {}".format(version))
+
+            customManifest = {
+                "firmware": {
+                    "URL": "https://{}/findupgrade?redirect=true&strict=true&type={}&model={}&version={}&download=true".format(
+                        constants.__DEFAULT_SW_HOST,
+                        fw_type,
+                        os.uname().sysname,
+                        version),
+                }
             }
-        }
-        self.write_firmware(customManifest)
+            self.write_firmware(customManifest)
+        else:
+            fileUrl = '{}://{}/firmware?'.format(constants.__DEFAULT_PYCONFIG_PROTOCOL, constants.__DEFAULT_PYCONFIG_DOMAIN)
+            customFirmwares = body['firmware']["customFirmwares"]
+            firmwareFilename = ''
+            for firmware in customFirmwares:
+                print_debug(1, "firmware['firmwareType']={} and os.uname().sysname.lower()={}".format(firmware['firmwareType'], os.uname().sysname.lower()))
+                print_debug(1, "firmware={}".format(firmware))
+                if (firmware['firmwareType'] == os.uname().sysname.lower()):
+                    firmwareFilename = firmware['firmwareFilename']
+            targetFileLocation = '{}application_id={}&target_ver={}&target_path={}'.format(
+                fileUrl,
+                applicationID,
+                firmwareFilename,
+                '/{}.bin'.format(firmwareFilename)
+            )
+            customManifest = {
+                "firmware": {
+                    "URL": targetFileLocation,
+                }
+            }
+            self.write_firmware(customManifest)
 
     def deploy_new_release(self, body):
         try:
@@ -783,12 +820,15 @@ def deploy_new_release(self, body):
         applicationID, currentReleaseID = self.get_application_details(body)
 
         letResp = self.get_update_manifest(applicationID, newReleaseID, currentReleaseID)
+
         if not letResp:
             return
 
+        fwtype = 'pygate' if hasattr(os.uname(), 'pygate') else 'pybytes'
+        fwtype = 'pymesh' if hasattr(os.uname(), 'pymesh') else fwtype
         self.update_files(letResp, applicationID, newReleaseID)
         self.delete_files(letResp)
         self.update_application_config(letResp, applicationID)
         self.update_network_config(letResp)
-        self.update_firmware(letResp)
+        self.update_firmware(letResp, applicationID, fw_type=fwtype)
         machine.reset()