logging cleanup, example skeleton for components

Author: gmorell

aqi/main.py

@@ -1,6 +1,4 @@
 import datetime
-import logging
-import sys
 from typing import Dict, List
 
 import treq
@@ -21,8 +19,6 @@
 API_KEY = "53295F0D-285C-4787-8336-4DA76A434E03"
 
 ns = "LA4-DATA-AQI"
-logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-logger = logging.Logger(ns)
 
 AQI_PREFIX = NAMESPACE_PREFIX + "aqi."
 
@@ -44,7 +40,7 @@ def as_obs_request(self):
 
 class AQIObservationRequestParams(BaseModel):
     zipCode: str = Field(alias="zip_code")
-    format: str = "application/json" #
+    format: str = "application/json"  #
     distance: int = 25
     API_KEY: str = Field(alias="api_key")
 
@@ -53,7 +49,7 @@ class AQIForecastRequestParams(BaseModel):
     date: datetime.date
 
     zipCode: str = Field(alias="zip_code")
-    format: str = "application/json" #
+    format: str = "application/json"  #
     distance: int = 25
     API_KEY: str = Field(alias="api_key")
 
@@ -104,18 +100,16 @@ class AQISession(LAMachineCompatMixin, ApplicationSession):
 
     @inlineCallbacks
     def onJoin(self, details):
-        logger.info("session ready")
-        print("session ready")
+        self.log.info("session ready")
         self.regs = yield self.register(self)
         self.subs = yield self.subscribe(self)
 
         self.ticker_aqi = LoopingCall(self.ticker_update_aqi)
         self.ticker_aqi.start(1800)  # 30m
 
         self.herald_aqi = LoopingCall(self.aqi_herald)
-        self.herald_aqi.start(10)
-        print("completed startup")
-        logger.info("completed startup")
+        self.herald_aqi.start(.05)
+        self.log.info("completed startup")
 
     @wamp.register(AQI_PREFIX + "register")
     def register_config(self):
@@ -134,35 +128,39 @@ def aqi_herald(self):
             for res in res_list:
                 id = f"{self.cconfig.machine_name.lower()}-{key}-{str(res.param_name).lower()}"
                 yield self.publish(f"{SRC_PREFIX}{id}",
-                                      self.run_brightness_on_val(observation_color_map[res.category.number] * LEDS_IN_ARRAY_DEFAULT),
-                                      id=id, options=options)
+                                   self.run_brightness_on_val(
+                                       observation_color_map[res.category.number] * LEDS_IN_ARRAY_DEFAULT),
+                                   id=id, options=options)
         if not self.aqi_results:
-            print("Herald: No data has been loaded, chilling")
+            self.log.info("Herald: No data has been loaded, chilling")
 
     @inlineCallbacks
     def do_update_aqi(self, config_id: str):
         try:
-            logger.info(f"Updating {config_id}")
             config = self.aqi_configs[config_id]
 
             result = yield treq.get(self.PATH_OBSERVATIONS, params=config.as_obs_request().dict())
             data = yield result.json()
             # pls forgive the fucking federal govt
             for i in data:
-                i['DateObserved'] = i['DateObserved'].replace(" ","")
+                i['DateObserved'] = i['DateObserved'].replace(" ", "")
             self.aqi_results[config_id] = [AirNowObservation(**i) for i in data]
         except Exception as e:
             import traceback
-            print(e)
-            print(traceback.format_exc())
+            self.log.error(e)
+            self.log.error(traceback.format_exc())
 
     def ticker_update_aqi(self):
         uninstantiated_results = set(self.aqi_configs.keys()) - set(self.aqi_results.keys())
+        other_results = set(self.aqi_configs.keys()) - uninstantiated_results
         for ur in uninstantiated_results:
+            self.log.info(f"Updating {ur} (1st)")
             self.do_update_aqi(ur)
 
-        for k in self.aqi_configs.keys():
-            self.do_update_aqi(ur)
+        for k in other_results:
+            self.log.info(f"Updating {ur}")
+            self.do_update_aqi(k)
+
 
 comp = Component(
     transports=u"ws://localhost:8083/ws",

compat.py

@@ -1,12 +1,11 @@
 import dataclasses
 import json
 import logging
+from dataclasses import dataclass
 from enum import Enum
 from typing import Dict, Any, Optional
 
 from autobahn import wamp
-from dataclasses import dataclass
-
 from autobahn.wamp import RegisterOptions
 from pydantic import BaseModel
 from pydantic.json import pydantic_encoder
@@ -134,7 +133,7 @@ def __init__(self, config=None):
         super().__init__(config)
 
         self.brightness_ticker = LoopingCall(self.ticker_brightness_ctrl)
-        self.brightness_ticker.start(.1)  # 30m
+        self.brightness_ticker.start(.1)
 
     def ticker_brightness_ctrl(self):
         if self.brightness_act != self.brightness_tgt.value:

example.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+from typing import Dict, Union, List
+
+from autobahn import wamp
+from autobahn.twisted import ApplicationSession
+from autobahn.twisted.component import Component
+from autobahn.twisted.component import run
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.task import LoopingCall
+
+from compat import LAMachineCompatMixin, CompatConfig
+from config import NAMESPACE_PREFIX
+
+# Some declarations, change these
+
+ns = "LA4-DATA-EXAMPLE"
+EXAMPLE_PREFIX = NAMESPACE_PREFIX + "example."
+
+
+@dataclass
+class SomeConfig:
+    pass
+
+
+@dataclass
+class SomeResult:
+    pass
+
+
+class ExampleSessionTwisted(LAMachineCompatMixin, ApplicationSession):
+    cconfig = CompatConfig(
+        machine_name=ns,
+        desc_blurb="An Example Description for the UI"
+    )
+
+    configs: Dict[str, SomeConfig] = {}
+    results: Union[Dict[str, SomeResult], Dict[str, List[SomeResult]]] = {}
+
+    @wamp.register(EXAMPLE_PREFIX + "register")
+    def register_config(self):
+        pass
+
+    @wamp.register(EXAMPLE_PREFIX + "register")
+    def remove_config(self):
+        pass
+
+    def update_data(self):
+        raise NotImplementedError("Need to configure a data update")
+
+    def herald_data(self):
+        raise NotImplementedError("Need to configure a data herald")
+
+    @inlineCallbacks
+    def onJoin(self, details):
+        self.log.info("session ready")
+        self.regs = yield self.register(self)
+        self.subs = yield self.subscribe(self)
+
+        self.ticker_ex = LoopingCall(self.update_data)
+        self.ticker_ex.start(1800)  # 30m
+
+        self.herald_ex = LoopingCall(self.herald_data)
+        self.herald_exs.start(10)
+
+
+comp = Component(
+    transports=u"ws://localhost:8083/ws",
+    realm=u"realm1",
+    session_factory=ExampleSessionTwisted
+)
+if __name__ == "__main__":
+    run([comp], log_level="info")

gbfs/__init__.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+from typing import Dict, Union, List
+
+from autobahn import wamp
+from autobahn.twisted import ApplicationSession
+from autobahn.twisted.component import Component
+from autobahn.twisted.component import run
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.task import LoopingCall
+
+from compat import LAMachineCompatMixin, CompatConfig
+from config import NAMESPACE_PREFIX
+
+ns = "LA4-DATA-GBFS"
+GBFS_PREFIX = NAMESPACE_PREFIX + "gbfs."
+
+
+@dataclass
+class SomeConfig:
+    pass
+
+
+@dataclass
+class SomeResult:
+    pass
+
+
+class GBFSSession(LAMachineCompatMixin, ApplicationSession):
+    cconfig = CompatConfig(
+        machine_name=ns,
+        desc_blurb="A GBFS Provider"
+    )
+
+    configs: Dict[str, SomeConfig] = {}
+    results: Union[Dict[str, SomeResult], Dict[str, List[SomeResult]]] = {}
+
+    @wamp.register(GBFS_PREFIX + "register")
+    def register_config(self):
+        pass
+
+    @wamp.register(GBFS_PREFIX + "register")
+    def remove_config(self):
+        pass
+
+    def update_data(self):
+        raise NotImplementedError("Need to configure a data update")
+
+    def herald_data(self):
+        raise NotImplementedError("Need to configure a data herald")
+
+    @inlineCallbacks
+    def onJoin(self, details):
+        self.log.info("session ready")
+        self.regs = yield self.register(self)
+        self.subs = yield self.subscribe(self)
+
+        self.ticker_ex = LoopingCall(self.update_data)
+        self.ticker_ex.start(1800)  # 30m
+
+        self.herald_ex = LoopingCall(self.herald_data)
+        self.herald_exs.start(10)
+
+
+comp = Component(
+    transports=u"ws://localhost:8083/ws",
+    realm=u"realm1",
+    session_factory=GBFSSession
+)
+if __name__ == "__main__":
+    run([comp], log_level="info")

gbfs/main.py

@@ -1,13 +1,12 @@
 import csv
 import datetime
-import logging
 import os
-import treq
 import zipfile
 from dataclasses import dataclass, field
 from typing import List, Dict, Optional, Any
 
 import pytz
+import treq
 from aenum import Enum
 from autobahn import wamp
 from autobahn.twisted import ApplicationSession
@@ -22,7 +21,6 @@
 
 GTFS_PREFIX = NAMESPACE_PREFIX + "gtfs."
 ns = "LA4-DATA-GTFS"
-logger = logging.Logger(ns)
 
 
 class GTFSParsers(Enum):
@@ -106,12 +104,11 @@ def get_next_schedules_for_stop(self, id: str, tz: str = "America/New_York", min
 
         stop_times = self._get_stop_times_by_id(id)
         nearby_times = self._filter_to_find_closest_times(stop_times, now, fut)
-        print(nearby_times)
         trips = self._get_trips_by_id([i.trip_id for i in nearby_times])
-        print(trips)
-        print("Upcoming")
+        self.log.info("Upcoming")
         for t in sorted(nearby_times, key=lambda x:x.arrival_time):
-            print(f"Line {trips[t.trip_id].route_id} Arrives at {t.arrival_time} {t.arrival_time_as_delta_seconds(now)} ")
+            self.log.info(
+                f"Line {trips[t.trip_id].route_id} Arrives at {t.arrival_time} {t.arrival_time_as_delta_seconds(now)} ")
 
 
 
@@ -142,7 +139,7 @@ class GTFSSession(ApplicationSession):
 
     @inlineCallbacks
     def onJoin(self, details):
-        print("session ready")
+        self.log.info("session ready")
         self.regs = yield self.register(self)
         self.subs = yield self.subscribe(self)
 
@@ -203,7 +200,7 @@ def parse_bundle(self, config_id: str):
             for file in os.listdir(f"cache/{config_id}/bundle"):
                 file_pre = file.replace('.txt', '')
                 if file_pre not in [i.name for i in GTFSParsers]:
-                    print(f"{file_pre} was not found in our parser list, may be a gtfs+/experimental entity")
+                    self.log.info(f"{file_pre} was not found in our parser list, may be a gtfs+/experimental entity")
                     continue
                 kwargs[file_pre] = []
                 with open(f"cache/{config_id}/bundle/{file}", 'r') as file_handle:
@@ -213,14 +210,14 @@ def parse_bundle(self, config_id: str):
                         kwargs[file_pre].append(getattr(GTFSParsers, file_pre).value(**dict(zip(header, row))))
 
             self.gtfs_results[config_id] = GTFSResults(**kwargs)
-            print("Completed Ingesting")
+            self.log.info(f"Completed Ingesting {config_id}")
 
         except Exception as e:
-            print(e)
+            self.log.error(e)
 
     @inlineCallbacks
     def do_update(self, config_id: str):
-        logger.info(f"Updating {config_id}")
+        self.log.info(f"Updating {config_id}")
         self.assert_directory_structure(config_id)
         yield self.download_and_write(config_id)
         self.unzip_bundle(config_id)
@@ -242,14 +239,14 @@ def gtfs_herald(self):
             for i in self.gtfs_results:
                 config = self.gtfs_configs[i]
                 for stop in config.relevant_stops:
-                    print(f"heralding {i} - {stop}")
+                    self.log.debug(f"heralding {i} - {stop}")
                     self.gtfs_results[i].get_next_schedules_for_stop(stop)
             if not self.gtfs_results:
-                print("Herald: No data has been loaded, chilling")
+                self.log.info("Herald: No data has been loaded, chilling")
         except Exception as e:
             import traceback
-            print(e)
-            print(traceback.format_exc())
+            self.log.error(e)
+            self.log.error(traceback.format_exc())
 
 comp = Component(
     transports=u"ws://localhost:8083/ws",