Fixes for PYTHONWARNINGS=error (#29381)

old-commit-hash: db287496d8

common/spinner.py

@@ -29,11 +29,11 @@ class Spinner():
 
   def close(self):
     if self.spinner_proc is not None:
+      self.spinner_proc.kill()
       try:
-        self.spinner_proc.stdin.close()
+        self.spinner_proc.communicate(timeout=2.)
-      except BrokenPipeError:
+      except subprocess.TimeoutExpired:
-        pass
+        print("WARNING: failed to kill spinner")
-      self.spinner_proc.terminate()
       self.spinner_proc = None
 
   def __del__(self):

selfdrive/boardd/tests/test_boardd_loopback.py

@@ -10,7 +10,6 @@ from pprint import pprint
 import cereal.messaging as messaging
 from cereal import car, log
 from common.params import Params
-from common.spinner import Spinner
 from common.timeout import Timeout
 from selfdrive.boardd.boardd import can_list_to_can_capnp
 from selfdrive.car import make_can_msg
@@ -24,11 +23,6 @@ class TestBoardd(unittest.TestCase):
   def setUpClass(cls):
     os.environ['STARTED'] = '1'
     os.environ['BOARDD_LOOPBACK'] = '1'
-    cls.spinner = Spinner()
-
-  @classmethod
-  def tearDownClass(cls):
-    cls.spinner.close()
 
   @phone_only
   @with_processes(['pandad'])
@@ -67,7 +61,6 @@ class TestBoardd(unittest.TestCase):
     n = 200
     for i in range(n):
       print(f"boardd loopback {i}/{n}")
-      self.spinner.update(f"boardd loopback {i}/{n}")
 
       sent_msgs = defaultdict(set)
       for _ in range(random.randrange(20, 100)):

selfdrive/statsd.py

@@ -23,6 +23,8 @@ class METRIC_TYPE:
 class StatLog:
   def __init__(self):
     self.pid = None
+    self.zctx = None
+    self.sock = None
 
   def connect(self) -> None:
     self.zctx = zmq.Context()
@@ -31,6 +33,12 @@ class StatLog:
     self.sock.connect(STATS_SOCKET)
     self.pid = os.getpid()
 
+  def __del__(self):
+    if self.sock is not None:
+      self.sock.close()
+    if self.zctx is not None:
+      self.zctx.term()
+
   def _send(self, metric: str) -> None:
     if os.getpid() != self.pid:
       self.connect()
@@ -68,7 +76,7 @@ def main() -> NoReturn:
     return res
 
   # open statistics socket
-  ctx = zmq.Context().instance()
+  ctx = zmq.Context.instance()
   sock = ctx.socket(zmq.PULL)
   sock.bind(STATS_SOCKET)
 
@@ -92,70 +100,74 @@ def main() -> NoReturn:
   last_flush_time = time.monotonic()
   gauges = {}
   samples: Dict[str, List[float]] = defaultdict(list)
-  while True:
+  try:
-    started_prev = sm['deviceState'].started
-    sm.update()
-
-    # Update metrics
     while True:
-      try:
+      started_prev = sm['deviceState'].started
-        metric = sock.recv_string(zmq.NOBLOCK)
+      sm.update()
+
+      # Update metrics
+      while True:
         try:
-          metric_type = metric.split('|')[1]
+          metric = sock.recv_string(zmq.NOBLOCK)
-          metric_name = metric.split(':')[0]
+          try:
-          metric_value = float(metric.split('|')[0].split(':')[1])
+            metric_type = metric.split('|')[1]
-
+            metric_name = metric.split(':')[0]
-          if metric_type == METRIC_TYPE.GAUGE:
+            metric_value = float(metric.split('|')[0].split(':')[1])
-            gauges[metric_name] = metric_value
+
-          elif metric_type == METRIC_TYPE.SAMPLE:
+            if metric_type == METRIC_TYPE.GAUGE:
-            samples[metric_name].append(metric_value)
+              gauges[metric_name] = metric_value
-          else:
+            elif metric_type == METRIC_TYPE.SAMPLE:
-            cloudlog.event("unknown metric type", metric_type=metric_type)
+              samples[metric_name].append(metric_value)
-        except Exception:
+            else:
-          cloudlog.event("malformed metric", metric=metric)
+              cloudlog.event("unknown metric type", metric_type=metric_type)
-      except zmq.error.Again:
+          except Exception:
-        break
+            cloudlog.event("malformed metric", metric=metric)
-
+        except zmq.error.Again:
-    # flush when started state changes or after FLUSH_TIME_S
+          break
-    if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
+
-      result = ""
+      # flush when started state changes or after FLUSH_TIME_S
-      current_time = datetime.utcnow().replace(tzinfo=timezone.utc)
+      if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
-      tags['started'] = sm['deviceState'].started
+        result = ""
-
+        current_time = datetime.utcnow().replace(tzinfo=timezone.utc)
-      for key, value in gauges.items():
+        tags['started'] = sm['deviceState'].started
-        result += get_influxdb_line(f"gauge.{key}", value, current_time, tags)
+
-
+        for key, value in gauges.items():
-      for key, values in samples.items():
+          result += get_influxdb_line(f"gauge.{key}", value, current_time, tags)
-        values.sort()
+
-        sample_count = len(values)
+        for key, values in samples.items():
-        sample_sum = sum(values)
+          values.sort()
-
+          sample_count = len(values)
-        stats = {
+          sample_sum = sum(values)
-          'count': sample_count,
+
-          'min': values[0],
+          stats = {
-          'max': values[-1],
+            'count': sample_count,
-          'mean': sample_sum / sample_count,
+            'min': values[0],
-        }
+            'max': values[-1],
-        for percentile in [0.05, 0.5, 0.95]:
+            'mean': sample_sum / sample_count,
-          value = values[int(round(percentile * (sample_count - 1)))]
+          }
-          stats[f"p{int(percentile * 100)}"] = value
+          for percentile in [0.05, 0.5, 0.95]:
-
+            value = values[int(round(percentile * (sample_count - 1)))]
-        result += get_influxdb_line(f"sample.{key}", stats, current_time, tags)
+            stats[f"p{int(percentile * 100)}"] = value
-
+
-      # clear intermediate data
+          result += get_influxdb_line(f"sample.{key}", stats, current_time, tags)
-      gauges.clear()
+
-      samples.clear()
+        # clear intermediate data
-      last_flush_time = time.monotonic()
+        gauges.clear()
-
+        samples.clear()
-      # check that we aren't filling up the drive
+        last_flush_time = time.monotonic()
-      if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT:
+
-        if len(result) > 0:
+        # check that we aren't filling up the drive
-          stats_path = os.path.join(STATS_DIR, f"{current_time.timestamp():.0f}_{idx}")
+        if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT:
-          with atomic_write_in_dir(stats_path) as f:
+          if len(result) > 0:
-            f.write(result)
+            stats_path = os.path.join(STATS_DIR, f"{current_time.timestamp():.0f}_{idx}")
-          idx += 1
+            with atomic_write_in_dir(stats_path) as f:
-      else:
+              f.write(result)
-        cloudlog.error("stats dir full")
+            idx += 1
+        else:
+          cloudlog.error("stats dir full")
+  finally:
+    sock.close()
+    ctx.term()
 
 
 if __name__ == "__main__":

system/loggerd/tests/test_loggerd.py

@@ -210,7 +210,8 @@ class TestLoggerd(unittest.TestCase):
     for fn in ["console-ramoops", "pmsg-ramoops-0"]:
       path = Path(os.path.join("/sys/fs/pstore/", fn))
       if path.is_file():
-        expected_val = open(path, "rb").read()
+        with open(path, "rb") as f:
+          expected_val = f.read()
         bootlog_val = [e.value for e in boot.pstore.entries if e.key == fn][0]
         self.assertEqual(expected_val, bootlog_val)
 

tools/bodyteleop/web.py

@@ -6,11 +6,16 @@ import ssl
 import uuid
 import time
 
-from common.basedir import BASEDIR
+# aiortc and its dependencies have lots of internal warnings :(
+import warnings
+warnings.resetwarnings()
+warnings.simplefilter("always")
+
 from aiohttp import web
 from aiortc import RTCPeerConnection, RTCSessionDescription
 
 import cereal.messaging as messaging
+from common.basedir import BASEDIR
 from tools.bodyteleop.bodyav import BodyMic, WebClientSpeaker, force_codec, play_sound, MediaBlackhole, EncodedBodyVideo
 
 logger = logging.getLogger("pc")