Merge branch 'master' of github.com:commaai/openpilot into prefix-imports

.github/workflows/selfdrive_tests.yaml

@@ -306,7 +306,7 @@ jobs:
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
-      timeout-minutes: 20
+      timeout-minutes: 30
       run: |
         ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                         coverage xml"

common/SConscript

@@ -31,5 +31,4 @@ if GetOption('test'):
   env.Program('tests/test_swaglog', ['tests/test_swaglog.cc'], LIBS=[_common, 'json11', 'zmq', 'pthread'])
 
 # Cython
-envCython.Program('clock.so', 'clock.pyx')
 envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])

common/clock.pyx

@@ -1,24 +0,0 @@
-# distutils: language = c++
-# cython: language_level = 3
-from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
-
-IF UNAME_SYSNAME == "Darwin":
-  # Darwin doesn't have a CLOCK_BOOTTIME
-  CLOCK_BOOTTIME = CLOCK_MONOTONIC_RAW
-ELSE:
-  from posix.time cimport CLOCK_BOOTTIME
-
-cdef double readclock(clockid_t clock_id):
-  cdef timespec ts
-  cdef double current
-
-  clock_gettime(clock_id, &ts)
-  current = ts.tv_sec + (ts.tv_nsec / 1000000000.)
-  return current
-
-def monotonic_time():
-  return readclock(CLOCK_MONOTONIC_RAW)
-
-def sec_since_boot():
-  return readclock(CLOCK_BOOTTIME)
-

common/realtime.py

@@ -5,9 +5,8 @@ import time
 from collections import deque
 from typing import Optional, List, Union
 
-from setproctitle import getproctitle  # pylint: disable=no-name-in-module
+from setproctitle import getproctitle
 
-from openpilot.common.clock import sec_since_boot  # pylint: disable=no-name-in-module, import-error
 from openpilot.system.hardware import PC
 
 
@@ -50,13 +49,13 @@ class Ratekeeper:
   def __init__(self, rate: float, print_delay_threshold: Optional[float] = 0.0) -> None:
     """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
     self._interval = 1. / rate
-    self._next_frame_time = sec_since_boot() + self._interval
+    self._next_frame_time = time.monotonic() + self._interval
     self._print_delay_threshold = print_delay_threshold
     self._frame = 0
     self._remaining = 0.0
     self._process_name = getproctitle()
     self._dts = deque([self._interval], maxlen=100)
-    self._last_monitor_time = sec_since_boot()
+    self._last_monitor_time = time.monotonic()
 
   @property
   def frame(self) -> int:
@@ -82,11 +81,11 @@ class Ratekeeper:
   # this only monitor the cumulative lag, but does not enforce a rate
   def monitor_time(self) -> bool:
     prev = self._last_monitor_time
-    self._last_monitor_time = sec_since_boot()
+    self._last_monitor_time = time.monotonic()
     self._dts.append(self._last_monitor_time - prev)
 
     lagged = False
-    remaining = self._next_frame_time - sec_since_boot()
+    remaining = self._next_frame_time - time.monotonic()
     self._next_frame_time += self._interval
     if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
       print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")

release/files_common

@@ -23,7 +23,6 @@ common/__init__.py
 common/conversions.py
 common/gpio.py
 common/realtime.py
-common/clock.pyx
 common/timeout.py
 common/ffi_wrapper.py
 common/file_helpers.py

scripts/waste.py

@@ -1,23 +1,23 @@
 #!/usr/bin/env python3
 import os
+import time
 import numpy as np
-from openpilot.common.realtime import sec_since_boot
 from multiprocessing import Process
-from setproctitle import setproctitle  # pylint: disable=no-name-in-module
+from setproctitle import setproctitle
 
 def waste(core):
-  os.sched_setaffinity(0, [core,])  # pylint: disable=no-member
+  os.sched_setaffinity(0, [core,])
 
   m1 = np.zeros((200, 200)) + 0.8
   m2 = np.zeros((200, 200)) + 1.2
 
   i = 1
-  st = sec_since_boot()
+  st = time.monotonic()
   j = 0
   while 1:
     if (i % 100) == 0:
       setproctitle("%3d: %8d" % (core, i))
-      lt = sec_since_boot()
+      lt = time.monotonic()
       print("%3d: %8d %f  %.2f" % (core, i, lt-st, j))
       st = lt
     i += 1

selfdrive/athena/athenad.py

@@ -34,7 +34,7 @@ from openpilot.common.api import Api
 from openpilot.common.basedir import PERSIST
 from openpilot.common.file_helpers import CallbackReader
 from openpilot.common.params import Params
-from openpilot.common.realtime import sec_since_boot, set_core_affinity
+from openpilot.common.realtime import set_core_affinity
 from openpilot.system.hardware import HARDWARE, PC, AGNOS
 from openpilot.system.loggerd.config import ROOT
 from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
@@ -593,10 +593,10 @@ def log_handler(end_event: threading.Event) -> None:
     return
 
   log_files = []
-  last_scan = 0
+  last_scan = 0.
   while not end_event.is_set():
     try:
-      curr_scan = sec_since_boot()
+      curr_scan = time.monotonic()
       if curr_scan - last_scan > 10:
         log_files = get_logs_to_send_sorted()
         last_scan = curr_scan
@@ -652,8 +652,8 @@ def log_handler(end_event: threading.Event) -> None:
 
 def stat_handler(end_event: threading.Event) -> None:
   while not end_event.is_set():
-    last_scan = 0
+    last_scan = 0.
-    curr_scan = sec_since_boot()
+    curr_scan = time.monotonic()
     try:
       if curr_scan - last_scan > 10:
         stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(STATS_DIR)))
@@ -721,7 +721,7 @@ def ws_proxy_send(ws: WebSocket, local_sock: socket.socket, signal_sock: socket.
 
 
 def ws_recv(ws: WebSocket, end_event: threading.Event) -> None:
-  last_ping = int(sec_since_boot() * 1e9)
+  last_ping = int(time.monotonic() * 1e9)
   while not end_event.is_set():
     try:
       opcode, data = ws.recv_data(control_frame=True)
@@ -730,10 +730,10 @@ def ws_recv(ws: WebSocket, end_event: threading.Event) -> None:
           data = data.decode("utf-8")
         recv_queue.put_nowait(data)
       elif opcode == ABNF.OPCODE_PING:
-        last_ping = int(sec_since_boot() * 1e9)
+        last_ping = int(time.monotonic() * 1e9)
         Params().put("LastAthenaPingTime", str(last_ping))
     except WebSocketTimeoutException:
-      ns_since_last_ping = int(sec_since_boot() * 1e9) - last_ping
+      ns_since_last_ping = int(time.monotonic() * 1e9) - last_ping
       if ns_since_last_ping > RECONNECT_TIMEOUT_S * 1e9:
         cloudlog.exception("athenad.ws_recv.timeout")
         end_event.set()

selfdrive/controls/controlsd.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 import os
 import math
+import time
 from typing import SupportsFloat
 
 from cereal import car, log
 from openpilot.common.numpy_fast import clip
-from openpilot.common.realtime import sec_since_boot, config_realtime_process, Priority, Ratekeeper, DT_CTRL
+from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper, DT_CTRL
 from openpilot.common.profiler import Profiler
 from openpilot.common.params import Params, put_nonblocking, put_bool_nonblocking
 import cereal.messaging as messaging
@@ -743,7 +744,7 @@ class Controls:
 
     if not self.read_only and self.initialized:
       # send car controls over can
-      now_nanos = self.can_log_mono_time if REPLAY else int(sec_since_boot() * 1e9)
+      now_nanos = self.can_log_mono_time if REPLAY else int(time.monotonic() * 1e9)
       self.last_actuators, can_sends = self.CI.apply(CC, now_nanos)
       self.pm.send('sendcan', can_list_to_can_capnp(can_sends, msgtype='sendcan', valid=CS.canValid))
       CC.actuatorsOutput = self.last_actuators
@@ -842,7 +843,7 @@ class Controls:
     self.CC = CC
 
   def step(self):
-    start_time = sec_since_boot()
+    start_time = time.monotonic()
     self.prof.checkpoint("Ratekeeper", ignore=True)
 
     self.is_metric = self.params.get_bool("IsMetric")

selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 import os
+import time
 import numpy as np
 
 from casadi import SX, vertcat, sin, cos
-from openpilot.common.realtime import sec_since_boot
 # WARNING: imports outside of constants will not trigger a rebuild
 from openpilot.selfdrive.modeld.constants import T_IDXS
 
@@ -182,9 +182,9 @@ class LateralMpc():
     self.solver.set(N, "p", p_cp[N])
     self.solver.cost_set(N, "yref", self.yref[N][:COST_E_DIM])
 
-    t = sec_since_boot()
+    t = time.monotonic()
     self.solution_status = self.solver.solve()
-    self.solve_time = sec_since_boot() - t
+    self.solve_time = time.monotonic() - t
 
     for i in range(N+1):
       self.x_sol[i] = self.solver.get(i, 'x')

selfdrive/controls/lib/lateral_planner.py

@@ -1,5 +1,6 @@
+import time
 import numpy as np
-from openpilot.common.realtime import sec_since_boot, DT_MDL
+from openpilot.common.realtime import DT_MDL
 from openpilot.common.numpy_fast import interp
 from openpilot.system.swaglog import cloudlog
 from openpilot.selfdrive.controls.lib.lateral_mpc_lib.lat_mpc import LateralMpc
@@ -108,7 +109,7 @@ class LateralPlanner:
 
     #  Check for infeasible MPC solution
     mpc_nans = np.isnan(self.lat_mpc.x_sol[:, 3]).any()
-    t = sec_since_boot()
+    t = time.monotonic()
     if mpc_nans or self.lat_mpc.solution_status != 0:
       self.reset_mpc()
       self.x0[3] = measured_curvature * self.v_ego

selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 import os
+import time
 import numpy as np
 from cereal import log
-from openpilot.common.realtime import sec_since_boot
 from openpilot.common.numpy_fast import clip
 from openpilot.system.swaglog import cloudlog
 # WARNING: imports outside of constants will not trigger a rebuild
@@ -411,7 +411,7 @@ class LongitudinalMpc:
         self.source = 'lead1'
 
   def run(self):
-    # t0 = sec_since_boot()
+    # t0 = time.monotonic()
     # reset = 0
     for i in range(N+1):
       self.solver.set(i, 'p', self.params[i])
@@ -442,14 +442,14 @@ class LongitudinalMpc:
 
     self.prev_a = np.interp(T_IDXS + 0.05, T_IDXS, self.a_solution)
 
-    t = sec_since_boot()
+    t = time.monotonic()
     if self.solution_status != 0:
       if t > self.last_cloudlog_t + 5.0:
         self.last_cloudlog_t = t
         cloudlog.warning(f"Long mpc reset, solution_status: {self.solution_status}")
       self.reset()
       # reset = 1
-    # print(f"long_mpc timings: total internal {self.solve_time:.2e}, external: {(sec_since_boot() - t0):.2e} qp {self.time_qp_solution:.2e}, \
+    # print(f"long_mpc timings: total internal {self.solve_time:.2e}, external: {(time.monotonic() - t0):.2e} qp {self.time_qp_solution:.2e}, \
     # lin {self.time_linearization:.2e} qp_iter {qp_iter}, reset {reset}")
 
 

selfdrive/debug/can_printer.py

@@ -1,17 +1,17 @@
 #!/usr/bin/env python3
 import argparse
 import binascii
+import time
 from collections import defaultdict
 
 import cereal.messaging as messaging
-from openpilot.common.realtime import sec_since_boot
 
 
 def can_printer(bus, max_msg, addr, ascii_decode):
   logcan = messaging.sub_sock('can', addr=addr)
 
-  start = sec_since_boot()
+  start = time.monotonic()
-  lp = sec_since_boot()
+  lp = time.monotonic()
   msgs = defaultdict(list)
   while 1:
     can_recv = messaging.drain_sock(logcan, wait_for_one=True)
@@ -20,17 +20,17 @@ def can_printer(bus, max_msg, addr, ascii_decode):
         if y.src == bus:
           msgs[y.address].append(y.dat)
 
-    if sec_since_boot() - lp > 0.1:
+    if time.monotonic() - lp > 0.1:
       dd = chr(27) + "[2J"
-      dd += f"{sec_since_boot() - start:5.2f}\n"
+      dd += f"{time.monotonic() - start:5.2f}\n"
       for addr in sorted(msgs.keys()):
         a = f"\"{msgs[addr][-1].decode('ascii', 'backslashreplace')}\"" if ascii_decode else ""
         x = binascii.hexlify(msgs[addr][-1]).decode('ascii')
-        freq = len(msgs[addr]) / (sec_since_boot() - start)
+        freq = len(msgs[addr]) / (time.monotonic() - start)
         if max_msg is None or addr < max_msg:
           dd += "%04X(%4d)(%6d)(%3dHz) %s %s\n" % (addr, addr, len(msgs[addr]), freq, x.ljust(20), a)
       print(dd)
-      lp = sec_since_boot()
+      lp = time.monotonic()
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="simple CAN data viewer",

selfdrive/debug/check_freq.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 import argparse
 import numpy as np
+import time
 from collections import defaultdict, deque
 from typing import DefaultDict, Deque, MutableSequence
 
-from openpilot.common.realtime import sec_since_boot
 import cereal.messaging as messaging
 
 
@@ -22,7 +22,7 @@ if __name__ == "__main__":
   rcv_times: DefaultDict[str, MutableSequence[float]] = defaultdict(lambda: deque(maxlen=100))
   valids: DefaultDict[str, Deque[bool]] = defaultdict(lambda: deque(maxlen=100))
 
-  t = sec_since_boot()
+  t = time.monotonic()
   for name in socket_names:
     sock = messaging.sub_sock(name, poller=poller)
     sockets[sock] = name
@@ -36,7 +36,7 @@ if __name__ == "__main__":
 
       name = msg.which()
 
-      t = sec_since_boot()
+      t = time.monotonic()
       rcv_times[name].append(msg.logMonoTime / 1e9)
       valids[name].append(msg.valid)
 

selfdrive/manager/process.py

@@ -10,14 +10,12 @@ from multiprocessing import Process
 
 from setproctitle import setproctitle  # pylint: disable=no-name-in-module
 
+from cereal import car, log
 import cereal.messaging as messaging
 import openpilot.selfdrive.sentry as sentry
-from cereal import car
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
-from openpilot.common.realtime import sec_since_boot
 from openpilot.system.swaglog import cloudlog
-from cereal import log
 
 WATCHDOG_FN = "/dev/shm/wd_"
 ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None
@@ -104,7 +102,7 @@ class ManagerProcess(ABC):
     except Exception:
       pass
 
-    dt = sec_since_boot() - self.last_watchdog_time / 1e9
+    dt = time.monotonic() - self.last_watchdog_time / 1e9
 
     if dt > self.watchdog_max_dt:
       if self.watchdog_seen and ENABLE_WATCHDOG:

selfdrive/thermald/power_monitoring.py

@@ -1,8 +1,8 @@
+import time
 import threading
 from typing import Optional
 
 from openpilot.common.params import Params, put_nonblocking
-from openpilot.common.realtime import sec_since_boot
 from openpilot.system.hardware import HARDWARE
 from openpilot.system.swaglog import cloudlog
 from openpilot.selfdrive.statsd import statlog
@@ -41,7 +41,7 @@ class PowerMonitoring:
   # Calculation tick
   def calculate(self, voltage: Optional[int], ignition: bool):
     try:
-      now = sec_since_boot()
+      now = time.monotonic()
 
       # If peripheralState is None, we're probably not in a car, so we don't care
       if voltage is None:
@@ -113,7 +113,7 @@ class PowerMonitoring:
     if offroad_timestamp is None:
       return False
 
-    now = sec_since_boot()
+    now = time.monotonic()
     should_shutdown = False
     offroad_time = (now - offroad_timestamp)
     low_voltage_shutdown = (self.car_voltage_mV < (VBATT_PAUSE_CHARGING * 1e3) and

selfdrive/thermald/tests/test_power_monitoring.py

@@ -7,12 +7,12 @@ params = Params()
 
 # Create fake time
 ssb = 0
-def mock_sec_since_boot():
+def mock_time_monotonic():
   global ssb
   ssb += 1
   return ssb
 
-with patch("common.realtime.sec_since_boot", new=mock_sec_since_boot):
+with patch("time.monotonic", new=mock_time_monotonic):
   with patch("common.params.put_nonblocking", new=params.put):
     from openpilot.selfdrive.thermald.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \
                                                     CAR_CHARGING_RATE_W, VBATT_PAUSE_CHARGING, DELAY_SHUTDOWN_TIME_S

selfdrive/thermald/thermald.py

@@ -17,7 +17,7 @@ from openpilot.common.dict_helpers import strip_deprecated_keys
 from openpilot.common.time import MIN_DATE
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.params import Params
-from openpilot.common.realtime import DT_TRML, sec_since_boot
+from openpilot.common.realtime import DT_TRML
 from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
 from openpilot.system.hardware import HARDWARE, TICI, AGNOS
 from openpilot.system.loggerd.config import get_available_percent
@@ -230,7 +230,7 @@ def thermald_thread(end_event, hw_queue):
         if TICI:
           fan_controller = TiciFanController()
 
-    elif (sec_since_boot() - sm.rcv_time['pandaStates']) > DISCONNECT_TIMEOUT:
+    elif (time.monotonic() - sm.rcv_time['pandaStates']) > DISCONNECT_TIMEOUT:
       if onroad_conditions["ignition"]:
         onroad_conditions["ignition"] = False
         cloudlog.error("panda timed out onroad")
@@ -273,7 +273,7 @@ def thermald_thread(end_event, hw_queue):
     if fan_controller is not None:
       msg.deviceState.fanSpeedPercentDesired = fan_controller.update(all_comp_temp, onroad_conditions["ignition"])
 
-    is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5))
+    is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (time.monotonic() - off_ts > 60 * 5))
     if is_offroad_for_5_min and offroad_comp_temp > OFFROAD_DANGER_TEMP:
       # if device is offroad and already hot without the extra onroad load,
       # we want to cool down first before increasing load
@@ -354,10 +354,10 @@ def thermald_thread(end_event, hw_queue):
     if should_start:
       off_ts = None
       if started_ts is None:
-        started_ts = sec_since_boot()
+        started_ts = time.monotonic()
         started_seen = True
         if startup_blocked_ts is not None:
-          cloudlog.event("Startup after block", block_duration=(sec_since_boot() - startup_blocked_ts),
+          cloudlog.event("Startup after block", block_duration=(time.monotonic() - startup_blocked_ts),
                          startup_conditions=startup_conditions, onroad_conditions=onroad_conditions,
                          startup_conditions_prev=startup_conditions_prev, error=True)
       startup_blocked_ts = None
@@ -365,11 +365,11 @@ def thermald_thread(end_event, hw_queue):
       if onroad_conditions["ignition"] and (startup_conditions != startup_conditions_prev):
         cloudlog.event("Startup blocked", startup_conditions=startup_conditions, onroad_conditions=onroad_conditions, error=True)
         startup_conditions_prev = startup_conditions.copy()
-        startup_blocked_ts = sec_since_boot()
+        startup_blocked_ts = time.monotonic()
 
       started_ts = None
       if off_ts is None:
-        off_ts = sec_since_boot()
+        off_ts = time.monotonic()
 
     # Offroad power monitoring
     voltage = None if peripheralState.pandaType == log.PandaState.PandaType.unknown else peripheralState.voltage