rebase

.python-version

@@ -1 +1 @@
-3.8.10
+3.11.4

Dockerfile.openpilot_base

@@ -13,7 +13,7 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 ENV POETRY_VIRTUALENVS_CREATE=false
-ENV PYENV_VERSION=3.8.10
+ENV PYENV_VERSION=3.11.4
 ENV PYENV_ROOT="/root/.pyenv"
 ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
 

SConstruct

@@ -263,6 +263,7 @@ py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
 envCython["CPPPATH"] += [py_include, np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":

cereal

@@ -1 +1 @@
-Subproject commit 6f7102581f57eb5074b816cc2cfd984218916773
+Subproject commit aed9fd278a704816aba11f4473aafefc281ed2bc

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.8
+python_version = 3.11
 plugins = numpy.typing.mypy_plugin
 files = body, common, docs, scripts, selfdrive, site_scons, system, tools
 exclude = ^(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)

opendbc

@@ -1 +1 @@
-Subproject commit 3ef35ed2298a3a9d199f9145409547710065884c
+Subproject commit 4231b0f12d8cf10d0554c4eb513ac984defc1f90

pyproject.toml

@@ -10,13 +10,13 @@ documentation = "https://docs.comma.ai"
 
 
 [tool.poetry.dependencies]
-python = "~3.8"
+python = "~3.11"
 atomicwrites = "^1.4.0"
 casadi = "==3.6.3"
 cffi = "^1.15.1"
 crcmod = "^1.7"
 cryptography = "^37.0.4"
-Cython = "^0.29.30"
+Cython = "^3.0.0"
 flake8 = "^4.0.1"
 Flask = "^2.1.2"
 future-fstrings = "^1.2.0"  # for acados
@@ -27,14 +27,14 @@ Jinja2 = "^3.1.2"
 json-rpc = "^1.13.0"
 libusb1 = "^3.0.0"
 nose = "^1.3.7"
-numpy = "^1.23.0"
+numpy = "==1.24.2"
 onnx = "^1.14.0"
 onnxruntime-gpu = { version = "^1.15.1", platform = "linux", markers = "platform_machine == 'x86_64'" }
 pillow = "^9.2.0"
 poetry = "==1.2.2"
 protobuf = "==3.20.3"
 psutil = "^5.9.1"
-pycapnp = "==1.1.0"
+pycapnp = "^1.3.0"
 pycryptodome = "^3.15.0"
 PyJWT = "^2.5.0"
 pyopencl = "^2022.2.4"
@@ -43,7 +43,7 @@ python-dateutil = "^2.8.2"
 PyYAML = "^6.0"
 pyzmq = "^23.2.0"
 requests = "^2.28.1"
-scons = "^4.3.0"
+scons = "^4.5.2"
 sentry-sdk = "^1.6.0"
 setproctitle = "^1.2.3"
 six = "^1.16.0"
@@ -51,8 +51,8 @@ smbus2 = "^0.4.2"
 sounddevice = "^0.4.5"
 spidev = { version = "^3.6", platform = "linux" }
 spidev2 = { version = "^0.9.0", platform = "linux" }
-sympy = "^1.10.1"
-timezonefinder = "^6.0.1"
+sympy = "^1.11.1"
+timezonefinder = "^6.2.0"
 tqdm = "^4.64.0"
 urllib3 = "^1.26.10"
 utm = "^0.7.0"
@@ -62,10 +62,9 @@ sconscontrib = {git = "https://github.com/SCons/scons-contrib.git"}
 
 
 [tool.poetry.group.dev.dependencies]
-av = "^9.2.0"
+av = "^10.0.0"
 azure-storage-blob = "~2.1"
 breathe = "^4.34.0"
-carla = { version = "==0.9.13", platform = "linux", markers = "platform_machine == 'x86_64'" }
 control = "^0.9.2"
 coverage = "^6.4.1"
 dictdiffer = "^0.9.0"
@@ -79,18 +78,17 @@ lxml = "^4.9.1"
 markdown-it-py = "^2.1.0"
 matplotlib = "^3.5.2"
 mpld3 = "^0.5.8"
-mypy = "^0.961"
+mypy = "^1.4.0"
 myst-parser = "^0.18.0"
 natsort = "^8.1.0"
-numpy = "^1.23.0"
-opencv-python-headless = { url = "https://github.com/commaai/opencv-python-builder/releases/download/4.5.5.64%2Bcu118/opencv_python_headless-4.5.5.64-cp38-cp38-manylinux_2_31_x86_64.whl", platform = "linux" }
+numpy = "==1.24.2"
 pandas = "^1.4.3"
 parameterized = "^0.8.1"
 paramiko = "^2.11.0"
 pprofile = "^2.1.0"
 pre-commit = "^2.19.0"
 pycurl = "^7.45.1"
-pygame = "^2.1.2"
+pygame = "^2.4.0"
 pylint = "^2.17.4"
 pyprof2calltree = "^1.4.5"
 pytest = "^7.1.2"
@@ -118,7 +116,6 @@ optional = true
 aenum = "^3.1.11"
 aiohttp = "^3.8.1"
 albumentations = "^1.2.1"
-apex = { url = "https://github.com/commaai/apex/releases/download/pytorch2.0.0%2Bcu11.8/apex-0.1-cp38-cp38-linux_x86_64.whl" }
 azure-cli-core = "^2.38.0"
 azure-common = "^1.1.28"
 azure-core = "^1.24.2"
@@ -167,9 +164,9 @@ scikit-learn = "^1.1.1"
 segmentation-models-pytorch = "==0.3.2"
 simplejson = "^3.17.6"
 SQLAlchemy = "^1.4.39"
-torch = { url = "https://download.pytorch.org/whl/cu118/torch-2.0.0%2Bcu118-cp38-cp38-linux_x86_64.whl" }
+torch = { url = "https://download.pytorch.org/whl/cu118/torch-2.0.1%2Bcu118-cp311-cp311-linux_x86_64.whl" }
 torchsummary = "^1.5.1"
-torchvision = { url = "https://download.pytorch.org/whl/cu118/torchvision-0.15.1%2Bcu118-cp38-cp38-linux_x86_64.whl" }
+torchvision = { url = "https://download.pytorch.org/whl/cu118/torchvision-0.15.2%2Bcu118-cp311-cp311-linux_x86_64.whl" }
 triton = "^2.0.0"
 Werkzeug = "^2.1.2"
 zerorpc = { git = "https://github.com/commaai/zerorpc-python.git", branch = "master" }

selfdrive/athena/athenad.py

@@ -798,7 +798,9 @@ def main(exit_event: Optional[threading.Event] = None):
     except (ConnectionError, TimeoutError, WebSocketException):
       conn_retries += 1
       params.remove("LastAthenaPingTime")
-    except socket.timeout:
+    # TODO: socket.timeout and TimeoutError are now the same exception since python3.10
+    # Remove the socket.timeout case once we have fully moved to python3.11
+    except socket.timeout: # pylint: disable=duplicate-except
       params.remove("LastAthenaPingTime")
     except Exception:
       cloudlog.exception("athenad.main.exception")

selfdrive/boardd/pandad.py

@@ -127,7 +127,7 @@ def main() -> NoReturn:
 
       # sort pandas to have deterministic order
       pandas.sort(key=cmp_to_key(panda_sort_cmp))
-      panda_serials = list(map(lambda p: p.get_usb_serial(), pandas))  # type: ignore
+      panda_serials = list(map(lambda p: p.get_usb_serial(), pandas))
 
       # log panda fw versions
       params.put("PandaSignatures", b','.join(p.get_signature() for p in pandas))

selfdrive/car/car_helpers.py

@@ -89,7 +89,7 @@ def fingerprint(logcan, sendcan, num_pandas):
 
     cached_params = params.get("CarParamsCache")
     if cached_params is not None:
-      cached_params = car.CarParams.from_bytes(cached_params)
+      with car.CarParams.from_bytes(cached_params) as cached_params:
         if cached_params.carName == "mock":
           cached_params = None
 

selfdrive/car/docs_definitions.py

@@ -149,7 +149,7 @@ class CarParts:
     return copy.deepcopy(self)
 
   @classmethod
-  def common(cls, add: List[EnumBase] = None, remove: List[EnumBase] = None):
+  def common(cls, add: Optiona[List[EnumBase]] = None, remove: Optiona[List[EnumBase]] = None):
     p = [part for part in (add or []) + DEFAULT_CAR_PARTS if part not in (remove or [])]
     return cls(p)
 

selfdrive/controls/controlsd.py

@@ -206,7 +206,7 @@ class Controls:
     if REPLAY:
       controls_state = Params().get("ReplayControlsState")
       if controls_state is not None:
-        controls_state = log.ControlsState.from_bytes(controls_state)
+        with log.ControlsState.from_bytes(controls_state) as controls_state:
           self.v_cruise_helper.v_cruise_kph = controls_state.vCruise
 
       if any(ps.controlsAllowed for ps in self.sm['pandaStates']):

selfdrive/controls/plannerd.py

@@ -32,7 +32,8 @@ def plannerd_thread(sm=None, pm=None):
 
   cloudlog.info("plannerd is waiting for CarParams")
   params = Params()
-  CP = car.CarParams.from_bytes(params.get("CarParams", block=True))
+  with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg:
+    CP = msg
   cloudlog.info("plannerd got CarParams: %s", CP.carName)
 
   debug_mode = bool(int(os.getenv("DEBUG", "0")))

selfdrive/controls/radard.py

@@ -185,7 +185,8 @@ def radard_thread(sm=None, pm=None, can_sock=None):
 
   # wait for stats about the car to come in from controls
   cloudlog.info("radard is waiting for CarParams")
-  CP = car.CarParams.from_bytes(Params().get("CarParams", block=True))
+  with car.CarParams.from_bytes(Params().get("CarParams", block=True)) as msg:
+    CP = msg
   cloudlog.info("radard got CarParams")
 
   # import the radar from the fingerprint

selfdrive/debug/dump.py

@@ -41,7 +41,8 @@ if __name__ == "__main__":
     polld = poller.poll(100)
     for sock in polld:
       msg = sock.receive()
-      evt = log.Event.from_bytes(msg)
+      with log.Event.from_bytes(msg) as log_evt:
+        evt = log_evt
 
       if not args.no_print:
         if args.pipe:

selfdrive/debug/internal/measure_torque_time_to_max.py

@@ -34,7 +34,8 @@ if __name__ == "__main__":
     polld = poller.poll(1000)
     for sock in polld:
       msg = sock.receive()
-      evt = log.Event.from_bytes(msg)
+      with log.Event.from_bytes(msg) as log_evt:
+        evt = log_evt
 
       for item in evt.can:
         if item.address == 0xe4 and item.src == 128:

selfdrive/debug/run_process_on_route.py

@@ -25,7 +25,7 @@ if __name__ == "__main__":
   # Remove message generated by the process under test and merge in the new messages
   produces = {o.which() for o in outputs}
   inputs = [i for i in inputs if i.which() not in produces]
-  outputs = sorted(inputs + outputs, key=lambda x: x.logMonoTime)  # type: ignore
+  outputs = sorted(inputs + outputs, key=lambda x: x.logMonoTime)
 
   fn = f"{args.route}_{args.process}.bz2"
   save_log(fn, outputs)

selfdrive/locationd/calibrationd.py

@@ -73,7 +73,7 @@ class Calibrator:
 
     if param_put and calibration_params:
       try:
-        msg = log.Event.from_bytes(calibration_params)
+        with log.Event.from_bytes(calibration_params) as msg:
           rpy_init = np.array(msg.liveCalibration.rpyCalib)
           valid_blocks = msg.liveCalibration.validBlocks
           wide_from_device_euler = np.array(msg.liveCalibration.wideFromDeviceEuler)

selfdrive/locationd/laikad.py

@@ -116,7 +116,7 @@ class Laikad:
 
     nav_dict = {}
     try:
-      ephem_cache = ephemeris_structs.EphemerisCache.from_bytes(cache_bytes)
+      with ephemeris_structs.EphemerisCache.from_bytes(cache_bytes) as ephem_cache:
         glonass_navs = [GLONASSEphemeris(data_struct, file_name=EPHEMERIS_CACHE) for data_struct in ephem_cache.glonassEphemerides]
         gps_navs = [GPSEphemeris(data_struct, file_name=EPHEMERIS_CACHE) for data_struct in ephem_cache.gpsEphemerides]
       for e in sum([glonass_navs, gps_navs], []):

selfdrive/locationd/paramsd.py

@@ -129,7 +129,8 @@ def main(sm=None, pm=None):
   params_reader = Params()
   # wait for stats about the car to come in from controls
   cloudlog.info("paramsd is waiting for CarParams")
-  CP = car.CarParams.from_bytes(params_reader.get("CarParams", block=True))
+  with car.CarParams.from_bytes(params_reader.get("CarParams", block=True)) as msg:
+    CP = msg
   cloudlog.info("paramsd got CarParams")
 
   min_sr, max_sr = 0.5 * CP.steerRatio, 2.0 * CP.steerRatio

selfdrive/locationd/test/_test_locationd_lib.py

@@ -40,7 +40,8 @@ void localizer_handle_msg_bytes(Localizer_t localizer, const char *data, size_t
 
   def localizer_get_msg(self, t=0, inputsOK=True, sensorsOK=True, gpsOK=True, msgValid=True):
     self.lib.localizer_get_message_bytes(self.localizer, inputsOK, sensorsOK, gpsOK, msgValid, self.ffi.addressof(self.msg_buff, 0), self.buff_size)
-    return log.Event.from_bytes(self.ffi.buffer(self.msg_buff), nesting_limit=self.buff_size // 8)
+    with log.Event.from_bytes(self.ffi.buffer(self.msg_buff), nesting_limit=self.buff_size // 8) as log_evt:
+      return log_evt
 
   def test_liblocalizer(self):
     msg = messaging.new_message('liveCalibration')

selfdrive/locationd/torqued.py

@@ -139,8 +139,10 @@ class TorqueEstimator:
     torque_cache = params.get("LiveTorqueParameters")
     if params_cache is not None and torque_cache is not None:
       try:
-        cache_ltp = log.Event.from_bytes(torque_cache).liveTorqueParameters
-        cache_CP = car.CarParams.from_bytes(params_cache)
+        with log.Event.from_bytes(torque_cache).liveTorqueParameters as log_evt:
+          cache_ltp = log_evt
+        with car.CarParams.from_bytes(params_cache) as msg:
+          cache_CP = msg
         if self.get_restore_key(cache_CP, cache_ltp.version) == self.get_restore_key(CP, VERSION):
           if cache_ltp.liveValid:
             initial_params = {
@@ -262,7 +264,8 @@ def main(sm=None, pm=None):
     pm = messaging.PubMaster(['liveTorqueParameters'])
 
   params = Params()
-  CP = car.CarParams.from_bytes(params.get("CarParams", block=True))
+  with car.CarParams.from_bytes(params.get("CarParams", block=True)) as CP:
+    estimator = TorqueEstimator(CP)
   estimator = TorqueEstimator(CP)
 
   def cache_params(sig, frame):

selfdrive/test/process_replay/helpers.py

@@ -7,7 +7,7 @@ from typing import List, Optional
 from common.params import Params
 
 class OpenpilotPrefix(object):
-  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True):
+  def __init__(self, prefix: Optional[str] = None, clean_dirs_on_exit: bool = True):
     self.prefix = prefix if prefix else str(uuid.uuid4())
     self.msgq_path = os.path.join('/dev/shm', self.prefix)
     self.clean_dirs_on_exit = clean_dirs_on_exit

system/sensord/rawgps/compare.py

@@ -56,7 +56,7 @@ if __name__ == "__main__":
     pr_err /= len(car)
     speed_err /= len(car)
     print("avg psuedorange err %f avg speed err %f" % (pr_err, speed_err))
-    for c in sorted(car, key=lambda x: abs(x[1] - x[3] - pr_err)):  # type: ignore
+    for c in sorted(car, key=lambda x: abs(x[1] - x[3] - pr_err)):
       svid, ublox_psuedorange, ublox_speed, qcom_psuedorange, qcom_speed, cno = c
       print("svid: %3d  pseudorange: %10.2f m  speed: %8.2f m/s   meas: %12.2f  speed: %10.2f   meas_err: %10.3f speed_err: %8.3f cno: %d" %
         (svid, ublox_psuedorange, ublox_speed, qcom_psuedorange, qcom_speed,