From 09d8327a14fc5364bb0a06cf260b2488639151a0 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 24 Jul 2025 23:56:04 -0700 Subject: [PATCH 01/10] bump opendbc (#35811) * bump * bump * bump * update refs --- opendbc_repo | 2 +- selfdrive/test/process_replay/ref_commit | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opendbc_repo b/opendbc_repo index 723b8e432a..bfbced838b 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit 723b8e432a489886eb7ff7e593c08bcf88529023 +Subproject commit bfbced838b6bbca4be09313d14366792e2785023 diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 81c203a919..73ff88e336 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -bb4d3d316bc20ca57e2753ac77f055517c31bfc7 \ No newline at end of file +afdbe3afb947a61f0d494a15d8e8137604987999 \ No newline at end of file From 938981dce9a84e0b4bcf77691c6faced199d4405 Mon Sep 17 00:00:00 2001 From: YassineYousfi Date: Fri, 25 Jul 2025 11:38:07 -0700 Subject: [PATCH 02/10] =?UTF-8?q?Space=20Lab=20Model=20=F0=9F=9B=B0?= =?UTF-8?q?=EF=B8=8F=20(#35804)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 95c34be5-4fa1-4875-8b1d-fb7162140f10/400 * 660235c5-1647-40c4-8493-880de8f662d0/400 --- selfdrive/modeld/models/driving_policy.onnx | 4 ++-- selfdrive/modeld/models/driving_vision.onnx | 4 ++-- selfdrive/modeld/parse_model_outputs.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/selfdrive/modeld/models/driving_policy.onnx b/selfdrive/modeld/models/driving_policy.onnx index ee9a9b6ec1..9bb9254a0f 100644 --- a/selfdrive/modeld/models/driving_policy.onnx +++ b/selfdrive/modeld/models/driving_policy.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef059460a95076f9a8600abb8c9d56c8c3b7384505b158064e2e875458b965bb -size 15701037 +oid sha256:02cf21ce7e9784c6009cc32a6f1ea22482911897b6d944419477973284052716 +size 15583374 diff --git a/selfdrive/modeld/models/driving_vision.onnx b/selfdrive/modeld/models/driving_vision.onnx index 6cdca2f93c..18f63358db 100644 --- a/selfdrive/modeld/models/driving_vision.onnx +++ b/selfdrive/modeld/models/driving_vision.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:afee02876ed51f21883e731990a252fd57afda6a78700c41fbf22d8cb288b4b5 -size 46147818 +oid sha256:c824f68646a3b94f117f01c70dc8316fb466e05fbd42ccdba440b8a8dc86914b +size 46265993 diff --git a/selfdrive/modeld/parse_model_outputs.py b/selfdrive/modeld/parse_model_outputs.py index d5a5dc1bb4..b89de27594 100644 --- a/selfdrive/modeld/parse_model_outputs.py +++ b/selfdrive/modeld/parse_model_outputs.py @@ -93,6 +93,9 @@ class Parser: self.parse_binary_crossentropy('lane_lines_prob', outs) self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN,ModelConstants.DESIRE_PRED_WIDTH)) self.parse_binary_crossentropy('meta', outs) + self.parse_binary_crossentropy('lead_prob', outs) + self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION, + out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH)) return outs def parse_policy_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: @@ -103,9 +106,6 @@ class Parser: if 'desired_curvature' in outs: self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) - self.parse_binary_crossentropy('lead_prob', outs) - self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION, - out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH)) return outs def parse_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: From 47f23828d291da629257c8ff3ecb1cfd7d4b1198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Fri, 25 Jul 2025 11:53:08 -0700 Subject: [PATCH 03/10] Tinygrad DEV=DEVICE (#35809) * bump tg * step one cleanup * cleanup * typo * cleaner * cleaner * Revert "cleaner" This reverts commit 9c1abd0dc06b4564e61dd32b0e93375badbc9ca5. * usbgpu * bit cleaner * cleaner sconscript --- selfdrive/modeld/SConscript | 8 ++++---- selfdrive/modeld/dmonitoringmodeld.py | 7 ++----- selfdrive/modeld/modeld.py | 10 +++------- tinygrad_repo | 2 +- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 0da3c24e19..6802b45e6e 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -50,9 +50,9 @@ def tg_compile(flags, model_name): # Compile small models for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']: flags = { - 'larch64': 'QCOM=1', - 'Darwin': 'CPU=1 IMAGE=0 JIT=2', - }.get(arch, 'LLVM=1 LLVMOPT=1 BEAM=0 IMAGE=0 JIT=2') + 'larch64': 'DEV=QCOM', + 'Darwin': 'DEV=CPU IMAGE=0', + }.get(arch, 'DEV=LLVM IMAGE=0') tg_compile(flags, model_name) # Compile BIG model if USB GPU is available @@ -62,7 +62,7 @@ if "USBGPU" in os.environ: devs = subprocess.check_output('python3 -c "from tinygrad import Device; print(list(Device.get_available_devices()))"', shell=True, cwd=env.Dir('#').abspath) if b"AMD" in devs: print("USB GPU detected... building") - flags = "AMD=1 AMD_IFACE=USB AMD_LLVM=1 NOLOCALS=0 IMAGE=0" + flags = "DEV=AMD AMD_IFACE=USB AMD_LLVM=1 NOLOCALS=0 IMAGE=0" bp = tg_compile(flags, "big_driving_policy") bv = tg_compile(flags, "big_driving_vision") lenv.SideEffect('lock', [bp, bv]) # tg doesn't support multi-process so build serially diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index 22d0057827..215056e6e6 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 import os from openpilot.system.hardware import TICI +os.environ['DEV'] = 'QCOM' if TICI else 'LLVM' from tinygrad.tensor import Tensor from tinygrad.dtype import dtypes -if TICI: - from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address - os.environ['QCOM'] = '1' -else: - os.environ['LLVM'] = '1' import math import time import pickle @@ -24,6 +20,7 @@ from openpilot.common.transformations.model import dmonitoringmodel_intrinsics, from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid +from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE CALIB_LEN = 3 diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 6767040c60..707b221bb9 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -1,16 +1,11 @@ #!/usr/bin/env python3 import os from openpilot.system.hardware import TICI +os.environ['DEV'] = 'QCOM' if TICI else 'LLVM' USBGPU = "USBGPU" in os.environ if USBGPU: - os.environ['AMD'] = '1' + os.environ['DEV'] = 'AMD' os.environ['AMD_IFACE'] = 'USB' -elif TICI: - from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address - os.environ['QCOM'] = '1' -else: - os.environ['LLVM'] = '1' - os.environ['JIT'] = '2' from tinygrad.tensor import Tensor from tinygrad.dtype import dtypes import time @@ -34,6 +29,7 @@ from openpilot.selfdrive.modeld.parse_model_outputs import Parser from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState from openpilot.selfdrive.modeld.constants import ModelConstants, Plan from openpilot.selfdrive.modeld.models.commonmodel_pyx import DrivingModelFrame, CLContext +from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address PROCESS_NAME = "selfdrive.modeld.modeld" diff --git a/tinygrad_repo b/tinygrad_repo index affd83961c..9da3f72495 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit affd83961cfe10cf07d9b306fcca76abfd30da61 +Subproject commit 9da3f7249535cec9254335abcc4edb3281409f67 From b99c73e1fa1c22ede2a1cd1bca8916efe8c83dc4 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 25 Jul 2025 12:50:15 -0700 Subject: [PATCH 04/10] bump opendbc (#35813) * bump opendbc * update tests --- cereal/messaging/tests/test_messaging.py | 2 +- opendbc_repo | 2 +- selfdrive/car/tests/test_models.py | 4 ++-- selfdrive/test/process_replay/ref_commit | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cereal/messaging/tests/test_messaging.py b/cereal/messaging/tests/test_messaging.py index 6234a11c08..2369229980 100644 --- a/cereal/messaging/tests/test_messaging.py +++ b/cereal/messaging/tests/test_messaging.py @@ -30,7 +30,7 @@ def zmq_sleep(t=1): # TODO: this should take any capnp struct and returrn a msg with random populated data def random_carstate(): - fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"] + fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"] msg = messaging.new_message("carState") cs = msg.carState for f in fields: diff --git a/opendbc_repo b/opendbc_repo index bfbced838b..21b6bcc6f0 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit bfbced838b6bbca4be09313d14366792e2785023 +Subproject commit 21b6bcc6f0bdb19266ee1479551b7a2b2662d2db diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index c734ca8cbb..8996ad6460 100644 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -356,7 +356,7 @@ class TestCarModelBase(unittest.TestCase): if self.safety.get_steering_disengage_prev() != prev_panda_steering_disengage: self.assertEqual(CS.steeringDisengage, self.safety.get_steering_disengage_prev()) - if self.safety.get_vehicle_moving() != prev_panda_vehicle_moving: + if self.safety.get_vehicle_moving() != prev_panda_vehicle_moving and not self.CP.notCar: self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving()) # check vehicle speed if angle control car or available @@ -413,7 +413,7 @@ class TestCarModelBase(unittest.TestCase): # TODO: check rest of panda's carstate (steering, ACC main on, etc.) checks['gasPressed'] += CS.gasPressed != self.safety.get_gas_pressed_prev() - checks['standstill'] += CS.standstill == self.safety.get_vehicle_moving() + checks['standstill'] += (CS.standstill == self.safety.get_vehicle_moving()) and not self.CP.notCar # check vehicle speed if angle control car or available if self.safety.get_vehicle_speed_min() > 0 or self.safety.get_vehicle_speed_max() > 0: diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 73ff88e336..2cff48b03f 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -afdbe3afb947a61f0d494a15d8e8137604987999 \ No newline at end of file +9e86884c9c94299f2dcbb1d7f6b30b081a74f2cc \ No newline at end of file From 5e07636d54c6205fd116bafa66b3cfecb554d0e3 Mon Sep 17 00:00:00 2001 From: Bruce Wayne Date: Fri, 25 Jul 2025 12:54:11 -0700 Subject: [PATCH 05/10] Revert "Tinygrad DEV=DEVICE (#35809)" This reverts commit 47f23828d291da629257c8ff3ecb1cfd7d4b1198. --- selfdrive/modeld/SConscript | 8 ++++---- selfdrive/modeld/dmonitoringmodeld.py | 7 +++++-- selfdrive/modeld/modeld.py | 10 +++++++--- tinygrad_repo | 2 +- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 6802b45e6e..0da3c24e19 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -50,9 +50,9 @@ def tg_compile(flags, model_name): # Compile small models for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']: flags = { - 'larch64': 'DEV=QCOM', - 'Darwin': 'DEV=CPU IMAGE=0', - }.get(arch, 'DEV=LLVM IMAGE=0') + 'larch64': 'QCOM=1', + 'Darwin': 'CPU=1 IMAGE=0 JIT=2', + }.get(arch, 'LLVM=1 LLVMOPT=1 BEAM=0 IMAGE=0 JIT=2') tg_compile(flags, model_name) # Compile BIG model if USB GPU is available @@ -62,7 +62,7 @@ if "USBGPU" in os.environ: devs = subprocess.check_output('python3 -c "from tinygrad import Device; print(list(Device.get_available_devices()))"', shell=True, cwd=env.Dir('#').abspath) if b"AMD" in devs: print("USB GPU detected... building") - flags = "DEV=AMD AMD_IFACE=USB AMD_LLVM=1 NOLOCALS=0 IMAGE=0" + flags = "AMD=1 AMD_IFACE=USB AMD_LLVM=1 NOLOCALS=0 IMAGE=0" bp = tg_compile(flags, "big_driving_policy") bv = tg_compile(flags, "big_driving_vision") lenv.SideEffect('lock', [bp, bv]) # tg doesn't support multi-process so build serially diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index 215056e6e6..22d0057827 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -1,9 +1,13 @@ #!/usr/bin/env python3 import os from openpilot.system.hardware import TICI -os.environ['DEV'] = 'QCOM' if TICI else 'LLVM' from tinygrad.tensor import Tensor from tinygrad.dtype import dtypes +if TICI: + from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address + os.environ['QCOM'] = '1' +else: + os.environ['LLVM'] = '1' import math import time import pickle @@ -20,7 +24,6 @@ from openpilot.common.transformations.model import dmonitoringmodel_intrinsics, from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid -from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE CALIB_LEN = 3 diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 707b221bb9..6767040c60 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -1,11 +1,16 @@ #!/usr/bin/env python3 import os from openpilot.system.hardware import TICI -os.environ['DEV'] = 'QCOM' if TICI else 'LLVM' USBGPU = "USBGPU" in os.environ if USBGPU: - os.environ['DEV'] = 'AMD' + os.environ['AMD'] = '1' os.environ['AMD_IFACE'] = 'USB' +elif TICI: + from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address + os.environ['QCOM'] = '1' +else: + os.environ['LLVM'] = '1' + os.environ['JIT'] = '2' from tinygrad.tensor import Tensor from tinygrad.dtype import dtypes import time @@ -29,7 +34,6 @@ from openpilot.selfdrive.modeld.parse_model_outputs import Parser from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState from openpilot.selfdrive.modeld.constants import ModelConstants, Plan from openpilot.selfdrive.modeld.models.commonmodel_pyx import DrivingModelFrame, CLContext -from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address PROCESS_NAME = "selfdrive.modeld.modeld" diff --git a/tinygrad_repo b/tinygrad_repo index 9da3f72495..affd83961c 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit 9da3f7249535cec9254335abcc4edb3281409f67 +Subproject commit affd83961cfe10cf07d9b306fcca76abfd30da61 From b4ae64358e1cf7261c4bdc6b515f2898947314e9 Mon Sep 17 00:00:00 2001 From: Simon Kuang Date: Fri, 25 Jul 2025 12:57:13 -0700 Subject: [PATCH 06/10] Update vehicle model reference to 2023 edition of Guiggiani's book (#35805) Update car_kf.py --- selfdrive/locationd/models/car_kf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/selfdrive/locationd/models/car_kf.py b/selfdrive/locationd/models/car_kf.py index 349897f371..6db749b949 100755 --- a/selfdrive/locationd/models/car_kf.py +++ b/selfdrive/locationd/models/car_kf.py @@ -93,8 +93,9 @@ class CarKalman(KalmanFilter): dim_state = CarKalman.initial_x.shape[0] name = CarKalman.name - # vehicle models comes from The Science of Vehicle Dynamics: Handling, Braking, and Ride of Road and Race Cars - # Model used is in 6.15 with formula from 6.198 + # Linearized single-track lateral dynamics, equations 7.211-7.213 + # Massimo Guiggiani, The Science of Vehicle Dynamics: Handling, Braking, and Ride of Road and Race Cars + # Springer Cham, 2023. doi: https://doi.org/10.1007/978-3-031-06461-6 # globals global_vars = [sp.Symbol(name) for name in CarKalman.global_vars] From 26a9760afc7d1994509e0fa81d082225341367ea Mon Sep 17 00:00:00 2001 From: Bao Date: Sat, 26 Jul 2025 03:53:37 +0700 Subject: [PATCH 07/10] util: fix read_file for files > 2GB (#35787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * util: fix read_file for files > 2GB * fix tellg on directory not returning -1 --------- Co-authored-by: Ngô Việt Hoài Bảo --- common/util.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/util.cc b/common/util.cc index a853faed04..3d03c96bbe 100644 --- a/common/util.cc +++ b/common/util.cc @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef __linux__ #include @@ -78,8 +79,9 @@ std::string read_file(const std::string& fn) { std::ifstream f(fn, std::ios::binary | std::ios::in); if (f.is_open()) { f.seekg(0, std::ios::end); - int size = f.tellg(); - if (f.good() && size > 0) { + std::streamsize size = f.tellg(); + // seekg and tellg on a directory doesn't return pos_type(-1) but max(streamsize) + if (f.good() && size > 0 && size < std::numeric_limits::max()) { std::string result(size, '\0'); f.seekg(0, std::ios::beg); f.read(result.data(), size); From e7b80b78cc242304da481c757b4cf42a04f2596f Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Fri, 25 Jul 2025 19:20:49 -0700 Subject: [PATCH 08/10] params: auto type cast on put (#35810) * start * fix * fix * more * more * more * fix * fix * [] * f * f * fix * lint * back * fix * yep * better msg * fix * fix * fix * fix * more * more --- common/params_keys.h | 6 +- common/params_pyx.pyx | 74 +++++++++++++------- common/tests/test_params.py | 21 +++--- selfdrive/locationd/paramsd.py | 8 +-- selfdrive/locationd/test/test_paramsd.py | 5 +- selfdrive/selfdrived/alertmanager.py | 2 +- selfdrive/selfdrived/selfdrived.py | 2 +- selfdrive/ui/layouts/settings/firehose.py | 3 +- selfdrive/ui/layouts/settings/toggles.py | 2 +- selfdrive/ui/lib/prime_state.py | 2 +- selfdrive/ui/tests/test_ui/run.py | 4 +- system/athena/athenad.py | 4 +- system/athena/tests/test_athenad.py | 10 +-- system/hardware/hardwared.py | 3 +- system/hardware/power_monitoring.py | 2 +- system/loggerd/tests/loggerd_tests_common.py | 2 +- system/loggerd/tests/test_loggerd.py | 2 +- system/manager/test/test_manager.py | 2 +- system/updated/updated.py | 8 +-- 19 files changed, 89 insertions(+), 73 deletions(-) diff --git a/common/params_keys.h b/common/params_keys.h index a08d69f0fc..7ccf5794ad 100644 --- a/common/params_keys.h +++ b/common/params_keys.h @@ -75,7 +75,7 @@ inline static std::unordered_map keys = { {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}}, {"LastUpdateTime", {PERSISTENT, TIME}}, {"LiveDelay", {PERSISTENT, BYTES}}, - {"LiveParameters", {PERSISTENT, BYTES}}, + {"LiveParameters", {PERSISTENT, JSON}}, {"LiveParametersV2", {PERSISTENT, BYTES}}, {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}}, {"LocationFilterInitialState", {PERSISTENT, BYTES}}, @@ -116,10 +116,10 @@ inline static std::unordered_map keys = { {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}}, {"UpdaterAvailableBranches", {PERSISTENT, STRING}}, {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}}, - {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, STRING}}, + {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}}, {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}}, {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}}, - {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, STRING}}, + {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}}, {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}}, {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}}, {"UpdaterLastFetchTime", {PERSISTENT, TIME}}, diff --git a/common/params_pyx.pyx b/common/params_pyx.pyx index d8fefd63ae..1c76e75846 100644 --- a/common/params_pyx.pyx +++ b/common/params_pyx.pyx @@ -1,5 +1,6 @@ # distutils: language = c++ # cython: language_level = 3 +import builtins import datetime import json from libcpp cimport bool @@ -7,6 +8,8 @@ from libcpp.string cimport string from libcpp.vector cimport vector from libcpp.optional cimport optional +from openpilot.common.swaglog import cloudlog + cdef extern from "common/params.h": cpdef enum ParamKeyFlag: PERSISTENT @@ -42,6 +45,25 @@ cdef extern from "common/params.h": void clearAll(ParamKeyFlag) vector[string] allKeys() +PYTHON_2_CPP = { + (str, STRING): lambda v: v, + (builtins.bool, BOOL): lambda v: "1" if v else "0", + (int, INT): str, + (float, FLOAT): str, + (datetime.datetime, TIME): lambda v: v.isoformat(), + (dict, JSON): json.dumps, + (list, JSON): json.dumps, + (bytes, BYTES): lambda v: v, +} +CPP_2_PYTHON = { + STRING: lambda v: v.decode("utf-8"), + BOOL: lambda v: v == b"1", + INT: int, + FLOAT: float, + TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")), + JSON: json.loads, + BYTES: lambda v: v, +} def ensure_bytes(v): return v.encode() if isinstance(v, str) else v @@ -74,45 +96,38 @@ cdef class Params: raise UnknownKeyName(key) return key - def cast(self, t, value, default): + def python2cpp(self, proposed_type, expected_type, value, key): + cast = PYTHON_2_CPP.get((proposed_type, expected_type)) + if cast: + return cast(value) + raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}") + + def cpp2python(self, t, value, default, key): if value is None: return None try: - if t == STRING: - return value.decode("utf-8") - elif t == BOOL: - return value == b"1" - elif t == INT: - return int(value) - elif t == FLOAT: - return float(value) - elif t == TIME: - return datetime.datetime.fromisoformat(value.decode("utf-8")) - elif t == JSON: - return json.loads(value) - elif t == BYTES: - return value - else: - raise TypeError() - except (TypeError, ValueError): - return self.cast(t, default, None) + return CPP_2_PYTHON[t](value) + except (KeyError, TypeError, ValueError): + cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}") + return self.cpp2python(t, default, None, key) def get(self, key, bool block=False, bool return_default=False): cdef string k = self.check_key(key) cdef ParamKeyType t = self.p.getKeyType(k) + cdef optional[string] default = self.p.getKeyDefaultValue(k) cdef string val with nogil: val = self.p.get(k, block) - default_val = self.get_default_value(k) if return_default else None + default_val = (default.value() if default.has_value() else None) if return_default else None if val == b"": if block: # If we got no value while running in blocked mode # it means we got an interrupt while waiting raise KeyboardInterrupt else: - return self.cast(t, default_val, None) - return self.cast(t, val, default_val) + return self.cpp2python(t, default_val, None, key) + return self.cpp2python(t, val, default_val, key) def get_bool(self, key, bool block=False): cdef string k = self.check_key(key) @@ -121,6 +136,11 @@ cdef class Params: r = self.p.getBool(k, block) return r + def _put_cast(self, key, dat): + cdef string k = self.check_key(key) + cdef ParamKeyType t = self.p.getKeyType(k) + return ensure_bytes(self.python2cpp(type(dat), t, dat, key)) + def put(self, key, dat): """ Warning: This function blocks until the param is written to disk! @@ -129,7 +149,7 @@ cdef class Params: in general try to avoid writing params as much as possible. """ cdef string k = self.check_key(key) - cdef string dat_bytes = ensure_bytes(dat) + cdef string dat_bytes = self._put_cast(key, dat) with nogil: self.p.put(k, dat_bytes) @@ -140,7 +160,7 @@ cdef class Params: def put_nonblocking(self, key, dat): cdef string k = self.check_key(key) - cdef string dat_bytes = ensure_bytes(dat) + cdef string dat_bytes = self._put_cast(key, dat) with nogil: self.p.putNonBlocking(k, dat_bytes) @@ -165,5 +185,7 @@ cdef class Params: return self.p.allKeys() def get_default_value(self, key): - cdef optional[string] default = self.p.getKeyDefaultValue(self.check_key(key)) - return default.value() if default.has_value() else None + cdef string k = self.check_key(key) + cdef ParamKeyType t = self.p.getKeyType(k) + cdef optional[string] default = self.p.getKeyDefaultValue(k) + return self.cpp2python(t, default.value(), None, key) if default.has_value() else None diff --git a/common/tests/test_params.py b/common/tests/test_params.py index 337f9da097..1f39769c2a 100644 --- a/common/tests/test_params.py +++ b/common/tests/test_params.py @@ -1,6 +1,5 @@ import pytest import datetime -import json import os import threading import time @@ -22,7 +21,7 @@ class TestParams: assert self.params.get("CarParams") == st def test_params_get_cleared_manager_start(self): - self.params.put("CarParams", "test") + self.params.put("CarParams", b"test") self.params.put("DongleId", "cb38263377b873ee") assert self.params.get("CarParams") == b"test" @@ -45,10 +44,10 @@ class TestParams: def test_params_get_block(self): def _delayed_writer(): time.sleep(0.1) - self.params.put("CarParams", "test") + self.params.put("CarParams", b"test") threading.Thread(target=_delayed_writer).start() assert self.params.get("CarParams") is None - assert self.params.get("CarParams", True) == b"test" + assert self.params.get("CarParams", block=True) == b"test" def test_params_unknown_key_fails(self): with pytest.raises(UnknownKeyName): @@ -78,17 +77,17 @@ class TestParams: self.params.put_bool("IsMetric", False) assert not self.params.get_bool("IsMetric") - self.params.put("IsMetric", "1") + self.params.put("IsMetric", True) assert self.params.get_bool("IsMetric") - self.params.put("IsMetric", "0") + self.params.put("IsMetric", False) assert not self.params.get_bool("IsMetric") def test_put_non_blocking_with_get_block(self): q = Params() def _delayed_writer(): time.sleep(0.1) - Params().put_nonblocking("CarParams", "test") + Params().put_nonblocking("CarParams", b"test") threading.Thread(target=_delayed_writer).start() assert q.get("CarParams") is None assert q.get("CarParams", True) == b"test" @@ -124,19 +123,19 @@ class TestParams: def test_params_get_type(self): # json - self.params.put("ApiCache_FirehoseStats", json.dumps({"a": 0})) + self.params.put("ApiCache_FirehoseStats", {"a": 0}) assert self.params.get("ApiCache_FirehoseStats") == {"a": 0} # int - self.params.put("BootCount", str(1441)) + self.params.put("BootCount", 1441) assert self.params.get("BootCount") == 1441 # bool - self.params.put("AdbEnabled", "1") + self.params.put("AdbEnabled", True) assert self.params.get("AdbEnabled") assert isinstance(self.params.get("AdbEnabled"), bool) # time now = datetime.datetime.now(datetime.UTC) - self.params.put("InstallDate", str(now)) + self.params.put("InstallDate", now) assert self.params.get("InstallDate") == now diff --git a/selfdrive/locationd/paramsd.py b/selfdrive/locationd/paramsd.py index ec15f501ae..b4084fe5bc 100755 --- a/selfdrive/locationd/paramsd.py +++ b/selfdrive/locationd/paramsd.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import json import numpy as np import capnp @@ -207,12 +206,11 @@ def migrate_cached_vehicle_params_if_needed(params: Params): return try: - last_parameters_dict = json.loads(last_parameters_data_old) last_parameters_msg = messaging.new_message('liveParameters') last_parameters_msg.liveParameters.valid = True - last_parameters_msg.liveParameters.steerRatio = last_parameters_dict['steerRatio'] - last_parameters_msg.liveParameters.stiffnessFactor = last_parameters_dict['stiffnessFactor'] - last_parameters_msg.liveParameters.angleOffsetAverageDeg = last_parameters_dict['angleOffsetAverageDeg'] + last_parameters_msg.liveParameters.steerRatio = last_parameters_data_old['steerRatio'] + last_parameters_msg.liveParameters.stiffnessFactor = last_parameters_data_old['stiffnessFactor'] + last_parameters_msg.liveParameters.angleOffsetAverageDeg = last_parameters_data_old['angleOffsetAverageDeg'] params.put("LiveParametersV2", last_parameters_msg.to_bytes()) except Exception as e: cloudlog.error(f"Failed to perform parameter migration: {e}") diff --git a/selfdrive/locationd/test/test_paramsd.py b/selfdrive/locationd/test/test_paramsd.py index 2129bf4386..dd496b7675 100644 --- a/selfdrive/locationd/test/test_paramsd.py +++ b/selfdrive/locationd/test/test_paramsd.py @@ -1,6 +1,5 @@ import random import numpy as np -import json from cereal import messaging from openpilot.selfdrive.locationd.paramsd import retrieve_initial_vehicle_params, migrate_cached_vehicle_params_if_needed @@ -47,7 +46,7 @@ class TestParamsd: CP = next(m for m in lr if m.which() == "carParams").carParams msg = get_random_live_parameters(CP) - params.put("LiveParameters", json.dumps(msg.liveParameters.to_dict())) + params.put("LiveParameters", msg.liveParameters.to_dict()) params.put("CarParamsPrevRoute", CP.as_builder().to_bytes()) params.remove("LiveParametersV2") @@ -60,7 +59,7 @@ class TestParamsd: def test_read_saved_params_corrupted_old_format(self): params = Params() - params.put("LiveParameters", b'\x00\x00\x02\x00\x01\x00:F\xde\xed\xae;') + params.put("LiveParameters", {}) params.remove("LiveParametersV2") migrate_cached_vehicle_params_if_needed(params) diff --git a/selfdrive/selfdrived/alertmanager.py b/selfdrive/selfdrived/alertmanager.py index ac1006ff25..251d32ba9a 100644 --- a/selfdrive/selfdrived/alertmanager.py +++ b/selfdrive/selfdrived/alertmanager.py @@ -17,7 +17,7 @@ def set_offroad_alert(alert: str, show_alert: bool, extra_text: str = None) -> N if show_alert: a = copy.copy(OFFROAD_ALERTS[alert]) a['extra'] = extra_text or '' - Params().put(alert, json.dumps(a)) + Params().put(alert, a) else: Params().remove(alert) diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 63ce471126..e43a25f409 100755 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -407,7 +407,7 @@ class SelfdriveD: if self.CP.openpilotLongitudinalControl: if any(not be.pressed and be.type == ButtonType.gapAdjustCruise for be in CS.buttonEvents): self.personality = (self.personality - 1) % 3 - self.params.put_nonblocking('LongitudinalPersonality', str(self.personality)) + self.params.put_nonblocking('LongitudinalPersonality', self.personality) self.events.add(EventName.personalityChanged) def data_sample(self): diff --git a/selfdrive/ui/layouts/settings/firehose.py b/selfdrive/ui/layouts/settings/firehose.py index 5f5b9792a7..74a8f317d5 100644 --- a/selfdrive/ui/layouts/settings/firehose.py +++ b/selfdrive/ui/layouts/settings/firehose.py @@ -1,5 +1,4 @@ import pyray as rl -import json import time import threading @@ -169,7 +168,7 @@ class FirehoseLayout(Widget): if response.status_code == 200: data = response.json() self.segment_count = data.get("firehose", 0) - self.params.put(self.PARAM_KEY, json.dumps(data)) + self.params.put(self.PARAM_KEY, data) except Exception as e: cloudlog.error(f"Failed to fetch firehose stats: {e}") diff --git a/selfdrive/ui/layouts/settings/toggles.py b/selfdrive/ui/layouts/settings/toggles.py index 86130b9014..58afcec5ef 100644 --- a/selfdrive/ui/layouts/settings/toggles.py +++ b/selfdrive/ui/layouts/settings/toggles.py @@ -92,4 +92,4 @@ class TogglesLayout(Widget): self._scroller.render(rect) def _set_longitudinal_personality(self, button_index: int): - self._params.put("LongitudinalPersonality", str(button_index)) + self._params.put("LongitudinalPersonality", button_index) diff --git a/selfdrive/ui/lib/prime_state.py b/selfdrive/ui/lib/prime_state.py index 30eb99a194..b6c0d88469 100644 --- a/selfdrive/ui/lib/prime_state.py +++ b/selfdrive/ui/lib/prime_state.py @@ -63,7 +63,7 @@ class PrimeState: with self._lock: if prime_type != self.prime_type: self.prime_type = prime_type - self._params.put("PrimeType", str(int(prime_type))) + self._params.put("PrimeType", int(prime_type)) cloudlog.info(f"Prime type updated to {prime_type}") def _worker_thread(self) -> None: diff --git a/selfdrive/ui/tests/test_ui/run.py b/selfdrive/ui/tests/test_ui/run.py index 2305b662b5..422183c5d5 100755 --- a/selfdrive/ui/tests/test_ui/run.py +++ b/selfdrive/ui/tests/test_ui/run.py @@ -299,9 +299,9 @@ def create_screenshots(): params = Params() params.put("DongleId", "123456789012345") if name == 'prime': - params.put('PrimeType', '1') + params.put('PrimeType', 1) elif name == 'pair_device': - params.put('ApiCache_Device', '{"is_paired":0, "prime_type":-1}') + params.put('ApiCache_Device', {"is_paired":0, "prime_type":-1}) t.test_ui(name, setup) diff --git a/system/athena/athenad.py b/system/athena/athenad.py index 7814905e60..6ed53b759c 100755 --- a/system/athena/athenad.py +++ b/system/athena/athenad.py @@ -161,7 +161,7 @@ class UploadQueueCache: try: queue: list[UploadItem | None] = list(upload_queue.queue) items = [asdict(i) for i in queue if i is not None and (i.id not in cancelled_uploads)] - Params().put("AthenadUploadQueue", json.dumps(items)) + Params().put("AthenadUploadQueue", items) except Exception: cloudlog.exception("athena.UploadQueueCache.cache.exception") @@ -748,7 +748,7 @@ def ws_recv(ws: WebSocket, end_event: threading.Event) -> None: recv_queue.put_nowait(data) elif opcode == ABNF.OPCODE_PING: last_ping = int(time.monotonic() * 1e9) - Params().put("LastAthenaPingTime", str(last_ping)) + Params().put("LastAthenaPingTime", last_ping) except WebSocketTimeoutException: ns_since_last_ping = int(time.monotonic() * 1e9) - last_ping if ns_since_last_ping > RECONNECT_TIMEOUT_S * 1e9: diff --git a/system/athena/tests/test_athenad.py b/system/athena/tests/test_athenad.py index 8fe095bc42..99ac3b1c6b 100644 --- a/system/athena/tests/test_athenad.py +++ b/system/athena/tests/test_athenad.py @@ -66,9 +66,9 @@ class TestAthenadMethods: def setup_method(self): self.default_params = { "DongleId": "0000000000000000", - "GithubSshKeys": b"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501 - "GithubUsername": b"commaci", - "AthenadUploadQueue": '[]', + "GithubSshKeys": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501 + "GithubUsername": "commaci", + "AthenadUploadQueue": [], } self.params = Params() @@ -400,11 +400,11 @@ class TestAthenadMethods: def test_get_ssh_authorized_keys(self): keys = dispatcher["getSshAuthorizedKeys"]() - assert keys == self.default_params["GithubSshKeys"].decode('utf-8') + assert keys == self.default_params["GithubSshKeys"] def test_get_github_username(self): keys = dispatcher["getGithubUsername"]() - assert keys == self.default_params["GithubUsername"].decode('utf-8') + assert keys == self.default_params["GithubUsername"] def test_get_version(self): resp = dispatcher["getVersion"]() diff --git a/system/hardware/hardwared.py b/system/hardware/hardwared.py index 17ac4d1618..2456eb919c 100755 --- a/system/hardware/hardwared.py +++ b/system/hardware/hardwared.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import fcntl import os -import json import queue import struct import threading @@ -440,7 +439,7 @@ def hardware_thread(end_event, hw_queue) -> None: # save last one before going onroad if rising_edge_started: try: - params.put("LastOffroadStatusPacket", json.dumps(dat)) + params.put("LastOffroadStatusPacket", dat) except Exception: cloudlog.exception("failed to save offroad status") diff --git a/system/hardware/power_monitoring.py b/system/hardware/power_monitoring.py index 22de6e0531..f8b0e8b629 100644 --- a/system/hardware/power_monitoring.py +++ b/system/hardware/power_monitoring.py @@ -56,7 +56,7 @@ class PowerMonitoring: self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0) self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh) if now - self.last_save_time >= 10: - self.params.put_nonblocking("CarBatteryCapacity", str(int(self.car_battery_capacity_uWh))) + self.params.put_nonblocking("CarBatteryCapacity", int(self.car_battery_capacity_uWh)) self.last_save_time = now # First measurement, set integration time diff --git a/system/loggerd/tests/loggerd_tests_common.py b/system/loggerd/tests/loggerd_tests_common.py index 15fd997eb8..87c3da65c2 100644 --- a/system/loggerd/tests/loggerd_tests_common.py +++ b/system/loggerd/tests/loggerd_tests_common.py @@ -76,7 +76,7 @@ class UploaderTestCase: self.seg_dir = self.seg_format.format(self.seg_num) self.params = Params() - self.params.put("IsOffroad", "1") + self.params.put("IsOffroad", True) self.params.put("DongleId", "0000000000000000") def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .1, lock: bool = False, diff --git a/system/loggerd/tests/test_loggerd.py b/system/loggerd/tests/test_loggerd.py index 8e2f52a731..8f9d92e104 100644 --- a/system/loggerd/tests/test_loggerd.py +++ b/system/loggerd/tests/test_loggerd.py @@ -182,7 +182,7 @@ class TestLoggerd: @pytest.mark.xdist_group("camera_encoder_tests") # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing def test_rotation(self): - Params().put("RecordFront", "1") + Params().put("RecordFront", True) expected_files = {"rlog.zst", "qlog.zst", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"} diff --git a/system/manager/test/test_manager.py b/system/manager/test/test_manager.py index 3dadacf32e..5e55648283 100644 --- a/system/manager/test/test_manager.py +++ b/system/manager/test/test_manager.py @@ -47,7 +47,7 @@ class TestManager: for k in params.all_keys(): default_value = params.get_default_value(k) if default_value: - assert params.get(k) == params.cast(params.get_type(k), default_value, None) + assert params.get(k) == default_value assert params.get("OpenpilotEnabledToggle") @pytest.mark.skip("this test is flaky the way it's currently written, should be moved to test_onroad") diff --git a/system/updated/updated.py b/system/updated/updated.py index 992e7c2269..b0c04b2f18 100755 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -61,7 +61,7 @@ class WaitTimeHelper: def write_time_to_param(params, param) -> None: t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - params.put(param, t.isoformat().encode('utf8')) + params.put(param, t) def run(cmd: list[str], cwd: str = None) -> str: return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8') @@ -264,7 +264,7 @@ class Updater: return run(["git", "rev-parse", "HEAD"], path).rstrip() def set_params(self, update_success: bool, failed_count: int, exception: str | None) -> None: - self.params.put("UpdateFailedCount", str(failed_count)) + self.params.put("UpdateFailedCount", failed_count) self.params.put("UpdaterTargetBranch", self.target_branch) self.params.put_bool("UpdaterFetchAvailable", self.update_available) @@ -421,8 +421,8 @@ def main() -> None: cloudlog.event("update installed") if not params.get("InstallDate"): - t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat() - params.put("InstallDate", t.encode('utf8')) + t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) + params.put("InstallDate", t) updater = Updater() update_failed_count = 0 # TODO: Load from param? From e461d18d54032ce207ea7b9fccd114adee11c239 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 25 Jul 2025 20:29:06 -0700 Subject: [PATCH 09/10] pytest: faster xdist startup (#35817) * faster * cmt --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b2433e340..7d84e28277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,8 +85,8 @@ testing = [ "pytest", "pytest-cpp", "pytest-subtests", - # https://github.com/pytest-dev/pytest-xdist/issues/1215 - "pytest-xdist @ git+https://github.com/sshane/pytest-xdist@909e97b49d12401c10608f9d777bfc9dab8a4413", + # https://github.com/pytest-dev/pytest-xdist/pull/1229 + "pytest-xdist @ git+https://github.com/sshane/pytest-xdist@b59e010821a25a2ebb95fc5c31c97022ed94221c", "pytest-timeout", "pytest-randomly", "pytest-asyncio", From 1123773fbde6c504129e66dfd5ef7b4b4c0df117 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 25 Jul 2025 20:49:32 -0700 Subject: [PATCH 10/10] uv lock for xdist (#35819) * xdist: latest version * lock --- pyproject.toml | 2 +- uv.lock | 70 +++++++++++++++++++++++++------------------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d84e28277..142b8b06d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ testing = [ "pytest-cpp", "pytest-subtests", # https://github.com/pytest-dev/pytest-xdist/pull/1229 - "pytest-xdist @ git+https://github.com/sshane/pytest-xdist@b59e010821a25a2ebb95fc5c31c97022ed94221c", + "pytest-xdist @ git+https://github.com/sshane/pytest-xdist@2b4372bd62699fb412c4fe2f95bf9f01bd2018da", "pytest-timeout", "pytest-randomly", "pytest-asyncio", diff --git a/uv.lock b/uv.lock index 439afcef03..1366703fd6 100644 --- a/uv.lock +++ b/uv.lock @@ -622,10 +622,10 @@ name = "gymnasium" version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cloudpickle" }, - { name = "farama-notifications" }, - { name = "numpy" }, - { name = "typing-extensions" }, + { name = "cloudpickle", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "farama-notifications", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/17/c2a0e15c2cd5a8e788389b280996db927b923410de676ec5c7b2695e9261/gymnasium-1.2.0.tar.gz", hash = "sha256:344e87561012558f603880baf264ebc97f8a5c997a957b0c9f910281145534b0", size = 821142, upload-time = "2025-06-27T08:21:20.262Z" } wheels = [ @@ -880,22 +880,22 @@ name = "metadrive-simulator" version = "0.4.2.4" source = { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl" } dependencies = [ - { name = "filelock" }, - { name = "gymnasium" }, - { name = "lxml" }, - { name = "matplotlib" }, - { name = "numpy" }, - { name = "opencv-python-headless" }, - { name = "panda3d" }, - { name = "panda3d-gltf" }, - { name = "pillow" }, - { name = "progressbar" }, - { name = "psutil" }, - { name = "pygments" }, - { name = "requests" }, - { name = "shapely" }, - { name = "tqdm" }, - { name = "yapf" }, + { name = "filelock", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "gymnasium", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "lxml", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "matplotlib", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "opencv-python-headless", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "panda3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "panda3d-gltf", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "pillow", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "progressbar", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "psutil", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "pygments", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "requests", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "shapely", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "tqdm", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "yapf", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl", hash = "sha256:fbf0ea9be67e65cd45d38ff930e3d49f705dd76c9ddbd1e1482e3f87b61efcef" }, @@ -1316,7 +1316,7 @@ requires-dist = [ { name = "pytest-repeat", marker = "extra == 'testing'" }, { name = "pytest-subtests", marker = "extra == 'testing'" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, - { name = "pytest-xdist", marker = "extra == 'testing'", git = "https://github.com/sshane/pytest-xdist?rev=909e97b49d12401c10608f9d777bfc9dab8a4413" }, + { name = "pytest-xdist", marker = "extra == 'testing'", git = "https://github.com/sshane/pytest-xdist?rev=2b4372bd62699fb412c4fe2f95bf9f01bd2018da" }, { name = "pytools", marker = "platform_machine != 'aarch64' and extra == 'dev'", specifier = "<2024.1.11" }, { name = "pywinctl", marker = "extra == 'dev'" }, { name = "pyzmq" }, @@ -1374,8 +1374,8 @@ name = "panda3d-gltf" version = "0.13" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "panda3d" }, - { name = "panda3d-simplepbr" }, + { name = "panda3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "panda3d-simplepbr", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/7f/9f18fc3fa843a080acb891af6bcc12262e7bdf1d194a530f7042bebfc81f/panda3d-gltf-0.13.tar.gz", hash = "sha256:d06d373bdd91cf530909b669f43080e599463bbf6d3ef00c3558bad6c6b19675", size = 25573, upload-time = "2021-05-21T05:46:32.738Z" } wheels = [ @@ -1387,8 +1387,8 @@ name = "panda3d-simplepbr" version = "0.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "panda3d" }, - { name = "typing-extensions" }, + { name = "panda3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/be/c4d1ded04c22b357277cf6e6a44c1ab4abb285a700bd1991460460e05b99/panda3d_simplepbr-0.13.1.tar.gz", hash = "sha256:c83766d7c8f47499f365a07fe1dff078fc8b3054c2689bdc8dceabddfe7f1a35", size = 6216055, upload-time = "2025-03-30T16:57:41.087Z" } wheels = [ @@ -4125,9 +4125,9 @@ name = "pyopencl" version = "2025.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "platformdirs" }, - { name = "pytools" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "platformdirs", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "pytools", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/88/0ac460d3e2def08b2ad6345db6a13613815f616bbbd60c6f4bdf774f4c41/pyopencl-2025.1.tar.gz", hash = "sha256:0116736d7f7920f87b8db4b66a03f27b1d930d2e37ddd14518407cc22dd24779", size = 422510, upload-time = "2025-01-22T00:16:58.421Z" } wheels = [ @@ -4303,8 +4303,8 @@ wheels = [ [[package]] name = "pytest-xdist" -version = "3.7.1.dev3+g909e97b" -source = { git = "https://github.com/sshane/pytest-xdist?rev=909e97b49d12401c10608f9d777bfc9dab8a4413#909e97b49d12401c10608f9d777bfc9dab8a4413" } +version = "3.7.1.dev24+g2b4372b" +source = { git = "https://github.com/sshane/pytest-xdist?rev=2b4372bd62699fb412c4fe2f95bf9f01bd2018da#2b4372bd62699fb412c4fe2f95bf9f01bd2018da" } dependencies = [ { name = "execnet" }, { name = "pytest" }, @@ -4345,9 +4345,9 @@ name = "pytools" version = "2024.1.10" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs" }, - { name = "siphash24" }, - { name = "typing-extensions" }, + { name = "platformdirs", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "siphash24", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ee/0f/56e109c0307f831b5d598ad73976aaaa84b4d0e98da29a642e797eaa940c/pytools-2024.1.10.tar.gz", hash = "sha256:9af6f4b045212c49be32bb31fe19606c478ee4b09631886d05a32459f4ce0a12", size = 81741, upload-time = "2024-07-17T18:47:38.287Z" } wheels = [ @@ -4670,7 +4670,7 @@ name = "shapely" version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ca/3c/2da625233f4e605155926566c0e7ea8dda361877f48e8b1655e53456f252/shapely-2.1.1.tar.gz", hash = "sha256:500621967f2ffe9642454808009044c21e5b35db89ce69f8a2042c2ffd0e2772", size = 315422, upload-time = "2025-05-19T11:04:41.265Z" } wheels = [ @@ -4899,7 +4899,7 @@ name = "yapf" version = "0.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs" }, + { name = "platformdirs", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907, upload-time = "2024-11-14T00:11:41.584Z" } wheels = [