import random
from collections . abc import Iterable
from hypothesis import settings , given , strategies as st
from parameterized import parameterized
selfdrive/car: ban cereal and capnp (#33208)
* ban cereal and msgq
* common too
* do toyota/values.py
* do all fingerprints
* example without builder
* this still works, but no type checking anymore
* stash
* wtf, how does this work
* okay actually not bad
* safe
* epic!
* stash data_structures.py
* some clean up
* hell yeah
* clean up old file
* add to delete
* delete
This reverts commit 90239b7797ace31ee647a2fdbd67e0c3faa98dcf.
* switch more CarParams stuff over
remove unused
* fix car tests by removing cereal! mypy forgets about dataclass if we wrap it :(
* fix this too
* fix this too
* remove more cereal and add some good hyundai tests
* bunch more typing
* override default with 20hz radar
* temp capnp converter helper
* more lateralTuning
* small union replicator is better than what i was trying, and fixes mypy dynamic typing issues
* can keep all this the same now!
* type ret: CarParams, add more missing structs, revert lateralTuning changes (smaller diff!)
* revert more
* get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :(
would have been `CarParams.NetworkLocation()`
* Revert "get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :("
This reverts commit bb28b228becba932052d2fc5a4389784027435b1.
* remove cereal from car_helpers (TODO: caching)
* remove a bunch of temp lines
* use dataclass_transform!
* remove some car.CarParams from the interfaces
* remove rest of car.CarParams from the interfaces
* same which() API
* sort
* from cereal/cache from fingerprinting!
* more typing
* dataclass to capnp helper for CarParams, cached it since it's kinda slow
* (partial) fix process replay fingerprintig for new API
* latcontrollers take capnp
* forgot this
* fix test_models
* fix unit tests
* not here
* VehicleModel and controller still takes capnp CP since they get it from Params()
* fix modeld test
* more fix
* need to namespace to structs, since CarState is both class and struct
* this was never in the base class?!
* clean that up again
* fix import error
fix import error
* cmts and more structs
* remove some more cereal from toyota + convert CarState to capnp
* bruh this was wrong
* replace more cereal
* EventName is one of the last things...
* replace a bunch more cereal.car
* missing imports
* more
* can fix this typing now
* proper toyota+others CS typing!
* mypy can detect return type of CS.update() now
* fix redeclaration of cruise_buttons type
* mypy is only complaining about events now
* temp fix
* add carControl struct
* replace CarControl
i hope there's no circular imports in hyundai's CC
* fine now
* lol this was wrong too
* fix crash
* include my failed attempts at recursively converting to dataclass (doesn't implicitly convert types/recursively :( )
but attrs does, maybe will switch in the future
* clean up
* try out attr.s for its converter (doesn't work recursively yet, but interesting!)
* Revert "try out attr.s for its converter (doesn't work recursively yet, but interesting!)"
This reverts commit ff2434f7bbd45a4d4bfb21f7d6712d1f1c3bcde9.
* test processes doesn't fail anymore (on toyota)!
* fix honda crash
* stash
* Revert "stash"
This reverts commit c1762af4e776790e4ad1322ad4ce0610157346e0.
* remove a bunch more cereal!
* LET'S GOOO
* fix these tests
* and these
* and that
* stash, something is wrong with hyundai enable
* Revert "stash, something is wrong with hyundai enable"
This reverts commit 39cf327def258e2959fe23cd7a550a858f6d8f03.
* forgot these
* remove cereal from fw_versions
* Revert "remove cereal from fw_versions"
This reverts commit 232b37cd409b55d04b1afc90d4a80c49e710eb56.
* remove rest of the cereal exceptions!
* fix that
* add typing to radard since I didn't realize RI.update() switched from cereal to structs
* and here too!
* add TODO for slots
* needed CS to be capnp, fix comparisons, and type hint car_specific so it's easier to catch type issues (capnp isn't detected by mypy :( )
* remove the struct converter
* save ~4-5% CPU at 100hz, we don't modify after so no need to deepcopy
btw pickle.loads(pickle.dumps()) is faster by ~1% CPU
* deepcopy -> copy: we can technically make a reference, but copy is almost free and less error-prone
saves ~1% CPU
* add non-copying asdict function
* should save ~3% CPU (still 4% above baseline)
* fix that, no dict support
* ~27% decrease in time for 20k iterations on 3X (3.37857 -> 2.4821s)
* give a better name
* fix
* dont support none, capitalize
* sheesh, this called type() on every field
* remove CS.events, clean up
* bump card %
* this was a bug on master!
* add a which enum
* default to pid
* revert
* update refs
* not needed, but consistent
* just Ecu
* don't need to do this in this pr
* clean up
* no cast
* consistent typing
* rm
* fix
* can do this if we're desperate for the last few %
* Revert "can do this if we're desperate for the last few %"
This reverts commit 18e11ac7883a0a56583750b1cc5a2b13011e7299.
* type this
* don't need to convert carControl
* i guess don't support set either
* fix CP type hint
* simplify that
old-commit-hash: 6a15c42143a4764e79df851d797f950a6212f464
8 months ago
from openpilot . selfdrive . car . structs import CarParams
from openpilot . selfdrive . car . fw_versions import build_fw_dict
from openpilot . selfdrive . car . ford . values import CAR , FW_QUERY_CONFIG , FW_PATTERN , get_platform_codes
from openpilot . selfdrive . car . ford . fingerprints import FW_VERSIONS
selfdrive/car: ban cereal and capnp (#33208)
* ban cereal and msgq
* common too
* do toyota/values.py
* do all fingerprints
* example without builder
* this still works, but no type checking anymore
* stash
* wtf, how does this work
* okay actually not bad
* safe
* epic!
* stash data_structures.py
* some clean up
* hell yeah
* clean up old file
* add to delete
* delete
This reverts commit 90239b7797ace31ee647a2fdbd67e0c3faa98dcf.
* switch more CarParams stuff over
remove unused
* fix car tests by removing cereal! mypy forgets about dataclass if we wrap it :(
* fix this too
* fix this too
* remove more cereal and add some good hyundai tests
* bunch more typing
* override default with 20hz radar
* temp capnp converter helper
* more lateralTuning
* small union replicator is better than what i was trying, and fixes mypy dynamic typing issues
* can keep all this the same now!
* type ret: CarParams, add more missing structs, revert lateralTuning changes (smaller diff!)
* revert more
* get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :(
would have been `CarParams.NetworkLocation()`
* Revert "get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :("
This reverts commit bb28b228becba932052d2fc5a4389784027435b1.
* remove cereal from car_helpers (TODO: caching)
* remove a bunch of temp lines
* use dataclass_transform!
* remove some car.CarParams from the interfaces
* remove rest of car.CarParams from the interfaces
* same which() API
* sort
* from cereal/cache from fingerprinting!
* more typing
* dataclass to capnp helper for CarParams, cached it since it's kinda slow
* (partial) fix process replay fingerprintig for new API
* latcontrollers take capnp
* forgot this
* fix test_models
* fix unit tests
* not here
* VehicleModel and controller still takes capnp CP since they get it from Params()
* fix modeld test
* more fix
* need to namespace to structs, since CarState is both class and struct
* this was never in the base class?!
* clean that up again
* fix import error
fix import error
* cmts and more structs
* remove some more cereal from toyota + convert CarState to capnp
* bruh this was wrong
* replace more cereal
* EventName is one of the last things...
* replace a bunch more cereal.car
* missing imports
* more
* can fix this typing now
* proper toyota+others CS typing!
* mypy can detect return type of CS.update() now
* fix redeclaration of cruise_buttons type
* mypy is only complaining about events now
* temp fix
* add carControl struct
* replace CarControl
i hope there's no circular imports in hyundai's CC
* fine now
* lol this was wrong too
* fix crash
* include my failed attempts at recursively converting to dataclass (doesn't implicitly convert types/recursively :( )
but attrs does, maybe will switch in the future
* clean up
* try out attr.s for its converter (doesn't work recursively yet, but interesting!)
* Revert "try out attr.s for its converter (doesn't work recursively yet, but interesting!)"
This reverts commit ff2434f7bbd45a4d4bfb21f7d6712d1f1c3bcde9.
* test processes doesn't fail anymore (on toyota)!
* fix honda crash
* stash
* Revert "stash"
This reverts commit c1762af4e776790e4ad1322ad4ce0610157346e0.
* remove a bunch more cereal!
* LET'S GOOO
* fix these tests
* and these
* and that
* stash, something is wrong with hyundai enable
* Revert "stash, something is wrong with hyundai enable"
This reverts commit 39cf327def258e2959fe23cd7a550a858f6d8f03.
* forgot these
* remove cereal from fw_versions
* Revert "remove cereal from fw_versions"
This reverts commit 232b37cd409b55d04b1afc90d4a80c49e710eb56.
* remove rest of the cereal exceptions!
* fix that
* add typing to radard since I didn't realize RI.update() switched from cereal to structs
* and here too!
* add TODO for slots
* needed CS to be capnp, fix comparisons, and type hint car_specific so it's easier to catch type issues (capnp isn't detected by mypy :( )
* remove the struct converter
* save ~4-5% CPU at 100hz, we don't modify after so no need to deepcopy
btw pickle.loads(pickle.dumps()) is faster by ~1% CPU
* deepcopy -> copy: we can technically make a reference, but copy is almost free and less error-prone
saves ~1% CPU
* add non-copying asdict function
* should save ~3% CPU (still 4% above baseline)
* fix that, no dict support
* ~27% decrease in time for 20k iterations on 3X (3.37857 -> 2.4821s)
* give a better name
* fix
* dont support none, capitalize
* sheesh, this called type() on every field
* remove CS.events, clean up
* bump card %
* this was a bug on master!
* add a which enum
* default to pid
* revert
* update refs
* not needed, but consistent
* just Ecu
* don't need to do this in this pr
* clean up
* no cast
* consistent typing
* rm
* fix
* can do this if we're desperate for the last few %
* Revert "can do this if we're desperate for the last few %"
This reverts commit 18e11ac7883a0a56583750b1cc5a2b13011e7299.
* type this
* don't need to convert carControl
* i guess don't support set either
* fix CP type hint
* simplify that
old-commit-hash: 6a15c42143a4764e79df851d797f950a6212f464
8 months ago
Ecu = CarParams . Ecu
ECU_ADDRESSES = {
Ecu . eps : 0x730 , # Power Steering Control Module (PSCM)
Ecu . abs : 0x760 , # Anti-Lock Brake System (ABS)
Ecu . fwdRadar : 0x764 , # Cruise Control Module (CCM)
Ecu . fwdCamera : 0x706 , # Image Processing Module A (IPMA)
Ecu . engine : 0x7E0 , # Powertrain Control Module (PCM)
Ecu . shiftByWire : 0x732 , # Gear Shift Module (GSM)
Ford: log interesting module configuration data (#31569)
* Ford: log interesting module configuration data
Ford ECUs have what is called "As-Built Data" which is configured at the
factory/workshop to set what packages/features are enabled on the car.
But they also contain vehicle specific information (VIN, make, model,
weight, wheel base...), DTC information and driver preferences.
I dumped the CAN traffic for the FORScan diagnostic tool to see how it
requests this information from the ECUs.
<details>
<summary>FORScan communication with IPMA (camera)</summary>
<pre>
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': '0200'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'TESTER_PRESENT', 'hex': '00'}
{'addr': 1806, 'type': 'positive_response', 'service': 'TESTER_PRESENT', 'hex': '00'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f190'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de00'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de00020799dbaa10296516a440000000000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f113'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1\x13JX7T-19H406-CH\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1134a5837542d3139483430362d434800000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f188'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1\x88JX7T-14F397-AH\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1884a5837542d3134463339372d414800000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f120'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1 JX7T-14F397-BF\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1204a5837542d3134463339372d424600000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f121'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f124'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1$JX7T-14F398-AG\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1244a5837542d3134463339382d414700000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f125'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1%JX7T-14F398-BF\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1254a5837542d3134463339382d424600000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f126'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f10a'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f111'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1\x11JX7T-14F403-CA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1114a5837542d3134463430332d434100000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f18c'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1\x8c182762191\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f18c31383237363231393100000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f162'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f110'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1\x10DS-JX7T-19H406-AD\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f11044532d4a5837542d3139483430362d414400000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': '0202'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'd100'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'd10001'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'd700'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'd70001010101'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'd701'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'd70101020000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'dd01'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'dd010102f8'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'f113'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'data': b'\xf1\x13JX7T-19H406-CH\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'hex': 'f1134a5837542d3139483430362d434800000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'fd08'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'fd0800000500500100300000000000000000000000300000000000000000200100400300100200001200f00300500000000000000300c00b00400200000000000000000000000000000000200f01201e01501400a00200200400700d02501d01700700e06405005e05503401100a000000002002002001000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'fd09'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'fd09ffec0001fef60002'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de00'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de00020799dbaa10296516a440000000000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DTC_INFORMATION', 'hex': '028f'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DTC_INFORMATION', 'hex': '02ff50019768c253002cc401862cc418862c'}
... skip DTC requests ...
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de00'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de00020799dbaa10296516a440000000000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de01'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de01fd5616db5fffff557fe1f842080000000800000008000000080000000819bfe00f7c00000000000000000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de02'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de02800000008000000080000000800000008337fc00800000008000000080000000800000008337fc0000000000000000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de03'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de03fffc26c3800000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de04'}
{'addr': 1806, 'type': 'positive_response', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de04546a8c0000000000'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de05'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de06'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de07'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de08'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de09'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de0a'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de0b'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de0c'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de0d'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
{'addr': 1798, 'type': 'request', 'service': 'READ_DATA_BY_IDENTIFIER', 'hex': 'de0e'}
{'addr': 1806, 'type': 'negative_response', 'hex': '2231'}
</pre>
</details>
Using UDS service `READ_DATA_BY_IDENTIFIER`, we can read the As-Built
blocks from `0xDExx` with no diagnostic session/security access necessary.
I used [Online As-Built databases](https://cyanlabs.net/asbuilt-db/) and
various coding spreadsheets shared online to find values we might be
interested in using for fingerprinting (both vehicle parameters and
identifying the platform).
ABS:
- Payload tier (Base, Mid Payload Upgrade, Heavy Duty Payload
Upgrade...)
- Wheelbase
- Steering ratio
- Cruise Control Mode (Normal, Adaptive)
- Enable Stop and Go
PSCM:
- Enable Lane Keeping Aid
- Enable Traffic Jam Assist
- Enable Lane Centering Assist
IPMA (Q4):
- Steering ratio
- Wheelbase
APIM (Sync 3 and Sync 4):
- Steering ratio
- Vehicle weight
- Wheelbase
There are more potentially useful signals which I haven't included
although they might not be necessary:
- Vehicle (Ford platform code, like "C344" or "C519" - although the
source of the mapping from index to code is FORScan and not Ford
themselves unless we can find a better source).
- Fuel type
- Vehicle length/height/front track/rear track
- Tire circumference (could be useful for converting wheel speed rad/s
to m/s)
- Steering angle source (Pinion, Wheel)
- Country code (letters, e.g. US, CA or UK)
- Transmission type
- CAN network architecture
- More feature flags (the APIM also stores settings for ACC, LCA, BLIS)
The full list of settings I have found is
[here](https://github.com/incognitojam/op-notebooks/blob/main/ford/settings.py).
* FwQueryConfig: add data_requests
* add car_data to CarInterface get_params
* Revert "add car_data to CarInterface get_params"
This reverts commit aa161a6b82082705db97bea2c4317e1888a74845.
* test_ford: add APIM ecu address
* Revert "FwQueryConfig: add data_requests"
This reverts commit dc5484a9b80be5bc61a7fdf55560b8813bc43ef7.
* fix block numbers and add extra queries
* bump test_fw_query_timing
* add missing query whitelists
* simplify asbuilt requests
* use forscan block ids
* formatting
---------
Co-authored-by: Shane Smiskol <shane@smiskol.com>
old-commit-hash: c724d1c86ce4044bfc03f1e7941871466f51a708
1 year ago
Ecu . debug : 0x7D0 , # Accessory Protocol Interface Module (APIM)
}
ECU_PART_NUMBER = {
Ecu . eps : [
b " 14D003 " ,
] ,
Ecu . abs : [
b " 2D053 " ,
] ,
Ecu . fwdRadar : [
b " 14D049 " ,
] ,
Ecu . fwdCamera : [
b " 14F397 " , # Ford Q3
b " 14H102 " , # Ford Q4
] ,
}
class TestFordFW :
def test_fw_query_config ( self ) :
for ( ecu , addr , subaddr ) in FW_QUERY_CONFIG . extra_ecus :
assert ecu in ECU_ADDRESSES , " Unknown ECU "
assert addr == ECU_ADDRESSES [ ecu ] , " ECU address mismatch "
assert subaddr is None , " Unexpected ECU subaddress "
@parameterized . expand ( FW_VERSIONS . items ( ) )
selfdrive/car: ban cereal and capnp (#33208)
* ban cereal and msgq
* common too
* do toyota/values.py
* do all fingerprints
* example without builder
* this still works, but no type checking anymore
* stash
* wtf, how does this work
* okay actually not bad
* safe
* epic!
* stash data_structures.py
* some clean up
* hell yeah
* clean up old file
* add to delete
* delete
This reverts commit 90239b7797ace31ee647a2fdbd67e0c3faa98dcf.
* switch more CarParams stuff over
remove unused
* fix car tests by removing cereal! mypy forgets about dataclass if we wrap it :(
* fix this too
* fix this too
* remove more cereal and add some good hyundai tests
* bunch more typing
* override default with 20hz radar
* temp capnp converter helper
* more lateralTuning
* small union replicator is better than what i was trying, and fixes mypy dynamic typing issues
* can keep all this the same now!
* type ret: CarParams, add more missing structs, revert lateralTuning changes (smaller diff!)
* revert more
* get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :(
would have been `CarParams.NetworkLocation()`
* Revert "get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :("
This reverts commit bb28b228becba932052d2fc5a4389784027435b1.
* remove cereal from car_helpers (TODO: caching)
* remove a bunch of temp lines
* use dataclass_transform!
* remove some car.CarParams from the interfaces
* remove rest of car.CarParams from the interfaces
* same which() API
* sort
* from cereal/cache from fingerprinting!
* more typing
* dataclass to capnp helper for CarParams, cached it since it's kinda slow
* (partial) fix process replay fingerprintig for new API
* latcontrollers take capnp
* forgot this
* fix test_models
* fix unit tests
* not here
* VehicleModel and controller still takes capnp CP since they get it from Params()
* fix modeld test
* more fix
* need to namespace to structs, since CarState is both class and struct
* this was never in the base class?!
* clean that up again
* fix import error
fix import error
* cmts and more structs
* remove some more cereal from toyota + convert CarState to capnp
* bruh this was wrong
* replace more cereal
* EventName is one of the last things...
* replace a bunch more cereal.car
* missing imports
* more
* can fix this typing now
* proper toyota+others CS typing!
* mypy can detect return type of CS.update() now
* fix redeclaration of cruise_buttons type
* mypy is only complaining about events now
* temp fix
* add carControl struct
* replace CarControl
i hope there's no circular imports in hyundai's CC
* fine now
* lol this was wrong too
* fix crash
* include my failed attempts at recursively converting to dataclass (doesn't implicitly convert types/recursively :( )
but attrs does, maybe will switch in the future
* clean up
* try out attr.s for its converter (doesn't work recursively yet, but interesting!)
* Revert "try out attr.s for its converter (doesn't work recursively yet, but interesting!)"
This reverts commit ff2434f7bbd45a4d4bfb21f7d6712d1f1c3bcde9.
* test processes doesn't fail anymore (on toyota)!
* fix honda crash
* stash
* Revert "stash"
This reverts commit c1762af4e776790e4ad1322ad4ce0610157346e0.
* remove a bunch more cereal!
* LET'S GOOO
* fix these tests
* and these
* and that
* stash, something is wrong with hyundai enable
* Revert "stash, something is wrong with hyundai enable"
This reverts commit 39cf327def258e2959fe23cd7a550a858f6d8f03.
* forgot these
* remove cereal from fw_versions
* Revert "remove cereal from fw_versions"
This reverts commit 232b37cd409b55d04b1afc90d4a80c49e710eb56.
* remove rest of the cereal exceptions!
* fix that
* add typing to radard since I didn't realize RI.update() switched from cereal to structs
* and here too!
* add TODO for slots
* needed CS to be capnp, fix comparisons, and type hint car_specific so it's easier to catch type issues (capnp isn't detected by mypy :( )
* remove the struct converter
* save ~4-5% CPU at 100hz, we don't modify after so no need to deepcopy
btw pickle.loads(pickle.dumps()) is faster by ~1% CPU
* deepcopy -> copy: we can technically make a reference, but copy is almost free and less error-prone
saves ~1% CPU
* add non-copying asdict function
* should save ~3% CPU (still 4% above baseline)
* fix that, no dict support
* ~27% decrease in time for 20k iterations on 3X (3.37857 -> 2.4821s)
* give a better name
* fix
* dont support none, capitalize
* sheesh, this called type() on every field
* remove CS.events, clean up
* bump card %
* this was a bug on master!
* add a which enum
* default to pid
* revert
* update refs
* not needed, but consistent
* just Ecu
* don't need to do this in this pr
* clean up
* no cast
* consistent typing
* rm
* fix
* can do this if we're desperate for the last few %
* Revert "can do this if we're desperate for the last few %"
This reverts commit 18e11ac7883a0a56583750b1cc5a2b13011e7299.
* type this
* don't need to convert carControl
* i guess don't support set either
* fix CP type hint
* simplify that
old-commit-hash: 6a15c42143a4764e79df851d797f950a6212f464
8 months ago
def test_fw_versions ( self , car_model : str , fw_versions : dict [ tuple [ Ecu , int , int | None ] , Iterable [ bytes ] ] ) :
for ( ecu , addr , subaddr ) , fws in fw_versions . items ( ) :
assert ecu in ECU_PART_NUMBER , " Unexpected ECU "
assert addr == ECU_ADDRESSES [ ecu ] , " ECU address mismatch "
assert subaddr is None , " Unexpected ECU subaddress "
for fw in fws :
assert len ( fw ) == 24 , " Expected ECU response to be 24 bytes "
match = FW_PATTERN . match ( fw )
assert match is not None , f " Unable to parse FW: { fw !r} "
if match :
part_number = match . group ( " part_number " )
assert part_number in ECU_PART_NUMBER [ ecu ] , f " Unexpected part number for { fw !r} "
codes = get_platform_codes ( [ fw ] )
assert 1 == len ( codes ) , f " Unable to parse FW: { fw !r} "
@settings ( max_examples = 100 )
@given ( data = st . data ( ) )
def test_platform_codes_fuzzy_fw ( self , data ) :
""" Ensure function doesn ' t raise an exception """
fw_strategy = st . lists ( st . binary ( ) )
fws = data . draw ( fw_strategy )
get_platform_codes ( fws )
def test_platform_codes_spot_check ( self ) :
# Asserts basic platform code parsing behavior for a few cases
results = get_platform_codes ( [
b " JX6A-14C204-BPL \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " NZ6T-14F397-AC \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " PJ6T-14H102-ABJ \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " LB5A-14C204-EAC \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
] )
assert results == { ( b " X6A " , b " J " ) , ( b " Z6T " , b " N " ) , ( b " J6T " , b " P " ) , ( b " B5A " , b " L " ) }
def test_fuzzy_match ( self ) :
for platform , fw_by_addr in FW_VERSIONS . items ( ) :
# Ensure there's no overlaps in platform codes
for _ in range ( 20 ) :
car_fw = [ ]
for ecu , fw_versions in fw_by_addr . items ( ) :
ecu_name , addr , sub_addr = ecu
fw = random . choice ( fw_versions )
selfdrive/car: ban cereal and capnp (#33208)
* ban cereal and msgq
* common too
* do toyota/values.py
* do all fingerprints
* example without builder
* this still works, but no type checking anymore
* stash
* wtf, how does this work
* okay actually not bad
* safe
* epic!
* stash data_structures.py
* some clean up
* hell yeah
* clean up old file
* add to delete
* delete
This reverts commit 90239b7797ace31ee647a2fdbd67e0c3faa98dcf.
* switch more CarParams stuff over
remove unused
* fix car tests by removing cereal! mypy forgets about dataclass if we wrap it :(
* fix this too
* fix this too
* remove more cereal and add some good hyundai tests
* bunch more typing
* override default with 20hz radar
* temp capnp converter helper
* more lateralTuning
* small union replicator is better than what i was trying, and fixes mypy dynamic typing issues
* can keep all this the same now!
* type ret: CarParams, add more missing structs, revert lateralTuning changes (smaller diff!)
* revert more
* get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :(
would have been `CarParams.NetworkLocation()`
* Revert "get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :("
This reverts commit bb28b228becba932052d2fc5a4389784027435b1.
* remove cereal from car_helpers (TODO: caching)
* remove a bunch of temp lines
* use dataclass_transform!
* remove some car.CarParams from the interfaces
* remove rest of car.CarParams from the interfaces
* same which() API
* sort
* from cereal/cache from fingerprinting!
* more typing
* dataclass to capnp helper for CarParams, cached it since it's kinda slow
* (partial) fix process replay fingerprintig for new API
* latcontrollers take capnp
* forgot this
* fix test_models
* fix unit tests
* not here
* VehicleModel and controller still takes capnp CP since they get it from Params()
* fix modeld test
* more fix
* need to namespace to structs, since CarState is both class and struct
* this was never in the base class?!
* clean that up again
* fix import error
fix import error
* cmts and more structs
* remove some more cereal from toyota + convert CarState to capnp
* bruh this was wrong
* replace more cereal
* EventName is one of the last things...
* replace a bunch more cereal.car
* missing imports
* more
* can fix this typing now
* proper toyota+others CS typing!
* mypy can detect return type of CS.update() now
* fix redeclaration of cruise_buttons type
* mypy is only complaining about events now
* temp fix
* add carControl struct
* replace CarControl
i hope there's no circular imports in hyundai's CC
* fine now
* lol this was wrong too
* fix crash
* include my failed attempts at recursively converting to dataclass (doesn't implicitly convert types/recursively :( )
but attrs does, maybe will switch in the future
* clean up
* try out attr.s for its converter (doesn't work recursively yet, but interesting!)
* Revert "try out attr.s for its converter (doesn't work recursively yet, but interesting!)"
This reverts commit ff2434f7bbd45a4d4bfb21f7d6712d1f1c3bcde9.
* test processes doesn't fail anymore (on toyota)!
* fix honda crash
* stash
* Revert "stash"
This reverts commit c1762af4e776790e4ad1322ad4ce0610157346e0.
* remove a bunch more cereal!
* LET'S GOOO
* fix these tests
* and these
* and that
* stash, something is wrong with hyundai enable
* Revert "stash, something is wrong with hyundai enable"
This reverts commit 39cf327def258e2959fe23cd7a550a858f6d8f03.
* forgot these
* remove cereal from fw_versions
* Revert "remove cereal from fw_versions"
This reverts commit 232b37cd409b55d04b1afc90d4a80c49e710eb56.
* remove rest of the cereal exceptions!
* fix that
* add typing to radard since I didn't realize RI.update() switched from cereal to structs
* and here too!
* add TODO for slots
* needed CS to be capnp, fix comparisons, and type hint car_specific so it's easier to catch type issues (capnp isn't detected by mypy :( )
* remove the struct converter
* save ~4-5% CPU at 100hz, we don't modify after so no need to deepcopy
btw pickle.loads(pickle.dumps()) is faster by ~1% CPU
* deepcopy -> copy: we can technically make a reference, but copy is almost free and less error-prone
saves ~1% CPU
* add non-copying asdict function
* should save ~3% CPU (still 4% above baseline)
* fix that, no dict support
* ~27% decrease in time for 20k iterations on 3X (3.37857 -> 2.4821s)
* give a better name
* fix
* dont support none, capitalize
* sheesh, this called type() on every field
* remove CS.events, clean up
* bump card %
* this was a bug on master!
* add a which enum
* default to pid
* revert
* update refs
* not needed, but consistent
* just Ecu
* don't need to do this in this pr
* clean up
* no cast
* consistent typing
* rm
* fix
* can do this if we're desperate for the last few %
* Revert "can do this if we're desperate for the last few %"
This reverts commit 18e11ac7883a0a56583750b1cc5a2b13011e7299.
* type this
* don't need to convert carControl
* i guess don't support set either
* fix CP type hint
* simplify that
old-commit-hash: 6a15c42143a4764e79df851d797f950a6212f464
8 months ago
car_fw . append ( CarParams . CarFw ( ecu = ecu_name , fwVersion = fw , address = addr ,
subAddress = 0 if sub_addr is None else sub_addr ) )
selfdrive/car: ban cereal and capnp (#33208)
* ban cereal and msgq
* common too
* do toyota/values.py
* do all fingerprints
* example without builder
* this still works, but no type checking anymore
* stash
* wtf, how does this work
* okay actually not bad
* safe
* epic!
* stash data_structures.py
* some clean up
* hell yeah
* clean up old file
* add to delete
* delete
This reverts commit 90239b7797ace31ee647a2fdbd67e0c3faa98dcf.
* switch more CarParams stuff over
remove unused
* fix car tests by removing cereal! mypy forgets about dataclass if we wrap it :(
* fix this too
* fix this too
* remove more cereal and add some good hyundai tests
* bunch more typing
* override default with 20hz radar
* temp capnp converter helper
* more lateralTuning
* small union replicator is better than what i was trying, and fixes mypy dynamic typing issues
* can keep all this the same now!
* type ret: CarParams, add more missing structs, revert lateralTuning changes (smaller diff!)
* revert more
* get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :(
would have been `CarParams.NetworkLocation()`
* Revert "get first enum automatically, but ofc mypy doesn't pick up the new metaclass so can't use :("
This reverts commit bb28b228becba932052d2fc5a4389784027435b1.
* remove cereal from car_helpers (TODO: caching)
* remove a bunch of temp lines
* use dataclass_transform!
* remove some car.CarParams from the interfaces
* remove rest of car.CarParams from the interfaces
* same which() API
* sort
* from cereal/cache from fingerprinting!
* more typing
* dataclass to capnp helper for CarParams, cached it since it's kinda slow
* (partial) fix process replay fingerprintig for new API
* latcontrollers take capnp
* forgot this
* fix test_models
* fix unit tests
* not here
* VehicleModel and controller still takes capnp CP since they get it from Params()
* fix modeld test
* more fix
* need to namespace to structs, since CarState is both class and struct
* this was never in the base class?!
* clean that up again
* fix import error
fix import error
* cmts and more structs
* remove some more cereal from toyota + convert CarState to capnp
* bruh this was wrong
* replace more cereal
* EventName is one of the last things...
* replace a bunch more cereal.car
* missing imports
* more
* can fix this typing now
* proper toyota+others CS typing!
* mypy can detect return type of CS.update() now
* fix redeclaration of cruise_buttons type
* mypy is only complaining about events now
* temp fix
* add carControl struct
* replace CarControl
i hope there's no circular imports in hyundai's CC
* fine now
* lol this was wrong too
* fix crash
* include my failed attempts at recursively converting to dataclass (doesn't implicitly convert types/recursively :( )
but attrs does, maybe will switch in the future
* clean up
* try out attr.s for its converter (doesn't work recursively yet, but interesting!)
* Revert "try out attr.s for its converter (doesn't work recursively yet, but interesting!)"
This reverts commit ff2434f7bbd45a4d4bfb21f7d6712d1f1c3bcde9.
* test processes doesn't fail anymore (on toyota)!
* fix honda crash
* stash
* Revert "stash"
This reverts commit c1762af4e776790e4ad1322ad4ce0610157346e0.
* remove a bunch more cereal!
* LET'S GOOO
* fix these tests
* and these
* and that
* stash, something is wrong with hyundai enable
* Revert "stash, something is wrong with hyundai enable"
This reverts commit 39cf327def258e2959fe23cd7a550a858f6d8f03.
* forgot these
* remove cereal from fw_versions
* Revert "remove cereal from fw_versions"
This reverts commit 232b37cd409b55d04b1afc90d4a80c49e710eb56.
* remove rest of the cereal exceptions!
* fix that
* add typing to radard since I didn't realize RI.update() switched from cereal to structs
* and here too!
* add TODO for slots
* needed CS to be capnp, fix comparisons, and type hint car_specific so it's easier to catch type issues (capnp isn't detected by mypy :( )
* remove the struct converter
* save ~4-5% CPU at 100hz, we don't modify after so no need to deepcopy
btw pickle.loads(pickle.dumps()) is faster by ~1% CPU
* deepcopy -> copy: we can technically make a reference, but copy is almost free and less error-prone
saves ~1% CPU
* add non-copying asdict function
* should save ~3% CPU (still 4% above baseline)
* fix that, no dict support
* ~27% decrease in time for 20k iterations on 3X (3.37857 -> 2.4821s)
* give a better name
* fix
* dont support none, capitalize
* sheesh, this called type() on every field
* remove CS.events, clean up
* bump card %
* this was a bug on master!
* add a which enum
* default to pid
* revert
* update refs
* not needed, but consistent
* just Ecu
* don't need to do this in this pr
* clean up
* no cast
* consistent typing
* rm
* fix
* can do this if we're desperate for the last few %
* Revert "can do this if we're desperate for the last few %"
This reverts commit 18e11ac7883a0a56583750b1cc5a2b13011e7299.
* type this
* don't need to convert carControl
* i guess don't support set either
* fix CP type hint
* simplify that
old-commit-hash: 6a15c42143a4764e79df851d797f950a6212f464
8 months ago
CP = CarParams ( carFw = car_fw )
matches = FW_QUERY_CONFIG . match_fw_to_car_fuzzy ( build_fw_dict ( CP . carFw ) , CP . carVin , FW_VERSIONS )
assert matches == { platform }
def test_match_fw_fuzzy ( self ) :
offline_fw = {
( Ecu . eps , 0x730 , None ) : [
b " L1MC-14D003-AJ \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " L1MC-14D003-AL \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
] ,
( Ecu . abs , 0x760 , None ) : [
b " L1MC-2D053-BA \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " L1MC-2D053-BD \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
] ,
( Ecu . fwdRadar , 0x764 , None ) : [
b " LB5T-14D049-AB \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " LB5T-14D049-AD \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
] ,
# We consider all model year hints for ECU, even with different platform codes
( Ecu . fwdCamera , 0x706 , None ) : [
b " LB5T-14F397-AD \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
b " NC5T-14F397-AF \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " ,
] ,
}
expected_fingerprint = CAR . FORD_EXPLORER_MK6
# ensure that we fuzzy match on all non-exact FW with changed revisions
live_fw = {
( 0x730 , None ) : { b " L1MC-14D003-XX \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " } ,
( 0x760 , None ) : { b " L1MC-2D053-XX \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " } ,
( 0x764 , None ) : { b " LB5T-14D049-XX \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " } ,
( 0x706 , None ) : { b " LB5T-14F397-XX \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " } ,
}
candidates = FW_QUERY_CONFIG . match_fw_to_car_fuzzy ( live_fw , ' ' , { expected_fingerprint : offline_fw } )
assert candidates == { expected_fingerprint }
# model year hint in between the range should match
live_fw [ ( 0x706 , None ) ] = { b " MB5T-14F397-XX \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " }
candidates = FW_QUERY_CONFIG . match_fw_to_car_fuzzy ( live_fw , ' ' , { expected_fingerprint : offline_fw , } )
assert candidates == { expected_fingerprint }
# unseen model year hint should not match
live_fw [ ( 0x760 , None ) ] = { b " M1MC-2D053-XX \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x00 " }
candidates = FW_QUERY_CONFIG . match_fw_to_car_fuzzy ( live_fw , ' ' , { expected_fingerprint : offline_fw } )
assert len ( candidates ) == 0 , " Should not match new model year hint "