From 3d48bd934d6c84a5352c2a1d56e6f93ed59e28c4 Mon Sep 17 00:00:00 2001 From: Greg Hogan Date: Thu, 25 Mar 2021 13:30:09 -0700 Subject: [PATCH] log to file and send through athena (#20250) * log to file and send through athena * rename logging level * pass thru log formatter * logMessage is TEXT * send queue always strings * switch to xattr and lower priority queue * enable cloud logging for devices * time or size based log rotation * basename -> dirname * remove HARDWARE.get_cloudlog_enabled * fix errors * fix another exception * xattrs need to be bytes * sending works * cleanup files at start * add id and adjust formatting * do not send active log file * better names * separate log formatters * fix formatter super init * fix log file order * ensure file always has file formatter * i see why there was no formatter * apply same formatting to cpp log msgs * apply same formatting to cpp log msgs * update queue names in tests * strip deprecated keys in STATUS_PACKET * strip DEPRECATED from dict recursively * athena log queue test * instanceof instead of type * isinstance instead of type * use super * remove logentries * last_scan param unused * comment about special log msg attr names * add dict_helpers.py to release files * use monotonic time and counter for log rotation * update for adjusted log file naming * use monotonic clock for tracking last log file scan --- Pipfile | 1 - Pipfile.lock | 375 ++++++++++++++----------- common/dict_helpers.py | 9 + common/logging_extra.py | 41 +++ release/files_common | 1 + selfdrive/athena/athenad.py | 115 +++++++- selfdrive/athena/tests/helpers.py | 2 +- selfdrive/athena/tests/test_athenad.py | 11 +- selfdrive/logmessaged.py | 23 +- selfdrive/manager/build.py | 4 +- selfdrive/manager/manager.py | 4 +- selfdrive/swaglog.py | 82 +++++- selfdrive/thermald/thermald.py | 7 +- 13 files changed, 455 insertions(+), 220 deletions(-) create mode 100644 common/dict_helpers.py diff --git a/Pipfile b/Pipfile index 0233cf056..47254a1b5 100644 --- a/Pipfile +++ b/Pipfile @@ -94,7 +94,6 @@ tqdm = "*" Cython = "*" PyYAML = "*" websocket_client = "*" -Logentries = {git = "https://github.com/commaai/le_python.git",ref = "feaeacb48f7f4bdb02c0a8fc092326d4e101b7f2"} urllib3 = "*" gunicorn = "*" utm = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 2e6b2c472..26d1e1410 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2183375c59225a990d3bc013efd68e0702765d1b340d98f11112875efdc9a03f" + "sha256": "fcb1581613337cb41eb3fb0382ca993caf9b71e469026c4acf35081d9b09cebe" }, "pipfile-spec": 6, "requires": { @@ -128,10 +128,10 @@ }, "cysignals": { "hashes": [ - "sha256:8107b67a0c5991f74b0e000c6fa9fe8efcb2a22c7ede5b017aac4c3e20fb7db2" + "sha256:5c4606c435775028316f725fdb7cb894e3cae0b6fd2f862a0d2971748469b43a" ], "index": "pypi", - "version": "==1.10.2" + "version": "==1.10.3" }, "cython": { "hashes": [ @@ -172,11 +172,11 @@ }, "flake8": { "hashes": [ - "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", - "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" + "sha256:12d05ab02614b6aee8df7c36b97d1a3b2372761222b19b58621355e82acddcff", + "sha256:78873e372b12b093da7b5e5ed302e8ad9e988b38b063b61ad937f26ca58fc5f0" ], "index": "pypi", - "version": "==3.8.4" + "version": "==3.9.0" }, "flask": { "hashes": [ @@ -211,11 +211,11 @@ }, "isort": { "hashes": [ - "sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e", - "sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc" + "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6", + "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d" ], "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==5.7.0" + "version": "==5.8.0" }, "itsdangerous": { "hashes": [ @@ -243,33 +243,31 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:1d33d6f789697f401b75ce08e73b1de567b947740f768376631079290118ad39", - "sha256:2f2de8f8ac0be3e40d17730e0600619d35c78c13a099ea91ef7fb4ad944ce694", - "sha256:3782931963dc89e0e9a0ae4348b44762e868ea280e4f8c233b537852a8996ab9", - "sha256:37d9c34b96cca6787fe014aeb651217944a967a5b165e2cacb6b858d2997ab84", - "sha256:38c3865bd220bd983fcaa9aa11462619e84a71233bafd9c880f7b1cb753ca7fa", - "sha256:429c4d1862f3fc37cd56304d880f2eae5bd0da83bdef889f3bd66458aac49128", - "sha256:522b7c94b524389f4a4094c4bf04c2b02228454ddd17c1a9b2801fac1d754871", - "sha256:57fb5c5504ddd45ed420b5b6461a78f58cbb0c1b0cbd9cd5a43ad30a4a3ee4d0", - "sha256:5944a9b95e97de1980c65f03b79b356f30a43de48682b8bdd90aa5089f0ec1f4", - "sha256:6f4e5e68b7af950ed7fdb594b3f19a0014a3ace0fedb86acb896e140ffb24302", - "sha256:71a1ef23f22fa8437974b2d60fedb947c99a957ad625f83f43fd3de70f77f458", - "sha256:8a44e9901c0555f95ac401377032f6e6af66d8fc1fbfad77a7a8b1a826e0b93c", - "sha256:b6577f15d5516d7d209c1a8cde23062c0f10625f19e8dc9fb59268859778d7d7", - "sha256:c8fe2d6ff0ff583784039d0255ea7da076efd08507f2be6f68583b0da32e3afb", - "sha256:cadfa2c2cf54d35d13dc8d231253b7985b97d629ab9ca6e7d672c35539d38163", - "sha256:cd1bdace1a8762534e9a36c073cd54e97d517a17d69a17985961265be6d22847", - "sha256:ddbdcd10eb999d7ab292677f588b658372aadb9a52790f82484a37127a390108", - "sha256:e7273c64bccfd9310e9601b8f4511d84730239516bada26a0c9846c9697617ef", - "sha256:e7428977763150b4cf83255625a80a23dfdc94d43be7791ce90799d446b4e26f", - "sha256:e960e8be509e8d6d618300a6c189555c24efde63e85acaf0b14b2cd1ac743315", - "sha256:ecb5dd5990cec6e7f5c9c1124a37cb2c710c6d69b0c1a5c4aa4b35eba0ada068", - "sha256:ef3f5e288aa57b73b034ce9c1f1ac753d968f9069cd0742d1d69c698a0167166", - "sha256:fa5b2dee0e231fa4ad117be114251bdfe6afe39213bd629d43deb117b6a6c40a", - "sha256:fa7fb7973c622b9e725bee1db569d2c2ee64d2f9a089201c5e8185d482c7352d" + "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653", + "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61", + "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2", + "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837", + "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3", + "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43", + "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726", + "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3", + "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587", + "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8", + "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a", + "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd", + "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f", + "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad", + "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4", + "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b", + "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf", + "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981", + "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741", + "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e", + "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93", + "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.5.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.6.0" }, "libusb1": { "hashes": [ @@ -284,10 +282,6 @@ "index": "pypi", "version": "==1.9.2" }, - "logentries": { - "git": "https://github.com/commaai/le_python.git", - "ref": "feaeacb48f7f4bdb02c0a8fc092326d4e101b7f2" - }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", @@ -567,11 +561,11 @@ }, "pycodestyle": { "hashes": [ - "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", - "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068", + "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.6.0" + "version": "==2.7.0" }, "pycparser": { "hashes": [ @@ -619,11 +613,11 @@ }, "pyflakes": { "hashes": [ - "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", - "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", + "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.2.0" + "version": "==2.3.1" }, "pyjwt": { "hashes": [ @@ -847,11 +841,11 @@ }, "urllib3": { "hashes": [ - "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", - "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], "index": "pypi", - "version": "==1.26.3" + "version": "==1.26.4" }, "utm": { "hashes": [ @@ -1021,11 +1015,11 @@ }, "azure-cli-core": { "hashes": [ - "sha256:a23bc53d222a3c76e3de5299c1d3c27d8d005926b9d44b0c98722cbdc30505ab", - "sha256:e11f9edba62905623d54a75388afd8d21eeeb31a8aec22197abd24337ff5fa75" + "sha256:6776d7e63644bb0ef5aa05bd70b1b67a9dffe92c66fd1d773747399c9feee70c", + "sha256:78e5f0066ca7a89aeed2e68999ce2ebdd9c1b64a69578e5631e4bec2fd688ae5" ], "index": "pypi", - "version": "==2.20.0" + "version": "==2.21.0" }, "azure-cli-telemetry": { "hashes": [ @@ -1143,19 +1137,19 @@ }, "boto3": { "hashes": [ - "sha256:64a8900b3a110e2d6ff4d87f4d8cd56f0c8527361d9fc9385fcb50efe7a4975a", - "sha256:8e9ff8006c41889ed8a11831dee62adf922e071f14d54c52946d1f7855ae7a8e" + "sha256:1e6e06b2f1eee5a76acdde1e7b4f57c93c1bf2905341207d74f2a140ce060cd8", + "sha256:40e84a5f7888924db74a2710dbe48d066b51fe1f5549efaffe90e6efe813f37b" ], "index": "pypi", - "version": "==1.17.26" + "version": "==1.17.35" }, "botocore": { "hashes": [ - "sha256:4a785847a351e59f2329627fc9a19cf50f07644ea68996a1595d5a20487a423f", - "sha256:d27cbe115a25bfa82b851861b62d71fc771c2883bf5645bf37a7c0114789407c" + "sha256:9119ffb231145ffadd55391c9356dcdb18e3de65c3a7c82844634e949f0ca5a0", + "sha256:e34bbb7d7de154c2ff2a73ae0691c601a69c5bda887374c8a6a23072380b07a4" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.20.26" + "version": "==1.20.35" }, "certifi": { "hashes": [ @@ -1241,10 +1235,10 @@ }, "control": { "hashes": [ - "sha256:fed8ca6c773175bea14b121c001eee20c48977629a498d9a2d19a85490112ba5" + "sha256:34eeca077cf002a2f22a9334c8998ec5b3bcc0fdae2aac790a923cf8bc80245a" ], "index": "pypi", - "version": "==0.8.4" + "version": "==0.9.0" }, "coverage": { "hashes": [ @@ -1369,11 +1363,11 @@ }, "elasticsearch": { "hashes": [ - "sha256:1e24b33a82bf381b42d3b0d390f76fdb9d6a9d47b310dea8eaeb0a5933c394c0", - "sha256:a113cfcee9ba8565cd48a67b60e9903b67a81b3b80ddc6d3fb2c16789a58b763" + "sha256:9a77172be02bc4855210d83f0f1346a1e7d421e3cb2ca47ba81ac0c5a717b3a0", + "sha256:c67b0f6541eda6de9f92eaea319c070aa2710c5d4d4ee5e3dfa3c21bd95aa378" ], "index": "pypi", - "version": "==7.11.0" + "version": "==7.12.0" }, "entrypoints": { "hashes": [ @@ -1465,6 +1459,55 @@ "index": "pypi", "version": "==2.5.1" }, + "greenlet": { + "hashes": [ + "sha256:0a77691f0080c9da8dfc81e23f4e3cffa5accf0f5b56478951016d7cfead9196", + "sha256:0ddd77586553e3daf439aa88b6642c5f252f7ef79a39271c25b1d4bf1b7cbb85", + "sha256:111cfd92d78f2af0bc7317452bd93a477128af6327332ebf3c2be7df99566683", + "sha256:122c63ba795fdba4fc19c744df6277d9cfd913ed53d1a286f12189a0265316dd", + "sha256:181300f826625b7fd1182205b830642926f52bd8cdb08b34574c9d5b2b1813f7", + "sha256:1a1ada42a1fd2607d232ae11a7b3195735edaa49ea787a6d9e6a53afaf6f3476", + "sha256:1bb80c71de788b36cefb0c3bb6bfab306ba75073dbde2829c858dc3ad70f867c", + "sha256:1d1d4473ecb1c1d31ce8fd8d91e4da1b1f64d425c1dc965edc4ed2a63cfa67b2", + "sha256:292e801fcb3a0b3a12d8c603c7cf340659ea27fd73c98683e75800d9fd8f704c", + "sha256:2c65320774a8cd5fdb6e117c13afa91c4707548282464a18cf80243cf976b3e6", + "sha256:4365eccd68e72564c776418c53ce3c5af402bc526fe0653722bc89efd85bf12d", + "sha256:5352c15c1d91d22902582e891f27728d8dac3bd5e0ee565b6a9f575355e6d92f", + "sha256:58ca0f078d1c135ecf1879d50711f925ee238fe773dfe44e206d7d126f5bc664", + "sha256:5d4030b04061fdf4cbc446008e238e44936d77a04b2b32f804688ad64197953c", + "sha256:5d69bbd9547d3bc49f8a545db7a0bd69f407badd2ff0f6e1a163680b5841d2b0", + "sha256:5f297cb343114b33a13755032ecf7109b07b9a0020e841d1c3cedff6602cc139", + "sha256:62afad6e5fd70f34d773ffcbb7c22657e1d46d7fd7c95a43361de979f0a45aef", + "sha256:647ba1df86d025f5a34043451d7c4a9f05f240bee06277a524daad11f997d1e7", + "sha256:719e169c79255816cdcf6dccd9ed2d089a72a9f6c42273aae12d55e8d35bdcf8", + "sha256:7cd5a237f241f2764324396e06298b5dee0df580cf06ef4ada0ff9bff851286c", + "sha256:875d4c60a6299f55df1c3bb870ebe6dcb7db28c165ab9ea6cdc5d5af36bb33ce", + "sha256:90b6a25841488cf2cb1c8623a53e6879573010a669455046df5f029d93db51b7", + "sha256:94620ed996a7632723a424bccb84b07e7b861ab7bb06a5aeb041c111dd723d36", + "sha256:b5f1b333015d53d4b381745f5de842f19fe59728b65f0fbb662dafbe2018c3a5", + "sha256:c5b22b31c947ad8b6964d4ed66776bcae986f73669ba50620162ba7c832a6b6a", + "sha256:c93d1a71c3fe222308939b2e516c07f35a849c5047f0197442a4d6fbcb4128ee", + "sha256:cdb90267650c1edb54459cdb51dab865f6c6594c3a47ebd441bc493360c7af70", + "sha256:cfd06e0f0cc8db2a854137bd79154b61ecd940dce96fad0cba23fe31de0b793c", + "sha256:d3789c1c394944084b5e57c192889985a9f23bd985f6d15728c745d380318128", + "sha256:da7d09ad0f24270b20f77d56934e196e982af0d0a2446120cb772be4e060e1a2", + "sha256:df3e83323268594fa9755480a442cabfe8d82b21aba815a71acf1bb6c1776218", + "sha256:df8053867c831b2643b2c489fe1d62049a98566b1646b194cc815f13e27b90df", + "sha256:e1128e022d8dce375362e063754e129750323b67454cac5600008aad9f54139e", + "sha256:e6e9fdaf6c90d02b95e6b0709aeb1aba5affbbb9ccaea5502f8638e4323206be", + "sha256:eac8803c9ad1817ce3d8d15d1bb82c2da3feda6bee1153eec5c58fa6e5d3f770", + "sha256:eb333b90036358a0e2c57373f72e7648d7207b76ef0bd00a4f7daad1f79f5203", + "sha256:ed1d1351f05e795a527abc04a0d82e9aecd3bdf9f46662c36ff47b0b00ecaf06", + "sha256:f3dc68272990849132d6698f7dc6df2ab62a88b0d36e54702a8fd16c0490e44f", + "sha256:f59eded163d9752fd49978e0bab7a1ff21b1b8d25c05f0995d140cc08ac83379", + "sha256:f5e2d36c86c7b03c94b8459c3bd2c9fe2c7dab4b258b8885617d44a22e453fb7", + "sha256:f6f65bf54215e4ebf6b01e4bb94c49180a589573df643735107056f7a910275b", + "sha256:f8450d5ef759dbe59f84f2c9f77491bb3d3c44bc1a573746daf086e70b14c243", + "sha256:f97d83049715fd9dec7911860ecf0e17b48d8725de01e45de07d8ac0bd5bc378" + ], + "markers": "python_version >= '3'", + "version": "==1.0.0" + }, "gunicorn": { "hashes": [ "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626", @@ -1506,11 +1549,11 @@ }, "identify": { "hashes": [ - "sha256:220169a38a0c977c8fef377dc808d6a3330641b5211ec7356c7bbe73cda487c7", - "sha256:da3d757c94596c50865aae63db6ba4e2e5e3f666c3ea6a6da0cd09a8b2d34abc" + "sha256:39c0b110c9d0cd2391b6c38cd0ff679ee4b4e98f8db8b06c5d9d9e502711a1e1", + "sha256:efbf090a619255bc31c4fbba709e2805f7d30913fd4854ad84ace52bd276e2f6" ], "markers": "python_full_version >= '3.6.1'", - "version": "==2.1.1" + "version": "==2.2.0" }, "idna": { "hashes": [ @@ -1567,11 +1610,11 @@ }, "isort": { "hashes": [ - "sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e", - "sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc" + "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6", + "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d" ], "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==5.7.0" + "version": "==5.8.0" }, "itsdangerous": { "hashes": [ @@ -1638,19 +1681,19 @@ }, "jupyter-client": { "hashes": [ - "sha256:5eaaa41df449167ebba5e1cf6ca9b31f7fd4f71625069836e2e4fee07fe3cb13", - "sha256:649ca3aca1e28f27d73ef15868a7c7f10d6e70f761514582accec3ca6bb13085" + "sha256:c4bca1d0846186ca8be97f4d2fa6d2bae889cce4892a167ffa1ba6bd1f73e782", + "sha256:e053a2c44b6fa597feebe2b3ecb5eea3e03d1d91cc94351a52931ee1426aecfc" ], "markers": "python_version >= '3.5'", - "version": "==6.1.11" + "version": "==6.1.12" }, "jupyter-console": { "hashes": [ - "sha256:1d80c06b2d85bfb10bd5cc731b3db18e9023bc81ab00491d3ac31f206490aee3", - "sha256:7f6194f4f4692d292da3f501c7f343ccd5e36c6a1becf7b7515e23e66d6bf1e9" + "sha256:242248e1685039cd8bff2c2ecb7ce6c1546eb50ee3b08519729e6e881aec19c7", + "sha256:7799c4ea951e0e96ba8260575423cb323ea5a03fcf5503560fa3e15748869e27" ], "markers": "python_version >= '3.6'", - "version": "==6.2.0" + "version": "==6.4.0" }, "jupyter-core": { "hashes": [ @@ -1730,33 +1773,31 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:1d33d6f789697f401b75ce08e73b1de567b947740f768376631079290118ad39", - "sha256:2f2de8f8ac0be3e40d17730e0600619d35c78c13a099ea91ef7fb4ad944ce694", - "sha256:3782931963dc89e0e9a0ae4348b44762e868ea280e4f8c233b537852a8996ab9", - "sha256:37d9c34b96cca6787fe014aeb651217944a967a5b165e2cacb6b858d2997ab84", - "sha256:38c3865bd220bd983fcaa9aa11462619e84a71233bafd9c880f7b1cb753ca7fa", - "sha256:429c4d1862f3fc37cd56304d880f2eae5bd0da83bdef889f3bd66458aac49128", - "sha256:522b7c94b524389f4a4094c4bf04c2b02228454ddd17c1a9b2801fac1d754871", - "sha256:57fb5c5504ddd45ed420b5b6461a78f58cbb0c1b0cbd9cd5a43ad30a4a3ee4d0", - "sha256:5944a9b95e97de1980c65f03b79b356f30a43de48682b8bdd90aa5089f0ec1f4", - "sha256:6f4e5e68b7af950ed7fdb594b3f19a0014a3ace0fedb86acb896e140ffb24302", - "sha256:71a1ef23f22fa8437974b2d60fedb947c99a957ad625f83f43fd3de70f77f458", - "sha256:8a44e9901c0555f95ac401377032f6e6af66d8fc1fbfad77a7a8b1a826e0b93c", - "sha256:b6577f15d5516d7d209c1a8cde23062c0f10625f19e8dc9fb59268859778d7d7", - "sha256:c8fe2d6ff0ff583784039d0255ea7da076efd08507f2be6f68583b0da32e3afb", - "sha256:cadfa2c2cf54d35d13dc8d231253b7985b97d629ab9ca6e7d672c35539d38163", - "sha256:cd1bdace1a8762534e9a36c073cd54e97d517a17d69a17985961265be6d22847", - "sha256:ddbdcd10eb999d7ab292677f588b658372aadb9a52790f82484a37127a390108", - "sha256:e7273c64bccfd9310e9601b8f4511d84730239516bada26a0c9846c9697617ef", - "sha256:e7428977763150b4cf83255625a80a23dfdc94d43be7791ce90799d446b4e26f", - "sha256:e960e8be509e8d6d618300a6c189555c24efde63e85acaf0b14b2cd1ac743315", - "sha256:ecb5dd5990cec6e7f5c9c1124a37cb2c710c6d69b0c1a5c4aa4b35eba0ada068", - "sha256:ef3f5e288aa57b73b034ce9c1f1ac753d968f9069cd0742d1d69c698a0167166", - "sha256:fa5b2dee0e231fa4ad117be114251bdfe6afe39213bd629d43deb117b6a6c40a", - "sha256:fa7fb7973c622b9e725bee1db569d2c2ee64d2f9a089201c5e8185d482c7352d" + "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653", + "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61", + "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2", + "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837", + "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3", + "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43", + "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726", + "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3", + "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587", + "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8", + "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a", + "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd", + "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f", + "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad", + "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4", + "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b", + "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf", + "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981", + "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741", + "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e", + "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93", + "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.5.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.6.0" }, "lru-dict": { "hashes": [ @@ -2038,11 +2079,11 @@ }, "notebook": { "hashes": [ - "sha256:0464b28e18e7a06cec37e6177546c2322739be07962dd13bf712bcb88361f013", - "sha256:25ad93c982b623441b491e693ef400598d1a46cdf11b8c9c0b3be6c61ebbb6cd" + "sha256:cb271af1e8134e3d6fc6d458bdc79c40cbfc84c1eb036a493f216d58f0880e92", + "sha256:cbc9398d6c81473e9cdb891d2cae9c0d3718fca289dda6d26df5cb660fcadc7d" ], - "markers": "python_version >= '3.5'", - "version": "==6.2.0" + "markers": "python_version >= '3.6'", + "version": "==6.3.0" }, "numpy": { "hashes": [ @@ -2286,11 +2327,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:4cea7d09e46723885cb8bc54678175453e5071e9449821dce6f017b1d1fbfc1a", - "sha256:9397a7162cf45449147ad6042fa37983a081b8a73363a5253dd4072666333137" + "sha256:bf00f22079f5fadc949f42ae8ff7f05702826a97059ffcc6281036ad40ac6f04", + "sha256:e1b4f11b9336a28fa11810bc623c357420f69dfdb6d2dac41ca2c21a55c033bc" ], "markers": "python_full_version >= '3.6.1'", - "version": "==3.0.17" + "version": "==3.0.18" }, "psutil": { "hashes": [ @@ -2517,11 +2558,11 @@ }, "pynmea2": { "hashes": [ - "sha256:4fbb2a293883b24932e8403730f654b479828c218b00470031478c53202e2f71", - "sha256:93079d80a850d5d334635a82b85f68a67bcbdaaf4989c7fc7cffee3e68aac376" + "sha256:2d415c586bf2f40aebf452d62105528428806a5333321bfcdcfadf16caccbd74", + "sha256:7b84ec29b727946b8812082824c8a5298c144cc8b77a875f53d92b6832b6cdfc" ], "index": "pypi", - "version": "==1.16.0" + "version": "==1.17.0" }, "pyopenssl": { "hashes": [ @@ -2731,11 +2772,11 @@ }, "qtconsole": { "hashes": [ - "sha256:0173486b9cd69e17df537fb4f1e0d62a88019f6661700a11fd7236fa89ed900b", - "sha256:404994edfe33c201d6bd0c4bd501b00c16125071573c938533224992bea0b30f" + "sha256:4a38053993ca2da058f76f8d75b3d8906efbf9183de516f92f222ac8e37d9614", + "sha256:c091a35607d2a2432e004c4a112d241ce908086570cf68594176dd52ccaa212d" ], "markers": "python_version >= '3.6'", - "version": "==5.0.2" + "version": "==5.0.3" }, "qtpy": { "hashes": [ @@ -2785,10 +2826,10 @@ }, "s3transfer": { "hashes": [ - "sha256:1e28620e5b444652ed752cf87c7e0cb15b0e578972568c6609f0f18212f259ed", - "sha256:7fdddb4f22275cf1d32129e21f056337fd2a80b6ccef1664528145b72c49e6d2" + "sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2", + "sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547" ], - "version": "==0.3.4" + "version": "==0.3.6" }, "scikit-image": { "hashes": [ @@ -2942,47 +2983,43 @@ }, "sqlalchemy": { "hashes": [ - "sha256:040bdfc1d76a9074717a3f43455685f781c581f94472b010cd6c4754754e1862", - "sha256:1fe5d8d39118c2b018c215c37b73fd6893c3e1d4895be745ca8ff6eb83333ed3", - "sha256:23927c3981d1ec6b4ea71eb99d28424b874d9c696a21e5fbd9fa322718be3708", - "sha256:24f9569e82a009a09ce2d263559acb3466eba2617203170e4a0af91e75b4f075", - "sha256:2578dbdbe4dbb0e5126fb37ffcd9793a25dcad769a95f171a2161030bea850ff", - "sha256:269990b3ab53cb035d662dcde51df0943c1417bdab707dc4a7e4114a710504b4", - "sha256:29cccc9606750fe10c5d0e8bd847f17a97f3850b8682aef1f56f5d5e1a5a64b1", - "sha256:37b83bf81b4b85dda273aaaed5f35ea20ad80606f672d94d2218afc565fb0173", - "sha256:63677d0c08524af4c5893c18dbe42141de7178001360b3de0b86217502ed3601", - "sha256:639940bbe1108ac667dcffc79925db2966826c270112e9159439ab6bb14f8d80", - "sha256:6a939a868fdaa4b504e8b9d4a61f21aac11e3fecc8a8214455e144939e3d2aea", - "sha256:6b8b8c80c7f384f06825612dd078e4a31f0185e8f1f6b8c19e188ff246334205", - "sha256:6c9e6cc9237de5660bcddea63f332428bb83c8e2015c26777281f7ffbd2efb84", - "sha256:6ec1044908414013ebfe363450c22f14698803ce97fbb47e53284d55c5165848", - "sha256:6fca33672578666f657c131552c4ef8979c1606e494f78cd5199742dfb26918b", - "sha256:751934967f5336a3e26fc5993ccad1e4fee982029f9317eb6153bc0bc3d2d2da", - "sha256:8be835aac18ec85351385e17b8665bd4d63083a7160a017bef3d640e8e65cadb", - "sha256:927ce09e49bff3104459e1451ce82983b0a3062437a07d883a4c66f0b344c9b5", - "sha256:94208867f34e60f54a33a37f1c117251be91a47e3bfdb9ab8a7847f20886ad06", - "sha256:94f667d86be82dd4cb17d08de0c3622e77ca865320e0b95eae6153faa7b4ecaf", - "sha256:9e9c25522933e569e8b53ccc644dc993cab87e922fb7e142894653880fdd419d", - "sha256:a0e306e9bb76fd93b29ae3a5155298e4c1b504c7cbc620c09c20858d32d16234", - "sha256:a8bfc1e1afe523e94974132d7230b82ca7fa2511aedde1f537ec54db0399541a", - "sha256:ac2244e64485c3778f012951fdc869969a736cd61375fde6096d08850d8be729", - "sha256:b4b0e44d586cd64b65b507fa116a3814a1a53d55dce4836d7c1a6eb2823ff8d1", - "sha256:baeb451ee23e264de3f577fee5283c73d9bbaa8cb921d0305c0bbf700094b65b", - "sha256:c7dc052432cd5d060d7437e217dd33c97025287f99a69a50e2dc1478dd610d64", - "sha256:d1a85dfc5dee741bf49cb9b6b6b8d2725a268e4992507cf151cba26b17d97c37", - "sha256:d90010304abb4102123d10cbad2cdf2c25a9f2e66a50974199b24b468509bad5", - "sha256:ddfb511e76d016c3a160910642d57f4587dc542ce5ee823b0d415134790eeeb9", - "sha256:e273367f4076bd7b9a8dc2e771978ef2bfd6b82526e80775a7db52bff8ca01dd", - "sha256:e5bb3463df697279e5459a7316ad5a60b04b0107f9392e88674d0ece70e9cf70", - "sha256:e8a1750b44ad6422ace82bf3466638f1aa0862dbb9689690d5f2f48cce3476c8", - "sha256:eab063a70cca4a587c28824e18be41d8ecc4457f8f15b2933584c6c6cccd30f0", - "sha256:ecce8c021894a77d89808222b1ff9687ad84db54d18e4bd0500ca766737faaf6", - "sha256:f4d972139d5000105fcda9539a76452039434013570d6059993120dc2a65e447", - "sha256:fd3b96f8c705af8e938eaa99cbd8fd1450f632d38cad55e7367c33b263bf98ec", - "sha256:fdd2ed7395df8ac2dbb10cefc44737b66c6a5cd7755c92524733d7a443e5b7e2" - ], - "index": "pypi", - "version": "==1.3.23" + "sha256:06125670280111e39014af87f14d74599fd4b39a512c74f1a10e21e5626eb158", + "sha256:09b08eb1bea621e47c2b0fcb0334fcbb00e1da2a3c2d45a98e56cd072b840719", + "sha256:0abab6d1044198993256f073340b14c459736777c550a7e914cd00444dcf9c30", + "sha256:0bb04fd7414718fb1f4dfa17efcb0be787363451cf99a5e992728925d298d9ae", + "sha256:1b9f3c7b281aa1c3d0c74ef12c4633e5f8358bb94f01be7b964887183fd53e5e", + "sha256:1ba6922331b3f38e116c9266206b044baf64576e5cebd87917b5ad872d7a025f", + "sha256:3b290ff34de625143a05d2d172a88a064bb04a7938265b09d4e4bf45f21948f6", + "sha256:3fa75c854dba3f9b9c28bc5d88d246f6bc6f20b7480367c65339bcb2864d4707", + "sha256:4d1447183356c9679853926e81c7ebce3fbca9b1c607ea439975298c72137a36", + "sha256:4e88549a5e58ba8c80c5ea071ac3b4e590236672a882bb80f56da4afcee45d96", + "sha256:5289cafee71037f15feeeaf736f01910b9e3572525b73b201bdd21816db010ed", + "sha256:59ec279f1bd55e1d703e3d4b651600cc463cc3eafa8d8e5a70ab844f736348d4", + "sha256:5fb8f6a391992dd6aafe4fdf1dffbf7934fba1f5938593f20b152aa7f9619f82", + "sha256:65c4df9517da9cce2c1255282d3e39f2afbc3a02deba60d99b0a3283ae80ec0b", + "sha256:6a8e4c2e65028933a6dc8643c8f5a4f295a367131195b3c708634925cb3e8ec1", + "sha256:6d6115edf1297bfa58994986ffe0dff21af18f0cba51dfa6d1769aa8a277be32", + "sha256:6e517126d3bc13d455826befdc35a89f82f01d163848f68db02caa80d25433fc", + "sha256:7e1b0ed6d720750f02333d2f52502dfc2a23185aacc2cc6ce6ec29d28c21397c", + "sha256:7eba42098a13a3bcd509080b5e44d73783d9129ba0383793979bf518d01e8bb3", + "sha256:8383292298bb85d7ad79a13c6571aff213b96c49737f3c3af129de63bbfb42c9", + "sha256:8cfcfcf2582b19c874fa20d0b75100abe17be80a4c637c0683b4eb919946dfee", + "sha256:920db115eb06fc507fe2c774fb5c82a898b05dffbdadc7fafad51ce2cfd8c549", + "sha256:9406b96a979ab8d6de5d89f58b1f103c9aeef6fb5367448537a8228619f11258", + "sha256:97e333260a99d989f2a131aa8aa74140636dfbd030987150cb3748da607ea7db", + "sha256:a6b4b7688fe7d251bbae3f9da4a487568bd584d13201bc7591c8639ad01fecdc", + "sha256:aed22be55a608787bb6875dbcf3561349a0e88fe33fd88c318c1e5b4eeb2306a", + "sha256:c6197c88ad53c31f58de5a8180936b8ef027356e788cd5f6514b3439d3d897ac", + "sha256:d3b2819f4d7ae56191efc6fc456eb1805ada2bd5ba93d918893bc24fa7a1e30c", + "sha256:da72e3499bde4548e8b7d7f2ab23ceed09a5bac307bf51057e066c406a0ba2e1", + "sha256:dcde5067a7dab1ff2eaea2f3622b2055c5225ce2aaf589c5a4c703d43519c4ba", + "sha256:e1692bdf1b95c97caab1201773a4576f59627997f598d30bdadc50dd9f897fec", + "sha256:edec945ed57d11a1123657e4066f0bf747aaa93c8a65ec1c2c98172d1f2a9b7d", + "sha256:facacaea95e0822f7bbeaa6909b30b2836b14cff8790209d52a0c866e240b673", + "sha256:ff76d7dbf33f62e30e5a1d1b095d46afcdc49e42cbe33ce12014110147466700" + ], + "index": "pypi", + "version": "==1.4.2" }, "subprocess32": { "hashes": [ @@ -3010,11 +3047,11 @@ }, "terminado": { "hashes": [ - "sha256:23a053e06b22711269563c8bb96b36a036a86be8b5353e85e804f89b84aaa23f", - "sha256:89e6d94b19e4bc9dce0ffd908dfaf55cc78a9bf735934e915a4a96f65ac9704c" + "sha256:261c0b7825fecf629666e1820b484a5380f7e54d6b8bd889fa482e99dcf9bde4", + "sha256:430e876ec9d4d93a4fd8a49e82dcfae0c25f846540d0c5ca774b397533e237e8" ], "markers": "python_version >= '3.6'", - "version": "==0.9.2" + "version": "==0.9.3" }, "testpath": { "hashes": [ @@ -3025,11 +3062,11 @@ }, "tifffile": { "hashes": [ - "sha256:8ae99b2d7a80512628dc7f7de5a178525b28cb50cc69e82578a67106c28beee6", - "sha256:f55a4c8bcbfdd28157529c2e2a7c066da1636a202144f1696092b9290f52685d" + "sha256:1b72c92ecd2273e52686c0f8792d1d1c4da4109b241dd1723dfe56ef4d1ad612", + "sha256:f6092aba910ed52b6087877c9a2f604ba67623a8703fa8079929b62992dcc69c" ], "markers": "python_version >= '3.7'", - "version": "==2021.3.5" + "version": "==2021.3.17" }, "toml": { "hashes": [ @@ -3139,19 +3176,19 @@ }, "urllib3": { "hashes": [ - "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", - "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], "index": "pypi", - "version": "==1.26.3" + "version": "==1.26.4" }, "virtualenv": { "hashes": [ - "sha256:147b43894e51dd6bba882cf9c282447f780e2251cd35172403745fc381a0a80d", - "sha256:2be72df684b74df0ea47679a7df93fd0e04e72520022c57b479d8f881485dbe3" + "sha256:49ec4eb4c224c6f7dd81bb6d0a28a09ecae5894f4e593c89b0db0885f565a107", + "sha256:83f95875d382c7abafe06bd2a4cdd1b363e1bb77e02f155ebe8ac082a916b37c" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.4.2" + "version": "==20.4.3" }, "wcwidth": { "hashes": [ diff --git a/common/dict_helpers.py b/common/dict_helpers.py new file mode 100644 index 000000000..62cff63b5 --- /dev/null +++ b/common/dict_helpers.py @@ -0,0 +1,9 @@ +# remove all keys that end in DEPRECATED +def strip_deprecated_keys(d): + for k in list(d.keys()): + if isinstance(k, str): + if k.endswith('DEPRECATED'): + d.pop(k) + elif isinstance(d[k], dict): + strip_deprecated_keys(d[k]) + return d diff --git a/common/logging_extra.py b/common/logging_extra.py index ce8889b41..a80a5a082 100644 --- a/common/logging_extra.py +++ b/common/logging_extra.py @@ -3,6 +3,7 @@ import os import sys import copy import json +import uuid import socket import logging import traceback @@ -62,8 +63,48 @@ class SwagFormatter(logging.Formatter): return record_dict def format(self, record): + if self.swaglogger is None: + raise Exception("must set swaglogger before calling format()") return json_robust_dumps(self.format_dict(record)) +class SwagLogFileFormatter(SwagFormatter): + def fix_kv(self, k, v): + # append type to names to preserve legacy naming in logs + # avoids overlapping key namespaces with different types + # e.g. log.info() creates 'msg' -> 'msg$s' + # log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i' + # because overlapping namespace 'msg' caused problems + if isinstance(v, (str, bytes)): + k += "$s" + elif isinstance(v, float): + k += "$f" + elif isinstance(v, bool): + k += "$b" + elif isinstance(v, int): + k += "$i" + elif isinstance(v, dict): + nv = {} + for ik, iv in v.items(): + ik, iv = self.fix_kv(ik, iv) + nv[ik] = iv + v = nv + elif isinstance(v, list): + k += "$a" + return k, v + + def format(self, record): + if isinstance(record, str): + v = json.loads(record) + else: + v = self.format_dict(record) + + mk, mv = self.fix_kv('msg', v['msg']) + del v['msg'] + v[mk] = mv + v['id'] = uuid.uuid4().hex + + return json_robust_dumps(v) + class SwagErrorFilter(logging.Filter): def filter(self, record): return record.levelno < logging.ERROR diff --git a/release/files_common b/release/files_common index 4cf62be81..71fca93bc 100644 --- a/release/files_common +++ b/release/files_common @@ -29,6 +29,7 @@ common/params_pyx.pyx common/xattr.py common/profiler.py common/basedir.py +common/dict_helpers.py common/filter_simple.py common/stat_live.py common/spinner.py diff --git a/selfdrive/athena/athenad.py b/selfdrive/athena/athenad.py index c3645ed8d..b5d486055 100755 --- a/selfdrive/athena/athenad.py +++ b/selfdrive/athena/athenad.py @@ -4,6 +4,7 @@ import hashlib import io import json import os +import sys import queue import random import select @@ -24,18 +25,24 @@ from common.api import Api from common.basedir import PERSIST from common.params import Params from common.realtime import sec_since_boot -from selfdrive.hardware import HARDWARE +from selfdrive.hardware import HARDWARE, PC from selfdrive.loggerd.config import ROOT -from selfdrive.swaglog import cloudlog +from selfdrive.loggerd.xattr_cache import getxattr, setxattr +from selfdrive.swaglog import cloudlog, SWAGLOG_DIR ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai') HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) LOCAL_PORT_WHITELIST = set([8022]) +LOG_ATTR_NAME = 'user.upload' +LOG_ATTR_VALUE_MAX_UNIX_TIME = int.to_bytes(2147483647, 4, sys.byteorder) + dispatcher["echo"] = lambda s: s -payload_queue: Any = queue.Queue() -response_queue: Any = queue.Queue() +recv_queue: Any = queue.Queue() +send_queue: Any = queue.Queue() upload_queue: Any = queue.Queue() +log_send_queue: Any = queue.Queue() +log_recv_queue: Any = queue.Queue() cancelled_uploads: Any = set() UploadItem = namedtuple('UploadItem', ['path', 'url', 'headers', 'created_at', 'id']) @@ -46,7 +53,8 @@ def handle_long_poll(ws): threads = [ threading.Thread(target=ws_recv, args=(ws, end_event)), threading.Thread(target=ws_send, args=(ws, end_event)), - threading.Thread(target=upload_handler, args=(end_event,)) + threading.Thread(target=upload_handler, args=(end_event,)), + threading.Thread(target=log_handler, args=(end_event,)), ] + [ threading.Thread(target=jsonrpc_handler, args=(end_event,)) for x in range(HANDLER_THREADS) @@ -64,19 +72,21 @@ def handle_long_poll(ws): for thread in threads: thread.join() - def jsonrpc_handler(end_event): dispatcher["startLocalProxy"] = partial(startLocalProxy, end_event) while not end_event.is_set(): try: - data = payload_queue.get(timeout=1) - response = JSONRPCResponseManager.handle(data, dispatcher) - response_queue.put_nowait(response) + data = recv_queue.get(timeout=1) + if "method" in data and "params" in data: + response = JSONRPCResponseManager.handle(data, dispatcher) + send_queue.put_nowait(response.json) + elif "result" in data and "id" in data: + log_recv_queue.put_nowait(data) except queue.Empty: pass except Exception as e: cloudlog.exception("athena jsonrpc handler failed") - response_queue.put_nowait(json.dumps({"error": str(e)})) + send_queue.put_nowait(json.dumps({"error": str(e)})) def upload_handler(end_event): @@ -244,6 +254,82 @@ def takeSnapshot(): raise Exception("not available while camerad is started") +def get_logs_to_send_sorted(): + # TODO: scan once then use inotify to detect file creation/deletion + curr_time = int(time.time()) + logs = [] + for log_entry in os.listdir(SWAGLOG_DIR): + log_path = os.path.join(SWAGLOG_DIR, log_entry) + try: + time_sent = int.from_bytes(getxattr(log_path, LOG_ATTR_NAME), sys.byteorder) + except (ValueError, TypeError): + time_sent = 0 + # assume send failed and we lost the response if sent more than one hour ago + if not time_sent or curr_time - time_sent > 3600: + logs.append(log_entry) + # return logs in order they should be sent + # excluding most recent (active) log file + return sorted(logs[:-1]) + + +def log_handler(end_event): + if PC: + return + + log_files = [] + last_scan = 0 + log_retries = 0 + while not end_event.is_set(): + try: + try: + result = json.loads(log_recv_queue.get(timeout=1)) + log_success = result.get("success") + log_entry = result.get("id") + log_path = os.path.join(SWAGLOG_DIR, log_entry) + if log_entry and log_success: + try: + setxattr(log_path, LOG_ATTR_NAME, LOG_ATTR_VALUE_MAX_UNIX_TIME) + except OSError: + pass # file could be deleted by log rotation + except queue.Empty: + pass + + curr_scan = sec_since_boot() + if curr_scan - last_scan > 10: + log_files = get_logs_to_send_sorted() + last_scan = curr_scan + + # never send last log file because it is the active log + # and only send one log file at a time (most recent first) + if not len(log_files) or not log_send_queue.empty(): + continue + + log_entry = log_files.pop() + try: + curr_time = int(time.time()) + log_path = os.path.join(SWAGLOG_DIR, log_entry) + setxattr(log_path, LOG_ATTR_NAME, int.to_bytes(curr_time, 4, sys.byteorder)) + with open(log_path, "r") as f: + jsonrpc = { + "method": "forwardLogs", + "params": { + "logs": f.read() + }, + "jsonrpc": "2.0", + "id": log_entry + } + log_send_queue.put_nowait(json.dumps(jsonrpc)) + except OSError: + pass # file could be deleted by log rotation + log_retries = 0 + except Exception: + cloudlog.exception("athena.log_handler.exception") + log_retries += 1 + + if log_retries != 0: + time.sleep(backoff(log_retries)) + + def ws_proxy_recv(ws, local_sock, ssock, end_event, global_end_event): while not (end_event.is_set() or global_end_event.is_set()): try: @@ -290,7 +376,7 @@ def ws_recv(ws, end_event): if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY): if opcode == ABNF.OPCODE_TEXT: data = data.decode("utf-8") - payload_queue.put_nowait(data) + recv_queue.put_nowait(data) elif opcode == ABNF.OPCODE_PING: Params().put("LastAthenaPingTime", str(int(sec_since_boot() * 1e9))) except WebSocketTimeoutException: @@ -303,8 +389,11 @@ def ws_recv(ws, end_event): def ws_send(ws, end_event): while not end_event.is_set(): try: - response = response_queue.get(timeout=1) - ws.send(response.json) + try: + data = send_queue.get_nowait() + except queue.Empty: + data = log_send_queue.get(timeout=1) + ws.send(data) except queue.Empty: pass except Exception: diff --git a/selfdrive/athena/tests/helpers.py b/selfdrive/athena/tests/helpers.py index ef64968f7..5a6fba362 100644 --- a/selfdrive/athena/tests/helpers.py +++ b/selfdrive/athena/tests/helpers.py @@ -91,7 +91,7 @@ def with_http_server(func): p.start() time.sleep(0.1) - with Timeout(2): + with Timeout(2, 'HTTP Server seeding failed'): while True: try: requests.put(f'http://{host}:{port}/qlog.bz2', data='') diff --git a/selfdrive/athena/tests/test_athenad.py b/selfdrive/athena/tests/test_athenad.py index 42eb5ca31..2e9e79b65 100755 --- a/selfdrive/athena/tests/test_athenad.py +++ b/selfdrive/athena/tests/test_athenad.py @@ -103,6 +103,7 @@ class TestAthenadMethods(unittest.TestCase): athenad.upload_queue.put_nowait(item) try: + time.sleep(1) # give it time to process to prevent shutdown before upload completes now = time.time() while time.time() - now < 5: if athenad.upload_queue.qsize() == 0: @@ -178,10 +179,14 @@ class TestAthenadMethods(unittest.TestCase): thread = threading.Thread(target=athenad.jsonrpc_handler, args=(end_event,)) thread.daemon = True thread.start() - athenad.payload_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0})) try: - resp = athenad.response_queue.get(timeout=3) - self.assertDictEqual(resp.data, {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'}) + athenad.recv_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0})) + resp = athenad.send_queue.get(timeout=3) + self.assertDictEqual(json.loads(resp), {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'}) + + athenad.recv_queue.put_nowait(json.dumps({'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'})) + resp = athenad.log_recv_queue.get(timeout=3) + self.assertDictEqual(json.loads(resp), {'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}) finally: end_event.set() thread.join() diff --git a/selfdrive/logmessaged.py b/selfdrive/logmessaged.py index 953f3d477..17ef42cd6 100755 --- a/selfdrive/logmessaged.py +++ b/selfdrive/logmessaged.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 import zmq import cereal.messaging as messaging -from selfdrive.swaglog import get_le_handler +from common.logging_extra import SwagLogFileFormatter +from selfdrive.swaglog import get_file_handler def main(): - le_handler = get_le_handler() - le_level = 20 # logging.INFO + log_handler = get_file_handler() + log_handler.setFormatter(SwagLogFileFormatter(None)) + log_level = 20 # logging.INFO ctx = zmq.Context().instance() sock = ctx.socket(zmq.PULL) @@ -17,19 +19,14 @@ def main(): while True: dat = b''.join(sock.recv_multipart()) - dat = dat.decode('utf8') - - levelnum = ord(dat[0]) - dat = dat[1:] - - if levelnum >= le_level: - # push to logentries - # TODO: push to athena instead - le_handler.emit_raw(dat) + level = dat[0] + record = dat[1:].decode("utf-8") + if level >= log_level: + log_handler.emit(record) # then we publish them msg = messaging.new_message() - msg.logMessage = dat + msg.logMessage = record pub_sock.send(msg.to_bytes()) diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index 4c12b3a31..9589cbee5 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -10,7 +10,7 @@ import textwrap from common.basedir import BASEDIR from common.spinner import Spinner from common.text_window import TextWindow -from selfdrive.swaglog import add_logentries_handler, cloudlog +from selfdrive.swaglog import cloudlog, add_file_handler from selfdrive.version import dirty TOTAL_SCONS_NODES = 1225 @@ -70,7 +70,7 @@ def build(spinner, dirty=False): errors = [line.decode('utf8', 'replace') for line in compile_output if any([err in line for err in [b'error: ', b'not found, needed by target']])] error_s = "\n".join(errors) - add_logentries_handler(cloudlog) + add_file_handler(cloudlog) cloudlog.error("scons build failed\n" + error_s) # Show TextWindow diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 9cb667d04..393b51393 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -16,7 +16,7 @@ from selfdrive.manager.helpers import unblock_stdout from selfdrive.manager.process import ensure_running from selfdrive.manager.process_config import managed_processes from selfdrive.registration import register -from selfdrive.swaglog import add_logentries_handler, cloudlog +from selfdrive.swaglog import cloudlog, add_file_handler from selfdrive.version import dirty, version @@ -183,7 +183,7 @@ if __name__ == "__main__": try: main() except Exception: - add_logentries_handler(cloudlog) + add_file_handler(cloudlog) cloudlog.exception("Manager failed to start") # Show last 3 lines of traceback diff --git a/selfdrive/swaglog.py b/selfdrive/swaglog.py index 48447527c..beacd5b87 100644 --- a/selfdrive/swaglog.py +++ b/selfdrive/swaglog.py @@ -1,19 +1,72 @@ import os +from pathlib import Path import logging +from logging.handlers import BaseRotatingHandler -from logentries import LogentriesHandler import zmq -from common.logging_extra import SwagLogger, SwagFormatter +from common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter +from common.realtime import sec_since_boot +from selfdrive.hardware import PC +if PC: + SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log") +else: + SWAGLOG_DIR = "/data/log/" -def get_le_handler(): - # setup logentries. we forward log messages to it - le_token = "e8549616-0798-4d7e-a2ca-2513ae81fa17" - return LogentriesHandler(le_token, use_tls=False, verbose=False) +def get_file_handler(): + Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True) + base_filename = os.path.join(SWAGLOG_DIR, "swaglog") + handler = SwaglogRotatingFileHandler(base_filename) + return handler +class SwaglogRotatingFileHandler(BaseRotatingHandler): + def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None): + super().__init__(base_filename, mode="a", encoding=encoding, delay=True) + self.base_filename = base_filename + self.interval = interval # seconds + self.max_bytes = max_bytes + self.backup_count = backup_count + self.log_files = self.get_existing_logfiles() + log_indexes = [f.split(".")[-1] for f in self.log_files] + self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1]) + self.last_rollover = None + self.doRollover() -class LogMessageHandler(logging.Handler): + def _open(self): + self.last_rollover = sec_since_boot() + self.last_file_idx += 1 + next_filename = f"{self.base_filename}.{self.last_file_idx:010}" + stream = open(next_filename, self.mode, encoding=self.encoding) + self.log_files.insert(0, next_filename) + return stream + + def get_existing_logfiles(self): + log_files = list() + base_dir = os.path.dirname(self.base_filename) + for fn in os.listdir(base_dir): + fp = os.path.join(base_dir, fn) + if fp.startswith(self.base_filename) and os.path.isfile(fp): + log_files.append(fp) + return sorted(log_files) + + def shouldRollover(self, record): + size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes + time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= sec_since_boot() + return size_exceeded or time_exceeded + + def doRollover(self): + if self.stream: + self.stream.close() + self.stream = self._open() + + if self.backup_count > 0: + while len(self.log_files) > self.backup_count: + to_delete = self.log_files.pop() + if os.path.exists(to_delete): # just being safe, should always exist + os.remove(to_delete) + +class UnixDomainSocketHandler(logging.Handler): def __init__(self, formatter): logging.Handler.__init__(self) self.setFormatter(formatter) @@ -40,11 +93,13 @@ class LogMessageHandler(logging.Handler): pass -def add_logentries_handler(log): - """Function to add the logentries handler to swaglog. - This can be used to send logs when logmessaged is not running.""" - handler = get_le_handler() - handler.setFormatter(SwagFormatter(log)) +def add_file_handler(log): + """ + Function to add the file log handler to swaglog. + This can be used to store logs when logmessaged is not running. + """ + handler = get_file_handler() + handler.setFormatter(SwagLogFileFormatter(log)) log.addHandler(handler) @@ -53,4 +108,5 @@ log.setLevel(logging.DEBUG) outhandler = logging.StreamHandler() log.addHandler(outhandler) -log.addHandler(LogMessageHandler(SwagFormatter(log))) +# logs are sent through IPC before writing to disk to prevent disk I/O blocking +log.addHandler(UnixDomainSocketHandler(SwagFormatter(log))) diff --git a/selfdrive/thermald/thermald.py b/selfdrive/thermald/thermald.py index 01268346d..6240d14e4 100755 --- a/selfdrive/thermald/thermald.py +++ b/selfdrive/thermald/thermald.py @@ -13,6 +13,7 @@ from common.filter_simple import FirstOrderFilter from common.numpy_fast import clip, interp from common.params import Params from common.realtime import DT_TRML, sec_since_boot +from common.dict_helpers import strip_deprecated_keys from selfdrive.controls.lib.alertmanager import set_offroad_alert from selfdrive.hardware import EON, TICI, HARDWARE from selfdrive.loggerd.config import get_available_percent @@ -417,9 +418,9 @@ def thermald_thread(): location = messaging.recv_sock(location_sock) cloudlog.event("STATUS_PACKET", count=count, - pandaState=(pandaState.to_dict() if pandaState else None), - location=(location.gpsLocationExternal.to_dict() if location else None), - deviceState=msg.to_dict()) + pandaState=(strip_deprecated_keys(pandaState.to_dict()) if pandaState else None), + location=(strip_deprecated_keys(location.gpsLocationExternal.to_dict()) if location else None), + deviceState=strip_deprecated_keys(msg.to_dict())) count += 1