From cf46de13d2934c52e9c87691c671fe9c78156ec9 Mon Sep 17 00:00:00 2001 From: grekiki Date: Tue, 22 Sep 2020 11:37:24 +0200 Subject: [PATCH] Uploader speedup (#2214) * use caching for getxattr * fix some git issues * Scheduled network checks * attempt optimization * Delete speed_test.py * Style fixes * Fix styling * fix spaces * fix spaces * fix naming * Update uploader.py * Update mark_all_uploaded.py * Add file to release * Update selfdrive/loggerd/tools/mark_all_uploaded.py Co-authored-by: Adeeb Shihadeh Co-authored-by: Adeeb Shihadeh --- release/files_common | 1 + selfdrive/loggerd/tools/mark_all_uploaded.py | 9 ++++++++ selfdrive/loggerd/uploader.py | 23 +++++++++++++------- selfdrive/loggerd/xattr_cache.py | 13 +++++++++++ 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 selfdrive/loggerd/tools/mark_all_uploaded.py create mode 100644 selfdrive/loggerd/xattr_cache.py diff --git a/release/files_common b/release/files_common index 4cea46873d..dc2dbc8b8e 100644 --- a/release/files_common +++ b/release/files_common @@ -315,6 +315,7 @@ selfdrive/loggerd/__init__.py selfdrive/loggerd/config.py selfdrive/loggerd/uploader.py selfdrive/loggerd/deleter.py +selfdrive/loggerd/xattr_cache.py selfdrive/sensord/SConscript selfdrive/sensord/gpsd.cc diff --git a/selfdrive/loggerd/tools/mark_all_uploaded.py b/selfdrive/loggerd/tools/mark_all_uploaded.py new file mode 100644 index 0000000000..ff6e3c1f17 --- /dev/null +++ b/selfdrive/loggerd/tools/mark_all_uploaded.py @@ -0,0 +1,9 @@ +import os +from common.xattr import setxattr +from selfdrive.loggerd.uploader import UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE + +from selfdrive.loggerd.config import ROOT +for folder in os.walk(ROOT): + for file1 in folder[2]: + full_path = os.path.join(folder[0], file1) + setxattr(full_path, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 9fddc83c25..08c1f323ec 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -16,7 +16,7 @@ from cereal import log from common.hardware import HARDWARE from common.api import Api from common.params import Params -from common.xattr import getxattr, setxattr +from selfdrive.loggerd.xattr_cache import getxattr, setxattr from selfdrive.loggerd.config import ROOT from selfdrive.swaglog import cloudlog @@ -26,6 +26,7 @@ UPLOAD_ATTR_VALUE = b'1' fake_upload = os.getenv("FAKEUPLOAD") is not None + def raise_on_thread(t, exctype): '''Raises an exception in the threads with id tid''' for ctid, tobj in threading._active.items(): @@ -77,9 +78,9 @@ def is_on_hotspot(): try: result = subprocess.check_output(["ifconfig", "wlan0"], stderr=subprocess.STDOUT, encoding='utf8') result = re.findall(r"inet addr:((\d+\.){3}\d+)", result)[0][0] - return (result.startswith('192.168.43.') or # android - result.startswith('172.20.10.') or # ios - result.startswith('10.0.2.')) # toyota entune + return (result.startswith('192.168.43.') or # android + result.startswith('172.20.10.') or # ios + result.startswith('10.0.2.')) # toyota entune except Exception: return False @@ -127,11 +128,11 @@ class Uploader(): is_uploaded = True # deleter could have deleted if is_uploaded: continue - yield (name, key, fn) def next_file_to_upload(self, with_raw): upload_files = list(self.gen_upload_files()) + # try to upload qlog files first for name, key, fn in upload_files: if name in self.immediate_priority: @@ -236,14 +237,19 @@ def uploader_fn(exit_event): uploader = Uploader(dongle_id, ROOT) backoff = 0.1 + counter = 0 + should_upload = False while not exit_event.is_set(): offroad = params.get("IsOffroad") == b'1' allow_raw_upload = (params.get("IsUploadRawEnabled") != b"0") and offroad - on_hotspot = is_on_hotspot() - on_wifi = is_on_wifi() - should_upload = on_wifi and not on_hotspot + check_network = (counter % 12 == 0 if offroad else True) + if check_network: + on_hotspot = is_on_hotspot() + on_wifi = is_on_wifi() + should_upload = on_wifi and not on_hotspot d = uploader.next_file_to_upload(with_raw=allow_raw_upload and should_upload) + counter += 1 if d is None: # Nothing to upload time.sleep(60 if offroad else 5) continue @@ -264,5 +270,6 @@ def uploader_fn(exit_event): def main(): uploader_fn(threading.Event()) + if __name__ == "__main__": main() diff --git a/selfdrive/loggerd/xattr_cache.py b/selfdrive/loggerd/xattr_cache.py new file mode 100644 index 0000000000..aa97f0c777 --- /dev/null +++ b/selfdrive/loggerd/xattr_cache.py @@ -0,0 +1,13 @@ +from common.xattr import getxattr as getattr1 +from common.xattr import setxattr as setattr1 + +cached_attributes = {} +def getxattr(path, attr_name): + if (path, attr_name) not in cached_attributes: + response = getattr1(path, attr_name) + cached_attributes[(path, attr_name)] = response + return cached_attributes[(path, attr_name)] + +def setxattr(path, attr_name, attr_value): + cached_attributes.pop((path, attr_name), None) + return setattr1(path, attr_name, attr_value)