diff --git a/selfdrive/common/params.cc b/selfdrive/common/params.cc index ea09ee0705..a405d21609 100644 --- a/selfdrive/common/params.cc +++ b/selfdrive/common/params.cc @@ -153,7 +153,6 @@ std::unordered_map keys = { {"TrainingVersion", PERSISTENT}, {"UpdateAvailable", CLEAR_ON_MANAGER_START}, {"UpdateFailedCount", CLEAR_ON_MANAGER_START}, - {"UploadRaw", PERSISTENT}, {"Version", PERSISTENT}, {"VisionRadarToggle", PERSISTENT}, {"ApiCache_Device", PERSISTENT}, diff --git a/selfdrive/loggerd/tests/loggerd_tests_common.py b/selfdrive/loggerd/tests/loggerd_tests_common.py index 579aaf6042..1a16e343e6 100644 --- a/selfdrive/loggerd/tests/loggerd_tests_common.py +++ b/selfdrive/loggerd/tests/loggerd_tests_common.py @@ -56,7 +56,6 @@ class MockParams(): def __init__(self): self.params = { "DongleId": b"0000000000000000", - "UploadRaw": b"1", "IsOffroad": b"1", } diff --git a/selfdrive/loggerd/tests/test_uploader.py b/selfdrive/loggerd/tests/test_uploader.py old mode 100644 new mode 100755 index 8ebb001c75..8562f5f21d --- a/selfdrive/loggerd/tests/test_uploader.py +++ b/selfdrive/loggerd/tests/test_uploader.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +import os import time import threading import unittest @@ -50,66 +52,72 @@ class TestUploader(UploaderTestCase): self.end_event.set() self.up_thread.join() - def gen_files(self, lock=False): + def gen_files(self, lock=False, boot=True): f_paths = list() - for t in ["bootlog.bz2", "qlog.bz2", "rlog.bz2", "dcamera.hevc", "fcamera.hevc"]: + for t in ["qlog.bz2", "rlog.bz2", "dcamera.hevc", "fcamera.hevc"]: f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock)) + + if boot: + f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}.bz2", 1, lock=lock)) return f_paths - def gen_order(self, seg1, seg2): - keys = [f"{self.seg_format.format(i)}/qlog.bz2" for i in seg1] + def gen_order(self, seg1, seg2, boot=True): + keys = [] + if boot: + keys += [f"boot/{self.seg_format.format(i)}.bz2" for i in seg1] + keys += [f"boot/{self.seg_format2.format(i)}.bz2" for i in seg2] + keys += [f"{self.seg_format.format(i)}/qlog.bz2" for i in seg1] keys += [f"{self.seg_format2.format(i)}/qlog.bz2" for i in seg2] - for i in seg1: - keys += [f"{self.seg_format.format(i)}/{f}" for f in ['rlog.bz2', 'fcamera.hevc', 'dcamera.hevc']] - for i in seg2: - keys += [f"{self.seg_format2.format(i)}/{f}" for f in ['rlog.bz2', 'fcamera.hevc', 'dcamera.hevc']] - keys += [f"{self.seg_format.format(i)}/bootlog.bz2" for i in seg1] - keys += [f"{self.seg_format2.format(i)}/bootlog.bz2" for i in seg2] return keys def test_upload(self): - f_paths = self.gen_files(lock=False) + self.gen_files(lock=False) self.start_thread() # allow enough time that files could upload twice if there is a bug in the logic time.sleep(5) self.join_thread() - self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored") - self.assertFalse(len(log_handler.upload_order) < len(f_paths), "Some files failed to upload") - self.assertFalse(len(log_handler.upload_order) > len(f_paths), "Some files were uploaded twice") - for f_path in f_paths: - self.assertTrue(getxattr(f_path, uploader.UPLOAD_ATTR_NAME), "All files not uploaded") exp_order = self.gen_order([self.seg_num], []) + + self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored") + self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") + self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") + for f_path in exp_order: + self.assertTrue(getxattr(os.path.join(self.root, f_path), uploader.UPLOAD_ATTR_NAME), "All files not uploaded") + self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") def test_upload_ignored(self): self.set_ignore() - f_paths = self.gen_files(lock=False) + self.gen_files(lock=False) self.start_thread() # allow enough time that files could upload twice if there is a bug in the logic time.sleep(5) self.join_thread() - self.assertTrue(len(log_handler.upload_order) == 0, "Some files were not ignored") - self.assertFalse(len(log_handler.upload_ignored) < len(f_paths), "Some files failed to ignore") - self.assertFalse(len(log_handler.upload_ignored) > len(f_paths), "Some files were ignored twice") - for f_path in f_paths: - self.assertTrue(getxattr(f_path, uploader.UPLOAD_ATTR_NAME), "All files not ignored") exp_order = self.gen_order([self.seg_num], []) + + self.assertTrue(len(log_handler.upload_order) == 0, "Some files were not ignored") + self.assertFalse(len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore") + self.assertFalse(len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice") + for f_path in exp_order: + self.assertTrue(getxattr(os.path.join(self.root, f_path), uploader.UPLOAD_ATTR_NAME), "All files not ignored") + self.assertTrue(log_handler.upload_ignored == exp_order, "Files ignored in wrong order") def test_upload_files_in_create_order(self): - f_paths = list() seg1_nums = [0, 1, 2, 10, 20] for i in seg1_nums: self.seg_dir = self.seg_format.format(i) - f_paths += self.gen_files() + self.gen_files(boot=False) seg2_nums = [5, 50, 51] for i in seg2_nums: self.seg_dir = self.seg_format2.format(i) - f_paths += self.gen_files() + self.gen_files(boot=False) + + exp_order = self.gen_order(seg1_nums, seg2_nums, boot=False) self.start_thread() # allow enough time that files could upload twice if there is a bug in the logic @@ -117,15 +125,15 @@ class TestUploader(UploaderTestCase): self.join_thread() self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored") - self.assertFalse(len(log_handler.upload_order) < len(f_paths), "Some files failed to upload") - self.assertFalse(len(log_handler.upload_order) > len(f_paths), "Some files were uploaded twice") - for f_path in f_paths: - self.assertTrue(getxattr(f_path, uploader.UPLOAD_ATTR_NAME), "All files not uploaded") - exp_order = self.gen_order(seg1_nums, seg2_nums) + self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") + self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") + for f_path in exp_order: + self.assertTrue(getxattr(os.path.join(self.root, f_path), uploader.UPLOAD_ATTR_NAME), "All files not uploaded") + self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") def test_no_upload_with_lock_file(self): - f_paths = self.gen_files(lock=True) + f_paths = self.gen_files(lock=True, boot=False) self.start_thread() # allow enough time that files should have been uploaded if they would be uploaded @@ -137,4 +145,4 @@ class TestUploader(UploaderTestCase): if __name__ == "__main__": - unittest.main() + unittest.main(failfast=True) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index e99725f0c0..10bf218b03 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -60,8 +60,6 @@ class Uploader(): self.last_resp = None self.last_exc = None - self.raw_size = 0 - self.raw_count = 0 self.immediate_size = 0 self.immediate_count = 0 @@ -72,21 +70,16 @@ class Uploader(): self.immediate_folders = ["crash/", "boot/"] self.immediate_priority = {"qlog.bz2": 0, "qcamera.ts": 1} - self.high_priority = {"rlog.bz2": 0, "fcamera.hevc": 1, "dcamera.hevc": 2, "ecamera.hevc": 3} def get_upload_sort(self, name): if name in self.immediate_priority: return self.immediate_priority[name] - if name in self.high_priority: - return self.high_priority[name] + 100 return 1000 def list_upload_files(self): if not os.path.isdir(self.root): return - self.raw_size = 0 - self.raw_count = 0 self.immediate_size = 0 self.immediate_count = 0 @@ -116,38 +109,27 @@ class Uploader(): if name in self.immediate_priority: self.immediate_count += 1 self.immediate_size += os.path.getsize(fn) - else: - self.raw_count += 1 - self.raw_size += os.path.getsize(fn) except OSError: pass yield (name, key, fn) - def next_file_to_upload(self, with_raw): + def next_file_to_upload(self): upload_files = list(self.list_upload_files()) - # try to upload qlog files first for name, key, fn in upload_files: - if name in self.immediate_priority or any(f in fn for f in self.immediate_folders): + if any(f in fn for f in self.immediate_folders): return (key, fn) - if with_raw: - # then upload the full log files, rear and front camera files - for name, key, fn in upload_files: - if name in self.high_priority: - return (key, fn) - - # then upload other files - for name, key, fn in upload_files: - if not name.endswith('.lock') and not name.endswith(".tmp"): - return (key, fn) + for name, key, fn in upload_files: + if name in self.immediate_priority: + return (key, fn) return None def do_upload(self, key, fn): try: - url_resp = self.api.get("v1.4/"+self.dongle_id+"/upload_url/", timeout=10, path=key, access_token=self.api.get_token()) + url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token()) if url_resp.status_code == 412: self.last_resp = url_resp return @@ -226,8 +208,6 @@ class Uploader(): def get_msg(self): msg = messaging.new_message("uploaderState") us = msg.uploaderState - us.rawQueueSize = int(self.raw_size / 1e6) - us.rawQueueCount = self.raw_count us.immediateQueueSize = int(self.immediate_size / 1e6) us.immediateQueueCount = self.immediate_count us.lastTime = self.last_time @@ -260,10 +240,7 @@ def uploader_fn(exit_event): time.sleep(60 if offroad else 5) continue - good_internet = network_type in [NetworkType.wifi, NetworkType.ethernet] - allow_raw_upload = params.get_bool("UploadRaw") - - d = uploader.next_file_to_upload(with_raw=allow_raw_upload and good_internet and offroad) + d = uploader.next_file_to_upload() if d is None: # Nothing to upload if allow_sleep: time.sleep(60 if offroad else 5) diff --git a/selfdrive/ui/qt/offroad/settings.cc b/selfdrive/ui/qt/offroad/settings.cc index ed696f5a4e..487ba56247 100644 --- a/selfdrive/ui/qt/offroad/settings.cc +++ b/selfdrive/ui/qt/offroad/settings.cc @@ -59,12 +59,6 @@ TogglesPanel::TogglesPanel(SettingsWindow *parent) : ListWidget(parent) { "Use features, such as community supported hardware, from the open source community that are not maintained or supported by comma.ai and have not been confirmed to meet the standard safety model. Be extra cautious when using these features", "../assets/offroad/icon_shell.png", }, - { - "UploadRaw", - "Upload Raw Logs", - "Upload full logs and full resolution video by default while on Wi-Fi. If not enabled, individual logs can be marked for upload at useradmin.comma.ai.", - "../assets/offroad/icon_network.png", - }, { "RecordFront", "Record and Upload Driver Camera",