diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 50557fffaa..87988171fb 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -23,6 +23,8 @@ NetworkType = log.DeviceState.NetworkType UPLOAD_ATTR_NAME = 'user.upload' UPLOAD_ATTR_VALUE = b'1' +UPLOAD_QLOG_QCAM_MAX_SIZE = 1e7 # 10 MB + allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1")) force_wifi = os.getenv("FORCEWIFI") is not None fake_upload = os.getenv("FAKEUPLOAD") is not None @@ -121,11 +123,11 @@ class Uploader(): for name, key, fn in upload_files: if any(f in fn for f in self.immediate_folders): - return (key, fn) + return (name, key, fn) for name, key, fn in upload_files: if name in self.immediate_priority: - return (key, fn) + return (name, key, fn) return None @@ -172,7 +174,7 @@ class Uploader(): return self.last_resp - def upload(self, key, fn, network_type, metered): + def upload(self, name, key, fn, network_type, metered): try: sz = os.path.getsize(fn) except OSError: @@ -182,22 +184,15 @@ class Uploader(): cloudlog.event("upload_start", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered) if sz == 0: - try: - # tag files of 0 size as uploaded - setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) - except OSError: - cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz) + # tag files of 0 size as uploaded + success = True + elif name in self.immediate_priority and sz > UPLOAD_QLOG_QCAM_MAX_SIZE: + cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz) success = True else: start_time = time.monotonic() stat = self.normal_upload(key, fn) if stat is not None and stat.status_code in (200, 201, 401, 403, 412): - try: - # tag file as uploaded - setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) - except OSError: - cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz) - self.last_filename = fn self.last_time = time.monotonic() - start_time self.last_speed = (sz / 1e6) / self.last_time @@ -207,6 +202,13 @@ class Uploader(): success = False cloudlog.event("upload_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered) + if success: + # tag file as uploaded + try: + setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) + except OSError: + cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz) + return success def get_msg(self): @@ -258,13 +260,13 @@ def uploader_fn(exit_event): time.sleep(60 if offroad else 5) continue - key, fn = d + name, key, fn = d # qlogs and bootlogs need to be compressed before uploading if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.bz2')): key += ".bz2" - success = uploader.upload(key, fn, sm['deviceState'].networkType.raw, sm['deviceState'].networkMetered) + success = uploader.upload(name, key, fn, sm['deviceState'].networkType.raw, sm['deviceState'].networkMetered) if success: backoff = 0.1 elif allow_sleep: