sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

loggerd: add types to tests and use pathlib (#28943)

Browse Source
pull/28944/head
Cameron Clough 2 years ago committed by GitHub
parent dd98dc48a1
commit 5d45d1fadc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 50 additions and 42 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      system/loggerd/config.py
  2. 12
      system/loggerd/tests/fill.py
  3. 31
      system/loggerd/tests/loggerd_tests_common.py
  4. 22
      system/loggerd/tests/test_deleter.py
  5. 21
      system/loggerd/tests/test_uploader.py
  6. 2
      system/loggerd/xattr_cache.py

4
system/loggerd/config.py
Unescape View File

@ -5,7 +5,7 @@ from system.hardware import PC
if os.environ.get('LOG_ROOT', False): if os.environ.get('LOG_ROOT', False):
ROOT = os.environ['LOG_ROOT'] ROOT = os.environ['LOG_ROOT']
elif PC: elif PC:
ROOT = os.path.join(str(Path.home()), ".comma", "media", "0", "realdata") ROOT = str(Path.home() / ".comma" / "media" / "0" / "realdata")
else: else:
ROOT = '/data/media/0/realdata/' ROOT = '/data/media/0/realdata/'
@ -16,7 +16,7 @@ SEGMENT_LENGTH = 60
STATS_DIR_FILE_LIMIT = 10000 STATS_DIR_FILE_LIMIT = 10000
STATS_SOCKET = "ipc:///tmp/stats" STATS_SOCKET = "ipc:///tmp/stats"
if PC: if PC:
STATS_DIR = os.path.join(str(Path.home()), ".comma", "stats") STATS_DIR = str(Path.home() / ".comma" / "stats")
else: else:
STATS_DIR = "/data/stats/" STATS_DIR = "/data/stats/"
STATS_FLUSH_TIME_S = 60 STATS_FLUSH_TIME_S = 60

12
system/loggerd/tests/fill_eon.py → system/loggerd/tests/fill.py
Unescape View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Script to fill up EON with fake data""" """Script to fill up storage with fake data"""
import os from pathlib import Path
from system.loggerd.config import ROOT, get_available_percent from system.loggerd.config import ROOT, get_available_percent
from system.loggerd.tests.loggerd_tests_common import create_random_file from system.loggerd.tests.loggerd_tests_common import create_random_file
@ -10,13 +10,13 @@ from system.loggerd.tests.loggerd_tests_common import create_random_file
if __name__ == "__main__": if __name__ == "__main__":
segment_idx = 0 segment_idx = 0
while True: while True:
seg_name = "1970-01-01--00-00-00--%d" % segment_idx seg_name = f"1970-01-01--00-00-00--{segment_idx}"
seg_path = os.path.join(ROOT, seg_name) seg_path = Path(ROOT) / seg_name
print(seg_path) print(seg_path)
create_random_file(os.path.join(seg_path, 'fcamera.hevc'), 36) create_random_file(seg_path / "fcamera.hevc", 36)
create_random_file(os.path.join(seg_path, 'rlog.bz2'), 2) create_random_file(seg_path / "rlog.bz2", 2)
segment_idx += 1 segment_idx += 1

31
system/loggerd/tests/loggerd_tests_common.py
Unescape View File

@ -4,29 +4,29 @@ import shutil
import random import random
import tempfile import tempfile
import unittest import unittest
from pathlib import Path
from typing import Optional
import system.loggerd.uploader as uploader import system.loggerd.uploader as uploader
def create_random_file(file_path, size_mb, lock=False, xattr=None):
try:
os.mkdir(os.path.dirname(file_path))
except OSError:
pass
lock_path = file_path + ".lock" def create_random_file(file_path: Path, size_mb: float, lock: bool = False, xattr: Optional[bytes] = None) -> None:
file_path.parent.mkdir(parents=True, exist_ok=True)
if lock: if lock:
lock_path = str(file_path) + ".lock"
os.close(os.open(lock_path, os.O_CREAT | os.O_EXCL)) os.close(os.open(lock_path, os.O_CREAT | os.O_EXCL))
chunks = 128 chunks = 128
chunk_bytes = int(size_mb * 1024 * 1024 / chunks) chunk_bytes = int(size_mb * 1024 * 1024 / chunks)
data = os.urandom(chunk_bytes) data = os.urandom(chunk_bytes)
with open(file_path, 'wb') as f: with open(file_path, "wb") as f:
for _ in range(chunks): for _ in range(chunks):
f.write(data) f.write(data)
if xattr is not None: if xattr is not None:
uploader.setxattr(file_path, uploader.UPLOAD_ATTR_NAME, xattr) uploader.setxattr(str(file_path), uploader.UPLOAD_ATTR_NAME, xattr)
class MockResponse(): class MockResponse():
def __init__(self, text, status_code): def __init__(self, text, status_code):
@ -75,12 +75,18 @@ class MockParams():
class UploaderTestCase(unittest.TestCase): class UploaderTestCase(unittest.TestCase):
f_type = "UNKNOWN" f_type = "UNKNOWN"
root: Path
seg_num: int
seg_format: str
seg_format2: str
seg_dir: str
def set_ignore(self): def set_ignore(self):
uploader.Api = MockApiIgnore uploader.Api = MockApiIgnore
def setUp(self): def setUp(self):
self.root = tempfile.mkdtemp() self.root = Path(tempfile.mkdtemp())
uploader.ROOT = self.root # Monkey patch root dir uploader.ROOT = str(self.root) # Monkey patch root dir
uploader.Api = MockApi uploader.Api = MockApi
uploader.Params = MockParams uploader.Params = MockParams
uploader.fake_upload = True uploader.fake_upload = True
@ -98,8 +104,9 @@ class UploaderTestCase(unittest.TestCase):
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
raise raise
def make_file_with_data(self, f_dir, fn, size_mb=.1, lock=False, xattr=None): def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .1, lock: bool = False,
file_path = os.path.join(self.root, f_dir, fn) xattr: Optional[bytes] = None) -> Path:
file_path = self.root / f_dir / fn
create_random_file(file_path, size_mb, lock, xattr) create_random_file(file_path, size_mb, lock, xattr)
return file_path return file_path

22
system/loggerd/tests/test_deleter.py
Unescape View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os
import time import time
import threading import threading
import unittest import unittest
@ -21,7 +20,7 @@ class TestDeleter(UploaderTestCase):
super().setUp() super().setUp()
self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096) self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096)
deleter.os.statvfs = self.fake_statvfs deleter.os.statvfs = self.fake_statvfs
deleter.ROOT = self.root deleter.ROOT = str(self.root)
def start_thread(self): def start_thread(self):
self.end_event = threading.Event() self.end_event = threading.Event()
@ -39,11 +38,11 @@ class TestDeleter(UploaderTestCase):
self.start_thread() self.start_thread()
with Timeout(5, "Timeout waiting for file to be deleted"): with Timeout(5, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path): while f_path.exists():
time.sleep(0.01) time.sleep(0.01)
self.join_thread() self.join_thread()
self.assertFalse(os.path.exists(f_path), "File not deleted") self.assertFalse(f_path.exists(), "File not deleted")
def test_delete_files_in_create_order(self): def test_delete_files_in_create_order(self):
f_path_1 = self.make_file_with_data(self.seg_dir, self.f_type) f_path_1 = self.make_file_with_data(self.seg_dir, self.f_type)
@ -55,14 +54,13 @@ class TestDeleter(UploaderTestCase):
self.start_thread() self.start_thread()
with Timeout(5, "Timeout waiting for file to be deleted"): with Timeout(5, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path_1) and os.path.exists(f_path_2): while f_path_1.exists() and f_path_2.exists():
time.sleep(0.01) time.sleep(0.01)
self.join_thread() self.join_thread()
self.assertFalse(os.path.exists(f_path_1), "Older file not deleted") self.assertFalse(f_path_1.exists(), "Older file not deleted")
self.assertTrue(f_path_2.exists(), "Newer file deleted before older file")
self.assertTrue(os.path.exists(f_path_2), "Newer file deleted before older file")
def test_no_delete_when_available_space(self): def test_no_delete_when_available_space(self):
f_path = self.make_file_with_data(self.seg_dir, self.f_type) f_path = self.make_file_with_data(self.seg_dir, self.f_type)
@ -75,14 +73,14 @@ class TestDeleter(UploaderTestCase):
try: try:
with Timeout(2, "Timeout waiting for file to be deleted"): with Timeout(2, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path): while f_path.exists():
time.sleep(0.01) time.sleep(0.01)
except TimeoutException: except TimeoutException:
pass pass
finally: finally:
self.join_thread() self.join_thread()
self.assertTrue(os.path.exists(f_path), "File deleted with available space") self.assertTrue(f_path.exists(), "File deleted with available space")
def test_no_delete_with_lock_file(self): def test_no_delete_with_lock_file(self):
f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True) f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True)
@ -91,14 +89,14 @@ class TestDeleter(UploaderTestCase):
try: try:
with Timeout(2, "Timeout waiting for file to be deleted"): with Timeout(2, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path): while f_path.exists():
time.sleep(0.01) time.sleep(0.01)
except TimeoutException: except TimeoutException:
pass pass
finally: finally:
self.join_thread() self.join_thread()
self.assertTrue(os.path.exists(f_path), "File deleted when locked") self.assertTrue(f_path.exists(), "File deleted when locked")
if __name__ == "__main__": if __name__ == "__main__":

21
system/loggerd/tests/test_uploader.py
Unescape View File

@ -5,6 +5,8 @@ import threading
import unittest import unittest
import logging import logging
import json import json
from pathlib import Path
from typing import List, Optional
from system.swaglog import cloudlog from system.swaglog import cloudlog
from system.loggerd.uploader import uploader_fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE from system.loggerd.uploader import uploader_fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
@ -50,8 +52,8 @@ class TestUploader(UploaderTestCase):
self.end_event.set() self.end_event.set()
self.up_thread.join() self.up_thread.join()
def gen_files(self, lock=False, xattr=None, boot=True): def gen_files(self, lock=False, xattr: Optional[bytes] = None, boot=True) -> List[Path]:
f_paths = list() f_paths = []
for t in ["qlog", "rlog", "dcamera.hevc", "fcamera.hevc"]: for t in ["qlog", "rlog", "dcamera.hevc", "fcamera.hevc"]:
f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock, xattr=xattr)) f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock, xattr=xattr))
@ -59,7 +61,7 @@ class TestUploader(UploaderTestCase):
f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}", 1, lock=lock, xattr=xattr)) f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}", 1, lock=lock, xattr=xattr))
return f_paths return f_paths
def gen_order(self, seg1, seg2, boot=True): def gen_order(self, seg1: List[int], seg2: List[int], boot=True) -> List[str]:
keys = [] keys = []
if boot: if boot:
keys += [f"boot/{self.seg_format.format(i)}.bz2" for i in seg1] keys += [f"boot/{self.seg_format.format(i)}.bz2" for i in seg1]
@ -82,7 +84,7 @@ class TestUploader(UploaderTestCase):
self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
for f_path in exp_order: for f_path in exp_order:
self.assertEqual(os.getxattr(os.path.join(self.root, f_path.replace('.bz2', '')), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded")
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
@ -100,7 +102,7 @@ class TestUploader(UploaderTestCase):
self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
for f_path in exp_order: for f_path in exp_order:
self.assertEqual(os.getxattr(os.path.join(self.root, f_path.replace('.bz2', '')), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded")
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
@ -119,7 +121,7 @@ class TestUploader(UploaderTestCase):
self.assertFalse(len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore") self.assertFalse(len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore")
self.assertFalse(len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice") self.assertFalse(len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice")
for f_path in exp_order: for f_path in exp_order:
self.assertEqual(os.getxattr(os.path.join(self.root, f_path.replace('.bz2', '')), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not ignored") self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not ignored")
self.assertTrue(log_handler.upload_ignored == exp_order, "Files ignored in wrong order") self.assertTrue(log_handler.upload_ignored == exp_order, "Files ignored in wrong order")
@ -144,7 +146,7 @@ class TestUploader(UploaderTestCase):
self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
for f_path in exp_order: for f_path in exp_order:
self.assertEqual(os.getxattr(os.path.join(self.root, f_path.replace('.bz2', '')), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded")
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
@ -159,7 +161,7 @@ class TestUploader(UploaderTestCase):
self.join_thread() self.join_thread()
for f_path in f_paths: for f_path in f_paths:
fn = f_path.replace('.bz2', '') fn = f_path.with_suffix(f_path.suffix.replace(".bz2", ""))
uploaded = UPLOAD_ATTR_NAME in os.listxattr(fn) and os.getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE uploaded = UPLOAD_ATTR_NAME in os.listxattr(fn) and os.getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
self.assertFalse(uploaded, "File upload when locked") self.assertFalse(uploaded, "File upload when locked")
@ -180,7 +182,8 @@ class TestUploader(UploaderTestCase):
self.join_thread() self.join_thread()
for f_path in f_paths: for f_path in f_paths:
self.assertFalse(os.path.isfile(f_path + ".lock"), "File lock not cleared on startup") lock_path = f_path.with_suffix(f_path.suffix + ".lock")
self.assertFalse(lock_path.is_file(), "File lock not cleared on startup")
if __name__ == "__main__": if __name__ == "__main__":

2
system/loggerd/xattr_cache.py
Unescape View File

@ -1,6 +1,6 @@
import os import os
import errno import errno
from typing import Dict, Tuple, Optional from typing import Dict, Optional, Tuple
_cached_attributes: Dict[Tuple, Optional[bytes]] = {} _cached_attributes: Dict[Tuple, Optional[bytes]] = {}

Write Preview
Loading…
Cancel
Save