Model + camerad test (#1622)
* kind of works
* move that
* hack to get camerad to reliably terminate
* not sure why SIGTERM wasn't working before
* compare bytes
* clean up some hacks
* gitignore
* fix that
* WIP
* no reboot
* comparison works
* pretty print
* fix build
* run in jenkins
* python path
* space
* raise timeout
* new eon
* skip the copy
* spinner
* spin less
* update model ref commit
* reenable that
* clean up
* fix jenkinsfile
* parallel
* wrap it in a stage
* fix linter
* better progress
* lower timeout
Co-authored-by: Comma Device <device@comma.ai>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 63c8e8439b
commatwo_master
Willem Melching
5 years ago
committed by
GitHub
parent
3d848e0d9e
commit
ce6dd8d4e9
18 changed files with 271 additions and 54 deletions
@ -1,2 +1,8 @@ |
||||
out/ |
||||
docker_out/ |
||||
|
||||
process_replay/diff.txt |
||||
process_replay/model_diff.txt |
||||
|
||||
*.bz2 |
||||
*.hevc |
||||
|
||||
@ -0,0 +1,26 @@ |
||||
#!/usr/bin/env python3 |
||||
import os |
||||
import sys |
||||
import subprocess |
||||
|
||||
BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" |
||||
|
||||
def get_url(route_name, segment_num, log_type="rlog"): |
||||
ext = "hevc" if log_type in ["fcamera", "dcamera"] else "bz2" |
||||
return BASE_URL + "%s/%s/%s.%s" % (route_name.replace("|", "/"), segment_num, log_type, ext) |
||||
|
||||
def upload_file(path, name): |
||||
from azure.storage.blob import BlockBlobService |
||||
sas_token = os.getenv("TOKEN", None) |
||||
if sas_token is None: |
||||
sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \ |
||||
--expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n") |
||||
service = BlockBlobService(account_name="commadataci", sas_token=sas_token) |
||||
service.create_blob_from_path("openpilotci", name, path) |
||||
return "https://commadataci.blob.core.windows.net/openpilotci/" + name |
||||
|
||||
if __name__ == "__main__": |
||||
for f in sys.argv[1:]: |
||||
name = os.path.basename(f) |
||||
url = upload_file(f, name) |
||||
print(url) |
||||
@ -1,22 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
import os |
||||
import sys |
||||
import subprocess |
||||
|
||||
|
||||
def upload_file(path, name): |
||||
from azure.storage.blob import BlockBlobService |
||||
sas_token = os.getenv("TOKEN", None) |
||||
if sas_token is None: |
||||
cmd = "az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions \ |
||||
lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv" |
||||
sas_token = subprocess.check_output(cmd, shell=True).decode().strip("\n") |
||||
service = BlockBlobService(account_name="commadataci", sas_token=sas_token) |
||||
service.create_blob_from_path("openpilotci", name, path) |
||||
return "https://commadataci.blob.core.windows.net/openpilotci/" + name |
||||
|
||||
if __name__ == "__main__": |
||||
for f in sys.argv[1:]: |
||||
name = os.path.basename(f) |
||||
url = upload_file(f, name) |
||||
print(url) |
||||
@ -0,0 +1,75 @@ |
||||
#!/usr/bin/env python3 |
||||
import paramiko # pylint: disable=import-error |
||||
import os |
||||
import sys |
||||
import re |
||||
import time |
||||
import socket |
||||
|
||||
TEST_DIR = "/data/openpilotci" |
||||
|
||||
def run_test(name, test_func): |
||||
ssh = paramiko.SSHClient() |
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
||||
|
||||
key_file = open(os.path.join(os.path.dirname(__file__), "../../release/id_rsa_public")) |
||||
key = paramiko.RSAKey.from_private_key(key_file) |
||||
|
||||
print("SSH to phone {}".format(name)) |
||||
|
||||
# Try connecting for one minute |
||||
t_start = time.time() |
||||
while True: |
||||
try: |
||||
ssh.connect(hostname=name, port=8022, pkey=key, timeout=10) |
||||
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError): |
||||
print("Connection failed") |
||||
if time.time() - t_start > 60: |
||||
raise |
||||
else: |
||||
break |
||||
time.sleep(1) |
||||
|
||||
conn = ssh.invoke_shell() |
||||
branch = os.environ['GIT_BRANCH'] |
||||
commit = os.environ.get('GIT_COMMIT', branch) |
||||
|
||||
conn.send("uname -a\n") |
||||
|
||||
conn.send(f"cd {TEST_DIR}\n") |
||||
conn.send("git reset --hard\n") |
||||
conn.send("git fetch origin\n") |
||||
conn.send("git checkout %s\n" % commit) |
||||
conn.send("git clean -xdf\n") |
||||
conn.send("git submodule update --init\n") |
||||
conn.send("git submodule foreach --recursive git reset --hard\n") |
||||
conn.send("git submodule foreach --recursive git clean -xdf\n") |
||||
conn.send("echo \"git took $SECONDS seconds\"\n") |
||||
|
||||
test_func(conn) |
||||
|
||||
conn.send('echo "RESULT:" $?\n') |
||||
conn.send("exit\n") |
||||
return conn |
||||
|
||||
def test_modeld(conn): |
||||
conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n") |
||||
|
||||
if __name__ == "__main__": |
||||
eon_name = os.environ.get('eon_name', None) |
||||
|
||||
conn = run_test(eon_name, test_modeld) |
||||
|
||||
dat = b"" |
||||
|
||||
while True: |
||||
recvd = conn.recv(4096) |
||||
if len(recvd) == 0: |
||||
break |
||||
|
||||
dat += recvd |
||||
sys.stdout.buffer.write(recvd) |
||||
sys.stdout.flush() |
||||
|
||||
returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE) |
||||
sys.exit(int(returns[0])) |
||||
@ -1,2 +0,0 @@ |
||||
*.bz2 |
||||
diff.txt |
||||
@ -0,0 +1,103 @@ |
||||
#!/usr/bin/env python3 |
||||
import os |
||||
import sys |
||||
import time |
||||
from typing import Any |
||||
from tqdm import tqdm |
||||
|
||||
from common.android import ANDROID |
||||
if ANDROID: |
||||
os.environ['QCOM_REPLAY'] = "1" |
||||
import selfdrive.manager as manager |
||||
|
||||
from common.spinner import Spinner |
||||
import cereal.messaging as messaging |
||||
from tools.lib.framereader import FrameReader |
||||
from tools.lib.logreader import LogReader |
||||
from selfdrive.test.openpilotci import BASE_URL, get_url |
||||
from selfdrive.test.process_replay.compare_logs import compare_logs, save_log |
||||
from selfdrive.test.process_replay.test_processes import format_diff |
||||
from selfdrive.version import get_git_commit |
||||
|
||||
TEST_ROUTE = "5b7c365c50084530|2020-04-15--16-13-24" |
||||
|
||||
def camera_replay(lr, fr): |
||||
|
||||
spinner = Spinner() |
||||
|
||||
pm = messaging.PubMaster(['frame', 'liveCalibration']) |
||||
sm = messaging.SubMaster(['model']) |
||||
|
||||
# TODO: add dmonitoringmodeld |
||||
print("preparing procs") |
||||
manager.prepare_managed_process("camerad") |
||||
manager.prepare_managed_process("modeld") |
||||
try: |
||||
print("starting procs") |
||||
manager.start_managed_process("camerad") |
||||
manager.start_managed_process("modeld") |
||||
time.sleep(5) |
||||
print("procs started") |
||||
|
||||
cal = [msg for msg in lr if msg.which() == "liveCalibration"] |
||||
for msg in cal[:5]: |
||||
pm.send(msg.which(), msg.as_builder()) |
||||
|
||||
log_msgs = [] |
||||
frame_idx = 0 |
||||
for msg in tqdm(lr): |
||||
if msg.which() == "liveCalibrationd": |
||||
pm.send(msg.which(), msg.as_builder()) |
||||
elif msg.which() == "frame": |
||||
f = msg.as_builder() |
||||
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1] |
||||
f.frame.image = img.flatten().tobytes() |
||||
frame_idx += 1 |
||||
|
||||
pm.send(msg.which(), f) |
||||
log_msgs.append(messaging.recv_one(sm.sock['model'])) |
||||
|
||||
spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count)) |
||||
|
||||
if frame_idx >= fr.frame_count: |
||||
break |
||||
except KeyboardInterrupt: |
||||
pass |
||||
|
||||
print("replay done") |
||||
spinner.close() |
||||
manager.kill_managed_process('modeld') |
||||
time.sleep(2) |
||||
manager.kill_managed_process('camerad') |
||||
return log_msgs |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
|
||||
update = "--update" in sys.argv |
||||
|
||||
lr = LogReader(get_url(TEST_ROUTE, 0)) |
||||
fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera")) |
||||
|
||||
log_msgs = camera_replay(list(lr), fr) |
||||
|
||||
if update: |
||||
ref_commit = get_git_commit() |
||||
log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit) |
||||
save_log(log_fn, log_msgs) |
||||
with open("model_replay_ref_commit", "w") as f: |
||||
f.write(ref_commit) |
||||
else: |
||||
ref_commit = open("model_replay_ref_commit").read().strip() |
||||
log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit) |
||||
cmp_log = LogReader(BASE_URL + log_fn) |
||||
results: Any = {TEST_ROUTE: {}} |
||||
results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid']) |
||||
diff1, diff2, failed = format_diff(results, ref_commit) |
||||
|
||||
print(diff1) |
||||
with open("model_diff.txt", "w") as f: |
||||
f.write(diff2) |
||||
|
||||
sys.exit(int(failed)) |
||||
|
||||
@ -0,0 +1 @@ |
||||
18e43945403c4022b1de72237d89736e1a8ab4c7 |
||||
Loading…
Reference in new issue