From c26f294218a72e4a42d90b6560bde32e1a9cf5cb Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 16:44:57 -0800 Subject: [PATCH] plotjuggler: add mac support (#23344) * plotjuggler: add mac support * fix test? * update readme * oops * fix * cleanup * typo * works * little more * fix test * little faster --- tools/plotjuggler/README.md | 8 +-- tools/plotjuggler/install.sh | 10 ---- tools/plotjuggler/juggle.py | 77 ++++++++++++++++++--------- tools/plotjuggler/test_plotjuggler.py | 23 +++----- 4 files changed, 65 insertions(+), 53 deletions(-) delete mode 100755 tools/plotjuggler/install.sh diff --git a/tools/plotjuggler/README.md b/tools/plotjuggler/README.md index 8ab150e821..e2370d5a80 100644 --- a/tools/plotjuggler/README.md +++ b/tools/plotjuggler/README.md @@ -4,17 +4,16 @@ ## Installation -**NOTE: this is Ubuntu only for now. Pull requests for macOS support are welcome.** - Once you've cloned and are in openpilot, this command will download PlotJuggler and install our plugins: -`cd tools/plotjuggler && ./install.sh` +`cd tools/plotjuggler && ./juggle.py --install` ## Usage ``` $ ./juggle.py -h -usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [route_name] [segment_number] [segment_count] +usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [--install] + [route_name] [segment_number] [segment_count] A helper to run PlotJuggler on openpilot routes @@ -30,6 +29,7 @@ optional arguments: --can Parse CAN data (default: False) --stream Start PlotJuggler in streaming mode (default: False) --layout [LAYOUT] Run PlotJuggler with a pre-defined layout (default: None) + --install Install or update PlotJuggler + plugins (default: False) ``` Example: diff --git a/tools/plotjuggler/install.sh b/tools/plotjuggler/install.sh deleted file mode 100755 index 27cb0dfb5d..0000000000 --- a/tools/plotjuggler/install.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -mkdir -p bin -cd bin - -for lib_name in libDataLoadRlog.so libDataStreamCereal.so plotjuggler; do - wget https://github.com/commaai/PlotJuggler/releases/download/latest/${lib_name}.tar.gz - tar -xf ${lib_name}.tar.gz - rm ${lib_name}.tar.gz -done diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index d488750701..c9bd1abdd7 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -2,9 +2,13 @@ import os import sys import multiprocessing +import platform +import shutil import subprocess +import tarfile +import tempfile +import requests import argparse -from tempfile import NamedTemporaryFile from common.basedir import BASEDIR from selfdrive.test.process_replay.compare_logs import save_log @@ -17,9 +21,32 @@ from urllib.parse import urlparse, parse_qs juggle_dir = os.path.dirname(os.path.realpath(__file__)) DEMO_ROUTE = "4cf7a6ad03080c90|2021-09-29--13-46-36" +RELEASES_URL="https://github.com/commaai/PlotJuggler/releases/download/latest" +INSTALL_DIR = os.path.join(juggle_dir, "bin") + + +def install(): + m = f"{platform.system()}-{platform.machine()}" + supported = ("Linux-x86_64", "Darwin-arm64", "Darwin-x86_64") + if m not in supported: + raise Exception(f"Unsupported platform: '{m}'. Supported platforms: {supported}") + + if os.path.exists(INSTALL_DIR): + shutil.rmtree(INSTALL_DIR) + os.mkdir(INSTALL_DIR) + + url = os.path.join(RELEASES_URL, m + ".tar.gz") + with requests.get(url, stream=True) as r, tempfile.NamedTemporaryFile() as tmp: + r.raise_for_status() + with open(tmp.name, 'wb') as tmpf: + for chunk in r.iter_content(chunk_size=1024*1024): + tmpf.write(chunk) + + with tarfile.open(tmp.name) as tar: + tar.extractall(path=INSTALL_DIR) + def load_segment(segment_name): - print(f"Loading {segment_name}") if segment_name is None: return [] @@ -29,25 +56,26 @@ def load_segment(segment_name): print(f"Error parsing {segment_name}: {e}") return [] + def start_juggler(fn=None, dbc=None, layout=None): env = os.environ.copy() env["BASEDIR"] = BASEDIR - pj = os.getenv("PLOTJUGGLER_PATH", os.path.join(juggle_dir, "bin/plotjuggler")) - + env["PATH"] = f"{INSTALL_DIR}:{os.getenv('PATH', '')}" if dbc: env["DBC_NAME"] = dbc - extra_args = [] + extra_args = "" if fn is not None: - extra_args.append(f'-d {fn}') - + extra_args += f"-d {fn}" if layout is not None: - extra_args.append(f'-l {layout}') + extra_args += f"-l {layout}" + + subprocess.call(f'plotjuggler --plugin_folders {INSTALL_DIR} {extra_args}', + shell=True, env=env, cwd=juggle_dir) - extra_args = " ".join(extra_args) - subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} {extra_args}', shell=True, env=env, cwd=juggle_dir) def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): + # TODO: abstract out the cabana stuff if 'cabana' in route_name: query = parse_qs(urlparse(route_name).query) api = CommaApi(get_token()) @@ -62,8 +90,8 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): logs = logs[segment_number:segment_number+segment_count] if None in logs: - fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ") - if fallback_answer == 'y': + ans = input(f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) ") + if ans == 'y': logs = r.qlog_paths() if segment_number is not None: logs = logs[segment_number:segment_number+segment_count] @@ -85,17 +113,17 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): try: DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values", fromlist=['DBC']).DBC dbc = DBC[cp.carParams.carFingerprint]['pt'] - except (ImportError, KeyError, AttributeError): + except Exception: pass break - tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) - save_log(tempfile.name, all_data, compress=False) - del all_data + with tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) as tmp: + save_log(tmp.name, all_data, compress=False) + del all_data + start_juggler(tmp.name, dbc, layout) - start_juggler(tempfile.name, dbc, layout) -def get_arg_parser(): +if __name__ == "__main__": parser = argparse.ArgumentParser(description="A helper to run PlotJuggler on openpilot routes", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -104,17 +132,18 @@ def get_arg_parser(): parser.add_argument("--can", action="store_true", help="Parse CAN data") parser.add_argument("--stream", action="store_true", help="Start PlotJuggler in streaming mode") parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout") + parser.add_argument("--install", action="store_true", help="Install or update PlotJuggler + plugins") parser.add_argument("route_name", nargs='?', help="The route name to plot (cabana share URL accepted)") parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment to plot") parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot", default=1) - return parser - -if __name__ == "__main__": - arg_parser = get_arg_parser() if len(sys.argv) == 1: - arg_parser.print_help() + parser.print_help() + sys.exit() + args = parser.parse_args() + + if args.install: + install() sys.exit() - args = arg_parser.parse_args(sys.argv[1:]) if args.stream: start_juggler(layout=args.layout) diff --git a/tools/plotjuggler/test_plotjuggler.py b/tools/plotjuggler/test_plotjuggler.py index 3e695c1990..edaec9c80a 100755 --- a/tools/plotjuggler/test_plotjuggler.py +++ b/tools/plotjuggler/test_plotjuggler.py @@ -7,27 +7,20 @@ import unittest from common.basedir import BASEDIR from common.timeout import Timeout -from selfdrive.test.openpilotci import get_url +from tools.plotjuggler.juggle import install class TestPlotJuggler(unittest.TestCase): - def test_install(self): - exit_code = os.system(os.path.join(BASEDIR, "tools/plotjuggler/install.sh")) - self.assertEqual(exit_code, 0) + def test_demo(self): + install() - def test_run(self): + pj = os.path.join(BASEDIR, "tools/plotjuggler/juggle.py") + p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1 --qlog', + stderr=subprocess.PIPE, shell=True, start_new_session=True) - test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0) - - # Launch PlotJuggler with the executable in the bin directory - os.environ["PLOTJUGGLER_PATH"] = f'{os.path.join(BASEDIR, "tools/plotjuggler/bin/plotjuggler")}' - p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} \ - "{test_url}"', stderr=subprocess.PIPE, shell=True, - start_new_session=True) - - # Wait max 60 seconds for the "Done reading Rlog data" signal from the plugin + # Wait for "Done reading Rlog data" signal from the plugin output = "\n" - with Timeout(120, error_msg=output): + with Timeout(180, error_msg=output): while output.splitlines()[-1] != "Done reading Rlog data": output += p.stderr.readline().decode("utf-8")