plotjuggler: add mac support (#23344)

* plotjuggler: add mac support

* fix test?

* update readme

* oops

* fix

* cleanup

* typo

* works

* little more

* fix test

* little faster
old-commit-hash: c26f294218

tools/plotjuggler/README.md

@@ -4,17 +4,16 @@
 
 ## Installation
 
-**NOTE: this is Ubuntu only for now. Pull requests for macOS support are welcome.**
-
 Once you've cloned and are in openpilot, this command will download PlotJuggler and install our plugins:
 
-`cd tools/plotjuggler && ./install.sh`
+`cd tools/plotjuggler && ./juggle.py --install`
 
 ## Usage
 
 ```
 $ ./juggle.py -h
-usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [route_name] [segment_number] [segment_count]
+usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [--install]
+                 [route_name] [segment_number] [segment_count]
 
 A helper to run PlotJuggler on openpilot routes
 
@@ -30,6 +29,7 @@ optional arguments:
   --can              Parse CAN data (default: False)
   --stream           Start PlotJuggler in streaming mode (default: False)
   --layout [LAYOUT]  Run PlotJuggler with a pre-defined layout (default: None)
+  --install          Install or update PlotJuggler + plugins (default: False)
 ```
 
 Example:

tools/plotjuggler/install.sh

@@ -1,10 +0,0 @@
-#!/bin/bash -e
-
-mkdir -p bin
-cd bin
-
-for lib_name in libDataLoadRlog.so libDataStreamCereal.so plotjuggler; do
-  wget https://github.com/commaai/PlotJuggler/releases/download/latest/${lib_name}.tar.gz
-  tar -xf ${lib_name}.tar.gz
-  rm ${lib_name}.tar.gz
-done

tools/plotjuggler/juggle.py

@@ -2,9 +2,13 @@
 import os
 import sys
 import multiprocessing
+import platform
+import shutil
 import subprocess
+import tarfile
+import tempfile
+import requests
 import argparse
-from tempfile import NamedTemporaryFile
 
 from common.basedir import BASEDIR
 from selfdrive.test.process_replay.compare_logs import save_log
@@ -17,9 +21,32 @@ from urllib.parse import urlparse, parse_qs
 juggle_dir = os.path.dirname(os.path.realpath(__file__))
 
 DEMO_ROUTE = "4cf7a6ad03080c90|2021-09-29--13-46-36"
+RELEASES_URL="https://github.com/commaai/PlotJuggler/releases/download/latest"
+INSTALL_DIR = os.path.join(juggle_dir, "bin")
+
+
+def install():
+  m = f"{platform.system()}-{platform.machine()}"
+  supported = ("Linux-x86_64", "Darwin-arm64", "Darwin-x86_64")
+  if m not in supported:
+    raise Exception(f"Unsupported platform: '{m}'. Supported platforms: {supported}")
+
+  if os.path.exists(INSTALL_DIR):
+    shutil.rmtree(INSTALL_DIR)
+  os.mkdir(INSTALL_DIR)
+
+  url = os.path.join(RELEASES_URL, m + ".tar.gz")
+  with requests.get(url, stream=True) as r, tempfile.NamedTemporaryFile() as tmp:
+    r.raise_for_status()
+    with open(tmp.name, 'wb') as tmpf:
+      for chunk in r.iter_content(chunk_size=1024*1024):
+        tmpf.write(chunk)
+
+    with tarfile.open(tmp.name) as tar:
+      tar.extractall(path=INSTALL_DIR)
+
 
 def load_segment(segment_name):
-  print(f"Loading {segment_name}")
   if segment_name is None:
     return []
 
@@ -29,25 +56,26 @@ def load_segment(segment_name):
     print(f"Error parsing {segment_name}: {e}")
     return []
 
+
 def start_juggler(fn=None, dbc=None, layout=None):
   env = os.environ.copy()
   env["BASEDIR"] = BASEDIR
-  pj = os.getenv("PLOTJUGGLER_PATH", os.path.join(juggle_dir, "bin/plotjuggler"))
+  env["PATH"] = f"{INSTALL_DIR}:{os.getenv('PATH', '')}"
-
   if dbc:
     env["DBC_NAME"] = dbc
 
-  extra_args = []
+  extra_args = ""
   if fn is not None:
-    extra_args.append(f'-d {fn}')
+    extra_args += f"-d {fn}"
-
   if layout is not None:
-    extra_args.append(f'-l {layout}')
+    extra_args += f"-l {layout}"
+
+  subprocess.call(f'plotjuggler --plugin_folders {INSTALL_DIR} {extra_args}',
+                  shell=True, env=env, cwd=juggle_dir)
 
-  extra_args = " ".join(extra_args)
-  subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} {extra_args}', shell=True, env=env, cwd=juggle_dir)
 
 def juggle_route(route_name, segment_number, segment_count, qlog, can, layout):
+  # TODO: abstract out the cabana stuff
   if 'cabana' in route_name:
     query = parse_qs(urlparse(route_name).query)
     api = CommaApi(get_token())
@@ -62,8 +90,8 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout):
     logs = logs[segment_number:segment_number+segment_count]
 
   if None in logs:
-    fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ")
+    ans = input(f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) ")
-    if fallback_answer == 'y':
+    if ans == 'y':
       logs = r.qlog_paths()
       if segment_number is not None:
         logs = logs[segment_number:segment_number+segment_count]
@@ -85,17 +113,17 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout):
     try:
       DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values", fromlist=['DBC']).DBC
       dbc = DBC[cp.carParams.carFingerprint]['pt']
-    except (ImportError, KeyError, AttributeError):
+    except Exception:
       pass
     break
 
-  tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir)
+  with tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) as tmp:
-  save_log(tempfile.name, all_data, compress=False)
+    save_log(tmp.name, all_data, compress=False)
     del all_data
+    start_juggler(tmp.name, dbc, layout)
 
-  start_juggler(tempfile.name, dbc, layout)
 
-def get_arg_parser():
+if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="A helper to run PlotJuggler on openpilot routes",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
@@ -104,17 +132,18 @@ def get_arg_parser():
   parser.add_argument("--can", action="store_true", help="Parse CAN data")
   parser.add_argument("--stream", action="store_true", help="Start PlotJuggler in streaming mode")
   parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout")
+  parser.add_argument("--install", action="store_true", help="Install or update PlotJuggler + plugins")
   parser.add_argument("route_name", nargs='?', help="The route name to plot (cabana share URL accepted)")
   parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment to plot")
   parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot", default=1)
-  return parser
-
-if __name__ == "__main__":
-  arg_parser = get_arg_parser()
   if len(sys.argv) == 1:
-    arg_parser.print_help()
+    parser.print_help()
+    sys.exit()
+  args = parser.parse_args()
+
+  if args.install:
+    install()
     sys.exit()
-  args = arg_parser.parse_args(sys.argv[1:])
 
   if args.stream:
     start_juggler(layout=args.layout)

tools/plotjuggler/test_plotjuggler.py

@@ -7,27 +7,20 @@ import unittest
 
 from common.basedir import BASEDIR
 from common.timeout import Timeout
-from selfdrive.test.openpilotci import get_url
+from tools.plotjuggler.juggle import install
 
 class TestPlotJuggler(unittest.TestCase):
 
-  def test_install(self):
+  def test_demo(self):
-    exit_code = os.system(os.path.join(BASEDIR, "tools/plotjuggler/install.sh"))
+    install()
-    self.assertEqual(exit_code, 0)
 
-  def test_run(self):
+    pj = os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")
+    p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1 --qlog',
+                         stderr=subprocess.PIPE, shell=True, start_new_session=True)
 
-    test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0)
+    # Wait for "Done reading Rlog data" signal from the plugin
-
-    # Launch PlotJuggler with the executable in the bin directory
-    os.environ["PLOTJUGGLER_PATH"] = f'{os.path.join(BASEDIR, "tools/plotjuggler/bin/plotjuggler")}'
-    p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} \
-    "{test_url}"', stderr=subprocess.PIPE, shell=True,
-    start_new_session=True)
-
-    # Wait max 60 seconds for the "Done reading Rlog data" signal from the plugin
     output = "\n"
-    with Timeout(120, error_msg=output):
+    with Timeout(180, error_msg=output):
       while output.splitlines()[-1] != "Done reading Rlog data":
         output += p.stderr.readline().decode("utf-8")