PlotJuggler CI with build from source (#20166)

* CI attempt

* fixing location

* fix

* fixes

* hardcode test_url for CI

* fixes

* cd fix

* Update tools/plotjuggler/juggle.py

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

* Update tools/plotjuggler/test_plotjuggler.py

* fixes

* fixes

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 9144dd7c2f

.github/workflows/tools_tests.yaml

@@ -3,6 +3,17 @@ on:
   push:
   pull_request:
 
+env:
+  BASE_IMAGE: openpilot-base
+  DOCKER_REGISTRY: ghcr.io/commaai
+
+  BUILD: |
+      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
+      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
+      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e \
+       GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
+
 jobs:
   plotjuggler:
     name: plotjuggler
@@ -12,7 +23,12 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: true
-    - name: Install
+    - name: Build docker image
-      run: cd tools/plotjuggler && ./install.sh
+      run: eval "$BUILD"
-    #- name: Juggle Demo
+    - name: Unit test
-    #  run: cd tools/plotjuggler && ./juggle.py
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal && \
+                        apt-get update && \
+                        apt-get install -y libdw-dev libqt5svg5-dev libqt5x11extras5-dev && \
+                        cd /tmp/openpilot/tools/plotjuggler && \
+                        ./test_plotjuggler.py"

tools/plotjuggler/README.md

@@ -1,7 +1,7 @@
 # PlotJuggler
 We've extended [PlotJuggler](https://github.com/facontidavide/PlotJuggler) to plot all of your openpilot logs
 
-Here's our fork: https://github.com/commaai/PlotJuggler 
+Here's our fork: https://github.com/commaai/PlotJuggler
 
 ## Installation
 

tools/plotjuggler/install.sh

@@ -1,6 +1,12 @@
 #!/bin/bash -e
 
-sudo snap install plotjuggler
+mkdir -p bin
+cd bin
+
 wget https://github.com/commaai/PlotJuggler/releases/download/latest/libDataLoadRlog.so.tar.gz
 tar -xf libDataLoadRlog.so.tar.gz
 rm libDataLoadRlog.so.tar.gz
+
+wget https://github.com/commaai/PlotJuggler/releases/download/latest/plotjuggler.tar.gz
+tar -xf plotjuggler.tar.gz
+rm plotjuggler.tar.gz

tools/plotjuggler/juggle.py

@@ -31,12 +31,17 @@ def juggle_file(fn, dbc=None):
   if dbc:
     env["DBC_NAME"] = dbc
 
-  subprocess.call(f"plotjuggler --plugin_folders {juggle_dir} -d {fn}", shell=True, env=env, cwd=juggle_dir)
+  pj = os.getenv("PLOTJUGGLER_PATH", "plotjuggler")
+  subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} -d {fn}', shell=True, env=env, cwd=juggle_dir)
 
 def juggle_route(route_name, segment_number, qlog):
-  r = Route(route_name)
 
-  logs = r.qlog_paths() if qlog else r.log_paths()
+  if route_name.startswith("http://") or route_name.startswith("https://"):
+    logs = [route_name]
+  else:
+    r = Route(route_name)
+    logs = r.qlog_paths() if qlog else r.log_paths()
+
   if segment_number is not None:
     logs = logs[segment_number:segment_number+1]
 
@@ -81,7 +86,6 @@ def get_arg_parser():
   return parser
 
 if __name__ == "__main__":
-
   arg_parser = get_arg_parser()
   if len(sys.argv) == 1:
     arg_parser.print_help()

tools/plotjuggler/test_plotjuggler.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import os
+import signal
+import subprocess
+import time
+import unittest
+
+from common.basedir import BASEDIR
+from common.timeout import Timeout
+from selfdrive.test.openpilotci import get_url
+
+class TestPlotJuggler(unittest.TestCase):
+
+  def test_install(self):
+    exit_code = os.system(os.path.join(BASEDIR, "tools/plotjuggler/install.sh"))
+    self.assertEqual(exit_code, 0)
+
+  def test_run(self):
+
+    test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0)
+
+    # Launch PlotJuggler with the executable in the bin directory
+    os.environ["PLOTJUGGLER_PATH"] = f'{os.path.join(BASEDIR, "tools/plotjuggler/bin/plotjuggler")}'
+    p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} \
+    "{test_url}"', stderr=subprocess.PIPE, shell=True,
+    start_new_session=True)
+
+    # Wait max 60 seconds for the "Done reading Rlog data" signal from the plugin
+    output = "\n"
+    with Timeout(60, error_msg=output):
+      while output.splitlines()[-1] != "Done reading Rlog data":
+        output += p.stderr.readline().decode("utf-8")
+
+    # ensure plotjuggler didn't crash after exiting the plugin
+    time.sleep(15)
+    self.assertEqual(p.poll(), None)
+    os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+
+if __name__ == "__main__":
+  unittest.main()