Support streaming in juggle.py (#20982)

* add --stream to juggle.py

* better

* rm old code

* for loopify

* clean up

* easier to pick out

* start by default

* don't exit

* Update tools/plotjuggler/juggle.py

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

* Revert "start by default"

This reverts commit 6cb481a6fd.

* Revert "don't exit"

This reverts commit 861b423eb8.

* rm

* move out of juggle_route

* Update PJ README.md

* Update tools/plotjuggler/README.md

* this is in the readme

* Updates

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

tools/plotjuggler/README.md

@@ -1,6 +1,6 @@
 # PlotJuggler
 
-We've extended [PlotJuggler](https://github.com/facontidavide/PlotJuggler) to plot all of your openpilot logs. Check out our plugin: https://github.com/commaai/PlotJuggler.
+We've extended [PlotJuggler](https://github.com/facontidavide/PlotJuggler) to plot all of your openpilot logs. Check out our plugins: https://github.com/commaai/PlotJuggler.
 
 ## Installation
 
@@ -16,7 +16,7 @@ Usage requires an installation of PlotJuggler. On systems with snap (e.g. Ubuntu
 
 ```
 batman@z840-openpilot:~/openpilot/tools/plotjuggler$ ./juggle.py -h
-usage: juggle.py [-h] [--qlog] [--layout [LAYOUT]] [route_name] [segment_number]
+usage: juggle.py [-h] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [route_name] [segment_number]
 
 PlotJuggler plugin for reading rlogs
 
@@ -27,6 +27,8 @@ positional arguments:
 optional arguments:
   -h, --help         show this help message and exit
   --qlog             Use qlogs (default: False)
+  --can              Parse CAN data (default: False)
+  --stream           Start PlotJuggler without a route to stream data using Cereal (default: False)
   --layout [LAYOUT]  Run PlotJuggler with a pre-defined layout (default: None)
 ```
 
@@ -34,7 +36,18 @@ Example:
 
 `./juggle.py "0982d79ebb0de295|2021-01-17--17-13-08"`
 
-## Demo:
+### Streaming
+
+To get started exploring and plotting data live in your car, you can start PlotJuggler in streaming mode: `./juggle.py --stream`.
+
+For this to work, you'll need a few things:
+- Enable tethering on your comma device and connect your laptop.
+- Run `export ZMQ=1` which tells the streaming plugin backend to use ZMQ. If you're streaming locally, you can omit this step as ZMQ is used to transport data over the network.
+- Most importantly: openpilot by default uses the MSGQ backend, so you'll need to [ssh into your device](https://github.com/commaai/openpilot/wiki/SSH) and run bridge. This simply re-broadcasts each message over ZMQ: `./cereal/messaging/bridge`
+
+Now start PlotJuggler using the above `juggle.py` command, and find the `Cereal Subscriber` plugin in the dropdown under Streaming. Click Start and enter the IP address of the comma two. You should now be seeing all the messages for each [service in openpilot](https://github.com/commaai/cereal/blob/master/services.py) received live from your car!
+
+## Demo
 
 For a quick demo, go through the installation step and run this command:
 

tools/plotjuggler/install.sh

@@ -3,10 +3,8 @@
 mkdir -p bin
 cd bin
 
-wget https://github.com/commaai/PlotJuggler/releases/download/latest/libDataLoadRlog.so.tar.gz
+for lib_name in libDataLoadRlog.so libDataStreamCereal.so plotjuggler; do
-tar -xf libDataLoadRlog.so.tar.gz
+  wget https://github.com/commaai/PlotJuggler/releases/download/latest/${lib_name}.tar.gz
-rm libDataLoadRlog.so.tar.gz
+  tar -xf ${lib_name}.tar.gz
-
+  rm ${lib_name}.tar.gz
-wget https://github.com/commaai/PlotJuggler/releases/download/latest/plotjuggler.tar.gz
+done
-tar -xf plotjuggler.tar.gz
-rm plotjuggler.tar.gz

tools/plotjuggler/juggle.py

@@ -24,21 +24,25 @@ def load_segment(segment_name):
     print(f"Error parsing {segment_name}: {e}")
     return []
 
-def juggle_file(fn, dbc=None, layout=None):
+def start_juggler(fn=None, dbc=None, layout=None):
   env = os.environ.copy()
   env["BASEDIR"] = BASEDIR
+  pj = os.getenv("PLOTJUGGLER_PATH", "plotjuggler")
 
   if dbc:
     env["DBC_NAME"] = dbc
 
-  pj = os.getenv("PLOTJUGGLER_PATH", "plotjuggler")
+  extra_args = []
-  extra_args = ""
+  if fn is not None:
+    extra_args.append(f'-d {fn}')
+
   if layout is not None:
-    extra_args += f'-l {layout}'
+    extra_args.append(f'-l {layout}')
-  subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} -d {fn} {extra_args}', shell=True, env=env, cwd=juggle_dir)
 
-def juggle_route(route_name, segment_number, qlog, can, layout):
+  extra_args = " ".join(extra_args)
+  subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} {extra_args}', shell=True, env=env, cwd=juggle_dir)
 
+def juggle_route(route_name, segment_number, qlog, can, layout):
   if route_name.startswith("http://") or route_name.startswith("https://") or os.path.isfile(route_name):
     logs = [route_name]
   else:
@@ -80,7 +84,7 @@ def juggle_route(route_name, segment_number, qlog, can, layout):
   save_log(tempfile.name, all_data, compress=False)
   del all_data
 
-  juggle_file(tempfile.name, dbc, layout)
+  start_juggler(tempfile.name, dbc, layout)
 
 def get_arg_parser():
   parser = argparse.ArgumentParser(description="PlotJuggler plugin for reading rlogs",
@@ -88,6 +92,7 @@ def get_arg_parser():
 
   parser.add_argument("--qlog", action="store_true", help="Use qlogs")
   parser.add_argument("--can", action="store_true", help="Parse CAN data")
+  parser.add_argument("--stream", action="store_true", help="Start PlotJuggler without a route to stream data using Cereal")
   parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout")
   parser.add_argument("route_name", nargs='?', help="The name of the route that will be plotted.")
   parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment that will be plotted")
@@ -99,4 +104,8 @@ if __name__ == "__main__":
     arg_parser.print_help()
     sys.exit()
   args = arg_parser.parse_args(sys.argv[1:])
-  juggle_route(args.route_name, args.segment_number, args.qlog, args.can, args.layout)
+
+  if args.stream:
+    start_juggler()
+  else:
+    juggle_route(args.route_name, args.segment_number, args.qlog, args.can, args.layout)