From d4feecd8032567c0e331713a8af7caab35b6495c Mon Sep 17 00:00:00 2001 From: iejMac <61431446+iejMac@users.noreply.github.com> Date: Wed, 17 Feb 2021 22:59:31 -0800 Subject: [PATCH] plotjuggler falling back to qlog (#20105) * fallback to qlogs * minor fixes * removed extra line * help on no args * Update tools/plotjuggler/juggle.py Co-authored-by: Adeeb Shihadeh --- tools/lib/route.py | 25 ++++++++++++++----- tools/plotjuggler/juggle.py | 49 +++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/tools/lib/route.py b/tools/lib/route.py index 3060741541..7b65897d6c 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -11,6 +11,7 @@ SEGMENT_NAME_RE = r'[a-z0-9]{16}[|_][0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2 EXPLORER_FILE_RE = r'^({})--([a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE) OP_SEGMENT_DIR_RE = r'^({})$'.format(SEGMENT_NAME_RE) +QLOG_FILENAMES = ['qlog.bz2'] LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2'] CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc'] @@ -21,20 +22,23 @@ class Route(object): self._segments = self._get_segments_local(data_dir) else: self._segments = self._get_segments_remote() + self.max_seg_number = self._segments[-1].canonical_name.segment_num @property def segments(self): return self._segments def log_paths(self): - max_seg_number = self._segments[-1].canonical_name.segment_num log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments} - return [log_path_by_seg_num.get(i, None) for i in range(max_seg_number+1)] + return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] + + def qlog_paths(self): + qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments} + return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def camera_paths(self): - max_seg_number = self._segments[-1].canonical_name.segment_num camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments} - return [camera_path_by_seg_num.get(i, None) for i in range(max_seg_number+1)] + return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def _get_segments_remote(self): api = CommaApi(get_token()) @@ -48,12 +52,14 @@ class Route(object): segments[segment_name] = RouteSegment( segment_name, url if fn in LOG_FILENAMES else segments[segment_name].log_path, + url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path, url if fn in CAMERA_FILENAMES else segments[segment_name].camera_path ) else: segments[segment_name] = RouteSegment( segment_name, url if fn in LOG_FILENAMES else None, + url if fn in QLOG_FILENAMES else None, url if fn in CAMERA_FILENAMES else None ) @@ -88,26 +94,33 @@ class Route(object): segments = [] for segment, files in segment_files.items(): + try: log_path = next(path for path, filename in files if filename in LOG_FILENAMES) except StopIteration: log_path = None + try: + qlog_path = next(path for path, filename in files if filename in QLOG_FILENAMES) + except StopIteration: + qlog_path = None + try: camera_path = next(path for path, filename in files if filename in CAMERA_FILENAMES) except StopIteration: camera_path = None - segments.append(RouteSegment(segment, log_path, camera_path)) + segments.append(RouteSegment(segment, log_path, qlog_path, camera_path)) if len(segments) == 0: raise ValueError('Could not find segments for route {} in data directory {}'.format(self.route_name, data_dir)) return sorted(segments, key=lambda seg: seg.canonical_name.segment_num) class RouteSegment(object): - def __init__(self, name, log_path, camera_path): + def __init__(self, name, log_path, qlog_path, camera_path): self._name = RouteSegmentName(name) self.log_path = log_path + self.qlog_path = qlog_path self.camera_path = camera_path @property diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 1f049b656d..70f53b847b 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -10,7 +10,6 @@ from common.basedir import BASEDIR from selfdrive.test.process_replay.compare_logs import save_log from tools.lib.route import Route from tools.lib.logreader import LogReader -from tools.lib.url_file import URLFile def load_segment(segment_name): @@ -26,14 +25,26 @@ def juggle_file(fn): juggle_dir = os.path.dirname(os.path.realpath(__file__)) subprocess.call(f"bin/plotjuggler -d {fn}", shell=True, env=env, cwd=juggle_dir) -def juggle_route(route_name): +def juggle_route(route_name, segment_number): r = Route(route_name) - all_data = [] - pool = multiprocessing.Pool(24) + logs = r.log_paths() + if segment_number is not None: + logs = logs[segment_number:segment_number+1] + + if None in logs: + fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ") + if fallback_answer == 'y': + logs = r.qlog_paths() + if segment_number is not None: + logs = logs[segment_number:segment_number+1] + else: + print(f"Please try a different {'segment' if segment_number is not None else 'route'}") + return all_data = [] - for d in pool.map(load_segment, r.log_paths()): + pool = multiprocessing.Pool(24) + for d in pool.map(load_segment, logs): all_data += d tempfile = NamedTemporaryFile(suffix='.rlog') @@ -42,29 +53,19 @@ def juggle_route(route_name): juggle_file(tempfile.name) -def juggle_segment(route_name, segment_nr): - r = Route(route_name) - lp = r.log_paths()[segment_nr] - - if lp is None: - print("This segment does not exist, please try a different one") - return - - uf = URLFile(lp) - juggle_file(uf.name) - def get_arg_parser(): parser = argparse.ArgumentParser(description="PlotJuggler plugin for reading rlogs", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument("route_name", nargs='?', help="The name of the route that will be plotted.") + + parser.add_argument("route_name", nargs='?', help="The name of the route that will be plotted.") parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment that will be plotted") return parser if __name__ == "__main__": - - args = get_arg_parser().parse_args(sys.argv[1:]) - if args.segment_number is None: - juggle_route(args.route_name) - else: - juggle_segment(args.route_name, args.segment_number) + + arg_parser = get_arg_parser() + if len(sys.argv) == 1: + arg_parser.print_help() + sys.exit() + args = arg_parser.parse_args(sys.argv[1:]) + juggle_route(args.route_name, args.segment_number)