You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
288 lines
12 KiB
288 lines
12 KiB
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import matplotlib.patches as mpatches
|
|
import matplotlib.pyplot as plt
|
|
import seaborn as sns
|
|
from tqdm import tqdm
|
|
import numpy as np
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
from openpilot.tools.lib.logreader import LogReader
|
|
|
|
DEMO_ROUTE = "9f583b1d93915c31|2022-05-18--10-49-51--0"
|
|
|
|
COLORS = ['blue', 'green', 'red', 'yellow', 'orange', 'purple']
|
|
PLOT_SERVICES = ['card', 'controlsd'] # , 'boardd']
|
|
|
|
|
|
def plot(lr):
|
|
seen = set()
|
|
aligned = False
|
|
|
|
start_time = None
|
|
# dict of services to events per inferred frame
|
|
times = {s: [[]] for s in PLOT_SERVICES}
|
|
|
|
first_event = None
|
|
# temp_times = {s: [] for s in PLOT_SERVICES} # holds only current frame of services
|
|
|
|
timestamps = [json.loads(msg.logMessage) for msg in lr if msg.which() == 'logMessage' and 'timestamp' in msg.logMessage]
|
|
# print(timestamps)
|
|
timestamps = sorted(timestamps, key=lambda m: float(m['msg']['timestamp']['time']))
|
|
|
|
# closely matches timestamp time
|
|
start_time = next(msg.logMonoTime for msg in lr)
|
|
|
|
for jmsg in timestamps:
|
|
if len(times[PLOT_SERVICES[0]]) > 1400:
|
|
continue
|
|
|
|
# print()
|
|
# print(msg.logMonoTime)
|
|
time = int(jmsg['msg']['timestamp']['time'])
|
|
service = jmsg['ctx']['daemon']
|
|
event = jmsg['msg']['timestamp']['event']
|
|
# print(jmsg)
|
|
# print(seen)
|
|
|
|
if service in PLOT_SERVICES and first_event is None:
|
|
first_event = event
|
|
|
|
# Align the best we can; wait for all to be seen and this is the first event
|
|
# TODO: detect first logMessage correctly by keeping track of events before aligned
|
|
aligned = aligned or (all(s in seen for s in PLOT_SERVICES) and event == first_event)
|
|
if not aligned:
|
|
seen.add(service)
|
|
continue
|
|
|
|
if service in PLOT_SERVICES:
|
|
|
|
# new frame when we've seen this event before
|
|
new_frame = event in {e[1] for e in times[service][-1]}
|
|
if new_frame:
|
|
times[service].append([])
|
|
|
|
# print(msg.logMonoTime, jmsg)
|
|
print('new_frame', new_frame)
|
|
times[service][-1].append(((time - start_time) * 1e-6, event))
|
|
|
|
points = {"x": [], "y": [], "labels": []}
|
|
height = 0.9
|
|
offsets = [[0, -10 * j] for j in range(len(PLOT_SERVICES))]
|
|
|
|
fig, ax = plt.subplots()
|
|
|
|
for idx, service_times in enumerate(zip(*times.values())):
|
|
print()
|
|
print('idx', idx)
|
|
service_bars = []
|
|
for j, (service, frame_times) in enumerate(zip(times.keys(), service_times)):
|
|
if idx + 1 == len(times[service]):
|
|
break
|
|
print(service, frame_times)
|
|
start = frame_times[0][0]
|
|
# use the first event time from next frame
|
|
end = times[service][idx + 1][0][0] # frame_times[-1][0]
|
|
print('start, end', start, end)
|
|
service_bars.append((start, end - start))
|
|
for event in frame_times:
|
|
points['x'].append(event[0])
|
|
points['y'].append(idx - j * 1)
|
|
points['labels'].append(event[1])
|
|
print(service_bars)
|
|
|
|
# offset = offset_services
|
|
# offset each service
|
|
for j, sb in enumerate(service_bars):
|
|
ax.broken_barh([sb], (idx - height / 2 - j * 1, height), facecolors=[COLORS[j]], alpha=0.5) # , offsets=offsets)
|
|
# ax.broken_barh(service_bars, (idx - height / 2, height), facecolors=COLORS, alpha=0.5, offsets=offsets)
|
|
|
|
scatter = ax.scatter(points['x'], points['y'], marker='d', edgecolor='black')
|
|
txt = ax.text(0, 0, '', ha='center', fontsize=8, color='red')
|
|
ax.set_xlabel('milliseconds')
|
|
|
|
plt.legend(handles=[mpatches.Patch(color=COLORS[i], label=PLOT_SERVICES[i]) for i in range(len(PLOT_SERVICES))])
|
|
|
|
def hover(event):
|
|
txt.set_text("")
|
|
status, pts = scatter.contains(event)
|
|
txt.set_visible(status)
|
|
if status:
|
|
pt_idx = pts['ind'][0]
|
|
txt.set_text(f"{points['labels'][pt_idx]} ({points['x'][pt_idx]:0.2f} ms)")
|
|
txt.set_position((event.xdata, event.ydata + 1))
|
|
event.canvas.draw()
|
|
|
|
fig.canvas.mpl_connect("motion_notify_event", hover)
|
|
|
|
plt.show()
|
|
# plt.pause(1000)
|
|
return times, points
|
|
|
|
|
|
def plot_dist(lr, poll):
|
|
|
|
lr = list(lr)
|
|
|
|
# carState_service_times = []
|
|
# prev_time = None
|
|
# for msg in lr:
|
|
# if msg.which() == 'carState':
|
|
# if prev_time is not None:
|
|
# carState_service_times.append((msg.logMonoTime - prev_time) * 1e-6)
|
|
# prev_time = msg.logMonoTime
|
|
|
|
# logMonoTime is from logmessaged, not when the timestamp was created. some messages are out of order
|
|
timestamps = [json.loads(msg.logMessage) for msg in tqdm(lr) if msg.which() == 'logMessage' and '"timestamp"' in msg.logMessage]
|
|
timestamps = sorted(timestamps, key=lambda m: float(m['msg']['timestamp']['time']))
|
|
timestamps = [m for m in tqdm(timestamps) if m['ctx']['daemon'] in PLOT_SERVICES]
|
|
|
|
initialized = False
|
|
ready = False
|
|
|
|
start_card_loop = None
|
|
received_can = None
|
|
state_updated = None
|
|
sent_carState = None
|
|
state_published = None
|
|
sent_carControl = None
|
|
|
|
card_e2e_loop_times = []
|
|
card_carInterface_update_times = []
|
|
carState_recv_times = []
|
|
carControl_recv_times = []
|
|
carState_to_carControl_times = []
|
|
card_controls_times = []
|
|
card_loop_times = []
|
|
|
|
for jmsg in tqdm(timestamps):
|
|
time = int(jmsg['msg']['timestamp']['time'])
|
|
service = jmsg['ctx']['daemon']
|
|
event = jmsg['msg']['timestamp']['event']
|
|
|
|
if event == 'Initialized' and service == 'card':
|
|
initialized = True
|
|
|
|
if initialized and event == 'Start card':
|
|
ready = True
|
|
|
|
if not ready:
|
|
continue
|
|
|
|
if event == 'Start card' and service == 'card':
|
|
if start_card_loop is not None:
|
|
card_e2e_loop_times.append((time - start_card_loop) * 1e-6)
|
|
start_card_loop = time
|
|
|
|
elif event == 'Received can' and service == 'card':
|
|
# measuring from this time does not include wait time for can packet, so this measures true card loop time taken
|
|
received_can = time
|
|
|
|
elif event == 'State updated' and service == 'card':
|
|
state_updated = time
|
|
card_carInterface_update_times.append((time - received_can) * 1e-6)
|
|
|
|
elif event == 'Sent carState' and service == 'card':
|
|
sent_carState = time
|
|
|
|
elif event == 'Got carState' and service == 'controlsd':
|
|
# TODO why none
|
|
if sent_carState is not None:
|
|
carState_recv_times.append((time - sent_carState) * 1e-6)
|
|
|
|
elif event == 'Logs published' and service == 'controlsd':
|
|
sent_carControl = time
|
|
|
|
elif event == 'State published' and service == 'card':
|
|
state_published = time
|
|
if poll: # only makes sense when polling
|
|
# from carState sent to carControl received
|
|
carControl_recv_times.append((time - sent_carControl) * 1e-6)
|
|
carState_to_carControl_times.append((time - sent_carState) * 1e-6)
|
|
|
|
elif event == 'Controls updated' and service == 'card':
|
|
card_controls_times.append((time - state_published) * 1e-6)
|
|
card_loop_times.append((time - received_can) * 1e-6) # this is time NOT spent waiting for can
|
|
|
|
|
|
fig, ax = plt.subplots(3)
|
|
|
|
fig.suptitle('Polling/waiting on carControl from controlsd' if poll else 'Not polling on carControl')
|
|
|
|
ax[0].set_title('cereal communication times')
|
|
ax[0].set_xlim(0, 16)
|
|
sns.histplot(carState_recv_times, kde=True, ax=ax[0],
|
|
label=f'carState->controlsd recv time: \n minmax: {min(carState_recv_times):0.2f}, {max(carState_recv_times):>5.2f}, ' +
|
|
f'med: {np.median(carState_recv_times):0.2f}, mean: {np.mean(carState_recv_times):0.2f}, ' +
|
|
f'95th: {np.percentile(carState_recv_times, 95):0.2f}')
|
|
if poll:
|
|
sns.histplot(carControl_recv_times, kde=True, ax=ax[0],
|
|
label=f'carControl->card recv time (polling on carControl): \n minmax: {min(carControl_recv_times):0.2f}, {max(carControl_recv_times):>5.2f}, ' +
|
|
f'med: {np.median(carControl_recv_times):0.2f}, mean: {np.mean(carControl_recv_times):0.2f}, ' +
|
|
f'95th: {np.percentile(carControl_recv_times, 95):0.2f}')
|
|
ax[0].legend()
|
|
|
|
ax[1].set_title('card loop times')
|
|
ax[1].set_xlim(0, 16)
|
|
if poll:
|
|
sns.histplot(carState_to_carControl_times, kde=True, ax=ax[1],
|
|
label=f'waiting on carControl (polling on carControl): \n minmax: {min(carState_to_carControl_times):0.2f}, {max(carState_to_carControl_times):>5.2f}, ' +
|
|
f'med: {np.median(carState_to_carControl_times):0.2f}, mean: {np.mean(carState_to_carControl_times):0.2f}, ' +
|
|
f'95th: {np.percentile(carState_to_carControl_times, 95):0.2f}')
|
|
sns.histplot(card_controls_times, kde=True, ax=ax[1], label=f'CI.apply(): \n minmax: {min(card_controls_times):0.2f}, {max(card_controls_times):>6.2f}, ' +
|
|
f'med: {np.median(card_controls_times):0.2f}, mean: {np.mean(card_controls_times):0.2f}, ' +
|
|
f'95th: {np.percentile(card_controls_times, 95):0.2f}')
|
|
sns.histplot(card_carInterface_update_times, kde=True, ax=ax[1],
|
|
label=f'CI.update(): \n minmax: {min(card_carInterface_update_times):0.2f}, {max(card_carInterface_update_times):>5.2f}, ' +
|
|
f'med: {np.median(card_carInterface_update_times):0.2f}, mean: {np.mean(card_carInterface_update_times):0.2f}, ' +
|
|
f'95th: {np.percentile(card_carInterface_update_times, 95):0.2f}')
|
|
ax[1].legend()
|
|
|
|
ax[2].set_title('total card loop time')
|
|
ax[2].set_xlim(0, 16)
|
|
sns.histplot(card_loop_times, kde=True, ax=ax[2], label=f'entire card loop time: \n minmax: {min(card_loop_times):0.2f}, {max(card_loop_times):>5.2f}, ' +
|
|
f'med: {np.median(card_loop_times):0.2f}, mean: {np.mean(card_loop_times):0.2f}, ' +
|
|
f'95th: {np.percentile(card_loop_times, 95):0.2f}')
|
|
ax[2].legend()
|
|
ax[2].set_xlabel('ms')
|
|
|
|
return timestamps, card_loop_times, carState_recv_times, carState_to_carControl_times
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# parser = argparse.ArgumentParser(description="A tool for analyzing openpilot's end-to-end latency",
|
|
# formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
# parser.add_argument("--demo", action="store_true", help="Use the demo route instead of providing one")
|
|
# parser.add_argument("route_or_segment_name", nargs='?', help="The route to print")
|
|
#
|
|
# if len(sys.argv) == 1:
|
|
# parser.print_help()
|
|
# sys.exit()
|
|
# args = parser.parse_args()
|
|
|
|
# r = DEMO_ROUTE if args.demo else args.route_or_segment_name.strip()
|
|
# lr = LogReader(r, sort_by_time=True)
|
|
# lr = LogReader('08e4c2a99df165b1/00000016--c3a4ca99ec/0', sort_by_time=True) # normal
|
|
# lr = LogReader('08e4c2a99df165b1/00000017--e2d24ab118/0', sort_by_time=True) # polls on carControl
|
|
# lr = LogReader('08e4c2a99df165b1/00000018--cf65e47c24/0', sort_by_time=True) # polls on carControl, sends it earlier
|
|
# lr = LogReader('08e4c2a99df165b1/00000019--e73e3ab4df/0', sort_by_time=True) # polls on carControl, more logging
|
|
|
|
# lr = LogReader('08e4c2a99df165b1/0000002c--b40eb82d6d/0:-1', sort_by_time=True) # polls on carControl
|
|
# lr = LogReader('08e4c2a99df165b1/0000002d--ccebe8b617/0:1', sort_by_time=True) # no poll on carControl
|
|
# lr = LogReader('08e4c2a99df165b1/0000002e--fd98f6603b/:7', sort_by_time=True) # no poll on carControl (no timestamps)
|
|
|
|
POLL = False # carControl polling or not
|
|
if POLL:
|
|
# lr = LogReader('08e4c2a99df165b1/00000032--2c1d57d894/0', sort_by_time=True) # carControl poll, w/ reduced timestamps
|
|
# lr = LogReader('08e4c2a99df165b1/00000033--1e2720e55b/0', sort_by_time=True) # carControl poll, w/ poll flag (FINAL)
|
|
lr = LogReader('08e4c2a99df165b1/00000036--4eb8126f04', sort_by_time=True) # carControl poll, w/ poll flag & Received can (FINAL v2)
|
|
else:
|
|
# lr = LogReader('08e4c2a99df165b1/00000031--f6f38d1ccf/0', sort_by_time=True) # no carControl poll, w/ reduced timestamps
|
|
# lr = LogReader('08e4c2a99df165b1/00000035--0abfde9c4a/0', sort_by_time=True) # no carControl poll, w/ poll flag (FINAL)
|
|
lr = LogReader('08e4c2a99df165b1/00000037--f6294815ac', sort_by_time=True) # no carControl poll, w/ poll flag & Received can (FINAL v2)
|
|
|
|
timestamps = plot_dist(lr, poll=POLL)
|
|
# times, points = plot(lr)
|
|
|