|
|
|
@ -1,6 +1,7 @@ |
|
|
|
|
import os |
|
|
|
|
import threading |
|
|
|
|
import time |
|
|
|
|
import unittest |
|
|
|
|
|
|
|
|
|
if "CI" in os.environ: |
|
|
|
|
def tqdm(x): |
|
|
|
@ -14,17 +15,7 @@ from tools.lib.logreader import LogReader |
|
|
|
|
|
|
|
|
|
ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'command', 'path']) |
|
|
|
|
|
|
|
|
|
class SimplePubMaster(): |
|
|
|
|
def __init__(self, services): # pylint: disable=super-init-not-called |
|
|
|
|
self.sock = {} |
|
|
|
|
for s in services: |
|
|
|
|
self.sock[s] = messaging.pub_sock(s) |
|
|
|
|
|
|
|
|
|
def send(self, s, dat): |
|
|
|
|
# print(dat) |
|
|
|
|
self.sock[s].send(dat.to_bytes()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" |
|
|
|
|
CONFIGS = [ |
|
|
|
|
ProcessConfig( |
|
|
|
|
proc_name="ubloxd", |
|
|
|
@ -37,53 +28,66 @@ CONFIGS = [ |
|
|
|
|
), |
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
def valgrindlauncher(arg, cwd): |
|
|
|
|
os.chdir(cwd) |
|
|
|
|
|
|
|
|
|
# Run valgrind on a process |
|
|
|
|
command = "valgrind --leak-check=full " + arg |
|
|
|
|
print(command) |
|
|
|
|
output = os.popen(command) |
|
|
|
|
while True: |
|
|
|
|
s = output.read() |
|
|
|
|
if s == "": |
|
|
|
|
break |
|
|
|
|
print(s) |
|
|
|
|
|
|
|
|
|
def replay_process(cfg, lr): |
|
|
|
|
pub_sockets = [s for s in cfg.pub_sub.keys() if s != 'can'] # We dump data from logs here |
|
|
|
|
|
|
|
|
|
pm = SimplePubMaster(pub_sockets) |
|
|
|
|
print("Sorting logs") |
|
|
|
|
all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime) |
|
|
|
|
pub_msgs = [msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())] |
|
|
|
|
print(len(pub_msgs)) |
|
|
|
|
thread = threading.Thread(target=valgrindlauncher, args=(cfg.command, cfg.path)) |
|
|
|
|
thread.daemon = True |
|
|
|
|
thread.start() |
|
|
|
|
time.sleep(5) # We give the process time to start |
|
|
|
|
for msg in tqdm(pub_msgs): |
|
|
|
|
pm.send(msg.which(), msg.as_builder()) |
|
|
|
|
|
|
|
|
|
BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" |
|
|
|
|
class SimplePubMaster(): |
|
|
|
|
def __init__(self, services): # pylint: disable=super-init-not-called |
|
|
|
|
self.sock = {} |
|
|
|
|
for s in services: |
|
|
|
|
self.sock[s] = messaging.pub_sock(s) |
|
|
|
|
|
|
|
|
|
def get_segment(segment_name, original=True): |
|
|
|
|
route_name, segment_num = segment_name.rsplit("--", 1) |
|
|
|
|
if original: |
|
|
|
|
rlog_url = BASE_URL + "%s/%s/rlog.bz2" % (route_name.replace("|", "/"), segment_num) |
|
|
|
|
else: |
|
|
|
|
process_replay_dir = os.path.dirname(os.path.abspath(__file__)) |
|
|
|
|
model_ref_commit = open(os.path.join(process_replay_dir, "model_ref_commit")).read().strip() |
|
|
|
|
rlog_url = BASE_URL + "%s/%s/rlog_%s.bz2" % (route_name.replace("|", "/"), segment_num, model_ref_commit) |
|
|
|
|
def send(self, s, dat): |
|
|
|
|
# print(dat) |
|
|
|
|
self.sock[s].send(dat.to_bytes()) |
|
|
|
|
|
|
|
|
|
return rlog_url |
|
|
|
|
class TestValgrind(unittest.TestCase): |
|
|
|
|
|
|
|
|
|
def valgrindlauncher(self, arg, cwd): |
|
|
|
|
os.chdir(cwd) |
|
|
|
|
|
|
|
|
|
# Run valgrind on a process |
|
|
|
|
command = "valgrind --leak-check=full " + arg |
|
|
|
|
# print(command) |
|
|
|
|
output = os.popen(command) |
|
|
|
|
while True: |
|
|
|
|
s = output.read() |
|
|
|
|
print(s) |
|
|
|
|
if s == "": |
|
|
|
|
break |
|
|
|
|
|
|
|
|
|
def replay_process(self, cfg, lr): |
|
|
|
|
pub_sockets = [s for s in cfg.pub_sub.keys() if s != 'can'] # We dump data from logs here |
|
|
|
|
|
|
|
|
|
pm = SimplePubMaster(pub_sockets) |
|
|
|
|
print("Sorting logs") |
|
|
|
|
all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime) |
|
|
|
|
pub_msgs = [msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())] |
|
|
|
|
print(len(pub_msgs)) |
|
|
|
|
thread = threading.Thread(target=self.valgrindlauncher, args=(cfg.command, cfg.path)) |
|
|
|
|
thread.daemon = True |
|
|
|
|
thread.start() |
|
|
|
|
time.sleep(5) # We give the process time to start |
|
|
|
|
for msg in tqdm(pub_msgs): |
|
|
|
|
pm.send(msg.which(), msg.as_builder()) |
|
|
|
|
|
|
|
|
|
def get_segment(self, segment_name, original=True): |
|
|
|
|
route_name, segment_num = segment_name.rsplit("--", 1) |
|
|
|
|
if original: |
|
|
|
|
rlog_url = BASE_URL + "%s/%s/rlog.bz2" % (route_name.replace("|", "/"), segment_num) |
|
|
|
|
else: |
|
|
|
|
process_replay_dir = os.path.dirname(os.path.abspath(__file__)) |
|
|
|
|
model_ref_commit = open(os.path.join(process_replay_dir, "model_ref_commit")).read().strip() |
|
|
|
|
rlog_url = BASE_URL + "%s/%s/rlog_%s.bz2" % (route_name.replace("|", "/"), segment_num, model_ref_commit) |
|
|
|
|
|
|
|
|
|
return rlog_url |
|
|
|
|
|
|
|
|
|
def test_config_0(self): |
|
|
|
|
cfg = CONFIGS[0] |
|
|
|
|
|
|
|
|
|
URL = self.get_segment("0375fdf7b1ce594d|2019-06-13--08-32-25--3") |
|
|
|
|
print(URL) |
|
|
|
|
lr = LogReader(URL) |
|
|
|
|
print(str(cfg)) |
|
|
|
|
self.replay_process(cfg, lr) |
|
|
|
|
time.sleep(30) |
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
|
cfg = CONFIGS[0] |
|
|
|
|
|
|
|
|
|
URL = get_segment("0375fdf7b1ce594d|2019-06-13--08-32-25--3") |
|
|
|
|
print(URL) |
|
|
|
|
lr = LogReader(URL) |
|
|
|
|
print(str(cfg)) |
|
|
|
|
replay_process(cfg, lr) |
|
|
|
|
time.sleep(30) |
|
|
|
|
unittest.main() |
|
|
|
|