diff --git a/Jenkinsfile b/Jenkinsfile index e0e8511251..c7ef88811f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -220,7 +220,7 @@ pipeline { steps { phone_steps("tici-common", [ ["build", "cd selfdrive/manager && ./build.py"], - ["model replay", "cd selfdrive/test/process_replay && NO_NAV=1 ./model_replay.py"], + ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"], ]) } } diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py index 1d1b6ec423..5f4b9f1f35 100755 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -11,10 +11,7 @@ def get_url(route_name, segment_num, log_type="rlog"): ext = "hevc" if log_type.endswith('camera') else "bz2" return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}" - -def upload_file(path, name): - from azure.storage.blob import BlockBlobService # pylint: disable=import-error - +def get_sas_token(): sas_token = os.environ.get("AZURE_TOKEN", None) if os.path.isfile(TOKEN_PATH): sas_token = open(TOKEN_PATH).read().strip() @@ -22,9 +19,20 @@ def upload_file(path, name): if sas_token is None: sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \ --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n") - service = BlockBlobService(account_name="commadataci", sas_token=sas_token) + + return sas_token + +def upload_bytes(data, name): + from azure.storage.blob import BlockBlobService # pylint: disable=import-error + service = BlockBlobService(account_name="commadataci", sas_token=get_sas_token()) + service.create_blob_from_bytes("openpilotci", name, data) + return BASE_URL + name + +def upload_file(path, name): + from azure.storage.blob import BlockBlobService # pylint: disable=import-error + service = BlockBlobService(account_name="commadataci", sas_token=get_sas_token()) service.create_blob_from_path("openpilotci", name, path) - return "https://commadataci.blob.core.windows.net/openpilotci/" + name + return BASE_URL + name if __name__ == "__main__": diff --git a/selfdrive/test/process_replay/model_replay.py b/selfdrive/test/process_replay/model_replay.py index 68cbb28aaf..9c1ec4e931 100755 --- a/selfdrive/test/process_replay/model_replay.py +++ b/selfdrive/test/process_replay/model_replay.py @@ -25,7 +25,7 @@ SEGMENT = 6 MAX_FRAMES = 100 if PC else 600 NAV_FRAMES = 50 -NO_NAV = "NO_NAV" in os.environ # TODO: make map renderer work in CI +NO_NAV = "NO_NAV" in os.environ SEND_EXTRA_INPUTS = bool(os.getenv("SEND_EXTRA_INPUTS", "0")) VIPC_STREAM = {"roadCameraState": VisionStreamType.VISION_STREAM_ROAD, "driverCameraState": VisionStreamType.VISION_STREAM_DRIVER, @@ -209,6 +209,31 @@ if __name__ == "__main__": 'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="ecamera"), readahead=True) } + # Update tile refs + if update: + import urllib + import requests + import threading + import http.server + from selfdrive.test.openpilotci import upload_bytes + os.environ['MAPS_HOST'] = 'http://localhost:5000' + + class HTTPRequestHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + assert len(self.path) > 10 # Sanity check on path length + r = requests.get(f'https://api.mapbox.com{self.path}', timeout=30) + upload_bytes(r.content, urllib.parse.urlparse(self.path).path.lstrip('/')) + self.send_response(r.status_code) + self.send_header('Content-type','text/html') + self.end_headers() + self.wfile.write(r.content) + + server = http.server.HTTPServer(("127.0.0.1", 5000), HTTPRequestHandler) + thread = threading.Thread(None, server.serve_forever, daemon=True) + thread.start() + else: + os.environ['MAPS_HOST'] = BASE_URL.rstrip('/') + # run replays log_msgs = model_replay(lr, frs) if not NO_NAV: diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit index 448c7b4291..e458f817d3 100644 --- a/selfdrive/test/process_replay/model_replay_ref_commit +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -1 +1 @@ -e3cf1856830902bede2d1c9ca3d0d60e5504ae20 +91bdaeae5a7141ff6bb6fd80672521c8d63f644c