diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 5ac23d57ec..4aec965f1a 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -3,9 +3,13 @@ from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") -def FileReader(fn, debug=False): +def resolve_name(fn): if fn.startswith("cd:/"): - fn = fn.replace("cd:/", DATA_ENDPOINT) + return fn.replace("cd:/", DATA_ENDPOINT) + return fn + +def FileReader(fn, debug=False): + fn = resolve_name(fn) if fn.startswith(("http://", "https://")): return URLFile(fn, debug=debug) return open(fn, "rb") diff --git a/tools/lib/framereader.py b/tools/lib/framereader.py index e0b24963ca..8ab9e10edc 100644 --- a/tools/lib/framereader.py +++ b/tools/lib/framereader.py @@ -17,7 +17,7 @@ from openpilot.tools.lib.exceptions import DataUnreadableError from openpilot.tools.lib.vidindex import hevc_index from openpilot.common.file_helpers import atomic_write_in_dir -from openpilot.tools.lib.filereader import FileReader +from openpilot.tools.lib.filereader import FileReader, resolve_name HEVC_SLICE_B = 0 HEVC_SLICE_P = 1 @@ -60,6 +60,7 @@ def fingerprint_video(fn): def ffprobe(fn, fmt=None): + fn = resolve_name(fn) cmd = ["ffprobe", "-v", "quiet", "-print_format", "json", diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 7612996223..2f933e3b7f 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -1,8 +1,6 @@ import os import time -import tempfile import threading -import urllib.parse import pycurl from hashlib import sha256 from io import BytesIO @@ -37,7 +35,8 @@ class URLFile: self._curl = self._tlocal.curl except AttributeError: self._curl = self._tlocal.curl = pycurl.Curl() - mkdirs_exists_ok(Paths.download_cache_root()) + if not self._force_download: + mkdirs_exists_ok(Paths.download_cache_root()) def __enter__(self): return self @@ -65,12 +64,13 @@ class URLFile: def get_length(self): if self._length is not None: return self._length + file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length") - if os.path.exists(file_length_path) and not self._force_download: + if not self._force_download and os.path.exists(file_length_path): with open(file_length_path) as file_length: - content = file_length.read() - self._length = int(content) - return self._length + content = file_length.read() + self._length = int(content) + return self._length self._length = self.get_length_online() if not self._force_download and self._length != -1: @@ -173,24 +173,4 @@ class URLFile: @property def name(self): - """Returns a local path to file with the URLFile's contents. - - This can be used to interface with modules that require local files. - """ - if self._local_file is None: - _, ext = os.path.splitext(urllib.parse.urlparse(self._url).path) - local_fd, local_path = tempfile.mkstemp(suffix=ext) - try: - os.write(local_fd, self.read()) - local_file = open(local_path, "rb") - except Exception: - os.remove(local_path) - raise - finally: - os.close(local_fd) - - self._local_file = local_file - self.read = self._local_file.read - self.seek = self._local_file.seek - - return self._local_file.name + return self._url