|
|
@ -13,7 +13,6 @@ import warnings |
|
|
|
import zstandard as zstd |
|
|
|
import zstandard as zstd |
|
|
|
|
|
|
|
|
|
|
|
from collections.abc import Callable, Iterable, Iterator |
|
|
|
from collections.abc import Callable, Iterable, Iterator |
|
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed |
|
|
|
|
|
|
|
from urllib.parse import parse_qs, urlparse |
|
|
|
from urllib.parse import parse_qs, urlparse |
|
|
|
|
|
|
|
|
|
|
|
from cereal import log as capnp_log |
|
|
|
from cereal import log as capnp_log |
|
|
@ -202,15 +201,9 @@ def direct_source(file_or_url: str) -> list[LogPath]: |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_invalid_files(files): |
|
|
|
def get_invalid_files(files): |
|
|
|
if not files: |
|
|
|
for f in files: |
|
|
|
return |
|
|
|
if f is None or not file_exists(f): |
|
|
|
|
|
|
|
yield f |
|
|
|
with ThreadPoolExecutor(max_workers=32) as executor: |
|
|
|
|
|
|
|
future_to_file = {executor.submit(file_exists, file): file for file in files} |
|
|
|
|
|
|
|
for future in as_completed(future_to_file): |
|
|
|
|
|
|
|
file = future_to_file[future] |
|
|
|
|
|
|
|
if not future.result(): |
|
|
|
|
|
|
|
yield file |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_source(source: Source, *args) -> list[LogPath]: |
|
|
|
def check_source(source: Source, *args) -> list[LogPath]: |
|
|
|