diff --git a/poetry.lock b/poetry.lock index 8443506e4c..d718a4910a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8fe4d2310328e34ec37f396db589ef55ad34a3d27c7add141abd9235fd38eb11 -size 434974 +oid sha256:004ffae57f0d3e8cdfea59b0aa549a2ce089adeb8e80fb5a64671828eaaf2f9a +size 434276 diff --git a/pyproject.toml b/pyproject.toml index 98091e0088..a519670d48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,6 @@ polyline = "*" # these should be removed markdown-it-py = "*" timezonefinder = "*" -pycurl = "*" setproctitle = "*" @@ -153,7 +152,6 @@ sphinx-rtd-theme = "*" sphinx-sitemap = "*" tabulate = "*" tenacity = "*" -types-pycurl = "*" types-requests = "*" types-tabulate = "*" tqdm = "*" diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 0105d4074f..3f72350234 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -5,7 +5,7 @@ import signal import itertools import math import time -import pycurl +import requests import shutil import subprocess import datetime @@ -102,27 +102,18 @@ def gps_enabled() -> bool: def download_assistance(): try: - c = pycurl.Curl() - c.setopt(pycurl.URL, ASSISTANCE_URL) - c.setopt(pycurl.NOBODY, 1) - c.setopt(pycurl.CONNECTTIMEOUT, 2) - c.perform() - bytes_n = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD) - c.close() - if bytes_n > 1e5: - cloudlog.error("Qcom assistance data larger than expected") - return + response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: - c = pycurl.Curl() - c.setopt(pycurl.URL, ASSISTANCE_URL) - c.setopt(pycurl.CONNECTTIMEOUT, 5) - - c.setopt(pycurl.WRITEDATA, fp) - c.perform() - c.close() - os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) - except pycurl.error: + for chunk in response.iter_content(chunk_size=8192): + fp.write(chunk) + if fp.tell() > 1e5: + cloudlog.error("Qcom assistance data larger than expected") + return + + os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) + + except requests.exceptions.RequestException: cloudlog.exception("Failed to download assistance file") return diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index a54552d241..3fb2135c08 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -1,10 +1,11 @@ import os import time import threading -import pycurl from hashlib import sha256 -from io import BytesIO +from urllib3 import PoolManager, Retry +from urllib3.util import Timeout from tenacity import retry, wait_random_exponential, stop_after_attempt + from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths # Cache chunk size @@ -35,13 +36,14 @@ class URLFile: if cache is not None: self._force_download = not cache - try: - self._curl = self._tlocal.curl - except AttributeError: - self._curl = self._tlocal.curl = pycurl.Curl() if not self._force_download: os.makedirs(Paths.download_cache_root(), exist_ok=True) + try: + self._http_client = URLFile._tlocal.http_client + except AttributeError: + self._http_client = URLFile._tlocal.http_client = PoolManager() + def __enter__(self): return self @@ -53,17 +55,10 @@ class URLFile: @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def get_length_online(self): - c = self._curl - c.reset() - c.setopt(pycurl.NOSIGNAL, 1) - c.setopt(pycurl.TIMEOUT_MS, 500000) - c.setopt(pycurl.FOLLOWLOCATION, True) - c.setopt(pycurl.URL, self._url) - c.setopt(c.NOBODY, 1) - c.perform() - length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD)) - c.reset() - return length + timeout = Timeout(connect=50.0, read=500.0) + response = self._http_client.request('HEAD', self._url, timeout=timeout, preload_content=False) + length = response.headers.get('content-length', 0) + return int(length) def get_length(self): if self._length is not None: @@ -117,7 +112,7 @@ class URLFile: @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def read_aux(self, ll=None): download_range = False - headers = ["Connection: keep-alive"] + headers = {'Connection': 'keep-alive'} if self._pos != 0 or ll is not None: if ll is None: end = self.get_length() - 1 @@ -125,50 +120,29 @@ class URLFile: end = min(self._pos + ll, self.get_length()) - 1 if self._pos >= end: return b"" - headers.append(f"Range: bytes={self._pos}-{end}") + headers['Range'] = f"bytes={self._pos}-{end}" download_range = True - dats = BytesIO() - c = self._curl - c.setopt(pycurl.URL, self._url) - c.setopt(pycurl.WRITEDATA, dats) - c.setopt(pycurl.NOSIGNAL, 1) - c.setopt(pycurl.TIMEOUT_MS, 500000) - c.setopt(pycurl.HTTPHEADER, headers) - c.setopt(pycurl.FOLLOWLOCATION, True) - if self._debug: - print("downloading", self._url) - - def header(x): - if b'MISS' in x: - print(x.strip()) - - c.setopt(pycurl.HEADERFUNCTION, header) - - def test(debug_type, debug_msg): - print(" debug(%d): %s" % (debug_type, debug_msg.strip())) - - c.setopt(pycurl.VERBOSE, 1) - c.setopt(pycurl.DEBUGFUNCTION, test) t1 = time.time() - c.perform() + timeout = Timeout(connect=50.0, read=500.0) + response = self._http_client.request('GET', self._url, timeout=timeout, preload_content=False, headers=headers) + ret = response.data if self._debug: t2 = time.time() if t2 - t1 > 0.1: - print(f"get {self._url} {headers!r} {t2 - t1:.f} slow") + print(f"get {self._url} {headers!r} {t2 - t1:.3f} slow") - response_code = c.getinfo(pycurl.RESPONSE_CODE) + response_code = response.status if response_code == 416: # Requested Range Not Satisfiable - raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(ret)[:500]}") if download_range and response_code != 206: # Partial Content - raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(ret)[:500]}") if (not download_range) and response_code != 200: # OK - raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(ret)[:500]}") - ret = dats.getvalue() self._pos += len(ret) return ret