sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

dependency: remove pycurl package (#30771)

Browse Source 
* Update qcomgpsd.py

* Update url_file.py

* remove pycurl

* requests -> urllib3

* unused

* redundant

* fix import

* Revert "requests -> urllib3"

This reverts commit 7ca39e6189.

* headless

* fix trail

* use requests.exceptions.RequestException

* use fp.tell

* fix indents

* reorder imports

* change timeout

* fix debug timing

* remove exception

* add timeout

* missing headers

* move to constructor

* move import

* unused import

* fix debug

* try

* no retries
pull/30786/head
royjr 1 year ago committed by GitHub
parent 8c1176ca83
commit 70624ffc81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 34 additions and 92 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 23
      poetry.lock
  2. 2
      pyproject.toml
  3. 31
      system/qcomgpsd/qcomgpsd.py
  4. 70
      tools/lib/url_file.py

23
poetry.lock generated
Unescape View File

@ -3206,16 +3206,6 @@ files = [
{file = "pycryptodome-3.19.0.tar.gz", hash = "sha256:bc35d463222cdb4dbebd35e0784155c81e161b9284e567e7e933d722e533331e"},
]
[[package]]
name = "pycurl"
version = "7.45.2"
description = "PycURL -- A Python Interface To The cURL library"
optional = false
python-versions = ">=3.5"
files = [
{file = "pycurl-7.45.2.tar.gz", hash = "sha256:5730590be0271364a5bddd9e245c9cc0fb710c4cbacbdd95264a3122d23224ca"},
]
[[package]]
name = "pyee"
version = "11.1.0"
@ -4655,17 +4645,6 @@ notebook = ["ipywidgets (>=6)"]
slack = ["slack-sdk"]
telegram = ["requests"]
[[package]]
name = "types-pycurl"
version = "7.45.2.5"
description = "Typing stubs for pycurl"
optional = false
python-versions = "*"
files = [
{file = "types-pycurl-7.45.2.5.tar.gz", hash = "sha256:cc07e4b1e388b98e1a935bc12ef33eec8e7294496d0a81ed50c46a31bf5df5ee"},
{file = "types_pycurl-7.45.2.5-py3-none-any.whl", hash = "sha256:e433f13a266245be770018eabc6a50139ea26150f75df5b0ee698d291575582b"},
]
[[package]]
name = "types-requests"
version = "2.31.0.10"
@ -4902,4 +4881,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = "~3.11"
content-hash = "eeb5efb6161eaec95fb863ddc9ec08afb540ed3cd5d521646a6a759ddf33ab2e"
content-hash = "26e963a3e9236b0e29466a0bca243215395a780c88485e9c17f6518969334890"

2
pyproject.toml
Unescape View File

@ -110,7 +110,6 @@ polyline = "*"
# these should be removed
markdown-it-py = "*"
timezonefinder = "*"
pycurl = "*"
setproctitle = "*"
@ -153,7 +152,6 @@ sphinx-rtd-theme = "*"
sphinx-sitemap = "*"
tabulate = "*"
tenacity = "*"
types-pycurl = "*"
types-requests = "*"
types-tabulate = "*"
tqdm = "*"

31
system/qcomgpsd/qcomgpsd.py
Unescape View File

@ -5,7 +5,7 @@ import signal
import itertools
import math
import time
import pycurl
import requests
import shutil
import subprocess
import datetime
@ -102,27 +102,18 @@ def gps_enabled() -> bool:
def download_assistance():
try:
c = pycurl.Curl()
c.setopt(pycurl.URL, ASSISTANCE_URL)
c.setopt(pycurl.NOBODY, 1)
c.setopt(pycurl.CONNECTTIMEOUT, 2)
c.perform()
bytes_n = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
c.close()
if bytes_n > 1e5:
cloudlog.error("Qcom assistance data larger than expected")
return
response = requests.get(ASSISTANCE_URL, timeout=5, stream=True)
with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp:
c = pycurl.Curl()
c.setopt(pycurl.URL, ASSISTANCE_URL)
c.setopt(pycurl.CONNECTTIMEOUT, 5)
c.setopt(pycurl.WRITEDATA, fp)
c.perform()
c.close()
os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE)
except pycurl.error:
for chunk in response.iter_content(chunk_size=8192):
fp.write(chunk)
if fp.tell() > 1e5:
cloudlog.error("Qcom assistance data larger than expected")
return
os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE)
except requests.exceptions.RequestException:
cloudlog.exception("Failed to download assistance file")
return

70
tools/lib/url_file.py
Unescape View File

@ -1,10 +1,11 @@
import os
import time
import threading
import pycurl
from hashlib import sha256
from io import BytesIO
from urllib3 import PoolManager, Retry
from urllib3.util import Timeout
from tenacity import retry, wait_random_exponential, stop_after_attempt
from openpilot.common.file_helpers import atomic_write_in_dir
from openpilot.system.hardware.hw import Paths
# Cache chunk size
@ -35,13 +36,14 @@ class URLFile:
if cache is not None:
self._force_download = not cache
try:
self._curl = self._tlocal.curl
except AttributeError:
self._curl = self._tlocal.curl = pycurl.Curl()
if not self._force_download:
os.makedirs(Paths.download_cache_root(), exist_ok=True)
try:
self._http_client = URLFile._tlocal.http_client
except AttributeError:
self._http_client = URLFile._tlocal.http_client = PoolManager()
def __enter__(self):
return self
@ -53,17 +55,10 @@ class URLFile:
@retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
def get_length_online(self):
c = self._curl
c.reset()
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.TIMEOUT_MS, 500000)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.URL, self._url)
c.setopt(c.NOBODY, 1)
c.perform()
length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD))
c.reset()
return length
timeout = Timeout(connect=50.0, read=500.0)
response = self._http_client.request('HEAD', self._url, timeout=timeout, preload_content=False)
length = response.headers.get('content-length', 0)
return int(length)
def get_length(self):
if self._length is not None:
@ -117,7 +112,7 @@ class URLFile:
@retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
def read_aux(self, ll=None):
download_range = False
headers = ["Connection: keep-alive"]
headers = {'Connection': 'keep-alive'}
if self._pos != 0 or ll is not None:
if ll is None:
end = self.get_length() - 1
@ -125,50 +120,29 @@ class URLFile:
end = min(self._pos + ll, self.get_length()) - 1
if self._pos >= end:
return b""
headers.append(f"Range: bytes={self._pos}-{end}")
headers['Range'] = f"bytes={self._pos}-{end}"
download_range = True
dats = BytesIO()
c = self._curl
c.setopt(pycurl.URL, self._url)
c.setopt(pycurl.WRITEDATA, dats)
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.TIMEOUT_MS, 500000)
c.setopt(pycurl.HTTPHEADER, headers)
c.setopt(pycurl.FOLLOWLOCATION, True)
if self._debug:
print("downloading", self._url)
def header(x):
if b'MISS' in x:
print(x.strip())
c.setopt(pycurl.HEADERFUNCTION, header)
def test(debug_type, debug_msg):
print(" debug(%d): %s" % (debug_type, debug_msg.strip()))
c.setopt(pycurl.VERBOSE, 1)
c.setopt(pycurl.DEBUGFUNCTION, test)
t1 = time.time()
c.perform()
timeout = Timeout(connect=50.0, read=500.0)
response = self._http_client.request('GET', self._url, timeout=timeout, preload_content=False, headers=headers)
ret = response.data
if self._debug:
t2 = time.time()
if t2 - t1 > 0.1:
print(f"get {self._url} {headers!r} {t2 - t1:.f} slow")
print(f"get {self._url} {headers!r} {t2 - t1:.3f} slow")
response_code = c.getinfo(pycurl.RESPONSE_CODE)
response_code = response.status
if response_code == 416: # Requested Range Not Satisfiable
raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(ret)[:500]}")
if download_range and response_code != 206: # Partial Content
raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(ret)[:500]}")
if (not download_range) and response_code != 200: # OK
raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(ret)[:500]}")
ret = dats.getvalue()
self._pos += len(ret)
return ret

Write Preview
Loading…
Cancel
Save