sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove wrappers

Browse Source
pull/35003/head
deanlee 8 months ago
parent e6742151aa
commit 900564fecc
5 changed files with 74 additions and 162 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 52
      common/spinner.py
  2. 63
      common/text_window.py
  3. 68
      system/athena/registration.py
  4. 48
      system/manager/build.py
  5. 5
      system/manager/manager.py

52
common/spinner.py
Unescape View File

@ -1,52 +0,0 @@
import os
import subprocess
from openpilot.common.basedir import BASEDIR
class Spinner:
def __init__(self):
try:
self.spinner_proc = subprocess.Popen(["./spinner.py"],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "system", "ui"),
close_fds=True)
except OSError:
self.spinner_proc = None
def __enter__(self):
return self
def update(self, spinner_text: str):
if self.spinner_proc is not None:
self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
try:
self.spinner_proc.stdin.flush()
except BrokenPipeError:
pass
def update_progress(self, cur: float, total: float):
self.update(str(round(100 * cur / total)))
def close(self):
if self.spinner_proc is not None:
self.spinner_proc.kill()
try:
self.spinner_proc.communicate(timeout=2.)
except subprocess.TimeoutExpired:
print("WARNING: failed to kill spinner")
self.spinner_proc = None
def __del__(self):
self.close()
def __exit__(self, exc_type, exc_value, traceback):
self.close()
if __name__ == "__main__":
import time
with Spinner() as s:
s.update("Spinner text")
time.sleep(5.0)
print("gone")
time.sleep(5.0)

63
common/text_window.py
Unescape View File

@ -1,63 +0,0 @@
#!/usr/bin/env python3
import os
import time
import subprocess
from openpilot.common.basedir import BASEDIR
class TextWindow:
def __init__(self, text):
try:
self.text_proc = subprocess.Popen(["./text.py", text],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "system", "ui"),
close_fds=True)
except OSError:
self.text_proc = None
def get_status(self):
if self.text_proc is not None:
self.text_proc.poll()
return self.text_proc.returncode
return None
def __enter__(self):
return self
def close(self):
if self.text_proc is not None:
self.text_proc.terminate()
self.text_proc = None
def wait_for_exit(self):
if self.text_proc is not None:
while True:
if self.get_status() == 1:
return
time.sleep(0.1)
def __del__(self):
self.close()
def __exit__(self, exc_type, exc_value, traceback):
self.close()
if __name__ == "__main__":
text = """Traceback (most recent call last):
File "./controlsd.py", line 608, in <module>
main()
File "./controlsd.py", line 604, in main
controlsd_thread(sm, pm, logcan)
File "./controlsd.py", line 455, in controlsd_thread
1/0
ZeroDivisionError: division by zero"""
print(text)
with TextWindow(text) as s:
for _ in range(100):
if s.get_status() == 1:
print("Got exit button")
break
time.sleep(0.1)
print("gone")

68
system/athena/registration.py
Unescape View File

@ -2,15 +2,18 @@
import time import time
import json import json
import jwt import jwt
import pyray as rl
from pathlib import Path from pathlib import Path
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, UTC from datetime import datetime, timedelta, UTC
from openpilot.common.api import api_get from openpilot.common.api import api_get
from openpilot.common.params import Params from openpilot.common.params import Params
from openpilot.common.spinner import Spinner
from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
from openpilot.system.hardware import HARDWARE, PC from openpilot.system.hardware import HARDWARE, PC
from openpilot.system.hardware.hw import Paths from openpilot.system.hardware.hw import Paths
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.spinner import Spinner
from openpilot.common.swaglog import cloudlog from openpilot.common.swaglog import cloudlog
@ -20,20 +23,8 @@ def is_registered_device() -> bool:
dongle = Params().get("DongleId", encoding='utf-8') dongle = Params().get("DongleId", encoding='utf-8')
return dongle not in (None, UNREGISTERED_DONGLE_ID) return dongle not in (None, UNREGISTERED_DONGLE_ID)
def _get_dongle_id() -> str | None:
def register(show_spinner=False) -> str | None: dongle_id: str | None = Params().get("DongleId", encoding='utf8')
"""
All devices built since March 2024 come with all
info stored in /persist/. This is kept around
only for devices built before then.
With a backend update to take serial number instead
of dongle ID to some endpoints, this can be removed
entirely.
"""
params = Params()
dongle_id: str | None = params.get("DongleId", encoding='utf8')
if dongle_id is None and Path(Paths.persist_root()+"/comma/dongle_id").is_file(): if dongle_id is None and Path(Paths.persist_root()+"/comma/dongle_id").is_file():
# not all devices will have this; added early in comma 3X production (2/28/24) # not all devices will have this; added early in comma 3X production (2/28/24)
with open(Paths.persist_root()+"/comma/dongle_id") as f: with open(Paths.persist_root()+"/comma/dongle_id") as f:
@ -43,11 +34,19 @@ def register(show_spinner=False) -> str | None:
if not pubkey.is_file(): if not pubkey.is_file():
dongle_id = UNREGISTERED_DONGLE_ID dongle_id = UNREGISTERED_DONGLE_ID
cloudlog.warning(f"missing public key: {pubkey}") cloudlog.warning(f"missing public key: {pubkey}")
elif dongle_id is None:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
return dongle_id
def do_register(spinner = None, done_event=None) -> str | None:
"""
All devices built since March 2024 come with all
info stored in /persist/. This is kept around
only for devices built before then.
With a backend update to take serial number instead
of dongle ID to some endpoints, this can be removed
entirely.
"""
# Create registration token, in the future, this key will make JWTs directly # Create registration token, in the future, this key will make JWTs directly
with open(Paths.persist_root()+"/comma/id_rsa.pub") as f1, open(Paths.persist_root()+"/comma/id_rsa") as f2: with open(Paths.persist_root()+"/comma/id_rsa.pub") as f1, open(Paths.persist_root()+"/comma/id_rsa") as f2:
public_key = f1.read() public_key = f1.read()
@ -65,9 +64,10 @@ def register(show_spinner=False) -> str | None:
cloudlog.exception("Error getting imei, trying again...") cloudlog.exception("Error getting imei, trying again...")
time.sleep(1) time.sleep(1)
if time.monotonic() - start_time > 60 and show_spinner: if time.monotonic() - start_time > 60 and spinner:
spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})") spinner.set_text(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})")
dongle_id = None
backoff = 0 backoff = 0
start_time = time.monotonic() start_time = time.monotonic()
while True: while True:
@ -89,14 +89,32 @@ def register(show_spinner=False) -> str | None:
backoff = min(backoff + 1, 15) backoff = min(backoff + 1, 15)
time.sleep(backoff) time.sleep(backoff)
if time.monotonic() - start_time > 60 and show_spinner: if time.monotonic() - start_time > 60 and spinner:
spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})") spinner.set_text(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})")
return dongle_id
def register(show_spinner=False) -> str | None:
dongle_id = _get_dongle_id()
if not dongle_id:
if show_spinner: if show_spinner:
spinner.close() with ThreadPoolExecutor(max_workers=1) as executor:
gui_app.init_window("Register")
spinner = Spinner()
spinner.set_text("registering device")
future = executor.submit(do_register, spinner)
while not future.done():
rl.begin_drawing()
rl.clear_background(rl.BLACK)
spinner.render()
rl.end_drawing()
gui_app.close()
dongle_id = future.result()
else:
dongle_id = do_register()
if dongle_id: if dongle_id:
params.put("DongleId", dongle_id) Params().put("DongleId", dongle_id)
set_offroad_alert("Offroad_UnofficialHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC) set_offroad_alert("Offroad_UnofficialHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC)
return dongle_id return dongle_id

48
system/manager/build.py
Unescape View File

@ -1,15 +1,18 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os import os
import subprocess import subprocess
import select
from pathlib import Path from pathlib import Path
import pyray as rl
# NOTE: Do NOT import anything here that needs be built (e.g. params) # NOTE: Do NOT import anything here that needs be built (e.g. params)
from openpilot.common.basedir import BASEDIR from openpilot.common.basedir import BASEDIR
from openpilot.common.spinner import Spinner
from openpilot.common.text_window import TextWindow
from openpilot.system.hardware import HARDWARE, AGNOS from openpilot.system.hardware import HARDWARE, AGNOS
from openpilot.common.swaglog import cloudlog, add_file_handler from openpilot.common.swaglog import cloudlog, add_file_handler
from openpilot.system.version import get_build_metadata from openpilot.system.version import get_build_metadata
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.spinner import Spinner
from openpilot.system.ui.text import TextWindow
MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
@ -17,39 +20,45 @@ CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
TOTAL_SCONS_NODES = 3130 TOTAL_SCONS_NODES = 3130
MAX_BUILD_PROGRESS = 100 MAX_BUILD_PROGRESS = 100
def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: def build(dirty: bool = False, minimal: bool = False) -> None:
env = os.environ.copy() env = os.environ.copy()
env['SCONS_PROGRESS'] = "1" env['SCONS_PROGRESS'] = "1"
nproc = os.cpu_count() nproc = os.cpu_count() or 2
if nproc is None:
nproc = 2
extra_args = ["--minimal"] if minimal else [] extra_args = ["--minimal"] if minimal else []
if AGNOS: if AGNOS:
HARDWARE.set_power_save(False) HARDWARE.set_power_save(False)
os.sched_setaffinity(0, range(8)) # ensure we can use the isolcpus cores os.sched_setaffinity(0, range(8)) # ensure we can use the isolcpus cores
# building with all cores can result in using too # building with all cores can result in using too
# much memory, so retry with less parallelism # much memory, so retry with less parallelism
gui_app.init_window("Spinner")
spinner = Spinner()
compile_output: list[bytes] = [] compile_output: list[bytes] = []
for n in (nproc, nproc/2, 1): for n in (nproc, nproc/2, 1):
compile_output.clear() compile_output.clear()
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
assert scons.stderr is not None assert scons.stderr is not None
os.set_blocking(scons.stderr.fileno(), False) # Non-blocking reads
# Read progress from stderr and update spinner # Read progress from stderr and update spinner
spinner.set_text("0")
while scons.poll() is None: while scons.poll() is None:
try: try:
line = scons.stderr.readline() rl.begin_drawing()
if line is None: rl.clear_background(rl.BLACK)
continue spinner.render()
rl.end_drawing()
if scons.stderr in select.select([scons.stderr], [], [], 0.02)[0]:
line = scons.stderr.readline()
if not line:
continue
line = line.rstrip() line = line.rstrip()
prefix = b'progress: ' prefix = b'progress: '
if line.startswith(prefix): if line.startswith(prefix):
i = int(line[len(prefix):]) i = int(line[len(prefix):])
spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.) spinner.set_text(str(int(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES))))
elif len(line): elif len(line):
compile_output.append(line) compile_output.append(line)
print(line.decode('utf8', 'replace')) print(line.decode('utf8', 'replace'))
@ -62,7 +71,7 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
if scons.returncode != 0: if scons.returncode != 0:
# Read remaining output # Read remaining output
if scons.stderr is not None: if scons.stderr is not None:
compile_output += scons.stderr.read().split(b'\n') compile_output += [line for line in scons.stderr.read().split(b'\n') if not line.startswith(b'progress')]
# Build failed log errors # Build failed log errors
error_s = b"\n".join(compile_output).decode('utf8', 'replace') error_s = b"\n".join(compile_output).decode('utf8', 'replace')
@ -70,10 +79,13 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
cloudlog.error("scons build failed\n" + error_s) cloudlog.error("scons build failed\n" + error_s)
# Show TextWindow # Show TextWindow
spinner.close()
if not os.getenv("CI"): if not os.getenv("CI"):
with TextWindow("openpilot failed to build\n \n" + error_s) as t: text_window = TextWindow("openpilot failed to build\n \n" + error_s)
t.wait_for_exit() while True:
rl.begin_drawing()
rl.clear_background(rl.BLACK)
text_window.render()
rl.end_drawing()
exit(1) exit(1)
# enforce max cache size # enforce max cache size
@ -88,7 +100,5 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
if __name__ == "__main__": if __name__ == "__main__":
spinner = Spinner()
spinner.update_progress(0, 100)
build_metadata = get_build_metadata() build_metadata = get_build_metadata()
build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS) build(build_metadata.openpilot.is_dirty, minimal = AGNOS)

5
system/manager/manager.py
Unescape View File

@ -9,7 +9,6 @@ from cereal import log
import cereal.messaging as messaging import cereal.messaging as messaging
import openpilot.system.sentry as sentry import openpilot.system.sentry as sentry
from openpilot.common.params import Params, ParamKeyType from openpilot.common.params import Params, ParamKeyType
from openpilot.common.text_window import TextWindow
from openpilot.system.hardware import HARDWARE from openpilot.system.hardware import HARDWARE
from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog
from openpilot.system.manager.process import ensure_running from openpilot.system.manager.process import ensure_running
@ -18,6 +17,7 @@ from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_I
from openpilot.common.swaglog import cloudlog, add_file_handler from openpilot.common.swaglog import cloudlog, add_file_handler
from openpilot.system.version import get_build_metadata, terms_version, training_version from openpilot.system.version import get_build_metadata, terms_version, training_version
from openpilot.system.hardware.hw import Paths from openpilot.system.hardware.hw import Paths
from openpilot.system.ui.text import show_text_in_window
def manager_init() -> None: def manager_init() -> None:
@ -221,8 +221,7 @@ if __name__ == "__main__":
# Show last 3 lines of traceback # Show last 3 lines of traceback
error = traceback.format_exc(-3) error = traceback.format_exc(-3)
error = "Manager failed to start\n\n" + error error = "Manager failed to start\n\n" + error
with TextWindow(error) as t: show_text_in_window(error)
t.wait_for_exit()
raise raise

Write Preview
Loading…
Cancel
Save