|
|
@ -1,18 +1,15 @@ |
|
|
|
#!/usr/bin/env python3 |
|
|
|
#!/usr/bin/env python3 |
|
|
|
import os |
|
|
|
import os |
|
|
|
import subprocess |
|
|
|
import subprocess |
|
|
|
import select |
|
|
|
|
|
|
|
from pathlib import Path |
|
|
|
from pathlib import Path |
|
|
|
import pyray as rl |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# NOTE: Do NOT import anything here that needs be built (e.g. params) |
|
|
|
# NOTE: Do NOT import anything here that needs be built (e.g. params) |
|
|
|
from openpilot.common.basedir import BASEDIR |
|
|
|
from openpilot.common.basedir import BASEDIR |
|
|
|
|
|
|
|
from openpilot.common.spinner import Spinner |
|
|
|
|
|
|
|
from openpilot.common.text_window import TextWindow |
|
|
|
from openpilot.system.hardware import HARDWARE, AGNOS |
|
|
|
from openpilot.system.hardware import HARDWARE, AGNOS |
|
|
|
from openpilot.common.swaglog import cloudlog, add_file_handler |
|
|
|
from openpilot.common.swaglog import cloudlog, add_file_handler |
|
|
|
from openpilot.system.version import get_build_metadata |
|
|
|
from openpilot.system.version import get_build_metadata |
|
|
|
from openpilot.system.ui.lib.application import gui_app |
|
|
|
|
|
|
|
from openpilot.system.ui.spinner import Spinner |
|
|
|
|
|
|
|
from openpilot.system.ui.text import TextWindow |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 |
|
|
|
MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 |
|
|
|
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") |
|
|
|
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") |
|
|
@ -20,50 +17,39 @@ CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") |
|
|
|
TOTAL_SCONS_NODES = 3130 |
|
|
|
TOTAL_SCONS_NODES = 3130 |
|
|
|
MAX_BUILD_PROGRESS = 100 |
|
|
|
MAX_BUILD_PROGRESS = 100 |
|
|
|
|
|
|
|
|
|
|
|
def build(dirty: bool = False, minimal: bool = False) -> None: |
|
|
|
def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: |
|
|
|
env = os.environ.copy() |
|
|
|
env = os.environ.copy() |
|
|
|
env['SCONS_PROGRESS'] = "1" |
|
|
|
env['SCONS_PROGRESS'] = "1" |
|
|
|
nproc = os.cpu_count() or 2 |
|
|
|
nproc = os.cpu_count() |
|
|
|
|
|
|
|
if nproc is None: |
|
|
|
|
|
|
|
nproc = 2 |
|
|
|
|
|
|
|
|
|
|
|
extra_args = ["--minimal"] if minimal else [] |
|
|
|
extra_args = ["--minimal"] if minimal else [] |
|
|
|
CI = os.getenv("CI") is not None |
|
|
|
|
|
|
|
if AGNOS: |
|
|
|
if AGNOS: |
|
|
|
HARDWARE.set_power_save(False) |
|
|
|
HARDWARE.set_power_save(False) |
|
|
|
os.sched_setaffinity(0, range(8)) # ensure we can use the isolcpus cores |
|
|
|
os.sched_setaffinity(0, range(8)) # ensure we can use the isolcpus cores |
|
|
|
|
|
|
|
|
|
|
|
# building with all cores can result in using too |
|
|
|
# building with all cores can result in using too |
|
|
|
# much memory, so retry with less parallelism |
|
|
|
# much memory, so retry with less parallelism |
|
|
|
if not CI: |
|
|
|
|
|
|
|
gui_app.init_window("Spinner") |
|
|
|
|
|
|
|
spinner = Spinner() |
|
|
|
|
|
|
|
compile_output: list[bytes] = [] |
|
|
|
compile_output: list[bytes] = [] |
|
|
|
for n in (nproc, nproc/2, 1): |
|
|
|
for n in (nproc, nproc/2, 1): |
|
|
|
compile_output.clear() |
|
|
|
compile_output.clear() |
|
|
|
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) |
|
|
|
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) |
|
|
|
assert scons.stderr is not None |
|
|
|
assert scons.stderr is not None |
|
|
|
os.set_blocking(scons.stderr.fileno(), False) # Non-blocking reads |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Read progress from stderr and update spinner |
|
|
|
# Read progress from stderr and update spinner |
|
|
|
if not CI: |
|
|
|
|
|
|
|
spinner.set_text("0") |
|
|
|
|
|
|
|
while scons.poll() is None: |
|
|
|
while scons.poll() is None: |
|
|
|
try: |
|
|
|
try: |
|
|
|
if not CI: |
|
|
|
line = scons.stderr.readline() |
|
|
|
rl.begin_drawing() |
|
|
|
if line is None: |
|
|
|
rl.clear_background(rl.BLACK) |
|
|
|
continue |
|
|
|
spinner.render() |
|
|
|
|
|
|
|
rl.end_drawing() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if scons.stderr in select.select([scons.stderr], [], [], 0.02)[0]: |
|
|
|
|
|
|
|
line = scons.stderr.readline() |
|
|
|
|
|
|
|
if not line: |
|
|
|
|
|
|
|
continue |
|
|
|
|
|
|
|
line = line.rstrip() |
|
|
|
line = line.rstrip() |
|
|
|
|
|
|
|
|
|
|
|
prefix = b'progress: ' |
|
|
|
prefix = b'progress: ' |
|
|
|
if line.startswith(prefix): |
|
|
|
if line.startswith(prefix): |
|
|
|
if not CI: |
|
|
|
i = int(line[len(prefix):]) |
|
|
|
i = int(line[len(prefix):]) |
|
|
|
spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.) |
|
|
|
spinner.set_text(str(int(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES)))) |
|
|
|
|
|
|
|
elif len(line): |
|
|
|
elif len(line): |
|
|
|
compile_output.append(line) |
|
|
|
compile_output.append(line) |
|
|
|
print(line.decode('utf8', 'replace')) |
|
|
|
print(line.decode('utf8', 'replace')) |
|
|
@ -76,7 +62,7 @@ def build(dirty: bool = False, minimal: bool = False) -> None: |
|
|
|
if scons.returncode != 0: |
|
|
|
if scons.returncode != 0: |
|
|
|
# Read remaining output |
|
|
|
# Read remaining output |
|
|
|
if scons.stderr is not None: |
|
|
|
if scons.stderr is not None: |
|
|
|
compile_output += [line for line in scons.stderr.read().split(b'\n') if not line.startswith(b'progress')] |
|
|
|
compile_output += scons.stderr.read().split(b'\n') |
|
|
|
|
|
|
|
|
|
|
|
# Build failed log errors |
|
|
|
# Build failed log errors |
|
|
|
error_s = b"\n".join(compile_output).decode('utf8', 'replace') |
|
|
|
error_s = b"\n".join(compile_output).decode('utf8', 'replace') |
|
|
@ -84,13 +70,10 @@ def build(dirty: bool = False, minimal: bool = False) -> None: |
|
|
|
cloudlog.error("scons build failed\n" + error_s) |
|
|
|
cloudlog.error("scons build failed\n" + error_s) |
|
|
|
|
|
|
|
|
|
|
|
# Show TextWindow |
|
|
|
# Show TextWindow |
|
|
|
if not CI: |
|
|
|
spinner.close() |
|
|
|
text_window = TextWindow("openpilot failed to build\n \n" + error_s) |
|
|
|
if not os.getenv("CI"): |
|
|
|
while True: |
|
|
|
with TextWindow("openpilot failed to build\n \n" + error_s) as t: |
|
|
|
rl.begin_drawing() |
|
|
|
t.wait_for_exit() |
|
|
|
rl.clear_background(rl.BLACK) |
|
|
|
|
|
|
|
text_window.render() |
|
|
|
|
|
|
|
rl.end_drawing() |
|
|
|
|
|
|
|
exit(1) |
|
|
|
exit(1) |
|
|
|
|
|
|
|
|
|
|
|
# enforce max cache size |
|
|
|
# enforce max cache size |
|
|
@ -105,5 +88,7 @@ def build(dirty: bool = False, minimal: bool = False) -> None: |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
if __name__ == "__main__": |
|
|
|
|
|
|
|
spinner = Spinner() |
|
|
|
|
|
|
|
spinner.update_progress(0, 100) |
|
|
|
build_metadata = get_build_metadata() |
|
|
|
build_metadata = get_build_metadata() |
|
|
|
build(build_metadata.openpilot.is_dirty, minimal = AGNOS) |
|
|
|
build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS) |
|
|
|