From 8d3a5970d4ee82ad1e138e1a47a154e3977186c9 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 10 Nov 2023 16:55:25 -0800 Subject: [PATCH] jenkins: use build.py to manage cache size (#30440) * jenkins: use build.py to manage cache size * label * set pp * double * cleanup * non mimimal build on PC --------- Co-authored-by: Justin Newberry old-commit-hash: 5f7143df02b857201e2615240ccd1be06daa32fe --- Jenkinsfile | 4 ++-- selfdrive/manager/build.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 314170341e..6857ce36e1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ def pcStage(String stageName, Closure body) { checkout scm - def dockerArgs = '--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache'; + def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE}"; docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .").inside(dockerArgs) { timeout(time: 20, unit: 'MINUTES') { try { @@ -228,7 +228,7 @@ node { }, 'car tests': { pcStage("car tests") { - sh "scons -j30" + sh label: "build", script: "selfdrive/manager/build.py" sh label: "test_models.py", script: "INTERNAL_SEG_CNT=250 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \ pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py" sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py" diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index be37b3e7e4..247f3c8fb8 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -19,19 +19,21 @@ TOTAL_SCONS_NODES = 2560 MAX_BUILD_PROGRESS = 100 PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt')) -def build(spinner: Spinner, dirty: bool = False) -> None: +def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: env = os.environ.copy() env['SCONS_PROGRESS'] = "1" nproc = os.cpu_count() if nproc is None: nproc = 2 + extra_args = ["--minimal"] if minimal else [] + # building with all cores can result in using too # much memory, so retry with less parallelism compile_output: List[bytes] = [] for n in (nproc, nproc/2, 1): compile_output.clear() - scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", "--minimal"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) + scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) assert scons.stderr is not None # Read progress from stderr and update spinner @@ -86,4 +88,4 @@ def build(spinner: Spinner, dirty: bool = False) -> None: if __name__ == "__main__" and not PREBUILT: spinner = Spinner() spinner.update_progress(0, 100) - build(spinner, is_dirty()) + build(spinner, is_dirty(), minimal = AGNOS)