jenkins: use build.py to manage cache size (#30440)

* jenkins: use build.py to manage cache size

* label

* set pp

* double

* cleanup

* non mimimal build on PC

---------

Co-authored-by: Justin Newberry <justin@comma.ai>
old-commit-hash: 5f7143df02

Jenkinsfile

@@ -85,7 +85,7 @@ def pcStage(String stageName, Closure body) {
 
     checkout scm
 
-    def dockerArgs = '--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache';
+    def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE}";
     docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .").inside(dockerArgs) {
       timeout(time: 20, unit: 'MINUTES') {
         try {
@@ -228,7 +228,7 @@ node {
       },
       'car tests': {
         pcStage("car tests") {
-          sh "scons -j30"
+          sh label: "build", script: "selfdrive/manager/build.py"
           sh label: "test_models.py", script: "INTERNAL_SEG_CNT=250 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \
               pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py"
           sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py"

selfdrive/manager/build.py

@@ -19,19 +19,21 @@ TOTAL_SCONS_NODES = 2560
 MAX_BUILD_PROGRESS = 100
 PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
 
-def build(spinner: Spinner, dirty: bool = False) -> None:
+def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
   env = os.environ.copy()
   env['SCONS_PROGRESS'] = "1"
   nproc = os.cpu_count()
   if nproc is None:
     nproc = 2
 
+  extra_args = ["--minimal"] if minimal else []
+
   # building with all cores can result in using too
   # much memory, so retry with less parallelism
   compile_output: List[bytes] = []
   for n in (nproc, nproc/2, 1):
     compile_output.clear()
-    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", "--minimal"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
     assert scons.stderr is not None
 
     # Read progress from stderr and update spinner
@@ -86,4 +88,4 @@ def build(spinner: Spinner, dirty: bool = False) -> None:
 if __name__ == "__main__" and not PREBUILT:
   spinner = Spinner()
   spinner.update_progress(0, 100)
-  build(spinner, is_dirty())
+  build(spinner, is_dirty(), minimal = AGNOS)