diff --git a/SConstruct b/SConstruct index 2e38e8bb44..992e7b57aa 100644 --- a/SConstruct +++ b/SConstruct @@ -229,10 +229,8 @@ if os.environ.get('SCONS_CACHE'): if TICI: cache_dir = '/data/scons_cache' - if QCOM_REPLAY: - cache_dir = '/tmp/scons_cache_qcom_replay' - CacheDir(cache_dir) + Clean(["."], cache_dir) node_interval = 5 node_count = 0 diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index 84f2e48a83..bda4ab8a1f 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -1,18 +1,22 @@ #!/usr/bin/env python3 import os -import shutil import subprocess import sys import time import textwrap +from pathlib import Path # NOTE: Do NOT import anything here that needs be built (e.g. params) from common.basedir import BASEDIR from common.spinner import Spinner from common.text_window import TextWindow +from selfdrive.hardware import TICI from selfdrive.swaglog import cloudlog, add_file_handler from selfdrive.version import dirty +MAX_CACHE_SIZE = 2e9 +CACHE_DIR = Path("/data/scons_cache" if TICI else "/tmp/scons_cache") + TOTAL_SCONS_NODES = 2405 MAX_BUILD_PROGRESS = 100 PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt')) @@ -26,7 +30,7 @@ def build(spinner, dirty=False): j_flag = "" if nproc is None else f"-j{nproc - 1}" for retry in [True, False]: - scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) + scons = subprocess.Popen(["scons", j_flag, "--cache-populate"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) compile_output = [] @@ -60,8 +64,6 @@ def build(spinner, dirty=False): print("....%d" % i) time.sleep(1) subprocess.check_call(["scons", "-c"], cwd=BASEDIR, env=env) - shutil.rmtree("/tmp/scons_cache", ignore_errors=True) - shutil.rmtree("/data/scons_cache", ignore_errors=True) else: print("scons build failed after retry") sys.exit(1) @@ -82,6 +84,16 @@ def build(spinner, dirty=False): else: break + # enforce max cache size + cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()] + cache_files.sort(key=lambda f: f.stat().st_mtime) + cache_size = sum(f.stat().st_size for f in cache_files) + for f in cache_files: + if cache_size < MAX_CACHE_SIZE: + break + cache_size -= f.stat().st_size + f.unlink() + if __name__ == "__main__" and not PREBUILT: spinner = Spinner() diff --git a/selfdrive/test/setup_device_ci.sh b/selfdrive/test/setup_device_ci.sh index cc04504ad9..fc4d4496a7 100755 --- a/selfdrive/test/setup_device_ci.sh +++ b/selfdrive/test/setup_device_ci.sh @@ -26,9 +26,6 @@ if [ ! -d "$SOURCE_DIR" ]; then git clone --depth 1 https://github.com/commaai/openpilot.git "$SOURCE_DIR" fi -# clear stale build cache -find /tmp/scons_cache*/* -mtime +1 -exec rm -rf '{}' \; || true - # this can get really big on the CI devices rm -rf /data/core