From cf5b743de64f4a95827c6da9bf6c42d7ae9ebf3a Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 25 Sep 2025 20:55:14 -0700 Subject: [PATCH] build system cleanups (#36202) * it's all common * never getting fixed * it's just tici * reorders * qcom2 -> tici * Revert "qcom2 -> tici" This reverts commit f4d849b2952cb0e662975805db6a1d32511ed392. * Reapply "qcom2 -> tici" This reverts commit 58b193cb8de872830f8a7821a339edca14e4a337. * is tici * lil more * Revert "is tici" This reverts commit a169be18d3fdcb3ef8317a63a89d8becadabfad8. * Revert "Reapply "qcom2 -> tici"" This reverts commit 26f9c0e7d068fc8a1a5f07383b3616e619cd4e8c. * qcom2 -> __tici__ * lil more * mv lenv * clean that up * lil more] * fix * lil more --- SConstruct | 250 ++++++++++--------------- common/SConscript | 11 +- selfdrive/modeld/SConscript | 4 +- selfdrive/ui/qt/qt_window.cc | 2 +- selfdrive/ui/qt/qt_window.h | 2 +- selfdrive/ui/qt/widgets/cameraview.cc | 12 +- selfdrive/ui/qt/widgets/cameraview.h | 4 +- system/camerad/SConscript | 4 +- system/camerad/cameras/camera_qcom2.cc | 2 +- system/hardware/hw.h | 2 +- system/loggerd/encoderd.cc | 2 +- 11 files changed, 117 insertions(+), 178 deletions(-) diff --git a/SConstruct b/SConstruct index 6f035e3045..80273db106 100644 --- a/SConstruct +++ b/SConstruct @@ -3,144 +3,52 @@ import subprocess import sys import sysconfig import platform +import shlex import numpy as np import SCons.Errors SCons.Warnings.warningAsException(True) -# pending upstream fix - https://github.com/SCons/scons/issues/4461 -#SetOption('warn', 'all') - -TICI = os.path.isfile('/TICI') -AGNOS = TICI - Decider('MD5-timestamp') SetOption('num_jobs', max(1, int(os.cpu_count()/2))) -AddOption('--kaitai', - action='store_true', - help='Regenerate kaitai struct parsers') - -AddOption('--asan', - action='store_true', - help='turn on ASAN') - -AddOption('--ubsan', - action='store_true', - help='turn on UBSan') - -AddOption('--ccflags', - action='store', - type='string', - default='', - help='pass arbitrary flags over the command line') - -AddOption('--mutation', - action='store_true', - help='generate mutation-ready code') - +AddOption('--kaitai', action='store_true', help='Regenerate kaitai struct parsers') +AddOption('--asan', action='store_true', help='turn on ASAN') +AddOption('--ubsan', action='store_true', help='turn on UBSan') +AddOption('--mutation', action='store_true', help='generate mutation-ready code') +AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line') AddOption('--minimal', action='store_false', dest='extras', default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS) help='the minimum build to run openpilot. no tests, tools, etc.') -## Architecture name breakdown (arch) -## - larch64: linux tici aarch64 -## - aarch64: linux pc aarch64 -## - x86_64: linux pc x64 -## - Darwin: mac x64 or arm64 -real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() +# Detect platform +arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() if platform.system() == "Darwin": arch = "Darwin" brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip() -elif arch == "aarch64" and AGNOS: +elif arch == "aarch64" and os.path.isfile('/TICI'): arch = "larch64" -assert arch in ["larch64", "aarch64", "x86_64", "Darwin"] - -lenv = { - "PATH": os.environ['PATH'], - "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath, - - "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath, - "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath, - "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer" -} - -rpath = [] - -if arch == "larch64": - cpppath = [ - "#third_party/opencl/include", - ] - - libpath = [ - "/usr/local/lib", - "/system/vendor/lib64", - f"#third_party/acados/{arch}/lib", - ] - - libpath += [ - "#third_party/libyuv/larch64/lib", - "/usr/lib/aarch64-linux-gnu" - ] - cflags = ["-DQCOM2", "-mcpu=cortex-a57"] - cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"] - rpath += ["/usr/local/lib"] -else: - cflags = [] - cxxflags = [] - cpppath = [] - rpath += [] - - # MacOS - if arch == "Darwin": - libpath = [ - f"#third_party/libyuv/{arch}/lib", - f"#third_party/acados/{arch}/lib", - f"{brew_prefix}/lib", - f"{brew_prefix}/opt/openssl@3.0/lib", - f"{brew_prefix}/opt/llvm/lib/c++", - "/System/Library/Frameworks/OpenGL.framework/Libraries", - ] - - cflags += ["-DGL_SILENCE_DEPRECATION"] - cxxflags += ["-DGL_SILENCE_DEPRECATION"] - cpppath += [ - f"{brew_prefix}/include", - f"{brew_prefix}/opt/openssl@3.0/include", - ] - # Linux - else: - libpath = [ - f"#third_party/acados/{arch}/lib", - f"#third_party/libyuv/{arch}/lib", - "/usr/lib", - "/usr/local/lib", - ] - -if GetOption('asan'): - ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"] - ldflags = ["-fsanitize=address"] -elif GetOption('ubsan'): - ccflags = ["-fsanitize=undefined"] - ldflags = ["-fsanitize=undefined"] -else: - ccflags = [] - ldflags = [] - -# no --as-needed on mac linker -if arch != "Darwin": - ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"] - -ccflags_option = GetOption('ccflags') -if ccflags_option: - ccflags += ccflags_option.split(' ') +assert arch in [ + "larch64", # linux tici arm64 + "aarch64", # linux pc arm64 + "x86_64", # linux pc x64 + "Darwin", # macOS arm64 (x86 not supported) +] env = Environment( - ENV=lenv, + ENV={ + "PATH": os.environ['PATH'], + "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath, + "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath, + "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath, + "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer" + }, + CC='clang', + CXX='clang++', CCFLAGS=[ "-g", "-fPIC", @@ -153,36 +61,31 @@ env = Environment( "-Wno-c99-designator", "-Wno-reorder-init-list", "-Wno-vla-cxx-extension", - ] + cflags + ccflags, - - CPPPATH=cpppath + [ + ], + CFLAGS=["-std=gnu11"], + CXXFLAGS=["-std=c++1z"], + CPPPATH=[ "#", + "#msgq", + "#third_party", + "#third_party/json11", + "#third_party/linux/include", "#third_party/acados/include", "#third_party/acados/include/blasfeo/include", "#third_party/acados/include/hpipm/include", "#third_party/catch2/include", "#third_party/libyuv/include", - "#third_party/json11", - "#third_party/linux/include", - "#third_party", - "#msgq", ], - - CC='clang', - CXX='clang++', - LINKFLAGS=ldflags, - - RPATH=rpath, - - CFLAGS=["-std=gnu11"] + cflags, - CXXFLAGS=["-std=c++1z"] + cxxflags, - LIBPATH=libpath + [ + LIBPATH=[ + "#common", "#msgq_repo", "#third_party", "#selfdrive/pandad", - "#common", "#rednose/helpers", + f"#third_party/libyuv/{arch}/lib", + f"#third_party/acados/{arch}/lib", ], + RPATH=[], CYTHONCFILESUFFIX=".cpp", COMPILATIONDB_USE_ABSPATH=True, REDNOSE_ROOT="#", @@ -190,29 +93,63 @@ env = Environment( toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"], ) -if arch == "Darwin": - # RPATH is not supported on macOS, instead use the linker flags - darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]] - env["LINKFLAGS"] += darwin_rpath_link_flags +# Arch-specific flags and paths +if arch == "larch64": + env.Append(CPPPATH=["#third_party/opencl/include"]) + env.Append(LIBPATH=[ + "/usr/local/lib", + "/system/vendor/lib64", + "/usr/lib/aarch64-linux-gnu", + ]) + arch_flags = ["-D__TICI__", "-mcpu=cortex-a57"] + env.Append(CCFLAGS=arch_flags) + env.Append(CXXFLAGS=arch_flags) +elif arch == "Darwin": + env.Append(LIBPATH=[ + f"{brew_prefix}/lib", + f"{brew_prefix}/opt/openssl@3.0/lib", + f"{brew_prefix}/opt/llvm/lib/c++", + "/System/Library/Frameworks/OpenGL.framework/Libraries", + ]) + env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"]) + env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"]) + env.Append(CPPPATH=[ + f"{brew_prefix}/include", + f"{brew_prefix}/opt/openssl@3.0/include", + ]) +else: + env.Append(LIBPATH=[ + "/usr/lib", + "/usr/local/lib", + ]) -env.CompilationDatabase('compile_commands.json') +# Sanitizers and extra CCFLAGS from CLI +if GetOption('asan'): + env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"]) + env.Append(LINKFLAGS=["-fsanitize=address"]) +elif GetOption('ubsan'): + env.Append(CCFLAGS=["-fsanitize=undefined"]) + env.Append(LINKFLAGS=["-fsanitize=undefined"]) -# Setup cache dir -cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' -CacheDir(cache_dir) -Clean(["."], cache_dir) +_extra_cc = shlex.split(GetOption('ccflags') or '') +if _extra_cc: + env.Append(CCFLAGS=_extra_cc) +# no --as-needed on mac linker +if arch != "Darwin": + env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"]) + +# progress output node_interval = 5 node_count = 0 def progress_function(node): global node_count node_count += node_interval sys.stderr.write("progress: %d\n" % node_count) - if os.environ.get('SCONS_PROGRESS'): Progress(progress_function, interval=node_interval) -# Cython build environment +# ********** Cython build environment ********** py_include = sysconfig.get_paths()['include'] envCython = env.Clone() envCython["CPPPATH"] += [py_include, np.get_include()] @@ -221,14 +158,14 @@ envCython["CCFLAGS"].remove("-Werror") envCython["LIBS"] = [] if arch == "Darwin": - envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags + envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"] else: envCython["LINKFLAGS"] = ["-pthread", "-shared"] np_version = SCons.Script.Value(np.__version__) Export('envCython', 'np_version') -# Qt build environment +# ********** Qt build environment ********** qt_env = env.Clone() qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"] @@ -278,16 +215,20 @@ qt_env['CXXFLAGS'] += qt_flags qt_env['LIBPATH'] += ['#selfdrive/ui', ] qt_env['LIBS'] = qt_libs -Export('env', 'qt_env', 'arch', 'real_arch') +Export('env', 'qt_env', 'arch') + +# Setup cache dir +cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache' +CacheDir(cache_dir) +Clean(["."], cache_dir) + +# ********** start building stuff ********** # Build common module SConscript(['common/SConscript']) -Import('_common', '_gpucommon') - +Import('_common') common = [_common, 'json11', 'zmq'] -gpucommon = [_gpucommon] - -Export('common', 'gpucommon') +Export('common') # Build messaging (cereal + msgq + socketmaster + their dependencies) # Enable swaglog include in submodules @@ -327,3 +268,6 @@ if Dir('#tools/cabana/').exists() and GetOption('extras'): SConscript(['tools/replay/SConscript']) if arch != "larch64": SConscript(['tools/cabana/SConscript']) + + +env.CompilationDatabase('compile_commands.json') diff --git a/common/SConscript b/common/SConscript index 3cdb6fc5a2..0891b79039 100644 --- a/common/SConscript +++ b/common/SConscript @@ -5,17 +5,12 @@ common_libs = [ 'swaglog.cc', 'util.cc', 'watchdog.cc', - 'ratekeeper.cc' -] - -_common = env.Library('common', common_libs, LIBS="json11") - -files = [ + 'ratekeeper.cc', 'clutil.cc', ] -_gpucommon = env.Library('gpucommon', files) -Export('_common', '_gpucommon') +_common = env.Library('common', common_libs, LIBS="json11") +Export('_common') if GetOption('extras'): env.Program('tests/test_common', diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 1a6df4bb9b..f20855c2cb 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -1,11 +1,11 @@ import os import glob -Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'transformations') +Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'visionipc', 'transformations') lenv = env.Clone() lenvCython = envCython.Clone() -libs = [cereal, messaging, visionipc, gpucommon, common, 'capnp', 'kj', 'pthread'] +libs = [cereal, messaging, visionipc, common, 'capnp', 'kj', 'pthread'] frameworks = [] common_src = [ diff --git a/selfdrive/ui/qt/qt_window.cc b/selfdrive/ui/qt/qt_window.cc index 8d3d7cf72e..cdd817ae2f 100644 --- a/selfdrive/ui/qt/qt_window.cc +++ b/selfdrive/ui/qt/qt_window.cc @@ -13,7 +13,7 @@ void setMainWindow(QWidget *w) { } w->show(); -#ifdef QCOM2 +#ifdef __TICI__ QPlatformNativeInterface *native = QGuiApplication::platformNativeInterface(); wl_surface *s = reinterpret_cast(native->nativeResourceForWindow("surface", w->windowHandle())); wl_surface_set_buffer_transform(s, WL_OUTPUT_TRANSFORM_270); diff --git a/selfdrive/ui/qt/qt_window.h b/selfdrive/ui/qt/qt_window.h index 6f16e00957..b9783477eb 100644 --- a/selfdrive/ui/qt/qt_window.h +++ b/selfdrive/ui/qt/qt_window.h @@ -6,7 +6,7 @@ #include #include -#ifdef QCOM2 +#ifdef __TICI__ #include #include #include diff --git a/selfdrive/ui/qt/widgets/cameraview.cc b/selfdrive/ui/qt/widgets/cameraview.cc index 81ef613393..5f533cd090 100644 --- a/selfdrive/ui/qt/widgets/cameraview.cc +++ b/selfdrive/ui/qt/widgets/cameraview.cc @@ -27,7 +27,7 @@ const char frame_vertex_shader[] = "}\n"; const char frame_fragment_shader[] = -#ifdef QCOM2 +#ifdef __TICI__ "#version 300 es\n" "#extension GL_OES_EGL_image_external_essl3 : enable\n" "precision mediump float;\n" @@ -79,7 +79,7 @@ CameraWidget::~CameraWidget() { glDeleteVertexArrays(1, &frame_vao); glDeleteBuffers(1, &frame_vbo); glDeleteBuffers(1, &frame_ibo); -#ifndef QCOM2 +#ifndef __TICI__ glDeleteTextures(2, textures); #endif } @@ -137,7 +137,7 @@ void CameraWidget::initializeGL() { glUseProgram(program->programId()); -#ifdef QCOM2 +#ifdef __TICI__ glUniform1i(program->uniformLocation("uTexture"), 0); #else glGenTextures(2, textures); @@ -165,7 +165,7 @@ void CameraWidget::stopVipcThread() { vipc_thread = nullptr; } -#ifdef QCOM2 +#ifdef __TICI__ EGLDisplay egl_display = eglGetCurrentDisplay(); assert(egl_display != EGL_NO_DISPLAY); for (auto &pair : egl_images) { @@ -226,7 +226,7 @@ void CameraWidget::paintGL() { glUseProgram(program->programId()); glPixelStorei(GL_UNPACK_ALIGNMENT, 1); -#ifdef QCOM2 +#ifdef __TICI__ // no frame copy glActiveTexture(GL_TEXTURE0); glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, egl_images[frame->idx]); @@ -263,7 +263,7 @@ void CameraWidget::vipcConnected(VisionIpcClient *vipc_client) { stream_height = vipc_client->buffers[0].height; stream_stride = vipc_client->buffers[0].stride; -#ifdef QCOM2 +#ifdef __TICI__ EGLDisplay egl_display = eglGetCurrentDisplay(); assert(egl_display != EGL_NO_DISPLAY); for (auto &pair : egl_images) { diff --git a/selfdrive/ui/qt/widgets/cameraview.h b/selfdrive/ui/qt/widgets/cameraview.h index 29aa8493c7..598603a08a 100644 --- a/selfdrive/ui/qt/widgets/cameraview.h +++ b/selfdrive/ui/qt/widgets/cameraview.h @@ -13,7 +13,7 @@ #include #include -#ifdef QCOM2 +#ifdef __TICI__ #define EGL_EGLEXT_PROTOTYPES #define EGL_NO_X11 #define GL_TEXTURE_EXTERNAL_OES 0x8D65 @@ -63,7 +63,7 @@ protected: std::unique_ptr program; QColor bg = QColor("#000000"); -#ifdef QCOM2 +#ifdef __TICI__ std::map egl_images; #endif diff --git a/system/camerad/SConscript b/system/camerad/SConscript index 734f748a2a..e288c6d8b0 100644 --- a/system/camerad/SConscript +++ b/system/camerad/SConscript @@ -1,6 +1,6 @@ -Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc') +Import('env', 'arch', 'messaging', 'common', 'visionipc') -libs = [common, 'OpenCL', messaging, visionipc, gpucommon] +libs = [common, 'OpenCL', messaging, visionipc] if arch != "Darwin": camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc', diff --git a/system/camerad/cameras/camera_qcom2.cc b/system/camerad/cameras/camera_qcom2.cc index 8c4602bb31..85f358977c 100644 --- a/system/camerad/cameras/camera_qcom2.cc +++ b/system/camerad/cameras/camera_qcom2.cc @@ -12,7 +12,7 @@ #include #include -#ifdef QCOM2 +#ifdef __TICI__ #include "CL/cl_ext_qcom.h" #else #define CL_PRIORITY_HINT_HIGH_QCOM NULL diff --git a/system/hardware/hw.h b/system/hardware/hw.h index d2083a5985..a9058401ce 100644 --- a/system/hardware/hw.h +++ b/system/hardware/hw.h @@ -5,7 +5,7 @@ #include "system/hardware/base.h" #include "common/util.h" -#if QCOM2 +#if __TICI__ #include "system/hardware/tici/hardware.h" #define Hardware HardwareTici #else diff --git a/system/loggerd/encoderd.cc b/system/loggerd/encoderd.cc index 3237d13074..9d4b81a3f9 100644 --- a/system/loggerd/encoderd.cc +++ b/system/loggerd/encoderd.cc @@ -3,7 +3,7 @@ #include "system/loggerd/loggerd.h" #include "system/loggerd/encoder/jpeg_encoder.h" -#ifdef QCOM2 +#ifdef __TICI__ #include "system/loggerd/encoder/v4l_encoder.h" #define Encoder V4LEncoder #else