From b1eb066965d706d887b458fd7a62e03f5157e981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20R=C4=85czy?= Date: Wed, 21 Jun 2023 22:14:20 +0200 Subject: [PATCH] macOS: fix build issues & re-enable CI (#28258) * Uncomment build_mac job * Add PYCURL_CURL_CONFIG * Acados universal binary * Remove x86 macos references from sconscript * Add missing include in cabana * Update cereal * Update workflow * Remove unnecessary libpaths for darwin * Add missing path to cached dirs * Fix path in cache preparation step * Add poetry caches to CI cache * Compile acados with lower deployment target for CI * Update cereal * Pass -rpath in linker flags as scons doed not support RPATH on macos * Use scons api instead of os * @rpath in install name of acados dylibs * SConstruct cleanup * fix liblocationd tests by adding lib suffix based on platform * Update cereal * Update opendbc * Update opendbc * Add acados gitignore to release files * Update cereal * Update cereal * Add scons_cache to build cache * Add Caskroom to cache * Fix typo * Link all packages at once, instead of one by one * Run cleanup stage using poetry * Remove casks from cache * Move scons cache to separate cache step * Save scons cache only on master * Remove restore-keys from save-scons-cache step * Uncomment if conditions for scons save * Add gcc-arm-embedded cask cache to cache * Custom handling of gcc-arm-embedded toolchain cache * Rename dep cache key * Exclude .fseventsd from cache * Fix glob pattern * Remove .feventsd before caching * Run mac_setup only if dependency cache-hit != true * Update cereal to master * Add overwrite flag to brew link * Remove manual casadi build from mac_setup * Remove restore-keys from dependency cache * Remove linux requirement for casadi * Restore restore-keys to dependency cache old-commit-hash: 45a6834a744e13833e702ed6eb3c5483391cf899 --- .github/workflows/selfdrive_tests.yaml | 136 +++++++++++------- SConstruct | 43 +++--- poetry.lock | 2 +- pyproject.toml | 2 +- release/files_common | 1 + .../locationd/test/_test_locationd_lib.py | 4 +- third_party/acados/.gitignore | 1 + third_party/acados/Darwin/lib/libacados.dylib | 4 +- .../acados/Darwin/lib/libblasfeo.dylib | 4 +- third_party/acados/Darwin/lib/libhpipm.dylib | 4 +- .../acados/Darwin/lib/libqpOASES_e.3.1.dylib | 4 +- third_party/acados/Darwin/t_renderer | 4 +- third_party/acados/build.sh | 17 +-- tools/cabana/util.cc | 2 + tools/mac_setup.sh | 24 +--- 15 files changed, 130 insertions(+), 122 deletions(-) diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 4de4dee575..c40a30af93 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -81,61 +81,87 @@ jobs: ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ scons -j$(nproc) --cache-populate" - #build_mac: - # name: build macos - # runs-on: macos-latest - # steps: - # - uses: actions/checkout@v3 - # with: - # submodules: true - # - name: Determine pre-existing Homebrew packages - # if: steps.dependency-cache.outputs.cache-hit != 'true' - # run: | - # echo 'EXISTING_CELLAR<> $GITHUB_ENV - # ls -1 /usr/local/Cellar >> $GITHUB_ENV - # echo 'EOF' >> $GITHUB_ENV - # - name: Cache dependencies - # id: dependency-cache - # uses: actions/cache@v2 - # with: - # path: | - # ~/.pyenv - # ~/.local/share/virtualenvs/ - # /usr/local/Cellar - # ~/github_brew_cache_entries.txt - # /tmp/scons_cache - # key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }} - # restore-keys: macos- - # - name: Brew link restored dependencies - # run: | - # if [ -f ~/github_brew_cache_entries.txt ]; then - # while read pkg; do - # brew link --force "$pkg" # `--force` for keg-only packages - # done < ~/github_brew_cache_entries.txt - # else - # echo "Cache entries not found" - # fi - # - name: Install dependencies - # run: ./tools/mac_setup.sh - # - name: Build openpilot - # run: | - # source tools/openpilot_env.sh - # poetry run selfdrive/manager/build.py - # - # # cleanup scons cache - # rm -rf /tmp/scons_cache/ - # poetry run scons -j$(nproc) --cache-populate - # - name: Remove pre-existing Homebrew packages for caching - # if: steps.dependency-cache.outputs.cache-hit != 'true' - # run: | - # cd /usr/local/Cellar - # new_cellar=$(ls -1) - # comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do - # if [[ $pkg != "zstd" ]]; then # caching step needs zstd - # rm -rf "$pkg" - # fi - # done - # comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt + build_mac: + name: build macos + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Determine pre-existing Homebrew packages + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: | + echo 'EXISTING_CELLAR<> $GITHUB_ENV + brew list --formula -1 >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Restore scons cache + id: scons-restore-cache + uses: actions/cache/restore@v3 + with: + path: /tmp/scons_cache + key: macos_scons-${{ github.sha }} + restore-keys: macos_scons- + - name: Cache dependencies + id: dependency-cache + uses: actions/cache@v2 + with: + path: | + ~/github_brew_cache_entries.txt + ~/.pyenv + ~/Library/Caches/pypoetry + /usr/local/Cellar + /usr/local/opt + /usr/local/Caskroom/gcc-arm-* + /opt/homebrew/Cellar + /opt/homebrew/opt + /opt/homebrew/Caskroom/gcc-arm-* + /Applications/ArmGNUToolchain/*/*/* + key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }} + restore-keys: macos_deps- + - name: Brew link restored dependencies + run: | + if [ -f ~/github_brew_cache_entries.txt ]; then + brew link --force --overwrite $(cat ~/github_brew_cache_entries.txt) # `--force` for keg-only packages + if [ -d /Applications/ArmGNUToolchain ]; then # link gcc-arm-embedded manually + GCC_TOOLCHAIN="$(echo /Applications/ArmGNUToolchain/**/**/bin)" + echo "$GCC_TOOLCHAIN" >> $GITHUB_PATH + fi + else + echo "Cache entries not found" + fi + - name: Install dependencies + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: ./tools/mac_setup.sh + - name: Build openpilot + run: | + source tools/openpilot_env.sh + poetry run scons -j$(nproc) + - name: Pre Cache - Cleanup scons cache + if: github.ref == 'refs/heads/master' + run: | + source tools/openpilot_env.sh + rm -rf /tmp/scons_cache/* + poetry run scons -j$(nproc) --cache-populate + - name: Pre Cache - Remove pre-existing Homebrew packages + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: | + new_cellar=$(brew list --formula -1) + comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do + if [[ $pkg != "zstd" ]]; then # caching step needs zstd + rm -rf "$(brew --cellar)/$pkg" + fi + done + comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt + # .fseventsd directory causes permission errors in dep caching step + # its used by the system to observe changes within the directory - toolchain works without it, just remove it + sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd + - name: Save scons cache + id: scons-save-cache + uses: actions/cache/save@v3 + if: github.ref == 'refs/heads/master' + with: + path: /tmp/scons_cache + key: macos_scons-${{ github.sha }} docker_push: name: docker push diff --git a/SConstruct b/SConstruct index 45b6fd8fdf..6de1d7cce0 100644 --- a/SConstruct +++ b/SConstruct @@ -67,17 +67,22 @@ AddOption('--no-test', real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() if platform.system() == "Darwin": arch = "Darwin" + brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip() if arch == "aarch64" and AGNOS: arch = "larch64" +# create symlink to lib dir of current arch, so the ACADOS_SOURCE_DIR would have valid structure +acados_lib_path = Dir(f"#third_party/acados/lib") +if not Dir(f"#third_party/acados/lib").exists(): + os.symlink(Dir(f"#third_party/acados/{arch}/lib").abspath, acados_lib_path.abspath) lenv = { "PATH": os.environ['PATH'], "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath], "PYTHONPATH": Dir("#").abspath, - "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath, + "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath, "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath, "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer" } @@ -110,26 +115,21 @@ else: cflags = [] cxxflags = [] cpppath = [] + rpath += [ + Dir("#cereal").abspath, + Dir("#common").abspath + ] # MacOS if arch == "Darwin": - if real_arch == "x86_64": - lenv["TERA_PATH"] = Dir("#").abspath + f"/third_party/acados/Darwin_x86_64/t_renderer" - - brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip() yuv_dir = "mac" if real_arch != "arm64" else "mac_arm64" libpath = [ f"#third_party/libyuv/{yuv_dir}/lib", + f"#third_party/acados/{arch}/lib", f"{brew_prefix}/lib", - f"{brew_prefix}/Library", f"{brew_prefix}/opt/openssl@3.0/lib", - f"{brew_prefix}/Cellar", "/System/Library/Frameworks/OpenGL.framework/Libraries", ] - if real_arch == "x86_64": - libpath.append(f"#third_party/acados/Darwin_x86_64/lib") - else: - libpath.append(f"#third_party/acados/{arch}/lib") cflags += ["-DGL_SILENCE_DEPRECATION"] cxxflags += ["-DGL_SILENCE_DEPRECATION"] @@ -137,6 +137,7 @@ else: f"{brew_prefix}/include", f"{brew_prefix}/opt/openssl@3.0/include", ] + lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"] # Linux 86_64 else: libpath = [ @@ -149,12 +150,9 @@ else: "/usr/lib", "/usr/local/lib", ] - - rpath += [ - Dir("#third_party/snpe/x86_64-linux-clang").abspath, - Dir("#cereal").abspath, - Dir("#common").abspath - ] + rpath += [ + Dir("#third_party/snpe/x86_64-linux-clang").abspath, + ] if GetOption('asan'): ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"] @@ -231,7 +229,9 @@ env = Environment( ) if arch == "Darwin": - env['RPATHPREFIX'] = "-rpath " + # RPATH is not supported on macOS, instead use the linker flags + darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]] + env["LINKFLAGS"] += darwin_rpath_link_flags if GetOption('compile_db'): env.CompilationDatabase('compile_commands.json') @@ -271,7 +271,7 @@ envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-decla envCython["LIBS"] = [] if arch == "Darwin": - envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags elif arch == "aarch64": envCython["LINKFLAGS"] = ["-shared"] envCython["LIBS"] = [os.path.basename(py_include)] @@ -286,10 +286,7 @@ qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", " qt_libs = [] if arch == "Darwin": - if real_arch == "arm64": - qt_env['QTDIR'] = "/opt/homebrew/opt/qt@5" - else: - qt_env['QTDIR'] = "/usr/local/opt/qt@5" + qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5" qt_dirs = [ os.path.join(qt_env['QTDIR'], "include"), ] diff --git a/poetry.lock b/poetry.lock index 1cae9b9e44..2ad3a7ea4c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fab73ac1cd3005510b2976fc38185e8b43adc52373716ea011aa872eaeacea58 +oid sha256:ba83bb6503e6d0bcbcef45dad151aa76f52bfd91e15660621d94e830cebda1b7 size 641319 diff --git a/pyproject.toml b/pyproject.toml index b1e0dfa36b..d8523d2e12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ documentation = "https://docs.comma.ai" [tool.poetry.dependencies] python = "~3.8" atomicwrites = "^1.4.0" -casadi = { version = "==3.6.3", platform = "linux" } +casadi = "==3.6.3" cffi = "^1.15.1" crcmod = "^1.7" cryptography = "^37.0.4" diff --git a/release/files_common b/release/files_common index 18596a4e79..1c4955b96f 100644 --- a/release/files_common +++ b/release/files_common @@ -439,6 +439,7 @@ third_party/libyuv/larch64/** third_party/snpe/include/** third_party/snpe/dsp** +third_party/acados/.gitignore third_party/acados/x86_64/** third_party/acados/larch64/** third_party/acados/include/** diff --git a/selfdrive/locationd/test/_test_locationd_lib.py b/selfdrive/locationd/test/_test_locationd_lib.py index 819bb1570e..bec086a767 100755 --- a/selfdrive/locationd/test/_test_locationd_lib.py +++ b/selfdrive/locationd/test/_test_locationd_lib.py @@ -10,10 +10,12 @@ from cffi import FFI import cereal.messaging as messaging from cereal import log +from common.ffi_wrapper import suffix + SENSOR_DECIMATION = 1 VISION_DECIMATION = 1 -LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd.so')) +LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd' + suffix())) class TestLocationdLib(unittest.TestCase): diff --git a/third_party/acados/.gitignore b/third_party/acados/.gitignore index 9787bd1b55..68858c62e4 100644 --- a/third_party/acados/.gitignore +++ b/third_party/acados/.gitignore @@ -1,4 +1,5 @@ acados_repo/ +lib !x86_64/ !larch64/ !aarch64/ diff --git a/third_party/acados/Darwin/lib/libacados.dylib b/third_party/acados/Darwin/lib/libacados.dylib index 69d070e93f..f484043a1b 100755 --- a/third_party/acados/Darwin/lib/libacados.dylib +++ b/third_party/acados/Darwin/lib/libacados.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3a9698ef12d376ec02ac740941ed8554cdd34a0a8053fbfbbca2f0bd86c219c5 -size 375488 +oid sha256:2926c48d96e20086823c87899e5d6d834fada0f616916c0edfe0f58c2aa4b555 +size 834016 diff --git a/third_party/acados/Darwin/lib/libblasfeo.dylib b/third_party/acados/Darwin/lib/libblasfeo.dylib index 92a6cd9bba..189a78079d 100755 --- a/third_party/acados/Darwin/lib/libblasfeo.dylib +++ b/third_party/acados/Darwin/lib/libblasfeo.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:288c5cd192c5d1b49b2589cd2fbdad453542250494d638e5c299410460b554db -size 897632 +oid sha256:8ee2734e9a056573efbf011f34f85be4d76841ad6c152b52bd9aa2ee24a390b7 +size 2010877 diff --git a/third_party/acados/Darwin/lib/libhpipm.dylib b/third_party/acados/Darwin/lib/libhpipm.dylib index df5041ac42..576669b421 100755 --- a/third_party/acados/Darwin/lib/libhpipm.dylib +++ b/third_party/acados/Darwin/lib/libhpipm.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5b5af77374594077186deef3369e9f7c9e4956c0629a75d09cf0b41fb504ab7a -size 1062416 +oid sha256:30249b5c6d5c14b064ff3bd69d7b0ebc201cd9484c9686181d99610268a59246 +size 2290624 diff --git a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib index c06ef84ac5..849dd58803 100755 --- a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib +++ b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ac2857e8bd630146c13f1382456dbcde65926433fedd7da502805e9a383e3f5c -size 218408 +oid sha256:d7153b9b69fb15c05a5e04afacc4b69d7b781da96381114390700d504dacd9c3 +size 496707 diff --git a/third_party/acados/Darwin/t_renderer b/third_party/acados/Darwin/t_renderer index 7f8730635b..eddc66ab46 100755 --- a/third_party/acados/Darwin/t_renderer +++ b/third_party/acados/Darwin/t_renderer @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:605ac85907005a3847a71b3f723ab55408645e470fbbe76da5a294e46d624e90 -size 3585208 +oid sha256:f2304f73cd27c821a8cc03cf990b74ca246a6a95bef46d28e2b5d5e6a3132fdb +size 7776232 diff --git a/third_party/acados/build.sh b/third_party/acados/build.sh index 9b18b2b67d..574a577b4d 100755 --- a/third_party/acados/build.sh +++ b/third_party/acados/build.sh @@ -13,13 +13,8 @@ fi ACADOS_FLAGS="-DACADOS_WITH_QPOASES=ON -UBLASFEO_TARGET -DBLASFEO_TARGET=$BLAS_TARGET" if [[ "$OSTYPE" == "darwin"* ]]; then - if [[ $(uname -m) == "x86_64" ]]; then - ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=x86_64" - ARCHNAME="Darwin_x86_64" - else - ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64;x86_64" - ARCHNAME="Darwin" - fi + ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64;x86_64 -DCMAKE_MACOSX_RPATH=1" + ARCHNAME="Darwin" fi if [ ! -d acados_repo/ ]; then @@ -51,5 +46,11 @@ cp -r $DIR/acados_repo/interfaces/acados_template/acados_template $DIR/ # build tera cd $DIR/acados_repo/interfaces/acados_template/tera_renderer/ -cargo build --verbose --release +if [[ "$OSTYPE" == "darwin"* ]]; then + cargo build --verbose --release --target aarch64-apple-darwin + cargo build --verbose --release --target x86_64-apple-darwin + lipo -create -output target/release/t_renderer target/x86_64-apple-darwin/release/t_renderer target/aarch64-apple-darwin/release/t_renderer +else + cargo build --verbose --release +fi cp target/release/t_renderer $INSTALL_DIR/ diff --git a/tools/cabana/util.cc b/tools/cabana/util.cc index 02fb7129b6..bb588ad05f 100644 --- a/tools/cabana/util.cc +++ b/tools/cabana/util.cc @@ -1,5 +1,7 @@ #include "tools/cabana/util.h" +#include + #include #include #include diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index 6e6538c5c5..d5b0f189bb 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -69,6 +69,7 @@ export CPPFLAGS="$CPPFLAGS -I${BREW_PREFIX}/opt/bzip2/include" # pycurl curl/openssl backend dependencies export LDFLAGS="$LDFLAGS -L${BREW_PREFIX}/opt/openssl@3/lib" export CPPFLAGS="$CPPFLAGS -I${BREW_PREFIX}/opt/openssl@3/include" +export PYCURL_CURL_CONFIG=/usr/bin/curl-config export PYCURL_SSL_LIBRARY=openssl # openpilot environment @@ -83,29 +84,6 @@ $ROOT/update_requirements.sh eval "$(pyenv init --path)" echo "[ ] installed python dependencies t=$SECONDS" -# install casadi -VENV=`poetry env info --path` -PYTHON_VER=3.8 -PYTHON_VERSION=$(cat $ROOT/.python-version) -if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then - echo "-- casadi manual install" - cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz - tar -xzf casadi.tar.gz - cd casadi-ge6/ && mkdir -p build && cd build - cmake .. \ - -DWITH_PYTHON=ON \ - -DWITH_EXAMPLES=OFF \ - -DCMAKE_INSTALL_PREFIX:PATH=$VENV \ - -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \ - -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \ - -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \ - -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \ - -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0" - CFLAGS="-ferror-limit=0" make -j$(nproc) && make install -else - echo "---- casadi found in venv. skipping build ----" -fi - echo echo "---- OPENPILOT SETUP DONE ----" echo "Open a new shell or configure your active shell env by running:"