macOS: fix build issues & re-enable CI (#28258)

* Uncomment build_mac job

* Add PYCURL_CURL_CONFIG

* Acados universal binary

* Remove x86 macos references from sconscript

* Add missing include in cabana

* Update cereal

* Update workflow

* Remove unnecessary libpaths for darwin

* Add missing path to cached dirs

* Fix path in cache preparation step

* Add poetry caches to CI cache

* Compile acados with lower deployment target for CI

* Update cereal

* Pass -rpath in linker flags as scons doed not support RPATH on macos

* Use scons api instead of os

* @rpath in install name of acados dylibs

* SConstruct cleanup

* fix liblocationd tests by adding lib suffix based on platform

* Update cereal

* Update opendbc

* Update opendbc

* Add acados gitignore to release files

* Update cereal

* Update cereal

* Add scons_cache to build cache

* Add Caskroom to cache

* Fix typo

* Link all packages at once, instead of one by one

* Run cleanup stage using poetry

* Remove casks from cache

* Move scons cache to separate cache step

* Save scons cache only on master

* Remove restore-keys from save-scons-cache step

* Uncomment if conditions for scons save

* Add gcc-arm-embedded cask cache to cache

* Custom handling of gcc-arm-embedded toolchain cache

* Rename dep cache key

* Exclude .fseventsd from cache

* Fix glob pattern

* Remove .feventsd before caching

* Run mac_setup only if dependency cache-hit != true

* Update cereal to master

* Add overwrite flag to brew link

* Remove manual casadi build from mac_setup

* Remove restore-keys from dependency cache

* Remove linux requirement for casadi

* Restore restore-keys to dependency cache
old-commit-hash: 45a6834a74

.github/workflows/selfdrive_tests.yaml

@@ -81,61 +81,87 @@ jobs:
           ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
                           scons -j$(nproc) --cache-populate"
 
-  #build_mac:
+  build_mac:
-  #  name: build macos
+    name: build macos
-  #  runs-on: macos-latest
+    runs-on: macos-latest
-  #  steps:
+    steps:
-  #  - uses: actions/checkout@v3
+    - uses: actions/checkout@v3
-  #    with:
+      with:
-  #      submodules: true
+        submodules: true
-  #  - name: Determine pre-existing Homebrew packages
+    - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
+      run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
+        brew list --formula -1 >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
+    - name: Restore scons cache
-  #    id: dependency-cache
+      id: scons-restore-cache
-  #    uses: actions/cache@v2
+      uses: actions/cache/restore@v3
-  #    with:
+      with:
-  #      path: |
+        path: /tmp/scons_cache
-  #        ~/.pyenv
+        key: macos_scons-${{ github.sha }}
-  #        ~/.local/share/virtualenvs/
+        restore-keys: macos_scons-
-  #        /usr/local/Cellar
+    - name: Cache dependencies
-  #        ~/github_brew_cache_entries.txt
+      id: dependency-cache
-  #        /tmp/scons_cache
+      uses: actions/cache@v2
-  #      key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
+      with:
-  #      restore-keys: macos-
+        path: |
-  #  - name: Brew link restored dependencies
+          ~/github_brew_cache_entries.txt
-  #    run: |
+          ~/.pyenv
-  #      if [ -f ~/github_brew_cache_entries.txt ]; then
+          ~/Library/Caches/pypoetry
-  #        while read pkg; do
+          /usr/local/Cellar
-  #          brew link --force "$pkg" # `--force` for keg-only packages
+          /usr/local/opt
-  #        done < ~/github_brew_cache_entries.txt
+          /usr/local/Caskroom/gcc-arm-*
-  #      else
+          /opt/homebrew/Cellar
-  #        echo "Cache entries not found"
+          /opt/homebrew/opt
-  #      fi
+          /opt/homebrew/Caskroom/gcc-arm-*
-  #  - name: Install dependencies
+          /Applications/ArmGNUToolchain/*/*/*
-  #    run: ./tools/mac_setup.sh
+        key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
-  #  - name: Build openpilot
+        restore-keys: macos_deps-
-  #    run: |
+    - name: Brew link restored dependencies
-  #      source tools/openpilot_env.sh
+      run: |
-  #      poetry run selfdrive/manager/build.py
+        if [ -f ~/github_brew_cache_entries.txt ]; then
-  #
+          brew link --force --overwrite $(cat ~/github_brew_cache_entries.txt) # `--force` for keg-only packages
-  #      # cleanup scons cache
+          if [ -d /Applications/ArmGNUToolchain ]; then # link gcc-arm-embedded manually
-  #      rm -rf /tmp/scons_cache/
+            GCC_TOOLCHAIN="$(echo /Applications/ArmGNUToolchain/**/**/bin)"
-  #      poetry run scons -j$(nproc) --cache-populate
+            echo "$GCC_TOOLCHAIN" >> $GITHUB_PATH
-  #  - name: Remove pre-existing Homebrew packages for caching
+          fi
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
+        else
-  #    run: |
+          echo "Cache entries not found"
-  #      cd /usr/local/Cellar
+        fi
-  #      new_cellar=$(ls -1)
+    - name: Install dependencies
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+      run: ./tools/mac_setup.sh
-  #          rm -rf "$pkg"
+    - name: Build openpilot
-  #        fi
+      run: |
-  #      done
+        source tools/openpilot_env.sh
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+        poetry run scons -j$(nproc)
+    - name: Pre Cache - Cleanup scons cache
+      if: github.ref == 'refs/heads/master'
+      run: |
+        source tools/openpilot_env.sh
+        rm -rf /tmp/scons_cache/*
+        poetry run scons -j$(nproc) --cache-populate
+    - name: Pre Cache - Remove pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        new_cellar=$(brew list --formula -1)
+        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+            rm -rf "$(brew --cellar)/$pkg"
+          fi
+        done
+        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+        # .fseventsd directory causes permission errors in dep caching step
+        # its used by the system to observe changes within the directory - toolchain works without it, just remove it
+        sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd
+    - name: Save scons cache
+      id: scons-save-cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/scons_cache
+        key: macos_scons-${{ github.sha }}
 
   docker_push:
     name: docker push

SConstruct

@@ -67,17 +67,22 @@ AddOption('--no-test',
 real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
 
 if arch == "aarch64" and AGNOS:
   arch = "larch64"
 
+# create symlink to lib dir of current arch, so the ACADOS_SOURCE_DIR would have valid structure
+acados_lib_path = Dir(f"#third_party/acados/lib")
+if not Dir(f"#third_party/acados/lib").exists():
+  os.symlink(Dir(f"#third_party/acados/{arch}/lib").abspath, acados_lib_path.abspath)
 
 lenv = {
   "PATH": os.environ['PATH'],
   "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
   "PYTHONPATH": Dir("#").abspath,
 
-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
   "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
   "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
@@ -110,26 +115,21 @@ else:
   cflags = []
   cxxflags = []
   cpppath = []
+  rpath += [
+    Dir("#cereal").abspath,
+    Dir("#common").abspath
+  ]
 
   # MacOS
   if arch == "Darwin":
-    if real_arch == "x86_64":
-      lenv["TERA_PATH"] = Dir("#").abspath + f"/third_party/acados/Darwin_x86_64/t_renderer"
-
-    brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
     yuv_dir = "mac" if real_arch != "arm64" else "mac_arm64"
     libpath = [
       f"#third_party/libyuv/{yuv_dir}/lib",
+      f"#third_party/acados/{arch}/lib",
       f"{brew_prefix}/lib",
-      f"{brew_prefix}/Library",
       f"{brew_prefix}/opt/openssl@3.0/lib",
-      f"{brew_prefix}/Cellar",
       "/System/Library/Frameworks/OpenGL.framework/Libraries",
     ]
-    if real_arch == "x86_64":
-      libpath.append(f"#third_party/acados/Darwin_x86_64/lib")
-    else:
-      libpath.append(f"#third_party/acados/{arch}/lib")
 
     cflags += ["-DGL_SILENCE_DEPRECATION"]
     cxxflags += ["-DGL_SILENCE_DEPRECATION"]
@@ -137,6 +137,7 @@ else:
       f"{brew_prefix}/include",
       f"{brew_prefix}/opt/openssl@3.0/include",
     ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
   # Linux 86_64
   else:
     libpath = [
@@ -149,11 +150,8 @@ else:
       "/usr/lib",
       "/usr/local/lib",
     ]
-
     rpath += [
       Dir("#third_party/snpe/x86_64-linux-clang").abspath,
-    Dir("#cereal").abspath,
-    Dir("#common").abspath
     ]
 
 if GetOption('asan'):
@@ -231,7 +229,9 @@ env = Environment(
 )
 
 if arch == "Darwin":
-  env['RPATHPREFIX'] = "-rpath "
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
 
 if GetOption('compile_db'):
   env.CompilationDatabase('compile_commands.json')
@@ -271,7 +271,7 @@ envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-decla
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 elif arch == "aarch64":
   envCython["LINKFLAGS"] = ["-shared"]
   envCython["LIBS"] = [os.path.basename(py_include)]
@@ -286,10 +286,7 @@ qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "
 
 qt_libs = []
 if arch == "Darwin":
-  if real_arch == "arm64":
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
-    qt_env['QTDIR'] = "/opt/homebrew/opt/qt@5"
-  else:
-    qt_env['QTDIR'] = "/usr/local/opt/qt@5"
   qt_dirs = [
     os.path.join(qt_env['QTDIR'], "include"),
   ]

poetry.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fab73ac1cd3005510b2976fc38185e8b43adc52373716ea011aa872eaeacea58
+oid sha256:ba83bb6503e6d0bcbcef45dad151aa76f52bfd91e15660621d94e830cebda1b7
 size 641319

pyproject.toml

@@ -12,7 +12,7 @@ documentation = "https://docs.comma.ai"
 [tool.poetry.dependencies]
 python = "~3.8"
 atomicwrites = "^1.4.0"
-casadi = { version = "==3.6.3", platform = "linux" }
+casadi = "==3.6.3"
 cffi = "^1.15.1"
 crcmod = "^1.7"
 cryptography = "^37.0.4"

release/files_common

@@ -439,6 +439,7 @@ third_party/libyuv/larch64/**
 third_party/snpe/include/**
 third_party/snpe/dsp**
 
+third_party/acados/.gitignore
 third_party/acados/x86_64/**
 third_party/acados/larch64/**
 third_party/acados/include/**

selfdrive/locationd/test/_test_locationd_lib.py

@@ -10,10 +10,12 @@ from cffi import FFI
 import cereal.messaging as messaging
 from cereal import log
 
+from common.ffi_wrapper import suffix
+
 SENSOR_DECIMATION = 1
 VISION_DECIMATION = 1
 
-LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd.so'))
+LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd' + suffix()))
 
 
 class TestLocationdLib(unittest.TestCase):

third_party/acados/.gitignore

@@ -1,4 +1,5 @@
 acados_repo/
+lib
 !x86_64/
 !larch64/
 !aarch64/

third_party/acados/Darwin/lib/libacados.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a9698ef12d376ec02ac740941ed8554cdd34a0a8053fbfbbca2f0bd86c219c5
+oid sha256:2926c48d96e20086823c87899e5d6d834fada0f616916c0edfe0f58c2aa4b555
-size 375488
+size 834016

third_party/acados/Darwin/lib/libblasfeo.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:288c5cd192c5d1b49b2589cd2fbdad453542250494d638e5c299410460b554db
+oid sha256:8ee2734e9a056573efbf011f34f85be4d76841ad6c152b52bd9aa2ee24a390b7
-size 897632
+size 2010877

third_party/acados/Darwin/lib/libhpipm.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b5af77374594077186deef3369e9f7c9e4956c0629a75d09cf0b41fb504ab7a
+oid sha256:30249b5c6d5c14b064ff3bd69d7b0ebc201cd9484c9686181d99610268a59246
-size 1062416
+size 2290624

third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac2857e8bd630146c13f1382456dbcde65926433fedd7da502805e9a383e3f5c
+oid sha256:d7153b9b69fb15c05a5e04afacc4b69d7b781da96381114390700d504dacd9c3
-size 218408
+size 496707

third_party/acados/Darwin/t_renderer

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:605ac85907005a3847a71b3f723ab55408645e470fbbe76da5a294e46d624e90
+oid sha256:f2304f73cd27c821a8cc03cf990b74ca246a6a95bef46d28e2b5d5e6a3132fdb
-size 3585208
+size 7776232

third_party/acados/build.sh

@@ -13,14 +13,9 @@ fi
 ACADOS_FLAGS="-DACADOS_WITH_QPOASES=ON -UBLASFEO_TARGET -DBLASFEO_TARGET=$BLAS_TARGET"
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
-  if [[  $(uname -m) == "x86_64" ]]; then
+  ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64;x86_64 -DCMAKE_MACOSX_RPATH=1"
-    ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=x86_64"
-    ARCHNAME="Darwin_x86_64"
-  else
-    ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64;x86_64"
   ARCHNAME="Darwin"
 fi
-fi
 
 if [ ! -d acados_repo/ ]; then
   git clone https://github.com/acados/acados.git $DIR/acados_repo
@@ -51,5 +46,11 @@ cp -r $DIR/acados_repo/interfaces/acados_template/acados_template $DIR/
 
 # build tera
 cd $DIR/acados_repo/interfaces/acados_template/tera_renderer/
+if [[ "$OSTYPE" == "darwin"* ]]; then
+  cargo build --verbose --release --target aarch64-apple-darwin
+  cargo build --verbose --release --target x86_64-apple-darwin
+  lipo -create -output target/release/t_renderer target/x86_64-apple-darwin/release/t_renderer target/aarch64-apple-darwin/release/t_renderer
+else 
   cargo build --verbose --release
+fi
 cp target/release/t_renderer $INSTALL_DIR/

tools/cabana/util.cc

@@ -1,5 +1,7 @@
 #include "tools/cabana/util.h"
 
+#include <array>
+
 #include <QColor>
 #include <QFontDatabase>
 #include <QLocale>

tools/mac_setup.sh

@@ -69,6 +69,7 @@ export CPPFLAGS="$CPPFLAGS -I${BREW_PREFIX}/opt/bzip2/include"
 # pycurl curl/openssl backend dependencies
 export LDFLAGS="$LDFLAGS -L${BREW_PREFIX}/opt/openssl@3/lib"
 export CPPFLAGS="$CPPFLAGS -I${BREW_PREFIX}/opt/openssl@3/include"
+export PYCURL_CURL_CONFIG=/usr/bin/curl-config
 export PYCURL_SSL_LIBRARY=openssl
 
 # openpilot environment
@@ -83,29 +84,6 @@ $ROOT/update_requirements.sh
 eval "$(pyenv init --path)"
 echo "[ ] installed python dependencies t=$SECONDS"
 
-# install casadi
-VENV=`poetry env info --path`
-PYTHON_VER=3.8
-PYTHON_VERSION=$(cat $ROOT/.python-version)
-if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then
-  echo "-- casadi manual install"
-  cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz
-  tar -xzf casadi.tar.gz
-  cd casadi-ge6/ && mkdir -p build && cd build
-  cmake .. \
-    -DWITH_PYTHON=ON \
-    -DWITH_EXAMPLES=OFF \
-    -DCMAKE_INSTALL_PREFIX:PATH=$VENV \
-    -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \
-    -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \
-    -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \
-    -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \
-    -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0"
-  CFLAGS="-ferror-limit=0" make -j$(nproc) && make install
-else
-  echo "----   casadi found in venv. skipping build   ----"
-fi
-
 echo
 echo "----   OPENPILOT SETUP DONE   ----"
 echo "Open a new shell or configure your active shell env by running:"