From 192978753d0da3eecc0bbb88f913820945ae168f Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 7 Jan 2022 15:15:18 -0800 Subject: [PATCH] mac: fix setup and re-enable CI (#23365) * rework mac_setup.sh * ignore casadi and onnx runtime in pipfile for Darwin * remove rust stuff * build and install casadi * add wget * explicit python3. dont do onnx just yet * resource compiler * replace acados Darwin lib w universal2 libs * relock Pipfile using linux machine * Update update_requirements.sh brackets * Update update_requirements.sh oops * ci: re-enable mac build * attempt to fix ci build: * pip stuff? * move rcc bin path to env script * oops * only mac * k im lazy. does ci like this?? * huh?? * * use curl * avoid casadi rebuilds * add comment to remove protobuf * host detect typo * python version * how did the builds not use swig? * better reinstall check * sometimes mac clang complain abt error limit * ci: build OP in venv * ci: bump to Big Sur * retry? idk why it failed @ acados link * * use macos-latest * move rcc path to scons * add extra paths for odd homebrew installs * acados source change * update macOS README * uh, maybe? * k nvm * ci: ok this is strange. might be scons bug? * fix conflicts: bump * just add cppcheck for pre-commit stuff * agane * cleanup * try that * fix path * no pyenv update on mac * source * fix rpath prefix? * no examples * fix exit * let's get this cached * add virtualenvs to cache * why did we cache that * let's see what's big * more * always ruyn * cache scons cache * better cache key * fix for partial hit * why so long Co-authored-by: Andrew Tec Co-authored-by: Adeeb Shihadeh Co-authored-by: Bruce Wayne old-commit-hash: 66644217100f17e0a0c4f95f86aed2fd8c09c28c --- .github/workflows/selfdrive_tests.yaml | 102 ++++++++++-------- Pipfile | 4 +- Pipfile.lock | 4 +- SConstruct | 9 +- third_party/acados/Darwin/lib/libacados.dylib | 4 +- .../acados/Darwin/lib/libblasfeo.dylib | 4 +- third_party/acados/Darwin/lib/libhpipm.dylib | 4 +- .../acados/Darwin/lib/libqpOASES_e.3.1.dylib | 4 +- tools/mac_setup.sh | 51 ++++++--- tools/openpilot_env.sh | 2 +- tools/ubuntu_setup.sh | 4 +- update_requirements.sh | 18 ++-- 12 files changed, 129 insertions(+), 81 deletions(-) diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 1bee59e458..f04eaad86f 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -69,52 +69,62 @@ jobs: rm -rf /tmp/scons_cache/* && \ scons -j$(nproc) --cache-populate" - #build_mac: - # name: build macos - # runs-on: macos-10.15 - # timeout-minutes: 60 - # steps: - # - uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Determine pre-existing Homebrew packages - # if: steps.dependency-cache.outputs.cache-hit != 'true' - # run: | - # echo 'EXISTING_CELLAR<> $GITHUB_ENV - # ls -1 /usr/local/Cellar >> $GITHUB_ENV - # echo 'EOF' >> $GITHUB_ENV - # - name: Cache dependencies - # id: dependency-cache - # uses: actions/cache@v2 - # with: - # path: | - # ~/.pyenv - # ~/Library/Caches/pip - # ~/Library/Caches/pipenv - # /usr/local/Cellar - # ~/github_brew_cache_entries.txt - # key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }} - # - name: Brew link restored dependencies - # if: steps.dependency-cache.outputs.cache-hit == 'true' - # run: | - # while read pkg; do - # brew link --force "$pkg" # `--force` for keg-only packages - # done < ~/github_brew_cache_entries.txt - # - name: Install dependencies - # run: ./tools/mac_setup.sh - # - name: Build openpilot - # run: eval "$(pyenv init -)" && scons -j$(nproc) - # - name: Remove pre-existing Homebrew packages for caching - # if: steps.dependency-cache.outputs.cache-hit != 'true' - # run: | - # cd /usr/local/Cellar - # new_cellar=$(ls -1) - # comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do - # if [[ $pkg != "zstd" ]]; then # caching step needs zstd - # rm -rf "$pkg" - # fi - # done - # comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt + build_mac: + name: build macos + runs-on: macos-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Determine pre-existing Homebrew packages + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: | + echo 'EXISTING_CELLAR<> $GITHUB_ENV + ls -1 /usr/local/Cellar >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Cache dependencies + id: dependency-cache + uses: actions/cache@v2 + with: + path: | + ~/.pyenv + ~/.local/share/virtualenvs/ + /usr/local/Cellar + ~/github_brew_cache_entries.txt + /tmp/scons_cache + key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }} + restore-keys: macos- + - name: Brew link restored dependencies + run: | + if [ -f ~/github_brew_cache_entries.txt ]; then + while read pkg; do + brew link --force "$pkg" # `--force` for keg-only packages + done < ~/github_brew_cache_entries.txt + else + echo "Cache entries not found" + fi + - name: Install dependencies + run: ./tools/mac_setup.sh + - name: Build openpilot + run: | + source tools/openpilot_env.sh + pipenv run selfdrive/manager/build.py + + # cleanup scons cache + rm -rf /tmp/scons_cache/ + pipenv run scons -j$(nproc) --cache-populate + - name: Remove pre-existing Homebrew packages for caching + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: | + cd /usr/local/Cellar + new_cellar=$(ls -1) + comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do + if [[ $pkg != "zstd" ]]; then # caching step needs zstd + rm -rf "$pkg" + fi + done + comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt build_webcam: name: build webcam diff --git a/Pipfile b/Pipfile index ccf05dea30..dd957a9dae 100644 --- a/Pipfile +++ b/Pipfile @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4a3baa202ee29371a3168613072d612a3239f368c0e9122a31b398f7db0eb7b6 -size 1282 +oid sha256:c7d458447a6077d0e58d47e4898fca9a30b4415e829693775e72ec7b2f80ed42 +size 1384 diff --git a/Pipfile.lock b/Pipfile.lock index b1efbaef18..93c9aeef4e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5aa6aaa2866c3bf2002532ac131e4dce9581adc1e8d3422dc37bba163e5fc874 -size 143236 +oid sha256:deb01290316c7cc8c24bf5a584c06bee95da54f935dd6792bdbe5485029a5e15 +size 143978 diff --git a/SConstruct b/SConstruct index 6b45d60f2e..832550b14b 100644 --- a/SConstruct +++ b/SConstruct @@ -66,7 +66,7 @@ lenv = { "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath], "PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath, - "ACADOS_SOURCE_DIR": Dir("#third_party/acados/acados").abspath, + "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath, "ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath, "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer", } @@ -125,8 +125,10 @@ else: f"#third_party/libyuv/{yuv_dir}/lib", "/usr/local/lib", "/opt/homebrew/lib", + "/usr/local/Homebrew/Library", "/usr/local/opt/openssl/lib", "/opt/homebrew/opt/openssl/lib", + "/usr/local/Cellar", f"#third_party/acados/{arch}/lib", "/System/Library/Frameworks/OpenGL.framework/Libraries", ] @@ -134,6 +136,7 @@ else: cxxflags += ["-DGL_SILENCE_DEPRECATION"] cpppath += [ "/opt/homebrew/include", + "/usr/local/include", "/usr/local/opt/openssl/include", "/opt/homebrew/opt/openssl/include" ] @@ -235,6 +238,9 @@ env = Environment( tools=["default", "cython", "compilation_db"], ) +if arch == "Darwin": + env['RPATHPREFIX'] = "-rpath " + if GetOption('compile_db'): env.CompilationDatabase('compile_commands.json') @@ -299,6 +305,7 @@ if arch == "Darwin": qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules] qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")] qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"] + qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin")) elif arch == "aarch64": qt_env['QTDIR'] = "/system/comma/usr" qt_dirs = [ diff --git a/third_party/acados/Darwin/lib/libacados.dylib b/third_party/acados/Darwin/lib/libacados.dylib index c6b0e97430..b09c571a64 100755 --- a/third_party/acados/Darwin/lib/libacados.dylib +++ b/third_party/acados/Darwin/lib/libacados.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:498f02aa0ec9a3f5018e0888b13eeacb64726289f6b9149bd4645252c19e8f2d -size 357804 +oid sha256:5906c1e63385d550a9d6658f32b280f0a95ac8253ac399365f58f75a40488cfe +size 816552 diff --git a/third_party/acados/Darwin/lib/libblasfeo.dylib b/third_party/acados/Darwin/lib/libblasfeo.dylib index 3b4a8d060e..036161f36d 100755 --- a/third_party/acados/Darwin/lib/libblasfeo.dylib +++ b/third_party/acados/Darwin/lib/libblasfeo.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2422711a5f711325947bb4cfd55368840ce57b33cffacf45fd0a6dcbf71cd8b3 -size 531149 +oid sha256:35174b2221aa673c80277cdf35d0b09663ca47f3a280c8b46dde81e28249f643 +size 2217032 diff --git a/third_party/acados/Darwin/lib/libhpipm.dylib b/third_party/acados/Darwin/lib/libhpipm.dylib index 349f204383..42c753c3b8 100755 --- a/third_party/acados/Darwin/lib/libhpipm.dylib +++ b/third_party/acados/Darwin/lib/libhpipm.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40530c77834e7efbf9f7e8b7f76d547d706459e4079a58fb5b93c2d68187a77a -size 1062411 +oid sha256:deb18d241a950e152d493069aad6c1376f7e790dc7dff12ab8877757dbd1941e +size 2291256 diff --git a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib index 480e41859a..4de1d47bbb 100755 --- a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib +++ b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ee3876b41af39f6815cb84ecabeada4e9866d34aa2a73874ab0a627a4651952b -size 218403 +oid sha256:cbc62731fec870121d65930ad8a39c414b895655442738f63a1b39141aef808c +size 513304 diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index 71ffe4a92a..fa1a23dd47 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/bin/bash + +set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" ROOT="$(cd $DIR/../ && pwd)" @@ -7,15 +9,16 @@ ROOT="$(cd $DIR/../ && pwd)" if [[ $(command -v brew) == "" ]]; then echo "Installing Hombrew" /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" + echo "[ ] installed brew t=$SECONDS" fi +# TODO: remove protobuf,protobuf-c,swig when casadi can be pip installed brew bundle --file=- <<-EOS brew "cmake" +brew "cppcheck" brew "git-lfs" brew "zlib" brew "bzip2" -brew "rust" -brew "rustup-init" brew "capnp" brew "coreutils" brew "eigen" @@ -29,21 +32,20 @@ brew "openssl" brew "pyenv" brew "qt@5" brew "zeromq" +brew "protobuf" +brew "protobuf-c" +brew "swig" cask "gcc-arm-embedded" EOS +echo "[ ] finished brew install t=$SECONDS" + if [[ $SHELL == "/bin/zsh" ]]; then RC_FILE="$HOME/.zshrc" elif [[ $SHELL == "/bin/bash" ]]; then RC_FILE="$HOME/.bash_profile" fi -# TODO: get rid of this somehow -# Build requirements for macOS -# https://github.com/pyenv/pyenv/issues/1740 -# https://github.com/pyca/cryptography/blob/main/docs/installation.rst -rustup-init -y - export LDFLAGS="$LDFLAGS -L/usr/local/opt/zlib/lib" export LDFLAGS="$LDFLAGS -L/usr/local/opt/bzip2/lib" export LDFLAGS="$LDFLAGS -L/usr/local/opt/openssl@1.1/lib" @@ -55,17 +57,40 @@ export PATH="$PATH:/usr/local/bin" # openpilot environment if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then - echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE - export PATH="$PATH:\"\$HOME/.cargo/bin\"" source "$ROOT/tools/openpilot_env.sh" echo "Added openpilot_env to RC file: $RC_FILE" fi # install python dependencies $ROOT/update_requirements.sh +eval "$(pyenv init --path)" +echo "[ ] installed python dependencies t=$SECONDS" + +# install casadi +VENV=`pipenv --venv` +PYTHON_VER=3.8 +PYTHON_VERSION=$(cat $ROOT/.python-version) +if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then + echo "-- casadi manual install" + cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz + tar -xzf casadi.tar.gz + cd casadi-ge6/ && mkdir -p build && cd build + cmake .. \ + -DWITH_PYTHON=ON \ + -DWITH_EXAMPLES=OFF \ + -DCMAKE_INSTALL_PREFIX:PATH=$VENV \ + -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \ + -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \ + -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \ + -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \ + -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0" + CFLAGS="-ferror-limit=0" make -j$(nproc) && make install +else + echo "---- casadi found in venv. skipping build ----" +fi echo -echo "---- FINISH OPENPILOT SETUP ----" -echo "Configure your active shell env by running:" +echo "---- OPENPILOT SETUP DONE ----" +echo "Open a new shell or configure your active shell env by running:" echo "source $RC_FILE" diff --git a/tools/openpilot_env.sh b/tools/openpilot_env.sh index ac73cc8305..59108312ac 100755 --- a/tools/openpilot_env.sh +++ b/tools/openpilot_env.sh @@ -9,13 +9,13 @@ if [ -z "$OPENPILOT_ENV" ]; then export PYENV_ROOT="$HOME/.pyenv" if [[ "$(uname)" == 'Linux' ]]; then - eval "$(pyenv init --path)" eval "$(pyenv virtualenv-init -)" elif [[ "$(uname)" == 'Darwin' ]]; then # msgq doesn't work on mac export ZMQ=1 export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES fi + eval "$(pyenv init --path)" eval "$(pyenv init -)" export OPENPILOT_ENV=1 diff --git a/tools/ubuntu_setup.sh b/tools/ubuntu_setup.sh index dd29c995ef..44ddb4f014 100755 --- a/tools/ubuntu_setup.sh +++ b/tools/ubuntu_setup.sh @@ -134,6 +134,6 @@ if [ -z "$OPENPILOT_ENV" ]; then fi echo -echo "---- FINISH OPENPILOT SETUP ----" -echo "Configure your active shell env by running:" +echo "---- OPENPILOT SETUP DONE ----" +echo "Open a new shell or configure your active shell env by running:" echo "source ~/.bashrc" diff --git a/update_requirements.sh b/update_requirements.sh index ebf99312b0..ac9472dca2 100755 --- a/update_requirements.sh +++ b/update_requirements.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/bin/bash + +set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" cd $DIR @@ -13,15 +15,18 @@ export MAKEFLAGS="-j$(nproc)" PYENV_PYTHON_VERSION=$(cat .python-version) if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then - echo "pyenv update ..." - pyenv update + # no pyenv update on mac + if [ "$(uname)" == "Linux" ]; then + echo "pyenv update ..." + pyenv update + fi echo "python ${PYENV_PYTHON_VERSION} install ..." CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION} fi +eval "$(pyenv init --path)" -echo "pip install ..." +echo "update pip" pip install pip==21.3.1 -echo "pipenv install ..." pip install pipenv==2021.11.23 if [ -d "./xx" ]; then @@ -36,7 +41,7 @@ else RUN="" fi -echo "pip packages install ..." +echo "pip packages install..." pipenv install --dev --deploy --clear pyenv rehash @@ -45,4 +50,5 @@ if [ -f "$DIR/.pre-commit-config.yaml" ]; then $RUN pre-commit install [ -d "./xx" ] && (cd xx && $RUN pre-commit install) [ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install) + echo "pre-commit hooks installed" fi