From dc72980c8364fb2bd69c96878dc6145b54131054 Mon Sep 17 00:00:00 2001
From: Mayfield <3902405+mayfieldiv@users.noreply.github.com>
Date: Sun, 3 Jan 2021 15:33:42 -0500
Subject: [PATCH] Make macos ci homebrew dep-caching fast (#19632)

* try out /usr/local for homebrew caching

* TEMP: disable non-macos jobs

* TEMP: disable more

* sizes

* temp: nothing change

* more

* try -1

* quotes

* delete existing downloads

* fix ~ expansion

* nothing

* try caching cellar

* move to yaml

* nothing

* nothing

* asdf

* nothing

* fixed cache key

* nothing

* stay in home dir

* keep zstd

* brew link

* fix cleaning?

* brew link in yaml

* fix zstd exclusion

* capnp

* revert mac_setup

* finish up

* done
old-commit-hash: cbc25cc0f1c639e948181e1218db60631cf9ff66
---
 .github/workflows/test.yaml | 36 ++++++++++++++++++++++++++++++++----
 tools/mac_setup.sh          |  2 +-
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 1f65f390ee..f013ce3270 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -54,6 +54,12 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: true
+    - name: Determine pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+        ls -1 /usr/local/Cellar >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
     - name: Cache dependencies
       id: dependency-cache
       uses: actions/cache@v2
@@ -62,14 +68,36 @@ jobs:
           ~/.pyenv
           ~/Library/Caches/pip
           ~/Library/Caches/pipenv
-          ~/Library/Caches/Homebrew
-        key: ${{ hashFiles('tools/mac_setup.sh') }}
+          /usr/local/Cellar
+          ~/github_brew_cache_entries.txt
+        key: macos-deps-${{ hashFiles('tools/mac_setup.sh') }}
+        restore-keys: macos-deps-
+    - name: Brew link restored dependencies
+      if: steps.dependency-cache.outputs.cache-hit == 'true'
+      run: |
+        while read package; do
+          brew link --force "$package" # `--force` for keg-only packages
+        done < ~/github_brew_cache_entries.txt
     - name: Install dependencies
       run: ./tools/mac_setup.sh
     - name: Build openpilot
       run: eval "$(pyenv init -)" && scons -j$(nproc)
-    - name: Brew cleanup
-      run: brew cleanup || true # keeps our cache small
+    - name: Remove pre-existing Homebrew packages for caching
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        new_cellar=$(ls -1 /usr/local/Cellar)
+        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read dir; do
+          if [[ $dir != "zstd" ]]; then # caching step needs zstd
+            echo "Removing /usr/local/Cellar/$dir"
+            rm -rf "/usr/local/Cellar/$dir"
+          fi
+        done
+
+        printf "\n\nNewly installed packages:\n"
+        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+
+        printf "\n\nThe following will be cached:\n"
+        (du -shc /usr/local/Cellar/* | sort -h) || true
 
   build_webcam:
     name: build webcam
diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh
index f2ce98f826..3afc32c165 100755
--- a/tools/mac_setup.sh
+++ b/tools/mac_setup.sh
@@ -15,7 +15,7 @@ brew install capnp \
              libusb \
              libtool \
              llvm \
-	     openssl \
+             openssl \
              pyenv \
              qt5 \
              zeromq