From 1eb938b8e81a3321adf1064e8ef348b8c7073d4c Mon Sep 17 00:00:00 2001
From: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Date: Sun, 26 May 2024 21:21:50 -0700
Subject: [PATCH] Revert no pyenv (#32547)

* Revert "`ubuntu_setup`: fix `No module apt_pkg` error when setting up (#32526)"

This reverts commit f4322666c6adc4dd2da569f483f6e108f18fe32a.

* Revert "Removal of pyenv (#32512)"

This reverts commit f5752121f8a2029050c63cfa04f2533901ae4f3a.
---
 .github/workflows/selfdrive_tests.yaml |  2 +-
 .python-version                        |  1 +
 Dockerfile.openpilot_base              | 13 ++++---
 selfdrive/test/ci_shell.sh             |  1 -
 tools/install_python_dependencies.sh   | 47 ++++++++++++++++++++++----
 tools/install_ubuntu_dependencies.sh   | 28 +--------------
 tools/mac_setup.sh                     |  5 +--
 7 files changed, 54 insertions(+), 43 deletions(-)
 create mode 100644 .python-version

diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
index e73e0df2a8..9001ca86fc 100644
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -19,7 +19,7 @@ env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1.5G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
 
diff --git a/.python-version b/.python-version
new file mode 100644
index 0000000000..0c7d5f5f5d
--- /dev/null
+++ b/.python-version
@@ -0,0 +1 @@
+3.11.4
diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base
index 2c5f9a035b..0789a39c3e 100644
--- a/Dockerfile.openpilot_base
+++ b/Dockerfile.openpilot_base
@@ -13,7 +13,7 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN INSTALL_EXTRA_PACKAGES=no INSTALL_DEADSNAKES_PPA=yes /tmp/tools/install_ubuntu_dependencies.sh && \
+RUN INSTALL_EXTRA_PACKAGES=no /tmp/tools/install_ubuntu_dependencies.sh && \
     rm -rf /var/lib/apt/lists/* /tmp/* && \
     cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \
     rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
@@ -64,16 +64,19 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER $USER
 
 ENV POETRY_VIRTUALENVS_CREATE=false
-ENV VIRTUAL_ENV_ROOT="/home/$USER/venv"
-ENV PATH="$VIRTUAL_ENV_ROOT/bin:$PATH"
+ENV PYENV_VERSION=3.11.4
+ENV PYENV_ROOT="/home/$USER/pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
 
-COPY --chown=$USER pyproject.toml poetry.lock /tmp/
+COPY --chown=$USER pyproject.toml poetry.lock .python-version /tmp/
 COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
 
 RUN cd /tmp && \
     tools/install_python_dependencies.sh && \
     rm -rf /tmp/* && \
-    rm -rf /home/$USER/.cache
+    rm -rf /home/$USER/.cache && \
+    find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \
+    rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test
 
 USER root
 RUN sudo git config --global --add safe.directory /tmp/openpilot
diff --git a/selfdrive/test/ci_shell.sh b/selfdrive/test/ci_shell.sh
index 1f3fa1f650..a5ff714b2d 100755
--- a/selfdrive/test/ci_shell.sh
+++ b/selfdrive/test/ci_shell.sh
@@ -11,7 +11,6 @@ fi
 
 docker run \
        -it \
-       --shm-size=100m \
        --rm \
        --volume $OP_ROOT:$OP_ROOT \
        --workdir $PWD \
diff --git a/tools/install_python_dependencies.sh b/tools/install_python_dependencies.sh
index eaa32a0a4b..df815b582f 100755
--- a/tools/install_python_dependencies.sh
+++ b/tools/install_python_dependencies.sh
@@ -10,18 +10,50 @@ if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then
   RC_FILE="$HOME/.bash_profile"
 fi
 
-export MAKEFLAGS="-j$(nproc)"
+if ! command -v "pyenv" > /dev/null 2>&1; then
+  echo "pyenv install ..."
+  curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+  PYENV_PATH_SETUP="export PATH=\$HOME/.pyenv/bin:\$HOME/.pyenv/shims:\$PATH"
+fi
 
+if [ -z "$PYENV_SHELL" ] || [ -n "$PYENV_PATH_SETUP" ]; then
+  echo "pyenvrc setup ..."
+  cat <<EOF > "${HOME}/.pyenvrc"
+if [ -z "\$PYENV_ROOT" ]; then
+  $PYENV_PATH_SETUP
+  export PYENV_ROOT="\$HOME/.pyenv"
+  eval "\$(pyenv init -)"
+  eval "\$(pyenv virtualenv-init -)"
+fi
+EOF
 
+  SOURCE_PYENVRC="source ~/.pyenvrc"
+  if ! grep "^$SOURCE_PYENVRC$" $RC_FILE > /dev/null; then
+    printf "\n$SOURCE_PYENVRC\n" >> $RC_FILE
+  fi
 
-echo "update pip"
-if [ ! -z "$VIRTUAL_ENV_ROOT" ] || [ ! -z "$INSTALL_DEADSNAKES_PPA" ] ; then
-  if [ -z "$VIRTUAL_ENV_ROOT" ]; then
-    export VIRTUAL_ENV_ROOT="venv"
+  eval "$SOURCE_PYENVRC"
+  # $(pyenv init -) produces a function which is broken on bash 3.2 which ships on macOS
+  if [ $(uname) == "Darwin" ]; then
+    unset -f pyenv
+  fi
+fi
+
+export MAKEFLAGS="-j$(nproc)"
+
+PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
+if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
+  # no pyenv update on mac
+  if [ "$(uname)" == "Linux" ]; then
+    echo "pyenv update ..."
+    pyenv update
   fi
-  python3 -m venv --system-site-packages $VIRTUAL_ENV_ROOT
-  source $VIRTUAL_ENV_ROOT/bin/activate
+  echo "python ${PYENV_PYTHON_VERSION} install ..."
+  CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
 fi
+eval "$(pyenv init --path)"
+
+echo "update pip"
 pip install pip==24.0
 pip install poetry==1.7.0
 
@@ -39,6 +71,7 @@ poetry self add poetry-dotenv-plugin@^0.1.0
 
 echo "pip packages install..."
 poetry install --no-cache --no-root
+pyenv rehash
 
 [ -n "$POETRY_VIRTUALENVS_CREATE" ] && RUN="" || RUN="poetry run"
 
diff --git a/tools/install_ubuntu_dependencies.sh b/tools/install_ubuntu_dependencies.sh
index 2469168c74..1f8ef24eaf 100755
--- a/tools/install_ubuntu_dependencies.sh
+++ b/tools/install_ubuntu_dependencies.sh
@@ -99,39 +99,13 @@ function install_ubuntu_lts_latest_requirements() {
 # Install Ubuntu 20.04 packages
 function install_ubuntu_focal_requirements() {
   install_ubuntu_common_requirements
-  install_deadsnakes_ppa
+
   $SUDO apt-get install -y --no-install-recommends \
     libavresample-dev \
     qt5-default \
     python-dev
 }
 
-# Remove once on Ubuntu 24.04
-function install_deadsnakes_ppa(){
-  if [[ -z "$INSTALL_DEADSNAKES_PPA" ]]; then
-    read -p "Do you want to use deadsnakes python@3.11? [Y/n]: " -n 1 -r
-    echo ""
-    if [[ $REPLY =~ ^[Yy]$ ]]; then
-      INSTALL_DEADSNAKES_PPA="yes"
-    fi
-  fi
-  if [[ "$INSTALL_DEADSNAKES_PPA" == "yes" ]]; then
-    # The reinstall ensures that apt_pkg.cpython-35m-x86_64-linux-gnu.so exists
-    # by reinstalling python3-minimal since it's not present in /usr/lib/python3/dist-packages.
-    $SUDO apt install -y --no-install-recommends --reinstall python3-minimal
-    $SUDO apt-get install -y --no-install-recommends python3-apt software-properties-common
-    $SUDO add-apt-repository ppa:deadsnakes/ppa
-    $SUDO apt-get install -y --no-install-recommends \
-      python3.11-dev \
-      python3.11 \
-      python3.11-venv \
-      python3.11-distutils \
-      python3-pip
-    $SUDO update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 20
-    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
-  fi
-}
-
 # Detect OS using /etc/os-release file
 if [ -f "/etc/os-release" ]; then
   source /etc/os-release
diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh
index b021866855..d26ec3cfe2 100755
--- a/tools/mac_setup.sh
+++ b/tools/mac_setup.sh
@@ -6,7 +6,7 @@ if [ -z "$SKIP_PROMPT" ]; then
   echo "---------------   macOS support   ---------------"
   echo "Running openpilot natively on macOS is not officially supported."
   echo "It might build, some parts of it might work, but it's not fully tested, so there might be some issues."
-  echo
+  echo 
   echo "Check out devcontainers for a seamless experience (see tools/README.md)."
   echo "-------------------------------------------------"
   echo -n "Are you sure you want to continue? [y/N] "
@@ -59,7 +59,8 @@ brew "libusb"
 brew "libtool"
 brew "llvm"
 brew "openssl@3.0"
-brew "python@3.11"
+brew "pyenv"
+brew "pyenv-virtualenv"
 brew "qt@5"
 brew "zeromq"
 cask "gcc-arm-embedded"