Revert no pyenv (#32547)

* Revert "`ubuntu_setup`: fix `No module apt_pkg` error when setting up (#32526)"

This reverts commit d850f07acb58e4fc6d530241a31c0917c2b6613e.

* Revert "Removal of pyenv (#32512)"

This reverts commit 41f72d5ca75aab9a9beb3181aca9f177e5a003f8.
old-commit-hash: 1eb938b8e8

.github/workflows/selfdrive_tests.yaml

@@ -19,7 +19,7 @@ env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1.5G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
 

.python-version

@@ -0,0 +1 @@
+3.11.4

Dockerfile.openpilot_base

@@ -13,7 +13,7 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN INSTALL_EXTRA_PACKAGES=no INSTALL_DEADSNAKES_PPA=yes /tmp/tools/install_ubuntu_dependencies.sh && \
+RUN INSTALL_EXTRA_PACKAGES=no /tmp/tools/install_ubuntu_dependencies.sh && \
     rm -rf /var/lib/apt/lists/* /tmp/* && \
     cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \
     rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
@@ -64,16 +64,19 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER $USER
 
 ENV POETRY_VIRTUALENVS_CREATE=false
-ENV VIRTUAL_ENV_ROOT="/home/$USER/venv"
+ENV PYENV_VERSION=3.11.4
-ENV PATH="$VIRTUAL_ENV_ROOT/bin:$PATH"
+ENV PYENV_ROOT="/home/$USER/pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
 
-COPY --chown=$USER pyproject.toml poetry.lock /tmp/
+COPY --chown=$USER pyproject.toml poetry.lock .python-version /tmp/
 COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
 
 RUN cd /tmp && \
     tools/install_python_dependencies.sh && \
     rm -rf /tmp/* && \
-    rm -rf /home/$USER/.cache
+    rm -rf /home/$USER/.cache && \
+    find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \
+    rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test
 
 USER root
 RUN sudo git config --global --add safe.directory /tmp/openpilot

selfdrive/test/ci_shell.sh

@@ -11,7 +11,6 @@ fi
 
 docker run \
        -it \
-       --shm-size=100m \
        --rm \
        --volume $OP_ROOT:$OP_ROOT \
        --workdir $PWD \

tools/install_python_dependencies.sh

@@ -10,18 +10,50 @@ if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then
   RC_FILE="$HOME/.bash_profile"
 fi
 
-export MAKEFLAGS="-j$(nproc)"
+if ! command -v "pyenv" > /dev/null 2>&1; then
+  echo "pyenv install ..."
+  curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+  PYENV_PATH_SETUP="export PATH=\$HOME/.pyenv/bin:\$HOME/.pyenv/shims:\$PATH"
+fi
 
+if [ -z "$PYENV_SHELL" ] || [ -n "$PYENV_PATH_SETUP" ]; then
+  echo "pyenvrc setup ..."
+  cat <<EOF > "${HOME}/.pyenvrc"
+if [ -z "\$PYENV_ROOT" ]; then
+  $PYENV_PATH_SETUP
+  export PYENV_ROOT="\$HOME/.pyenv"
+  eval "\$(pyenv init -)"
+  eval "\$(pyenv virtualenv-init -)"
+fi
+EOF
 
+  SOURCE_PYENVRC="source ~/.pyenvrc"
+  if ! grep "^$SOURCE_PYENVRC$" $RC_FILE > /dev/null; then
+    printf "\n$SOURCE_PYENVRC\n" >> $RC_FILE
+  fi
 
-echo "update pip"
+  eval "$SOURCE_PYENVRC"
-if [ ! -z "$VIRTUAL_ENV_ROOT" ] || [ ! -z "$INSTALL_DEADSNAKES_PPA" ] ; then
+  # $(pyenv init -) produces a function which is broken on bash 3.2 which ships on macOS
-  if [ -z "$VIRTUAL_ENV_ROOT" ]; then
+  if [ $(uname) == "Darwin" ]; then
-    export VIRTUAL_ENV_ROOT="venv"
+    unset -f pyenv
+  fi
 fi
-  python3 -m venv --system-site-packages $VIRTUAL_ENV_ROOT
+
-  source $VIRTUAL_ENV_ROOT/bin/activate
+export MAKEFLAGS="-j$(nproc)"
+
+PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
+if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
+  # no pyenv update on mac
+  if [ "$(uname)" == "Linux" ]; then
+    echo "pyenv update ..."
+    pyenv update
   fi
+  echo "python ${PYENV_PYTHON_VERSION} install ..."
+  CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
+fi
+eval "$(pyenv init --path)"
+
+echo "update pip"
 pip install pip==24.0
 pip install poetry==1.7.0
 
@@ -39,6 +71,7 @@ poetry self add poetry-dotenv-plugin@^0.1.0
 
 echo "pip packages install..."
 poetry install --no-cache --no-root
+pyenv rehash
 
 [ -n "$POETRY_VIRTUALENVS_CREATE" ] && RUN="" || RUN="poetry run"
 

tools/install_ubuntu_dependencies.sh

@@ -99,39 +99,13 @@ function install_ubuntu_lts_latest_requirements() {
 # Install Ubuntu 20.04 packages
 function install_ubuntu_focal_requirements() {
   install_ubuntu_common_requirements
-  install_deadsnakes_ppa
+
   $SUDO apt-get install -y --no-install-recommends \
     libavresample-dev \
     qt5-default \
     python-dev
 }
 
-# Remove once on Ubuntu 24.04
-function install_deadsnakes_ppa(){
-  if [[ -z "$INSTALL_DEADSNAKES_PPA" ]]; then
-    read -p "Do you want to use deadsnakes python@3.11? [Y/n]: " -n 1 -r
-    echo ""
-    if [[ $REPLY =~ ^[Yy]$ ]]; then
-      INSTALL_DEADSNAKES_PPA="yes"
-    fi
-  fi
-  if [[ "$INSTALL_DEADSNAKES_PPA" == "yes" ]]; then
-    # The reinstall ensures that apt_pkg.cpython-35m-x86_64-linux-gnu.so exists
-    # by reinstalling python3-minimal since it's not present in /usr/lib/python3/dist-packages.
-    $SUDO apt install -y --no-install-recommends --reinstall python3-minimal
-    $SUDO apt-get install -y --no-install-recommends python3-apt software-properties-common
-    $SUDO add-apt-repository ppa:deadsnakes/ppa
-    $SUDO apt-get install -y --no-install-recommends \
-      python3.11-dev \
-      python3.11 \
-      python3.11-venv \
-      python3.11-distutils \
-      python3-pip
-    $SUDO update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 20
-    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
-  fi
-}
-
 # Detect OS using /etc/os-release file
 if [ -f "/etc/os-release" ]; then
   source /etc/os-release

tools/mac_setup.sh

@@ -59,7 +59,8 @@ brew "libusb"
 brew "libtool"
 brew "llvm"
 brew "openssl@3.0"
-brew "python@3.11"
+brew "pyenv"
+brew "pyenv-virtualenv"
 brew "qt@5"
 brew "zeromq"
 cask "gcc-arm-embedded"