Removal of pyenv (#32512)

* initial removal of pyenv

* remove .python-version copy in dockerfile

* successful image build with ppa

* update prompt

* pip install scons

* apt install scons

* finally fix dockerfile to work with venv

* cleanup userflow

* increase memory to 100m

* typos

* wrong variable

* lmao
old-commit-hash: f5752121f8

.python-version

@@ -1 +0,0 @@
-3.11.4

Dockerfile.openpilot_base

@@ -13,7 +13,7 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN INSTALL_EXTRA_PACKAGES=no /tmp/tools/install_ubuntu_dependencies.sh && \
+RUN INSTALL_EXTRA_PACKAGES=no INSTALL_DEADSNAKES_PPA=yes /tmp/tools/install_ubuntu_dependencies.sh && \
     rm -rf /var/lib/apt/lists/* /tmp/* && \
     cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \
     rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
@@ -64,19 +64,16 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER $USER
 
 ENV POETRY_VIRTUALENVS_CREATE=false
-ENV PYENV_VERSION=3.11.4
+ENV VIRTUAL_ENV_ROOT="/home/$USER/venv"
-ENV PYENV_ROOT="/home/$USER/pyenv"
+ENV PATH="$VIRTUAL_ENV_ROOT/bin:$PYENV_ROOT/lib:$PATH"
-ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
 
-COPY --chown=$USER pyproject.toml poetry.lock .python-version /tmp/
+COPY --chown=$USER pyproject.toml poetry.lock /tmp/
 COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
 
 RUN cd /tmp && \
     tools/install_python_dependencies.sh && \
     rm -rf /tmp/* && \
-    rm -rf /home/$USER/.cache && \
+    rm -rf /home/$USER/.cache
-    find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \
-    rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test
 
 USER root
 RUN sudo git config --global --add safe.directory /tmp/openpilot

selfdrive/test/ci_shell.sh

@@ -11,6 +11,7 @@ fi
 
 docker run \
        -it \
+       --shm-size=100m \
        --rm \
        --volume $OP_ROOT:$OP_ROOT \
        --workdir $PWD \

tools/install_python_dependencies.sh

@@ -10,50 +10,15 @@ if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then
   RC_FILE="$HOME/.bash_profile"
 fi
 
-if ! command -v "pyenv" > /dev/null 2>&1; then
-  echo "pyenv install ..."
-  curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-  PYENV_PATH_SETUP="export PATH=\$HOME/.pyenv/bin:\$HOME/.pyenv/shims:\$PATH"
-fi
-
-if [ -z "$PYENV_SHELL" ] || [ -n "$PYENV_PATH_SETUP" ]; then
-  echo "pyenvrc setup ..."
-  cat <<EOF > "${HOME}/.pyenvrc"
-if [ -z "\$PYENV_ROOT" ]; then
-  $PYENV_PATH_SETUP
-  export PYENV_ROOT="\$HOME/.pyenv"
-  eval "\$(pyenv init -)"
-  eval "\$(pyenv virtualenv-init -)"
-fi
-EOF
-
-  SOURCE_PYENVRC="source ~/.pyenvrc"
-  if ! grep "^$SOURCE_PYENVRC$" $RC_FILE > /dev/null; then
-    printf "\n$SOURCE_PYENVRC\n" >> $RC_FILE
-  fi
-
-  eval "$SOURCE_PYENVRC"
-  # $(pyenv init -) produces a function which is broken on bash 3.2 which ships on macOS
-  if [ $(uname) == "Darwin" ]; then
-    unset -f pyenv
-  fi
-fi
-
 export MAKEFLAGS="-j$(nproc)"
 
-PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
+
-if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
-  # no pyenv update on mac
-  if [ "$(uname)" == "Linux" ]; then
-    echo "pyenv update ..."
-    pyenv update
-  fi
-  echo "python ${PYENV_PYTHON_VERSION} install ..."
-  CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
-fi
-eval "$(pyenv init --path)"
 
 echo "update pip"
+if [ ! -z "\$VIRTUAL_ENV_ROOT" ] || [ ! -z "$INSTALL_DEADSNAKES_PPA" ] ; then
+  python3 -m venv --system-site-packages $VIRTUAL_ENV_ROOT
+  source $VIRTUAL_ENV_ROOT/bin/activate
+fi
 pip install pip==24.0
 pip install poetry==1.7.0
 
@@ -71,7 +36,6 @@ poetry self add poetry-dotenv-plugin@^0.1.0
 
 echo "pip packages install..."
 poetry install --no-cache --no-root
-pyenv rehash
 
 [ -n "$POETRY_VIRTUALENVS_CREATE" ] && RUN="" || RUN="poetry run"
 

tools/install_ubuntu_dependencies.sh

@@ -99,13 +99,37 @@ function install_ubuntu_lts_latest_requirements() {
 # Install Ubuntu 20.04 packages
 function install_ubuntu_focal_requirements() {
   install_ubuntu_common_requirements
-
+  install_deadsnakes_ppa
   $SUDO apt-get install -y --no-install-recommends \
     libavresample-dev \
     qt5-default \
     python-dev
 }
 
+# Remove once on Ubuntu 24.04
+function install_deadsnakes_ppa(){
+  if [[ -z "$INSTALL_DEADSNAKES_PPA" ]]; then
+    read -p "Do you want to use deadsnakes python@3.11? [Y/n]: " -n 1 -r
+    echo ""
+    if [[ $REPLY =~ ^[Yy]$ ]]; then
+      INSTALL_DEADSNAKES_PPA="yes"
+      export VIRTUAL_ENV_ROOT=".venv"
+    fi
+  fi
+  if [[ "$INSTALL_DEADSNAKES_PPA" == "yes" ]]; then
+    $SUDO apt-get install software-properties-common -y --no-install-recommends
+    $SUDO add-apt-repository ppa:deadsnakes/ppa
+    $SUDO apt-get install -y --no-install-recommends \
+      python3.11-dev \
+      python3.11 \
+      python3.11-venv \
+      python3.11-distutils \
+      python3-pip
+    $SUDO update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 20
+    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
+  fi
+}
+
 # Detect OS using /etc/os-release file
 if [ -f "/etc/os-release" ]; then
   source /etc/os-release

tools/mac_setup.sh

@@ -59,8 +59,7 @@ brew "libusb"
 brew "libtool"
 brew "llvm"
 brew "openssl@3.0"
-brew "pyenv"
+brew "python@3.11"
-brew "pyenv-virtualenv"
 brew "qt@5"
 brew "zeromq"
 cask "gcc-arm-embedded"