From 3ca9b7f02ee4955d216dd8f1a22cce54a40697b6 Mon Sep 17 00:00:00 2001 From: HaraldSchafer Date: Fri, 21 Oct 2022 16:23:56 -0700 Subject: [PATCH] Pipenv -> poetry (#24858) * Try poetry * Remove casadi for now * Update docker * Copy pipfiles * add casadi back * Too many slashes * New poetry api * Install system * Try again * No more pipenv * new dependencies * updates * poetry 1.2.1, install dev dependencies * keep install pipenv for xx for now? * add pre-commit checks for poetry * poetry lock is too slow * update pip * migrate to poetry groups * update lockfile * don't need to specify dev group unless it is made optional * always install poetry * set POETRY_VIRTUALENVS_CREATE instead, and use pipenv for xx * use env for docs docker image * alphabetical * poetry 1.2.2 * add dev dependencies for typing added in aacf6ae3 * remove constraint * fix PIPENV_SYSTEM * remove constraint * don't need this here * bump * bump pipenv adds support for installing local pyprojects (can add openpilot as dependency of xx) * README improvements * probably not necessary * bump pip * maybe not necessary? * revert * don't install openpilot itself into the virtual env * remove PySide2 and shiboken2 reverts 3e41c775cbf670740ac648a8614d2d6b433312fe * remove Pipenv, add xx dependencies, sync system python * add pipenv as xx dep * semver package constraints, use old lockfile versions * fix casadi * remove whitespace Co-authored-by: Adeeb Shihadeh * disable poetry cache * cleanup * prefer config file Co-authored-by: Cameron Clough Co-authored-by: Adeeb Shihadeh old-commit-hash: a98d105cb95a445f5ec69e783585c56604b09449 --- .github/workflows/selfdrive_tests.yaml | 6 +- .gitignore | 2 + .pre-commit-config.yaml | 4 + Dockerfile.openpilot_base | 6 +- Pipfile | 3 - Pipfile.lock | 3 - docs/docker/Dockerfile | 7 +- poetry.lock | 3 + pyproject.toml | 176 +++++++++++++++++++++++++ tools/README.md | 9 +- tools/mac_setup.sh | 2 +- update_requirements.sh | 30 +++-- 12 files changed, 213 insertions(+), 38 deletions(-) delete mode 100644 Pipfile delete mode 100644 Pipfile.lock create mode 100644 poetry.lock create mode 100644 pyproject.toml diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 20bf8c3d8e..0693a66164 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -106,7 +106,7 @@ jobs: # /usr/local/Cellar # ~/github_brew_cache_entries.txt # /tmp/scons_cache - # key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }} + # key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }} # restore-keys: macos- # - name: Brew link restored dependencies # run: | @@ -122,11 +122,11 @@ jobs: # - name: Build openpilot # run: | # source tools/openpilot_env.sh - # pipenv run selfdrive/manager/build.py + # poetry run selfdrive/manager/build.py # # # cleanup scons cache # rm -rf /tmp/scons_cache/ - # pipenv run scons -j$(nproc) --cache-populate + # poetry run scons -j$(nproc) --cache-populate # - name: Remove pre-existing Homebrew packages for caching # if: steps.dependency-cache.outputs.cache-hit != 'true' # run: | diff --git a/.gitignore b/.gitignore index 062358ef24..31cef94222 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,5 @@ selfdrive/modeld/models/*.thneed build/ !**/.gitkeep + +poetry.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85c24b911b..91b9c61628 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,3 +78,7 @@ repos: entry: selfdrive/ui/tests/test_translations.py language: script pass_filenames: false +- repo: https://github.com/python-poetry/poetry + rev: '1.2.2' + hooks: + - id: poetry-check diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 0de3008baf..4cd82259e2 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -12,19 +12,19 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -ENV PIPENV_SYSTEM=1 +ENV POETRY_VIRTUALENVS_CREATE=false ENV PYENV_VERSION=3.8.10 ENV PYENV_ROOT="/root/.pyenv" ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH" -COPY Pipfile Pipfile.lock .python-version update_requirements.sh /tmp/ +COPY pyproject.toml poetry.lock .python-version update_requirements.sh /tmp/ COPY tools/ubuntu_setup.sh /tmp/tools/ RUN cd /tmp && \ tools/ubuntu_setup.sh && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /tmp/* && \ rm -rf /root/.cache && \ - pip uninstall -y pipenv && \ + pip uninstall -y poetry && \ # remove unused architectures from gcc for panda cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \ rm -rf arm/ && \ diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 0d37a136aa..0000000000 --- a/Pipfile +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:67078d38037d44571e909dee2288c4fe9f7a1cc3691fe3165c1628ff3d19cde6 -size 1736 diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 4db1a8b5e5..0000000000 --- a/Pipfile.lock +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:848091d4b13abe8c0c97c997fc8641953e1685582560a45841bac625eb0991b4 -size 205318 diff --git a/docs/docker/Dockerfile b/docs/docker/Dockerfile index cefe9be855..0d5883f566 100644 --- a/docs/docker/Dockerfile +++ b/docs/docker/Dockerfile @@ -4,16 +4,11 @@ ENV PYTHONUNBUFFERED 1 ENV OPENPILOT_PATH /home/batman/openpilot/ ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH} +ENV POETRY_VIRUALENVS_CREATE false RUN mkdir -p ${OPENPILOT_PATH} WORKDIR ${OPENPILOT_PATH} -COPY Pipfile Pipfile.lock $OPENPILOT_PATH -RUN pip install --no-cache-dir pipenv==2021.5.29 pip==21.3.1 && \ - pipenv install --system --deploy --dev --clear && \ - pip uninstall -y pipenv - - COPY SConstruct ${OPENPILOT_PATH} COPY ./pyextra ${OPENPILOT_PATH}/pyextra diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000000..e1973534bb --- /dev/null +++ b/poetry.lock @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb2af7efeaeba324c900772307764df05b4923d37771bcea9d08fdcaa50eba56 +size 560592 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..7855ad0124 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,176 @@ +[tool.poetry] +name = "openpilot" +version = "0.1.0" +description = "an open source driver assistance system" +authors = ["Vehicle Researcher "] +license = "MIT" +readme = "README.md" +repository = "https://github.com/commaai/openpilot" +documentation = "https://docs.comma.ai" + + +[tool.poetry.dependencies] +python = "~3.8" +atomicwrites = "^1.4.0" +casadi = { version = "==3.5.5", markers = "platform_system != 'Darwin'" } +cffi = "^1.15.1" +crcmod = "^1.7" +cryptography = "^37.0.4" +Cython = "^0.29.30" +flake8 = "^4.0.1" +Flask = "^2.1.2" +future-fstrings = "^1.2.0" # for acados +gunicorn = "^20.1.0" +hatanaka = "==2.4" +hexdump = "^3.3" +Jinja2 = "^3.1.2" +json-rpc = "^1.13.0" +libusb1 = "^3.0.0" +nose = "^1.3.7" +numpy = "^1.23.0" +onnx = "^1.12.0" +onnxruntime-gpu = { version = "^1.11.1", markers = "platform_system != 'Darwin'" } +pillow = "^9.2.0" +poetry = "==1.2.2" +protobuf = "==3.20.1" +psutil = "^5.9.1" +pycapnp = "==1.1.0" +pycryptodome = "^3.15.0" +PyJWT = "^2.5.0" +pylint = "^2.15.4" +pyopencl = "^2022.2.4" +pyserial = "^3.5" +python-dateutil = "^2.8.2" +PyYAML = "^6.0" +pyzmq = "^23.2.0" +requests = "^2.28.1" +scons = "^4.3.0" +sentry-sdk = "^1.6.0" +setproctitle = "^1.2.3" +six = "^1.16.0" +smbus2 = "^0.4.2" +sympy = "^1.10.1" +timezonefinder = "^6.0.1" +tqdm = "^4.64.0" +urllib3 = "^1.26.10" +utm = "^0.7.0" +websocket_client = "^1.3.3" + + +[tool.poetry.group.dev.dependencies] +av = "^9.2.0" +azure-storage-blob = "~2.1" +breathe = "^4.34.0" +carla = "==0.9.13" +control = "^0.9.2" +coverage = "^6.4.1" +dictdiffer = "^0.9.0" +fastcluster = "^1.2.6" +ft4222 = "^1.4.1" +hexdump = "^3.3" +hypothesis = "==6.46.7" +inputs = "^0.5" +lru-dict = "^1.1.7" +lxml = "^4.9.1" +markdown-it-py = "^2.1.0" +matplotlib = "^3.5.2" +mpld3 = "^0.5.8" +mypy = "^0.961" +myst-parser = "^0.18.0" +natsort = "^8.1.0" +numpy = "^1.23.0" +opencv-python-headless = { url = "https://github.com/commaai/opencv-python-builder/releases/download/4.5.5.64%2Bcu113/opencv_python_headless-4.5.5.64-cp38-cp38-manylinux_2_31_x86_64.whl" } +pandas = "^1.4.3" +parameterized = "^0.8.1" +paramiko = "^2.11.0" +pprofile = "^2.1.0" +pre-commit = "^2.19.0" +pycurl = "^7.45.1" +pygame = "^2.1.2" +pyprof2calltree = "^1.4.5" +pytest = "^7.1.2" +pytest-xdist = "^2.5.0" +reverse_geocoder = "^1.5.1" +scipy = "^1.8.1" +sphinx = "^5.0.2" +sphinx-rtd-theme = "^1.0.0" +sphinx-sitemap = "^2.2.0" +subprocess32 = "^3.5.4" +tabulate = "^0.8.10" +tenacity = "^8.0.1" +types-atomicwrites = "^1.4.5" +types-certifi = "^2021.10.8" +types-pycurl = "^7.45.1" +types-PyYAML = "^6.0" +types-requests = "^2.28.11" + + +[tool.poetry.group.xx] +optional = true + +[tool.poetry.group.xx.dependencies] +aenum = "^3.1.11" +aiohttp = "^3.8.1" +albumentations = "^1.2.1" +apex = { url = "https://github.com/commaai/apex/releases/download/pytorch1.10.0%2Bcu11.1/apex-0.1-cp38-cp38-linux_x86_64.whl" } +azure-cli-core = "^2.38.0" +azure-common = "^1.1.28" +azure-core = "^1.24.2" +azure-nspkg = "~3.0" +azure-storage-blob = "~2.1" +azure-storage-common = "~2.1" +azure-storage-nspkg = "~3.1" +blosc = "==1.9.2" +cloudpickle = "^2.1.0" +configargparse = "^1.5.3" +cupy-cuda113 = "^10.6.0" +datadog = "^0.44.0" +dotmap = "^1.3.30" +elasticsearch = "^8.3.1" +Flask-Cors = "^3.0.10" +Flask-SocketIO = "^5.2.0" +GeoAlchemy2 = "^0.12.1" +imageio = "^2.19.5" +influxdb-client = "^1.30.0" +ipykernel = "^6.15.1" +ipython = "^8.4.0" +joblib = "^1.1.0" +json-logging-py = "^0.2" +jupyter = "^1.0.0" +jupyterlab = "^3.4.4" +jupyterlab-vim = "^0.15.1" +Markdown = "^3.4.1" +mpld3 = "^0.5.8" +msgpack-python = "^0.5.6" +networkx = "~2.3" +nvidia-ml-py3 = "^7.352.0" +onnxoptimizer = "^0.2.6" +opencv-python-headless = { url = "https://github.com/commaai/opencv-python-builder/releases/download/4.5.5.64%2Bcu113/opencv_python_headless-4.5.5.64-cp38-cp38-manylinux_2_31_x86_64.whl" } +osmium = "^3.3.0" +pandas = "^1.4.3" +pillow-avif-plugin = "^1.2.2" +pipenv = "==2022.10.12" +plotly = "^5.9.0" +pycuda = "^2022.1" +Pygments = "^2.12.0" +PyMySQL = "~0.9" +pyproj = "^3.3.1" +python-logstash = "^0.4.8" +redis = "^4.3.4" +s2sphere = "^0.2.5" +scikit-image = "^0.19.3" +scikit-learn = "^1.1.1" +segmentation-models-pytorch = "==0.2.1" +simplejson = "^3.17.6" +SQLAlchemy = "^1.4.39" +torch = { url = "https://download.pytorch.org/whl/cu113/torch-1.11.0%2Bcu113-cp38-cp38-linux_x86_64.whl" } +torchsummary = "^1.5.1" +torchvision = { url = "https://download.pytorch.org/whl/cu113/torchvision-0.12.0%2Bcu113-cp38-cp38-linux_x86_64.whl" } +triton = "^1.1.1" +Werkzeug = "^2.1.2" +zerorpc = { git = "git@github.com:commaai/zerorpc-python.git", branch = "master" } + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/README.md b/tools/README.md index 59eea3230d..3969deb24d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -6,13 +6,12 @@ openpilot is developed and tested on **Ubuntu 20.04**, which is the primary deve ## Setup your PC - First, clone openpilot: ``` bash cd ~ git clone https://github.com/commaai/openpilot.git -cd openpilot +cd openpilot git submodule update --init ``` @@ -26,10 +25,10 @@ tools/ubuntu_setup.sh tools/mac_setup.sh ``` -Activate a shell with the install Python dependencies: +Activate a shell with the Python dependencies installed: ``` bash -cd openpilot && pipenv shell +cd openpilot && poetry shell ``` Build openpilot with this command: @@ -43,7 +42,7 @@ Neither openpilot nor any of the tools are developed or tested on Windows, but t Follow [these instructions](https://docs.microsoft.com/en-us/windows/wsl/install) to setup the WSL and install the `Ubuntu-20.04` distribution. Once your Ubuntu WSL environment is setup, follow the Linux setup instructions to finish setting up your environment. -GUI applications do not work with WSL out of the box. You will have to either [upgrade your system to Windows 11](https://docs.microsoft.com/en-us/windows/wsl/tutorials/gui-apps) or [set up an Xorg server](https://techcommunity.microsoft.com/t5/windows-dev-appconsult/running-wsl-gui-apps-on-windows-10/ba-p/1493242). +GUI applications do not work with WSL out of the box. You will have to either [upgrade your system to Windows 11](https://docs.microsoft.com/en-us/windows/wsl/tutorials/gui-apps) or [set up an Xorg server](https://techcommunity.microsoft.com/t5/windows-dev-appconsult/running-wsl-gui-apps-on-windows-10/ba-p/1493242). ## CTF diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index 214a917f8b..e85292da03 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -88,7 +88,7 @@ eval "$(pyenv init --path)" echo "[ ] installed python dependencies t=$SECONDS" # install casadi -VENV=`pipenv --venv` +VENV=`poetry env info --path` PYTHON_VER=3.8 PYTHON_VERSION=$(cat $ROOT/.python-version) if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then diff --git a/update_requirements.sh b/update_requirements.sh index 060dca76ef..8571726e10 100755 --- a/update_requirements.sh +++ b/update_requirements.sh @@ -40,27 +40,29 @@ fi eval "$(pyenv init --path)" echo "update pip" -pip install pip==21.3.1 -pip install pipenv==2021.11.23 +pip install pip==22.3 +pip install poetry==1.2.2 -if [ -d "./xx" ]; then - echo "WARNING: using xx Pipfile ******" - export PIPENV_SYSTEM=1 - export PIPENV_PIPFILE=./xx/Pipfile -fi +poetry config virtualenvs.prefer-active-python true --local -if [ -z "$PIPENV_SYSTEM" ]; then - echo "PYTHONPATH=${PWD}" > .env - RUN="pipenv run" -else - RUN="" +POETRY_INSTALL_ARGS="" +if [ -d "./xx" ]; then + echo "WARNING: using xx dependency group, installing globally" + poetry config virtualenvs.create false --local + POETRY_INSTALL_ARGS="--with xx --sync" fi echo "pip packages install..." -pipenv sync --dev -pipenv --clear +poetry install --no-cache --no-root $POETRY_INSTALL_ARGS pyenv rehash +if [ -d "./xx" ] || [ -n "$POETRY_VIRTUALENVS_CREATE" ]; then + RUN="" +else + echo "PYTHONPATH=${PWD}" > .env + RUN="poetry run" +fi + echo "pre-commit hooks install..." shopt -s nullglob for f in .pre-commit-config.yaml */.pre-commit-config.yaml; do