Pipenv -> poetry (#24858)

* Try poetry

* Remove casadi for now

* Update docker

* Copy pipfiles

* add casadi back

* Too many slashes

* New poetry api

* Install system

* Try again

* No more pipenv

* new dependencies

* updates

* poetry 1.2.1, install dev dependencies

* keep install pipenv for xx for now?

* add pre-commit checks for poetry

* poetry lock is too slow

* update pip

* migrate to poetry groups

* update lockfile

* don't need to specify dev group unless it is made optional

* always install poetry

* set POETRY_VIRTUALENVS_CREATE instead, and use pipenv for xx

* use env for docs docker image

* alphabetical

* poetry 1.2.2

* add dev dependencies for typing

added in 4e310b80

* remove constraint

* fix PIPENV_SYSTEM

* remove constraint

* don't need this here

* bump

* bump pipenv

adds support for installing local pyprojects (can add openpilot as dependency of xx)

* README improvements

* probably not necessary

* bump pip

* maybe not necessary?

* revert

* don't install openpilot itself into the virtual env

* remove PySide2 and shiboken2

reverts c6b749fb96

* remove Pipenv, add xx dependencies, sync system python

* add pipenv as xx dep

* semver package constraints, use old lockfile versions

* fix casadi

* remove whitespace

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

* disable poetry cache

* cleanup

* prefer config file

Co-authored-by: Cameron Clough <cameronjclough@gmail.com>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
pull/26202/head
HaraldSchafer 3 years ago committed by GitHub
parent 68e7f42d68
commit a98d105cb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      .github/workflows/selfdrive_tests.yaml
  2. 2
      .gitignore
  3. 4
      .pre-commit-config.yaml
  4. 6
      Dockerfile.openpilot_base
  5. 100
      Pipfile
  6. 3072
      Pipfile.lock
  7. 7
      docs/docker/Dockerfile
  8. 7963
      poetry.lock
  9. 176
      pyproject.toml
  10. 5
      tools/README.md
  11. 2
      tools/mac_setup.sh
  12. 30
      update_requirements.sh

@ -106,7 +106,7 @@ jobs:
# /usr/local/Cellar
# ~/github_brew_cache_entries.txt
# /tmp/scons_cache
# key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
# key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
# restore-keys: macos-
# - name: Brew link restored dependencies
# run: |
@ -122,11 +122,11 @@ jobs:
# - name: Build openpilot
# run: |
# source tools/openpilot_env.sh
# pipenv run selfdrive/manager/build.py
# poetry run selfdrive/manager/build.py
#
# # cleanup scons cache
# rm -rf /tmp/scons_cache/
# pipenv run scons -j$(nproc) --cache-populate
# poetry run scons -j$(nproc) --cache-populate
# - name: Remove pre-existing Homebrew packages for caching
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |

2
.gitignore vendored

@ -83,3 +83,5 @@ selfdrive/modeld/models/*.thneed
build/
!**/.gitkeep
poetry.toml

@ -78,3 +78,7 @@ repos:
entry: selfdrive/ui/tests/test_translations.py
language: script
pass_filenames: false
- repo: https://github.com/python-poetry/poetry
rev: '1.2.2'
hooks:
- id: poetry-check

@ -12,19 +12,19 @@ ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
ENV PIPENV_SYSTEM=1
ENV POETRY_VIRTUALENVS_CREATE=false
ENV PYENV_VERSION=3.8.10
ENV PYENV_ROOT="/root/.pyenv"
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
COPY Pipfile Pipfile.lock .python-version update_requirements.sh /tmp/
COPY pyproject.toml poetry.lock .python-version update_requirements.sh /tmp/
COPY tools/ubuntu_setup.sh /tmp/tools/
RUN cd /tmp && \
tools/ubuntu_setup.sh && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /tmp/* && \
rm -rf /root/.cache && \
pip uninstall -y pipenv && \
pip uninstall -y poetry && \
# remove unused architectures from gcc for panda
cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \
rm -rf arm/ && \

@ -1,100 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
av = "*"
azure-storage-blob = "~=2.1"
control = "*"
coverage = "*"
dictdiffer = "*"
fastcluster = "*"
hexdump = "*"
hypothesis = "==6.46.7"
inputs = "*"
lru-dict = "*"
markdown-it-py = "*"
matplotlib = "*"
mypy = "*"
myst-parser = "*"
natsort = "*"
numpy = "*"
opencv-python-headless = "*"
parameterized = "*"
paramiko = "*"
pprofile = "*"
pre-commit = "*"
pycurl = "*"
pygame = "*"
pyprof2calltree = "*"
pytest = "*"
pytest-xdist = "*"
reverse_geocoder = "*"
scipy = "*"
sphinx = "*"
sphinx-sitemap = "*"
sphinx-rtd-theme = "*"
breathe = "*"
subprocess32 = "*"
tenacity = "*"
mpld3 = "*"
carla = {version = "==0.9.13", markers="platform_system != 'Darwin'"}
ft4222 = "*"
pandas = "*"
tabulate = "*"
types-pyyaml = "*"
lxml = "*"
types-atomicwrites = "*"
types-pycurl = "*"
types-requests = "*"
types-certifi = "*"
[packages]
atomicwrites = "*"
casadi = {version = "*", markers="platform_system != 'Darwin'"}
cffi = "*"
crcmod = "*"
cryptography = "*"
Cython = "*"
flake8 = "*"
Flask = "*"
future-fstrings = "*" # for acados
gunicorn = "*"
hexdump = "*"
Jinja2 = "*"
json-rpc = "*"
libusb1 = "*"
nose = "*"
numpy = "*"
protobuf = "==3.20.1"
onnx = "*"
onnxruntime-gpu = {version = "*", markers="platform_system != 'Darwin'"}
pillow = "*"
psutil = "*"
pycapnp = "==1.1.0"
pycryptodome = "*"
PyJWT = "*"
pylint = "*"
pyopencl = "*"
pyserial = "*"
python-dateutil = "*"
PyYAML = "*"
pyzmq = "*"
requests = "*"
scons = "*"
sentry-sdk = "*"
setproctitle = "*"
six = "*"
smbus2 = "*"
sympy = "!=1.6.1"
timezonefinder = "*"
tqdm = "*"
urllib3 = "*"
utm = "*"
websocket_client = "*"
hatanaka = "==2.4"
PySide2 = "*"
[requires]
python_version = "3.8"

3072
Pipfile.lock generated

File diff suppressed because it is too large Load Diff

@ -4,16 +4,11 @@ ENV PYTHONUNBUFFERED 1
ENV OPENPILOT_PATH /home/batman/openpilot/
ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
ENV POETRY_VIRUALENVS_CREATE false
RUN mkdir -p ${OPENPILOT_PATH}
WORKDIR ${OPENPILOT_PATH}
COPY Pipfile Pipfile.lock $OPENPILOT_PATH
RUN pip install --no-cache-dir pipenv==2021.5.29 pip==21.3.1 && \
pipenv install --system --deploy --dev --clear && \
pip uninstall -y pipenv
COPY SConstruct ${OPENPILOT_PATH}
COPY ./pyextra ${OPENPILOT_PATH}/pyextra

7963
poetry.lock generated

File diff suppressed because it is too large Load Diff

@ -0,0 +1,176 @@
[tool.poetry]
name = "openpilot"
version = "0.1.0"
description = "an open source driver assistance system"
authors = ["Vehicle Researcher <user@comma.ai>"]
license = "MIT"
readme = "README.md"
repository = "https://github.com/commaai/openpilot"
documentation = "https://docs.comma.ai"
[tool.poetry.dependencies]
python = "~3.8"
atomicwrites = "^1.4.0"
casadi = { version = "==3.5.5", markers = "platform_system != 'Darwin'" }
cffi = "^1.15.1"
crcmod = "^1.7"
cryptography = "^37.0.4"
Cython = "^0.29.30"
flake8 = "^4.0.1"
Flask = "^2.1.2"
future-fstrings = "^1.2.0" # for acados
gunicorn = "^20.1.0"
hatanaka = "==2.4"
hexdump = "^3.3"
Jinja2 = "^3.1.2"
json-rpc = "^1.13.0"
libusb1 = "^3.0.0"
nose = "^1.3.7"
numpy = "^1.23.0"
onnx = "^1.12.0"
onnxruntime-gpu = { version = "^1.11.1", markers = "platform_system != 'Darwin'" }
pillow = "^9.2.0"
poetry = "==1.2.2"
protobuf = "==3.20.1"
psutil = "^5.9.1"
pycapnp = "==1.1.0"
pycryptodome = "^3.15.0"
PyJWT = "^2.5.0"
pylint = "^2.15.4"
pyopencl = "^2022.2.4"
pyserial = "^3.5"
python-dateutil = "^2.8.2"
PyYAML = "^6.0"
pyzmq = "^23.2.0"
requests = "^2.28.1"
scons = "^4.3.0"
sentry-sdk = "^1.6.0"
setproctitle = "^1.2.3"
six = "^1.16.0"
smbus2 = "^0.4.2"
sympy = "^1.10.1"
timezonefinder = "^6.0.1"
tqdm = "^4.64.0"
urllib3 = "^1.26.10"
utm = "^0.7.0"
websocket_client = "^1.3.3"
[tool.poetry.group.dev.dependencies]
av = "^9.2.0"
azure-storage-blob = "~2.1"
breathe = "^4.34.0"
carla = "==0.9.13"
control = "^0.9.2"
coverage = "^6.4.1"
dictdiffer = "^0.9.0"
fastcluster = "^1.2.6"
ft4222 = "^1.4.1"
hexdump = "^3.3"
hypothesis = "==6.46.7"
inputs = "^0.5"
lru-dict = "^1.1.7"
lxml = "^4.9.1"
markdown-it-py = "^2.1.0"
matplotlib = "^3.5.2"
mpld3 = "^0.5.8"
mypy = "^0.961"
myst-parser = "^0.18.0"
natsort = "^8.1.0"
numpy = "^1.23.0"
opencv-python-headless = { url = "https://github.com/commaai/opencv-python-builder/releases/download/4.5.5.64%2Bcu113/opencv_python_headless-4.5.5.64-cp38-cp38-manylinux_2_31_x86_64.whl" }
pandas = "^1.4.3"
parameterized = "^0.8.1"
paramiko = "^2.11.0"
pprofile = "^2.1.0"
pre-commit = "^2.19.0"
pycurl = "^7.45.1"
pygame = "^2.1.2"
pyprof2calltree = "^1.4.5"
pytest = "^7.1.2"
pytest-xdist = "^2.5.0"
reverse_geocoder = "^1.5.1"
scipy = "^1.8.1"
sphinx = "^5.0.2"
sphinx-rtd-theme = "^1.0.0"
sphinx-sitemap = "^2.2.0"
subprocess32 = "^3.5.4"
tabulate = "^0.8.10"
tenacity = "^8.0.1"
types-atomicwrites = "^1.4.5"
types-certifi = "^2021.10.8"
types-pycurl = "^7.45.1"
types-PyYAML = "^6.0"
types-requests = "^2.28.11"
[tool.poetry.group.xx]
optional = true
[tool.poetry.group.xx.dependencies]
aenum = "^3.1.11"
aiohttp = "^3.8.1"
albumentations = "^1.2.1"
apex = { url = "https://github.com/commaai/apex/releases/download/pytorch1.10.0%2Bcu11.1/apex-0.1-cp38-cp38-linux_x86_64.whl" }
azure-cli-core = "^2.38.0"
azure-common = "^1.1.28"
azure-core = "^1.24.2"
azure-nspkg = "~3.0"
azure-storage-blob = "~2.1"
azure-storage-common = "~2.1"
azure-storage-nspkg = "~3.1"
blosc = "==1.9.2"
cloudpickle = "^2.1.0"
configargparse = "^1.5.3"
cupy-cuda113 = "^10.6.0"
datadog = "^0.44.0"
dotmap = "^1.3.30"
elasticsearch = "^8.3.1"
Flask-Cors = "^3.0.10"
Flask-SocketIO = "^5.2.0"
GeoAlchemy2 = "^0.12.1"
imageio = "^2.19.5"
influxdb-client = "^1.30.0"
ipykernel = "^6.15.1"
ipython = "^8.4.0"
joblib = "^1.1.0"
json-logging-py = "^0.2"
jupyter = "^1.0.0"
jupyterlab = "^3.4.4"
jupyterlab-vim = "^0.15.1"
Markdown = "^3.4.1"
mpld3 = "^0.5.8"
msgpack-python = "^0.5.6"
networkx = "~2.3"
nvidia-ml-py3 = "^7.352.0"
onnxoptimizer = "^0.2.6"
opencv-python-headless = { url = "https://github.com/commaai/opencv-python-builder/releases/download/4.5.5.64%2Bcu113/opencv_python_headless-4.5.5.64-cp38-cp38-manylinux_2_31_x86_64.whl" }
osmium = "^3.3.0"
pandas = "^1.4.3"
pillow-avif-plugin = "^1.2.2"
pipenv = "==2022.10.12"
plotly = "^5.9.0"
pycuda = "^2022.1"
Pygments = "^2.12.0"
PyMySQL = "~0.9"
pyproj = "^3.3.1"
python-logstash = "^0.4.8"
redis = "^4.3.4"
s2sphere = "^0.2.5"
scikit-image = "^0.19.3"
scikit-learn = "^1.1.1"
segmentation-models-pytorch = "==0.2.1"
simplejson = "^3.17.6"
SQLAlchemy = "^1.4.39"
torch = { url = "https://download.pytorch.org/whl/cu113/torch-1.11.0%2Bcu113-cp38-cp38-linux_x86_64.whl" }
torchsummary = "^1.5.1"
torchvision = { url = "https://download.pytorch.org/whl/cu113/torchvision-0.12.0%2Bcu113-cp38-cp38-linux_x86_64.whl" }
triton = "^1.1.1"
Werkzeug = "^2.1.2"
zerorpc = { git = "git@github.com:commaai/zerorpc-python.git", branch = "master" }
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

@ -6,7 +6,6 @@ openpilot is developed and tested on **Ubuntu 20.04**, which is the primary deve
## Setup your PC
First, clone openpilot:
``` bash
cd ~
@ -26,10 +25,10 @@ tools/ubuntu_setup.sh
tools/mac_setup.sh
```
Activate a shell with the install Python dependencies:
Activate a shell with the Python dependencies installed:
``` bash
cd openpilot && pipenv shell
cd openpilot && poetry shell
```
Build openpilot with this command:

@ -88,7 +88,7 @@ eval "$(pyenv init --path)"
echo "[ ] installed python dependencies t=$SECONDS"
# install casadi
VENV=`pipenv --venv`
VENV=`poetry env info --path`
PYTHON_VER=3.8
PYTHON_VERSION=$(cat $ROOT/.python-version)
if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then

@ -40,27 +40,29 @@ fi
eval "$(pyenv init --path)"
echo "update pip"
pip install pip==21.3.1
pip install pipenv==2021.11.23
pip install pip==22.3
pip install poetry==1.2.2
if [ -d "./xx" ]; then
echo "WARNING: using xx Pipfile ******"
export PIPENV_SYSTEM=1
export PIPENV_PIPFILE=./xx/Pipfile
fi
poetry config virtualenvs.prefer-active-python true --local
if [ -z "$PIPENV_SYSTEM" ]; then
echo "PYTHONPATH=${PWD}" > .env
RUN="pipenv run"
else
RUN=""
POETRY_INSTALL_ARGS=""
if [ -d "./xx" ]; then
echo "WARNING: using xx dependency group, installing globally"
poetry config virtualenvs.create false --local
POETRY_INSTALL_ARGS="--with xx --sync"
fi
echo "pip packages install..."
pipenv sync --dev
pipenv --clear
poetry install --no-cache --no-root $POETRY_INSTALL_ARGS
pyenv rehash
if [ -d "./xx" ] || [ -n "$POETRY_VIRTUALENVS_CREATE" ]; then
RUN=""
else
echo "PYTHONPATH=${PWD}" > .env
RUN="poetry run"
fi
echo "pre-commit hooks install..."
shopt -s nullglob
for f in .pre-commit-config.yaml */.pre-commit-config.yaml; do

Loading…
Cancel
Save