move all third party stuff into third_party/ (#26853)

* mv fastcluster

* move msm_kgsl.h

* camerad include

* update path

* mv pyextra

* fix tici build

* add acados_template to release build

Co-authored-by: Comma Device <device@comma.ai>

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
     rev: v4.1.0
     hooks:
     -   id: check-ast
-        exclude: '^(pyextra)/'
+        exclude: '^(third_party)/'
     -   id: check-json
     -   id: check-xml
     -   id: check-yaml
@@ -19,7 +19,7 @@ repos:
     rev: v2.2.1
     hooks:
     -   id: codespell
-        exclude: '^(pyextra/)|(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(include/)|(selfdrive/ui/translations/.*.ts)|(selfdrive/controls/lib/cluster)'
+        exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
         - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup
@@ -31,12 +31,12 @@ repos:
         entry: mypy
         language: system
         types: [python]
-        exclude: '^(pyextra/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
+        exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
 -   repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:
     -   id: flake8
-        exclude: '^(pyextra/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(selfdrive/debug/)/'
+        exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(selfdrive/debug/)/'
         additional_dependencies: ['flake8-no-implicit-concat']
         args:
         - --indent-size=2
@@ -51,7 +51,7 @@ repos:
         entry: pylint
         language: system
         types: [python]
-        exclude: '^(pyextra/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
+        exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
         args:
         - -j0
         - -rn
@@ -64,7 +64,7 @@ repos:
         entry: cppcheck
         language: system
         types: [c++]
-        exclude: '^(third_party/)|(pyextra/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)'
+        exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)'
         args:
         - --error-exitcode=1
         - --language=c++

Dockerfile.openpilot

@@ -10,7 +10,6 @@ WORKDIR ${OPENPILOT_PATH}
 
 COPY SConstruct ${OPENPILOT_PATH}
 
-COPY ./pyextra ${OPENPILOT_PATH}/pyextra
 COPY ./third_party ${OPENPILOT_PATH}/third_party
 COPY ./site_scons ${OPENPILOT_PATH}/site_scons
 COPY ./laika ${OPENPILOT_PATH}/laika

README.md

@@ -103,7 +103,6 @@ Directory Structure
     ├── opendbc             # Files showing how to interpret data from cars
     ├── panda               # Code used to communicate on CAN
     ├── third_party         # External libraries
-    ├── pyextra             # Extra python packages
     └── system              # Generic services
         ├── camerad         # Driver to capture images from the camera sensors
         ├── clocksd         # Broadcasts current time

SConstruct

@@ -71,10 +71,10 @@ if arch == "aarch64" and AGNOS:
 lenv = {
   "PATH": os.environ['PATH'],
   "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
-  "PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath,
+  "PYTHONPATH": Dir("#").abspath,
 
   "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
-  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
   "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
 
@@ -422,7 +422,6 @@ SConscript(['common/transformations/SConscript'])
 
 SConscript(['selfdrive/modeld/SConscript'])
 
-SConscript(['selfdrive/controls/lib/cluster/SConscript'])
 SConscript(['selfdrive/controls/lib/lateral_mpc_lib/SConscript'])
 SConscript(['selfdrive/controls/lib/longitudinal_mpc_lib/SConscript'])
 

docs/Makefile

@@ -41,7 +41,7 @@ clean:
 
 	@echo "Building rst files..."
 	sphinx-apidoc -o "$(DOCSBUILDDIR)" ../ \
-		../xx ../laika_repo ../rednose_repo ../pyextra ../notebooks ../panda_jungle \
+		../xx ../laika_repo ../rednose_repo ../notebooks ../panda_jungle \
 		../third_party \
 		../panda/examples \
 		../scripts \

docs/docker/Dockerfile

@@ -11,7 +11,6 @@ WORKDIR ${OPENPILOT_PATH}
 
 COPY SConstruct ${OPENPILOT_PATH}
 
-COPY ./pyextra ${OPENPILOT_PATH}/pyextra
 COPY ./third_party ${OPENPILOT_PATH}/third_party
 COPY ./site_scons ${OPENPILOT_PATH}/site_scons
 COPY ./laika ${OPENPILOT_PATH}/laika

launch_chffrplus.sh

@@ -74,7 +74,7 @@ function launch {
 
   # handle pythonpath
   ln -sfn $(pwd) /data/pythonpath
-  export PYTHONPATH="$PWD:$PWD/pyextra"
+  export PYTHONPATH="$PWD"
 
   # hardware specific init
   agnos_init

mypy.ini

@@ -2,7 +2,7 @@
 python_version = 3.8
 plugins = numpy.typing.mypy_plugin
 files = body, common, docs, scripts, selfdrive, site_scons, system, tools
-exclude = ^(pyextra/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)
+exclude = ^(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)
 
 ; third-party packages
 ignore_missing_imports = True

release/files_common

@@ -192,8 +192,6 @@ selfdrive/controls/lib/pid.py
 selfdrive/controls/lib/radar_helpers.py
 selfdrive/controls/lib/vehicle_model.py
 
-selfdrive/controls/lib/cluster/*
-
 selfdrive/controls/lib/lateral_mpc_lib/.gitignore
 selfdrive/controls/lib/longitudinal_mpc_lib/.gitignore
 selfdrive/controls/lib/lateral_mpc_lib/*
@@ -328,7 +326,6 @@ system/camerad/SConscript
 system/camerad/main.cc
 
 system/camerad/snapshot/*
-system/camerad/include/*
 system/camerad/cameras/camera_common.h
 system/camerad/cameras/camera_common.cc
 system/camerad/cameras/sensor2_i2c.h
@@ -384,7 +381,6 @@ selfdrive/modeld/thneed/thneed.h
 selfdrive/modeld/thneed/thneed_common.cc
 selfdrive/modeld/thneed/thneed_qcom2.cc
 selfdrive/modeld/thneed/serialize.cc
-selfdrive/modeld/thneed/include/*
 
 selfdrive/modeld/runners/snpemodel.cc
 selfdrive/modeld/runners/snpemodel.h
@@ -412,8 +408,11 @@ selfdrive/assets/offroad/*
 selfdrive/assets/sounds/*
 selfdrive/assets/training/*
 
+third_party/.gitignore
 third_party/SConscript
 
+third_party/cluster/*
+
 third_party/linux/**
 third_party/opencl/**
 
@@ -436,15 +435,13 @@ third_party/snpe/dsp**
 third_party/acados/x86_64/**
 third_party/acados/larch64/**
 third_party/acados/include/**
+third_party/acados/acados_template/**
 
 third_party/qt5/larch64/bin/**
 
 scripts/update_now.sh
 scripts/stop_updater.sh
 
-pyextra/.gitignore
-pyextra/acados_template/**
-
 rednose/.gitignore
 rednose/**
 laika/**

selfdrive/controls/lib/lateral_mpc_lib/SConscript

@@ -44,7 +44,7 @@ generated_files = [
 ] + build_files
 
 acados_dir = '#third_party/acados'
-acados_templates_dir = '#pyextra/acados_template/c_templates_tera'
+acados_templates_dir = '#third_party/acados/acados_template/c_templates_tera'
 
 source_list = ['lat_mpc.py',
   f'{acados_dir}/include/acados_c/ocp_nlp_interface.h',
@@ -70,8 +70,8 @@ lib_solver = lenv.SharedLibrary(f"{gen}/acados_ocp_solver_lat",
                                 LIBS=['m', 'acados', 'hpipm', 'blasfeo', 'qpOASES_e'])
 
 # generate cython stuff
-acados_ocp_solver_pyx = File("#pyextra/acados_template/acados_ocp_solver_pyx.pyx")
+acados_ocp_solver_pyx = File("#third_party/acados/acados_template/acados_ocp_solver_pyx.pyx")
-acados_ocp_solver_common = File("#pyextra/acados_template/acados_solver_common.pxd")
+acados_ocp_solver_common = File("#third_party/acados/acados_template/acados_solver_common.pxd")
 libacados_ocp_solver_pxd = File(f'{gen}/acados_solver.pxd')
 libacados_ocp_solver_c = File(f'{gen}/acados_ocp_solver_pyx.c')
 

selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py

@@ -8,7 +8,7 @@ from common.realtime import sec_since_boot
 from selfdrive.modeld.constants import T_IDXS
 
 if __name__ == '__main__':  # generating code
-  from pyextra.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver
+  from third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver
 else:
   from selfdrive.controls.lib.lateral_mpc_lib.c_generated_code.acados_ocp_solver_pyx import AcadosOcpSolverCython  # pylint: disable=no-name-in-module, import-error
 

selfdrive/controls/lib/longitudinal_mpc_lib/SConscript

@@ -51,7 +51,7 @@ generated_files = [
 ] + build_files
 
 acados_dir = '#third_party/acados'
-acados_templates_dir = '#pyextra/acados_template/c_templates_tera'
+acados_templates_dir = '#third_party/acados/acados_template/c_templates_tera'
 
 source_list = ['long_mpc.py',
   f'{acados_dir}/include/acados_c/ocp_nlp_interface.h',
@@ -77,8 +77,8 @@ lib_solver = lenv.SharedLibrary(f"{gen}/acados_ocp_solver_long",
                                 LIBS=['m', 'acados', 'hpipm', 'blasfeo', 'qpOASES_e'])
 
 # generate cython stuff
-acados_ocp_solver_pyx = File("#pyextra/acados_template/acados_ocp_solver_pyx.pyx")
+acados_ocp_solver_pyx = File("#third_party/acados/acados_template/acados_ocp_solver_pyx.pyx")
-acados_ocp_solver_common = File("#pyextra/acados_template/acados_solver_common.pxd")
+acados_ocp_solver_common = File("#third_party/acados/acados_template/acados_solver_common.pxd")
 libacados_ocp_solver_pxd = File(f'{gen}/acados_solver.pxd')
 libacados_ocp_solver_c = File(f'{gen}/acados_ocp_solver_pyx.c')
 

selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py

@@ -9,7 +9,7 @@ from selfdrive.modeld.constants import index_function
 from selfdrive.controls.lib.radar_helpers import _LEAD_ACCEL_TAU
 
 if __name__ == '__main__':  # generating code
-  from pyextra.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver
+  from third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver
 else:
   from selfdrive.controls.lib.longitudinal_mpc_lib.c_generated_code.acados_ocp_solver_pyx import AcadosOcpSolverCython  # pylint: disable=no-name-in-module, import-error
 

selfdrive/controls/radard.py

@@ -8,9 +8,9 @@ from cereal import car
 from common.numpy_fast import interp
 from common.params import Params
 from common.realtime import Ratekeeper, Priority, config_realtime_process
-from selfdrive.controls.lib.cluster.fastcluster_py import cluster_points_centroid
 from selfdrive.controls.lib.radar_helpers import Cluster, Track, RADAR_TO_CAMERA
 from system.swaglog import cloudlog
+from third_party.cluster.fastcluster_py import cluster_points_centroid
 
 
 class KalmanParams():

selfdrive/controls/tests/test_clustering.py

@@ -7,8 +7,8 @@ from fastcluster import linkage_vector
 from scipy.cluster import _hierarchy
 from scipy.spatial.distance import pdist
 
-from selfdrive.controls.lib.cluster.fastcluster_py import hclust, ffi
+from third_party.cluster.fastcluster_py import hclust, ffi
-from selfdrive.controls.lib.cluster.fastcluster_py import cluster_points_centroid
+from third_party.cluster.fastcluster_py import cluster_points_centroid
 
 
 def fcluster(Z, t, criterion='inconsistent', depth=2, R=None, monocrit=None):

selfdrive/manager/manager.py

@@ -23,8 +23,6 @@ from system.version import is_dirty, get_commit, get_version, get_origin, get_sh
                               terms_version, training_version, is_tested_branch
 
 
-sys.path.append(os.path.join(BASEDIR, "pyextra"))
-
 
 def manager_init() -> None:
   # update system time from panda

selfdrive/modeld/thneed/thneed.h

@@ -12,7 +12,7 @@
 
 #include <CL/cl.h>
 
-#include "selfdrive/modeld/thneed/include/msm_kgsl.h"
+#include "msm_kgsl.h"
 
 using namespace std;
 

system/camerad/SConscript

@@ -2,17 +2,13 @@ Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc')
 
 libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon, 'atomic']
 
-cenv = env.Clone()
+camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/camera_util.cc'])
-cenv['CPPPATH'].append('include/')
+env.Program('camerad', [
-
-camera_obj = cenv.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/camera_util.cc'])
-cenv.Program('camerad', [
     'main.cc',
     camera_obj,
   ], LIBS=libs)
 
 if GetOption("test") and arch == "x86_64":
-  cenv.Program('test/ae_gray_test', [
+  env.Program('test/ae_gray_test',
-      'test/ae_gray_test.cc',
+              ['test/ae_gray_test.cc', camera_obj],
-      camera_obj,
+              LIBS=libs)
-    ], LIBS=libs)

third_party/SConscript

@@ -4,3 +4,5 @@ env.Library('json11', ['json11/json11.cpp'], CCFLAGS=env['CCFLAGS'] + ['-Wno-unq
 env.Append(CPPPATH=[Dir('json11')])
 
 env.Library('kaitai', ['kaitai/kaitaistream.cpp'], CPPDEFINES=['KS_STR_ENCODING_NONE'])
+
+SConscript(['cluster/SConscript'])

third_party/acados/build.sh

@@ -43,11 +43,10 @@ mkdir -p $INSTALL_DIR
 
 rm $DIR/acados_repo/lib/*.json
 
-rm -rf $DIR/include
+rm -rf $DIR/include $DIR/acados_template
 cp -r $DIR/acados_repo/include $DIR
 cp -r $DIR/acados_repo/lib $INSTALL_DIR
-rm -rf $DIR/../../pyextra/acados_template
+cp -r $DIR/acados_repo/interfaces/acados_template/acados_template $DIR/
-cp -r $DIR/acados_repo/interfaces/acados_template/acados_template $DIR/../../pyextra
 #pip3 install -e $DIR/acados/interfaces/acados_template
 
 # build tera