Scons builder for cython extensions (#2485)

* complie boardd without python

* not good, but don't want to lose the file, because it works

* clean a bit

* update dbc

* should build on CI

* not good, but don't want to lose the file, because it works

* clean a bit

* should build on CI

* remove unneeded path

* reorder paths

* reduce diff

* and now it works?!

* ... should work in CI

* add kj, 30% chance to fix macos

* pydebug

* new way to find path

* fix :)

* tested

* sanity check

* repl. MacOS flags

* hope it works

* need more logs

* need more logs2

* test if it works

* should work on CI

* correct python file

* should not work

* cleanup

* real cleanup

* more removals

* 50% of file

* transformations

* fixed a hardcoded variable

* more logs

* simpl.

* kalman

* all donw if it passes tests

* cleanup

* reduce code by 20 lines if this works

* fix bugs

* cleanup

* SharedLibrary

* cleanup

* ...

* remove unused

* CI fix maybe?

* add more valid path

* more logs

* ...:

* fix webcam CI

* remove WError flag

* deprecated is not an error

* more Wno things

* reduce diff, add Wno to env

* don't import nonexistent stuff

* SharedLibrary v2

* less custom env

* renaming, remove SharedLibs

* pack libs in envCython

* experiment

* better docker caching

* whitespace

* more docker caching

* improvement

* improvements

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

Dockerfile.openpilotci

@@ -12,6 +12,7 @@ COPY SConstruct \
 
 COPY ./pyextra /tmp/openpilot/pyextra
 COPY ./phonelibs /tmp/openpilot/phonelibs
+COPY ./site_scons /tmp/openpilot/site_scons
 COPY ./laika /tmp/openpilot/laika
 COPY ./laika_repo /tmp/openpilot/laika_repo
 COPY ./rednose /tmp/openpilot/rednose

SConstruct

@@ -5,6 +5,8 @@ import shutil
 import subprocess
 import sys
 import platform
+import numpy as np
+from sysconfig import get_paths
 
 TICI = os.path.isfile('/TICI')
 Decider('MD5-timestamp')
@@ -128,6 +130,10 @@ else:
 # change pythonpath to this
 lenv["PYTHONPATH"] = Dir("#").path
 
+#Get the path for Python.h for cython linking
+python_path = get_paths()['include']
+numpy_path = np.get_include()
+
 env = Environment(
   ENV=lenv,
   CCFLAGS=[
@@ -159,6 +165,7 @@ env = Environment(
     "#phonelibs/linux/include",
     "#phonelibs/snpe/include",
     "#phonelibs/nanovg",
+    "#selfdrive/boardd",
     "#selfdrive/common",
     "#selfdrive/camerad",
     "#selfdrive/camerad/include",
@@ -181,9 +188,12 @@ env = Environment(
   CXXFLAGS=["-std=c++1z"] + cxxflags,
   LIBPATH=libpath + [
     "#cereal",
+    "#selfdrive/boardd",
     "#selfdrive/common",
     "#phonelibs",
-  ]
+  ],
+  CYTHONCFILESUFFIX=".cpp",
+  tools=["default", "cython"]
 )
   
 if os.environ.get('SCONS_CACHE'):
@@ -222,6 +232,25 @@ def abspath(x):
     # rpath works elsewhere
     return x[0].path.rsplit("/", 1)[1][:-3]
 
+#Cython build enviroment
+envCython = env.Clone()
+envCython["CPPPATH"] += [python_path, numpy_path]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"]
+
+python_libs = []
+if arch == "Darwin":
+  envCython["LINKFLAGS"]=["-bundle", "-undefined", "dynamic_lookup"]
+elif arch == "aarch64":
+  envCython["LINKFLAGS"]=["-shared"]
+
+  python_libs.append(os.path.basename(python_path))
+else:
+  envCython["LINKFLAGS"]=["-pthread", "-shared"]
+
+envCython["LIBS"] = python_libs
+
+Export('envCython')
+
 # still needed for apks
 zmq = 'zmq'
 Export('env', 'arch', 'real_arch', 'zmq', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY')
@@ -277,6 +306,5 @@ SConscript(['selfdrive/ui/SConscript'])
 if arch != "Darwin":
   SConscript(['selfdrive/logcatd/SConscript'])
 
-
 if arch == "x86_64":
   SConscript(['tools/lib/index_log/SConscript'])

common/SConscript

@@ -1,14 +1,4 @@
-Import('env', 'cython_dependencies')
+Import('envCython')
 
-# Build cython clock module
+envCython.Program('clock.so', 'clock.pyx')
-env.Command(['common_pyx.so', 'clock.cpp'],
+envCython.Program('params_pyx.so', 'params_pyx.pyx')
-            cython_dependencies + ['common_pyx_setup.py', 'clock.pyx'],
-            "cd common && python3 common_pyx_setup.py build_ext --inplace")
-
-# Build cython params module
-env.Command(['params_pyx.so', 'params_pyx.cpp'],
-            cython_dependencies + [
-                'params_pyx_setup.py', 'params_pyx.pyx', 'params_pxd.pxd',
-                '#selfdrive/common/params.cc', '#selfdrive/common/params.h',
-                '#selfdrive/common/util.c', '#selfdrive/common/util.h'],
-            "cd common && python3 params_pyx_setup.py build_ext --inplace")

common/clock.pyx

@@ -1,3 +1,4 @@
+# distutils: language = c++
 # cython: language_level = 3
 from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
 

common/common_pyx_setup.py

@@ -1,20 +0,0 @@
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-from Cython.Build import cythonize
-
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-
-sourcefiles = ['clock.pyx']
-extra_compile_args = ["-std=c++1z"]
-
-setup(name='common',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "common_pyx",
-          language="c++",
-          sources=sourcefiles,
-          extra_compile_args=extra_compile_args,
-        ),
-        nthreads=4,
-      ),
-)

common/kalman/SConscript

@@ -1,6 +1,3 @@
-Import('env', 'cython_dependencies')
+Import('envCython')
-
-env.Command(['simple_kalman_impl.so'],
-            cython_dependencies + ['simple_kalman_impl.pyx', 'simple_kalman_impl.pxd', 'simple_kalman_setup.py'],
-            "cd common/kalman && python3 simple_kalman_setup.py build_ext --inplace")
 
+envCython.Program('simple_kalman_impl.so', 'simple_kalman_impl.pyx')

common/kalman/simple_kalman_impl.pyx

@@ -1,3 +1,4 @@
+# distutils: language = c++
 # cython: language_level=3
 
 cdef class KF1D:

common/kalman/simple_kalman_setup.py

@@ -1,10 +0,0 @@
-from distutils.core import Extension, setup
-
-from Cython.Build import cythonize
-
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-
-setup(name='Simple Kalman Implementation',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(Extension("simple_kalman_impl",
-                                      ["simple_kalman_impl.pyx"])))

common/kalman/tests/test_simple_kalman.py

@@ -82,3 +82,6 @@ kf = KF1D(x0=[[x0_0], [x1_0]],
     kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000)
     kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000)
     self.assertTrue(kf_speed < kf_old_speed / 4)
+
+if __name__ == "__main__":
+  unittest.main()

common/params_pyx.pyx

@@ -2,7 +2,7 @@
 # cython: language_level = 3
 from libcpp cimport bool
 from libcpp.string cimport string
-from params_pxd cimport Params as c_Params
+from common.params_pxd cimport Params as c_Params
 
 import os
 import threading

common/params_pyx_setup.py

@@ -1,33 +0,0 @@
-import os
-import subprocess
-from distutils.core import Extension, setup
-from Cython.Build import cythonize
-
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-from common.basedir import BASEDIR
-from common.hardware import TICI
-
-ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()  # pylint: disable=unexpected-keyword-arg
-
-sourcefiles = ['params_pyx.pyx']
-extra_compile_args = ["-std=c++1z"]
-
-if ARCH == "aarch64":
-  if TICI:
-    extra_compile_args += ["-DQCOM2"]
-  else:
-    extra_compile_args += ["-DQCOM"]
-
-
-setup(name='common',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "params_pyx",
-          language="c++",
-          sources=sourcefiles,
-          include_dirs=[BASEDIR, os.path.join(BASEDIR, 'selfdrive')],
-          extra_compile_args=extra_compile_args
-        )
-      )
-)

common/realtime.py

@@ -5,7 +5,7 @@ import time
 import multiprocessing
 
 from common.hardware import PC
-from common.common_pyx import sec_since_boot  # pylint: disable=no-name-in-module, import-error
+from common.clock import sec_since_boot  # pylint: disable=no-name-in-module, import-error
 
 
 # time step for each process

common/transformations/SConscript

@@ -1,8 +1,3 @@
-Import('env', 'cython_dependencies')
+Import('envCython')
 
-d = Dir('.')
+envCython.Program('transformations.so', 'transformations.pyx')
-
-env.Command(['transformations.so'],
-            cython_dependencies + ['transformations.pxd', 'transformations.pyx',
-             'coordinates.cc', 'orientation.cc', 'coordinates.hpp', 'orientation.hpp'],
-            'cd ' + d.path + ' && python3 setup.py build_ext --inplace')

common/transformations/setup.py

@@ -1,20 +0,0 @@
-import numpy
-
-from Cython.Build import cythonize
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-
-setup(
-  name='Cython transformations wrapper',
-  cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-  ext_modules=cythonize(
-    Extension(
-      "transformations",
-      sources=["transformations.pyx"],
-      language="c++",
-      extra_compile_args=["-std=c++1z", "-Wno-cpp"],
-      include_dirs=[numpy.get_include()],
-    ),
-    nthreads=4,
-  )
-)

common/transformations/transformations.pyx

@@ -1,18 +1,20 @@
-from transformations cimport Matrix3, Vector3, Quaternion
+# distutils: language = c++
-from transformations cimport ECEF, NED, Geodetic
+# cython: language_level = 3
-
+from common.transformations.transformations cimport Matrix3, Vector3, Quaternion
-from transformations cimport euler2quat as euler2quat_c
+from common.transformations.transformations cimport ECEF, NED, Geodetic
-from transformations cimport quat2euler as quat2euler_c
+
-from transformations cimport quat2rot as quat2rot_c
+from common.transformations.transformations cimport euler2quat as euler2quat_c
-from transformations cimport rot2quat as rot2quat_c
+from common.transformations.transformations cimport quat2euler as quat2euler_c
-from transformations cimport euler2rot as euler2rot_c
+from common.transformations.transformations cimport quat2rot as quat2rot_c
-from transformations cimport rot2euler as rot2euler_c
+from common.transformations.transformations cimport rot2quat as rot2quat_c
-from transformations cimport rot_matrix as rot_matrix_c
+from common.transformations.transformations cimport euler2rot as euler2rot_c
-from transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from common.transformations.transformations cimport rot2euler as rot2euler_c
-from transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from common.transformations.transformations cimport rot_matrix as rot_matrix_c
-from transformations cimport geodetic2ecef as geodetic2ecef_c
+from common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
-from transformations cimport ecef2geodetic as ecef2geodetic_c
+from common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
-from transformations cimport LocalCoord_c
+from common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
+from common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
+from common.transformations.transformations cimport LocalCoord_c
 
 
 import cython

release/files_common

@@ -12,6 +12,7 @@ CONTRIBUTING.md
 README.md
 RELEASES.md
 SAFETY.md
+site_scons/site_tools/cython.py
 
 apk/ai.comma*.apk
 
@@ -32,7 +33,6 @@ common/numpy_fast.py
 common/params.py
 common/params_pxd.pxd
 common/params_pyx.pyx
-common/params_pyx_setup.py
 common/xattr.py
 common/profiler.py
 common/basedir.py
@@ -43,7 +43,6 @@ common/text_window.py
 common/cython_hacks.py
 common/apk.py
 common/SConscript
-common/common_pyx_setup.py
 
 common/kalman/.gitignore
 common/kalman/*
@@ -53,7 +52,6 @@ common/transformations/camera.py
 common/transformations/model.py
 
 common/transformations/SConscript
-common/transformations/setup.py
 common/transformations/coordinates.py
 common/transformations/coordinates.cc
 common/transformations/coordinates.hpp
@@ -95,7 +93,6 @@ selfdrive/boardd/__init__.py
 selfdrive/boardd/boardd.cc
 selfdrive/boardd/boardd.py
 selfdrive/boardd/boardd_api_impl.pyx
-selfdrive/boardd/boardd_setup.py
 selfdrive/boardd/can_list_to_can_capnp.cc
 selfdrive/boardd/panda.cc
 selfdrive/boardd/panda.h

selfdrive/boardd/SConscript

@@ -1,8 +1,6 @@
-Import('env', 'common', 'cereal', 'messaging', 'cython_dependencies')
+Import('env', 'envCython', 'common', 'cereal', 'messaging')
 
 env.Program('boardd', ['boardd.cc', 'panda.cc', 'pigeon.cc'], LIBS=['usb-1.0', common, cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj'])
 env.Library('libcan_list_to_can_capnp', ['can_list_to_can_capnp.cc'])
 
-env.Command(['boardd_api_impl.so', 'boardd_api_impl.cpp'],
+envCython.Program('boardd_api_impl.so', 'boardd_api_impl.pyx', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"])
-            cython_dependencies + ['libcan_list_to_can_capnp.a', 'boardd_api_impl.pyx', 'boardd_setup.py'],
-            "cd selfdrive/boardd && python3 boardd_setup.py build_ext --inplace")

selfdrive/boardd/boardd_setup.py

@@ -1,22 +0,0 @@
-from distutils.core import Extension, setup
-from Cython.Build import cythonize
-
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-
-libraries = ['can_list_to_can_capnp', 'capnp', 'kj']
-
-setup(name='Boardd API Implementation',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "boardd_api_impl",
-          libraries=libraries,
-          library_dirs=[
-            './',
-          ],
-          sources=['boardd_api_impl.pyx'],
-          language="c++",
-          extra_compile_args=["-std=c++1z", "-Wno-nullability-completeness"],
-        )
-      )
-)

site_scons/site_tools/cython.py

@@ -0,0 +1,37 @@
+import SCons
+from SCons.Action import Action
+
+cythonAction = Action("$CYTHONCOM")
+
+def create_builder(env):
+    try:
+        cython = env['BUILDERS']['Cython']
+    except KeyError:
+        cython = SCons.Builder.Builder(
+                  action = cythonAction,
+                  emitter = {},
+                  suffix = cython_suffix_emitter,
+                  single_source = 1)
+        env['BUILDERS']['Cython'] = cython
+    return cython
+
+def cython_suffix_emitter(env, source):
+    return "$CYTHONCFILESUFFIX"
+
+def generate(env):
+    env["CYTHON"] = "cythonize"
+    env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
+    env["CYTHONCFILESUFFIX"] = ".cpp"
+
+    c_file, _ = SCons.Tool.createCFileBuilders(env)
+
+    c_file.suffix['.pyx'] = cython_suffix_emitter
+    c_file.add_action('.pyx', cythonAction)
+
+    c_file.suffix['.py'] = cython_suffix_emitter
+    c_file.add_action('.py', cythonAction)
+
+    create_builder(env)
+
+def exists(env):
+  return True

tools/webcam/Dockerfile

@@ -33,6 +33,7 @@ COPY SConstruct \
 
 COPY ./pyextra /tmp/openpilot/pyextra
 COPY ./phonelibs /tmp/openpilot/phonelibs
+COPY ./site_scons /tmp/openpilot/site_scons
 COPY ./laika /tmp/openpilot/laika
 COPY ./laika_repo /tmp/openpilot/laika_repo
 COPY ./rednose /tmp/openpilot/rednose