Running pre-commit in CI (#1590)

* Running pre-commit in CI

* fix dockerfile syntax

* dont run on submodule repo folders

* Fix some import errors in ci

* more stuff

* That should be the last one

.github/workflows/test.yaml

@@ -37,7 +37,7 @@ jobs:
 
         # need these so docker copy won't fail
         cp Pipfile Pipfile.lock flake8_openpilot.sh pylint_openpilot.sh .pylintrc \
-           cppcheck_openpilot.sh .coveragerc-app $TEST_DIR
+           cppcheck_openpilot.sh .coveragerc-app .pre-commit-config.yaml $TEST_DIR
         cd $TEST_DIR
         mkdir pyextra laika laika_repo tools release
     - name: Build Docker image
@@ -77,6 +77,8 @@ jobs:
         run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh"
       - name: pylint
         run: $RUN "cd /tmp/openpilot/ && ./pylint_openpilot.sh"
+      - name: pre-commit
+        run: $RUN "cd /tmp/openpilot/ && git init && git add -A && pre-commit run --all"
       - name: cppcheck
         run: $PERSIST "cd /tmp/openpilot/ && ./cppcheck_openpilot.sh 2> cppcheck_report.txt"
       - name: Print cppcheck report

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     rev: master
     hooks:
     -   id: flake8
-        exclude: '^(pyextra)|(external)/'
+        exclude: '^(pyextra)|(external)|(cereal)|(rednose)|(panda)|(laika)|(laika_repo)|(rednose_repo)/'
         args:
         - --select=F
 -   repo: local
@@ -25,6 +25,6 @@ repos:
         entry: pylint
         language: system
         types: [python]
-        exclude: '^(pyextra)|(external)/'
+        exclude: '^(pyextra)|(external)|(cereal)|(rednose)|(panda)|(laika)|(laika_repo)|(rednose_repo)/'
         args:
         - --disable=R,C,W

Dockerfile.openpilot

@@ -71,7 +71,8 @@ RUN pyenv install 3.8.2 && \
       azure-storage-common==2.1.0 \
       azure-storage-nspkg==3.1.0 \
       pycurl==7.43.0.3 \
-      coverage==5.1
+      coverage==5.1 \
+      pre-commit==2.4.0
 
 RUN mkdir -p /tmp/openpilot
 
@@ -80,6 +81,7 @@ COPY SConstruct \
      flake8_openpilot.sh \
      pylint_openpilot.sh \
      .pylintrc \
+     .pre-commit-config.yaml \
      .coveragerc-app \
      /tmp/openpilot/
 

common/window.py

@@ -1,6 +1,6 @@
 import sys
-import pygame
+import pygame  # pylint: disable=import-error
-import cv2
+import cv2  # pylint: disable=import-error
 
 class Window():
   def __init__(self, w, h, caption="window", double=False):

release/remote_build.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python2
-import paramiko
+import paramiko # pylint: disable=import-error
 import os
 import sys
 import re

selfdrive/camerad/test/yuv_bench/cnv.py

@@ -1,5 +1,5 @@
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 
 # img_bgr = np.zeros((874, 1164, 3), dtype=np.uint8)
 # for y in range(874):

selfdrive/debug/internal/design_lqr.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import numpy as np
-import control
+import control # pylint: disable=import-error
 
 dt = 0.01
 A = np.array([[ 0.        ,  1.        ],       [-0.78823806,  1.78060701]])

selfdrive/modeld/runners/keras_runner.py

@@ -2,13 +2,13 @@
 # TODO: why are the keras models saved with python 2?
 from __future__ import print_function
 
-import tensorflow as tf
+import tensorflow as tf  # pylint: disable=import-error
 import os
 import sys
-import tensorflow.keras as keras
+import tensorflow.keras as keras # pylint: disable=import-error
 import numpy as np
-from tensorflow.keras.models import Model
+from tensorflow.keras.models import Model # pylint: disable=import-error
-from tensorflow.keras.models import load_model
+from tensorflow.keras.models import load_model # pylint: disable=import-error
 
 def read(sz):
   dd = []
@@ -55,4 +55,3 @@ if __name__ == "__main__":
   no = keras.layers.Concatenate()(m(tii))
   m = Model(inputs=ri, outputs=[no])
   run_loop(m)
-

selfdrive/modeld/test/tf_test/pb_loader.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 import sys
-import tensorflow as tf
+import tensorflow as tf  # pylint: disable=import-error
 
 with open(sys.argv[1], "rb") as f:
   graph_def = tf.compat.v1.GraphDef()
   graph_def.ParseFromString(f.read())
   #tf.io.write_graph(graph_def, '', sys.argv[1]+".try")
-

selfdrive/test/profiling/controlsd.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
 import os
-import cProfile
+import cProfile  # pylint: disable=import-error
-import pprofile
+import pprofile  # pylint: disable=import-error
-import pyprof2calltree
+import pyprof2calltree  # pylint: disable=import-error
 
 from tools.lib.logreader import LogReader
 from selfdrive.controls.controlsd import main as controlsd_thread

tools/carcontrols/joystick_test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import pygame
+import pygame  # pylint: disable=import-error
 
 # Define some colors
 BLACK    = (   0,   0,   0)

tools/carcontrols/joystickd.py

@@ -6,7 +6,7 @@
 
 ### this process needs pygame and can't run on the EON ###
 
-import pygame
+import pygame  # pylint: disable=import-error
 import cereal.messaging as messaging
 
 def joystick_thread():

tools/livedm/helpers.py

@@ -1,5 +1,5 @@
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 
 def rot_matrix(roll, pitch, yaw):
   cr, sr = np.cos(roll), np.sin(roll)
@@ -27,4 +27,3 @@ def draw_pose(img, pose, loc, W=160, H=320, xyoffset=(0,0), faceprob=0):
   else:
     color = (64,64,64)
   cv2.circle(img, tr, 7, color=color)
-  

tools/livedm/livedm.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 import os
 import argparse
-import pygame
+import pygame  # pylint: disable=import-error
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 
 from cereal import log
 import cereal.messaging as messaging

tools/replay/camera.py

@@ -6,9 +6,9 @@ os.environ['BASEDIR'] = BASEDIR
 SCALE = float(os.getenv("SCALE", 1.0))
 
 import argparse
-import pygame
+import pygame  # pylint: disable=import-error
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 import sys
 import cereal.messaging as messaging
 

tools/replay/lib/ui_helpers.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, Tuple
 import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
-import pygame
+import pygame  # pylint: disable=import-error
 
 from selfdrive.config import RADAR_TO_CAMERA
 from selfdrive.config import UIParams as UP

tools/replay/sensorium.py

@@ -3,7 +3,7 @@
 # Question: Can a human drive from this data?
 
 import os
-import cv2
+import cv2  # pylint: disable=import-error
 import numpy as np
 import cereal.messaging as messaging
 from common.window import Window
@@ -49,5 +49,3 @@ if __name__ == "__main__":
         flags=cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
 
       win.draw(imgw)
-
-

tools/replay/ui.py

@@ -5,9 +5,9 @@ import sys
 
 os.environ["OMP_NUM_THREADS"] = "1"
 
-import cv2
+import cv2  # pylint: disable=import-error
 import numpy as np
-import pygame
+import pygame  # pylint: disable=import-error
 
 from common.basedir import BASEDIR
 from common.transformations.camera import FULL_FRAME_SIZE, eon_intrinsics

tools/sim/lib/can.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import cereal.messaging as messaging
 from opendbc.can.packer import CANPacker
-from selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp  # pylint: disable=no-name-in-module
+from selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp  # pylint: disable=no-name-in-module,import-error
 from selfdrive.car.honda.values import FINGERPRINTS, CAR
 from selfdrive.car import crc8_pedal
 import math

tools/webcam/front_mount_helper.py

@@ -18,7 +18,7 @@ webcam_intrinsics = np.array([
 cam_id = 2
 
 if __name__ == "__main__":
-  import cv2
+  import cv2  # pylint: disable=import-error
 
   trans_webcam_to_eon_front = np.dot(eon_dcam_intrinsics,np.linalg.inv(webcam_intrinsics))
 

tools/webcam/warp_vis.py

@@ -23,7 +23,7 @@ webcam_intrinsics = np.array([
   [  0.,    0.,     1.]])
 
 if __name__ == "__main__":
-  import cv2
+  import cv2  # pylint: disable=import-error
   trans_webcam_to_eon_rear = np.dot(eon_intrinsics,np.linalg.inv(webcam_intrinsics))
   trans_webcam_to_eon_front = np.dot(eon_dcam_intrinsics,np.linalg.inv(webcam_intrinsics))
   print("trans_webcam_to_eon_rear:\n", trans_webcam_to_eon_rear)
@@ -41,5 +41,3 @@ if __name__ == "__main__":
       print(img.shape, end='\r')
       cv2.imshow('preview', img)
       cv2.waitKey(10)
-
-