Running pre-commit in CI (#1590)

* Running pre-commit in CI

* fix dockerfile syntax

* dont run on submodule repo folders

* Fix some import errors in ci

* more stuff

* That should be the last one

.github/workflows/test.yaml

@@ -37,7 +37,7 @@ jobs:
 
         # need these so docker copy won't fail
         cp Pipfile Pipfile.lock flake8_openpilot.sh pylint_openpilot.sh .pylintrc \
-           cppcheck_openpilot.sh .coveragerc-app $TEST_DIR
+           cppcheck_openpilot.sh .coveragerc-app .pre-commit-config.yaml $TEST_DIR
         cd $TEST_DIR
         mkdir pyextra laika laika_repo tools release
     - name: Build Docker image
@@ -77,6 +77,8 @@ jobs:
         run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh"
       - name: pylint
         run: $RUN "cd /tmp/openpilot/ && ./pylint_openpilot.sh"
+      - name: pre-commit
+        run: $RUN "cd /tmp/openpilot/ && git init && git add -A && pre-commit run --all"
       - name: cppcheck
         run: $PERSIST "cd /tmp/openpilot/ && ./cppcheck_openpilot.sh 2> cppcheck_report.txt"
       - name: Print cppcheck report

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     rev: master
     hooks:
     -   id: flake8
-        exclude: '^(pyextra)|(external)/'
+        exclude: '^(pyextra)|(external)|(cereal)|(rednose)|(panda)|(laika)|(laika_repo)|(rednose_repo)/'
         args:
         - --select=F
 -   repo: local
@@ -25,6 +25,6 @@ repos:
         entry: pylint
         language: system
         types: [python]
-        exclude: '^(pyextra)|(external)/'
+        exclude: '^(pyextra)|(external)|(cereal)|(rednose)|(panda)|(laika)|(laika_repo)|(rednose_repo)/'
         args:
         - --disable=R,C,W

Dockerfile.openpilot

@@ -71,7 +71,8 @@ RUN pyenv install 3.8.2 && \
       azure-storage-common==2.1.0 \
       azure-storage-nspkg==3.1.0 \
       pycurl==7.43.0.3 \
-      coverage==5.1
+      coverage==5.1 \
+      pre-commit==2.4.0
 
 RUN mkdir -p /tmp/openpilot
 
@@ -80,6 +81,7 @@ COPY SConstruct \
      flake8_openpilot.sh \
      pylint_openpilot.sh \
      .pylintrc \
+     .pre-commit-config.yaml \
      .coveragerc-app \
      /tmp/openpilot/
 

common/window.py

@@ -1,6 +1,6 @@
 import sys
-import pygame
+import pygame  # pylint: disable=import-error
-import cv2
+import cv2  # pylint: disable=import-error
 
 class Window():
   def __init__(self, w, h, caption="window", double=False):

release/remote_build.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python2
-import paramiko
+import paramiko # pylint: disable=import-error
 import os
 import sys
 import re

selfdrive/camerad/test/yuv_bench/cnv.py

@@ -1,5 +1,5 @@
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 
 # img_bgr = np.zeros((874, 1164, 3), dtype=np.uint8)
 # for y in range(874):

selfdrive/debug/internal/design_lqr.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import numpy as np
-import control
+import control # pylint: disable=import-error
 
 dt = 0.01
 A = np.array([[ 0.        ,  1.        ],       [-0.78823806,  1.78060701]])

selfdrive/modeld/runners/keras_runner.py

@@ -2,13 +2,13 @@
 # TODO: why are the keras models saved with python 2?
 from __future__ import print_function
 
-import tensorflow as tf
+import tensorflow as tf  # pylint: disable=import-error
 import os
 import sys
-import tensorflow.keras as keras
+import tensorflow.keras as keras # pylint: disable=import-error
 import numpy as np
-from tensorflow.keras.models import Model
+from tensorflow.keras.models import Model # pylint: disable=import-error
-from tensorflow.keras.models import load_model
+from tensorflow.keras.models import load_model # pylint: disable=import-error
 
 def read(sz):
   dd = []
@@ -55,4 +55,3 @@ if __name__ == "__main__":
   no = keras.layers.Concatenate()(m(tii))
   m = Model(inputs=ri, outputs=[no])
   run_loop(m)
-

selfdrive/modeld/test/tf_test/pb_loader.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 import sys
-import tensorflow as tf
+import tensorflow as tf  # pylint: disable=import-error
 
 with open(sys.argv[1], "rb") as f:
   graph_def = tf.compat.v1.GraphDef()
   graph_def.ParseFromString(f.read())
   #tf.io.write_graph(graph_def, '', sys.argv[1]+".try")
-

selfdrive/test/profiling/controlsd.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
 import os
-import cProfile
+import cProfile  # pylint: disable=import-error
-import pprofile
+import pprofile  # pylint: disable=import-error
-import pyprof2calltree
+import pyprof2calltree  # pylint: disable=import-error
 
 from tools.lib.logreader import LogReader
 from selfdrive.controls.controlsd import main as controlsd_thread

tools/carcontrols/joystick_test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import pygame
+import pygame  # pylint: disable=import-error
 
 # Define some colors
 BLACK    = (   0,   0,   0)
@@ -17,21 +17,21 @@ class TextPrint:
         textBitmap = self.font.render(textString, True, BLACK)
         screen.blit(textBitmap, [self.x, self.y])
         self.y += self.line_height
-        
+
     def reset(self):
         self.x = 10
         self.y = 10
         self.line_height = 15
-        
+
     def indent(self):
         self.x += 10
-        
+
     def unindent(self):
         self.x -= 10
-    
+
 
 pygame.init()
- 
+
 # Set the width and height of the screen [width,height]
 size = [500, 700]
 screen = pygame.display.set_mode(size)
@@ -46,7 +46,7 @@ clock = pygame.time.Clock()
 
 # Initialize the joysticks
 pygame.joystick.init()
-    
+
 # Get ready to print
 textPrint = TextPrint()
 
@@ -56,14 +56,14 @@ while done==False:
     for event in pygame.event.get(): # User did something
         if event.type == pygame.QUIT: # If user clicked close
             done=True # Flag that we are done so we exit this loop
-        
+
         # Possible joystick actions: JOYAXISMOTION JOYBALLMOTION JOYBUTTONDOWN JOYBUTTONUP JOYHATMOTION
         if event.type == pygame.JOYBUTTONDOWN:
             print("Joystick button pressed.")
         if event.type == pygame.JOYBUTTONUP:
             print("Joystick button released.")
-            
+
- 
+
     # DRAWING STEP
     # First, clear the screen to white. Don't put other drawing commands
     # above this, or they will be erased with this command.
@@ -75,29 +75,29 @@ while done==False:
 
     textPrint.printf(screen, "Number of joysticks: {}".format(joystick_count) )
     textPrint.indent()
-    
+
     # For each joystick:
     joystick = pygame.joystick.Joystick(0)
     joystick.init()
-    
+
     textPrint.printf(screen, "Joystick {}".format(0) )
     textPrint.indent()
-    
+
     # Get the name from the OS for the controller/joystick
     name = joystick.get_name()
     textPrint.printf(screen, "Joystick name: {}".format(name) )
-    
+
     # Usually axis run in pairs, up/down for one, and left/right for
     # the other.
     axes = joystick.get_numaxes()
     textPrint.printf(screen, "Number of axes: {}".format(axes) )
     textPrint.indent()
-    
+
     for i in range( axes ):
         axis = joystick.get_axis( i )
         textPrint.printf(screen, "Axis {} value: {:>6.3f}".format(i, axis) )
     textPrint.unindent()
-        
+
     buttons = joystick.get_numbuttons()
     textPrint.printf(screen, "Number of buttons: {}".format(buttons) )
     textPrint.indent()
@@ -110,15 +110,15 @@ while done==False:
 
     textPrint.unindent()
 
-    
+
     # ALL CODE TO DRAW SHOULD GO ABOVE THIS COMMENT
-    
+
     # Go ahead and update the screen with what we've drawn.
     pygame.display.flip()
 
     # Limit to 20 frames per second
     clock.tick(20)
-    
+
 # Close the window and quit.
 # If you forget this line, the program will 'hang'
 # on exit if running from IDLE.

tools/carcontrols/joystickd.py

@@ -6,7 +6,7 @@
 
 ### this process needs pygame and can't run on the EON ###
 
-import pygame
+import pygame  # pylint: disable=import-error
 import cereal.messaging as messaging
 
 def joystick_thread():

tools/livedm/helpers.py

@@ -1,5 +1,5 @@
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 
 def rot_matrix(roll, pitch, yaw):
   cr, sr = np.cos(roll), np.sin(roll)
@@ -27,4 +27,3 @@ def draw_pose(img, pose, loc, W=160, H=320, xyoffset=(0,0), faceprob=0):
   else:
     color = (64,64,64)
   cv2.circle(img, tr, 7, color=color)
-  

tools/livedm/livedm.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 import os
 import argparse
-import pygame
+import pygame  # pylint: disable=import-error
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 
 from cereal import log
 import cereal.messaging as messaging

tools/replay/camera.py

@@ -6,9 +6,9 @@ os.environ['BASEDIR'] = BASEDIR
 SCALE = float(os.getenv("SCALE", 1.0))
 
 import argparse
-import pygame
+import pygame  # pylint: disable=import-error
 import numpy as np
-import cv2
+import cv2  # pylint: disable=import-error
 import sys
 import cereal.messaging as messaging
 

tools/replay/lib/ui_helpers.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, Tuple
 import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
-import pygame
+import pygame  # pylint: disable=import-error
 
 from selfdrive.config import RADAR_TO_CAMERA
 from selfdrive.config import UIParams as UP

tools/replay/sensorium.py

@@ -3,7 +3,7 @@
 # Question: Can a human drive from this data?
 
 import os
-import cv2
+import cv2  # pylint: disable=import-error
 import numpy as np
 import cereal.messaging as messaging
 from common.window import Window
@@ -49,5 +49,3 @@ if __name__ == "__main__":
         flags=cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
 
       win.draw(imgw)
-
-

tools/replay/ui.py

@@ -5,9 +5,9 @@ import sys
 
 os.environ["OMP_NUM_THREADS"] = "1"
 
-import cv2
+import cv2  # pylint: disable=import-error
 import numpy as np
-import pygame
+import pygame  # pylint: disable=import-error
 
 from common.basedir import BASEDIR
 from common.transformations.camera import FULL_FRAME_SIZE, eon_intrinsics

tools/sim/lib/can.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import cereal.messaging as messaging
 from opendbc.can.packer import CANPacker
-from selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp  # pylint: disable=no-name-in-module
+from selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp  # pylint: disable=no-name-in-module,import-error
 from selfdrive.car.honda.values import FINGERPRINTS, CAR
 from selfdrive.car import crc8_pedal
 import math

tools/webcam/front_mount_helper.py

@@ -18,7 +18,7 @@ webcam_intrinsics = np.array([
 cam_id = 2
 
 if __name__ == "__main__":
-  import cv2
+  import cv2  # pylint: disable=import-error
 
   trans_webcam_to_eon_front = np.dot(eon_dcam_intrinsics,np.linalg.inv(webcam_intrinsics))
 
@@ -32,4 +32,4 @@ if __name__ == "__main__":
       img = cv2.warpPerspective(img, trans_webcam_to_eon_front, (1152,864), borderMode=cv2.BORDER_CONSTANT, borderValue=0)
       img = img[:,-864//2:,:]
       cv2.imshow('preview', img)
-      cv2.waitKey(10)
+      cv2.waitKey(10)

tools/webcam/warp_vis.py

@@ -23,7 +23,7 @@ webcam_intrinsics = np.array([
   [  0.,    0.,     1.]])
 
 if __name__ == "__main__":
-  import cv2
+  import cv2  # pylint: disable=import-error
   trans_webcam_to_eon_rear = np.dot(eon_intrinsics,np.linalg.inv(webcam_intrinsics))
   trans_webcam_to_eon_front = np.dot(eon_dcam_intrinsics,np.linalg.inv(webcam_intrinsics))
   print("trans_webcam_to_eon_rear:\n", trans_webcam_to_eon_rear)
@@ -41,5 +41,3 @@ if __name__ == "__main__":
       print(img.shape, end='\r')
       cv2.imshow('preview', img)
       cv2.waitKey(10)
-
-