Direct logentries logging from manager (#1306)

* add function to add direct logentries handler

* Whitespace

* Add manager crash handler

* Log scons build errors

* Add line to format exception

* Small cleanup

* That is now a list

* Read all data from stderr

selfdrive/logmessaged.py

@@ -3,12 +3,16 @@ import zmq
 from logentries import LogentriesHandler
 import cereal.messaging as messaging
 
-def main():
+
+def get_le_handler():
   # setup logentries. we forward log messages to it
   le_token = "e8549616-0798-4d7e-a2ca-2513ae81fa17"
-  le_handler = LogentriesHandler(le_token, use_tls=False, verbose=False)
+  return LogentriesHandler(le_token, use_tls=False, verbose=False)
 
-  le_level = 20 #logging.INFO
+
+def main():
+  le_handler = get_le_handler()
+  le_level = 20  # logging.INFO
 
   ctx = zmq.Context().instance()
   sock = ctx.socket(zmq.PULL)
@@ -36,5 +40,6 @@ def main():
     msg.logMessage = dat
     pub_sock.send(msg.to_bytes())
 
+
 if __name__ == "__main__":
   main()

selfdrive/manager.py

@@ -8,6 +8,7 @@ import signal
 import shutil
 import subprocess
 import datetime
+from selfdrive.swaglog import cloudlog, add_logentries_handler
 
 from common.basedir import BASEDIR, PARAMS
 from common.android import ANDROID
@@ -88,25 +89,32 @@ if not prebuilt:
     j_flag = "" if nproc is None else "-j%d" % (nproc - 1)
     scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
 
+    compile_output = []
+
     # Read progress from stderr and update spinner
     while scons.poll() is None:
       try:
         line = scons.stderr.readline()
         if line is None:
           continue
-
         line = line.rstrip()
+
         prefix = b'progress: '
         if line.startswith(prefix):
           i = int(line[len(prefix):])
           if spinner is not None:
             spinner.update("%d" % (50.0 * (i / TOTAL_SCONS_NODES)))
         elif len(line):
-          print(line.decode('utf8'))
+          compile_output.append(line)
+          print(line.decode('utf8', 'replace'))
       except Exception:
         pass
 
     if scons.returncode != 0:
+      # Read remaining output
+      r = scons.stderr.read().split(b'\n')
+      compile_output += r
+
       if retry:
         print("scons build failed, cleaning in")
         for i in range(3,-1,-1):
@@ -115,7 +123,15 @@ if not prebuilt:
         subprocess.check_call(["scons", "-c"], cwd=BASEDIR, env=env)
         shutil.rmtree("/tmp/scons_cache")
       else:
-        raise RuntimeError("scons build failed")
+        # Build failed log errors
+        errors = [line.decode('utf8', 'replace') for line in compile_output
+                  if any([err in line for err in [b'error: ', b'not found, needed by target']])]
+        errors = "\n".join(errors)
+        add_logentries_handler(cloudlog)
+        cloudlog.error("scons build failed\n" + errors)
+
+        # TODO: Show errors in TextWindow
+        exit(1)
     else:
       break
 
@@ -124,7 +140,6 @@ import cereal.messaging as messaging
 
 from common.params import Params
 import selfdrive.crash as crash
-from selfdrive.swaglog import cloudlog
 from selfdrive.registration import register
 from selfdrive.version import version, dirty
 from selfdrive.loggerd.config import ROOT
@@ -567,7 +582,17 @@ def main():
   if params.get("DoUninstall", encoding='utf8') == "1":
     uninstall()
 
+
 if __name__ == "__main__":
+  try:
     main()
+  except Exception:
+    add_logentries_handler(cloudlog)
+    cloudlog.exception("Manager failed to start")
+    # TODO: Show exception using TextWindow
+    # error = traceback.format_exc()
+
+    raise
+
   # manual exit because we are forked
   sys.exit(0)

selfdrive/swaglog.py

@@ -5,6 +5,7 @@ import zmq
 
 from common.logging_extra import SwagLogger, SwagFormatter
 
+
 class LogMessageHandler(logging.Handler):
   def __init__(self, formatter):
     logging.Handler.__init__(self)
@@ -31,10 +32,19 @@ class LogMessageHandler(logging.Handler):
       # drop :/
       pass
 
+
+def add_logentries_handler(log):
+  """Function to add the logentries handler to swaglog.
+  This can be used to send logs when logmessaged is not running."""
+  from selfdrive.logmessaged import get_le_handler
+  handler = get_le_handler()
+  handler.setFormatter(SwagFormatter(log))
+  log.addHandler(handler)
+
+
 cloudlog = log = SwagLogger()
 log.setLevel(logging.DEBUG)
 
 outhandler = logging.StreamHandler()
 log.addHandler(outhandler)
-
 log.addHandler(LogMessageHandler(SwagFormatter(log)))