diff --git a/selfdrive/logmessaged.py b/selfdrive/logmessaged.py index 693d259a78..56e13e8028 100755 --- a/selfdrive/logmessaged.py +++ b/selfdrive/logmessaged.py @@ -3,12 +3,16 @@ import zmq from logentries import LogentriesHandler import cereal.messaging as messaging -def main(): + +def get_le_handler(): # setup logentries. we forward log messages to it le_token = "e8549616-0798-4d7e-a2ca-2513ae81fa17" - le_handler = LogentriesHandler(le_token, use_tls=False, verbose=False) + return LogentriesHandler(le_token, use_tls=False, verbose=False) - le_level = 20 #logging.INFO + +def main(): + le_handler = get_le_handler() + le_level = 20 # logging.INFO ctx = zmq.Context().instance() sock = ctx.socket(zmq.PULL) @@ -36,5 +40,6 @@ def main(): msg.logMessage = dat pub_sock.send(msg.to_bytes()) + if __name__ == "__main__": main() diff --git a/selfdrive/manager.py b/selfdrive/manager.py index 59c98a5d5b..2bbc5cde67 100755 --- a/selfdrive/manager.py +++ b/selfdrive/manager.py @@ -8,6 +8,7 @@ import signal import shutil import subprocess import datetime +from selfdrive.swaglog import cloudlog, add_logentries_handler from common.basedir import BASEDIR, PARAMS from common.android import ANDROID @@ -88,25 +89,32 @@ if not prebuilt: j_flag = "" if nproc is None else "-j%d" % (nproc - 1) scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE) + compile_output = [] + # Read progress from stderr and update spinner while scons.poll() is None: try: line = scons.stderr.readline() if line is None: continue - line = line.rstrip() + prefix = b'progress: ' if line.startswith(prefix): i = int(line[len(prefix):]) if spinner is not None: spinner.update("%d" % (50.0 * (i / TOTAL_SCONS_NODES))) elif len(line): - print(line.decode('utf8')) + compile_output.append(line) + print(line.decode('utf8', 'replace')) except Exception: pass if scons.returncode != 0: + # Read remaining output + r = scons.stderr.read().split(b'\n') + compile_output += r + if retry: print("scons build failed, cleaning in") for i in range(3,-1,-1): @@ -115,7 +123,15 @@ if not prebuilt: subprocess.check_call(["scons", "-c"], cwd=BASEDIR, env=env) shutil.rmtree("/tmp/scons_cache") else: - raise RuntimeError("scons build failed") + # Build failed log errors + errors = [line.decode('utf8', 'replace') for line in compile_output + if any([err in line for err in [b'error: ', b'not found, needed by target']])] + errors = "\n".join(errors) + add_logentries_handler(cloudlog) + cloudlog.error("scons build failed\n" + errors) + + # TODO: Show errors in TextWindow + exit(1) else: break @@ -124,7 +140,6 @@ import cereal.messaging as messaging from common.params import Params import selfdrive.crash as crash -from selfdrive.swaglog import cloudlog from selfdrive.registration import register from selfdrive.version import version, dirty from selfdrive.loggerd.config import ROOT @@ -567,7 +582,17 @@ def main(): if params.get("DoUninstall", encoding='utf8') == "1": uninstall() + if __name__ == "__main__": - main() + try: + main() + except Exception: + add_logentries_handler(cloudlog) + cloudlog.exception("Manager failed to start") + # TODO: Show exception using TextWindow + # error = traceback.format_exc() + + raise + # manual exit because we are forked sys.exit(0) diff --git a/selfdrive/swaglog.py b/selfdrive/swaglog.py index 90962cb5d8..ef94fb91dc 100644 --- a/selfdrive/swaglog.py +++ b/selfdrive/swaglog.py @@ -5,6 +5,7 @@ import zmq from common.logging_extra import SwagLogger, SwagFormatter + class LogMessageHandler(logging.Handler): def __init__(self, formatter): logging.Handler.__init__(self) @@ -31,10 +32,19 @@ class LogMessageHandler(logging.Handler): # drop :/ pass + +def add_logentries_handler(log): + """Function to add the logentries handler to swaglog. + This can be used to send logs when logmessaged is not running.""" + from selfdrive.logmessaged import get_le_handler + handler = get_le_handler() + handler.setFormatter(SwagFormatter(log)) + log.addHandler(handler) + + cloudlog = log = SwagLogger() log.setLevel(logging.DEBUG) outhandler = logging.StreamHandler() log.addHandler(outhandler) - log.addHandler(LogMessageHandler(SwagFormatter(log)))