athena: uploads, show which items fail on error (#23412)

* athena: uploads, show which items fail on error

* fix upload-id

* no more 404

* Update selfdrive/athena/athenad.py

Co-authored-by: Willem Melching <willem.melching@gmail.com>

selfdrive/athena/athenad.py

@@ -269,23 +269,29 @@ def uploadFileToUrl(fn, url, headers):
 @dispatcher.add_method
 def uploadFilesToUrls(files_data):
   items = []
+  failed = []
   for fn, url, headers in files_data:
     if len(fn) == 0 or fn[0] == '/' or '..' in fn:
-      return 500
+      failed.append(fn)
+      continue
     path = os.path.join(ROOT, fn)
     if not os.path.exists(path):
-      return 404
+      failed.append(fn)
+      continue
 
     item = UploadItem(path=path, url=url, headers=headers, created_at=int(time.time() * 1000), id=None)
     upload_id = hashlib.sha1(str(item).encode()).hexdigest()
-    items.append(item._replace(id=upload_id))
+    item = item._replace(id=upload_id)
-
-  for item in items:
     upload_queue.put_nowait(item)
+    items.append(item._asdict())
 
   UploadQueueCache.cache(upload_queue)
 
-  return {"enqueued": len(items), "items": [i._asdict() for i in items]}
+  resp = {"enqueued": len(items), "items": items}
+  if failed:
+    resp["failed"] = failed
+
+  return resp
 
 
 @dispatcher.add_method

selfdrive/athena/tests/test_athenad.py

@@ -134,13 +134,14 @@ class TestAthenadMethods(unittest.TestCase):
   @with_http_server
   def test_uploadFileToUrl(self, host):
     not_exists_resp = dispatcher["uploadFileToUrl"]("does_not_exist.bz2", "http://localhost:1238", {})
-    self.assertEqual(not_exists_resp, 404)
+    self.assertEqual(not_exists_resp, {'enqueued': 0, 'items': [], 'failed': ['does_not_exist.bz2']})
 
     fn = os.path.join(athenad.ROOT, 'qlog.bz2')
     Path(fn).touch()
 
     resp = dispatcher["uploadFileToUrl"]("qlog.bz2", f"{host}/qlog.bz2", {})
     self.assertEqual(resp['enqueued'], 1)
+    self.assertNotIn('failed', resp)
     self.assertDictContainsSubset({"path": fn, "url": f"{host}/qlog.bz2", "headers": {}}, resp['items'][0])
     self.assertIsNotNone(resp['items'][0].get('id'))
     self.assertEqual(athenad.upload_queue.qsize(), 1)