Car docs diff bot: detect name + year changes (#25224)

* These cars have TSS-P standard

* temp

* debug close matching

* check everything

* clean up

* combine and use a dict

* programmatic

* revert

* clean up match_cars

* comment

* clean up

* clean up

* restore car changes

* test

* Revert "test"

This reverts commit e96f693681.

* we don't need this

* fix footnotes on model column

selfdrive/debug/print_docs_diff.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 import argparse
+from collections import defaultdict
+import difflib
 import pickle
 
 from selfdrive.car.docs import get_all_car_info
 from selfdrive.car.docs_definitions import Column
 
+FOOTNOTE_TAG = "<sup>{}</sup>"
 STAR_ICON = '<a href="##"><img valign="top" src="https://raw.githubusercontent.com/commaai/openpilot/master/docs/assets/icon-star-{}.svg" width="22" /></a>'
 COLUMNS = "|" + "|".join([column.value for column in Column]) + "|"
-COLUMN_HEADER = "|---|---|---|:---:|:---:|:---:|:---:|:---:|"
+COLUMN_HEADER = "|---|---|---|:---:|:---:|:---:|:---:|"
 ARROW_SYMBOL = "➡️"
 
 
@@ -16,69 +19,84 @@ def load_base_car_info(path):
     return pickle.load(f)
 
 
-def get_star_diff(base_car, new_car):
+def match_cars(base_cars, new_cars):
-  return [column for column, value in base_car.row.items() if value != new_car.row[column]]
+  """Matches CarInfo by name similarity and finds additions and removals"""
-
+  changes = []
+  additions = []
+  for new in new_cars:
+    closest_match = difflib.get_close_matches(new.name, [b.name for b in base_cars], cutoff=0.)[0]
 
-def format_row(builder):
+    if closest_match not in [c[1].name for c in changes]:
-  return "|" + "|".join(builder) + "|"
+      changes.append((new, next(car for car in base_cars if car.name == closest_match)))
+    else:
+      additions.append(new)
 
+  removals = [b for b in base_cars if b.name not in [c[1].name for c in changes]]
+  return changes, additions, removals
 
-def print_car_info_diff(path):
-  base_car_info = {f"{i.make} {i.model}": i for i in load_base_car_info(path)}
-  new_car_info = {f"{i.make} {i.model}": i for i in get_all_car_info()}
 
-  tier_changes = []
+def build_column_diff(base_car, new_car):
-  star_changes = []
+  row_builder = []
-  removals = []
+  for column in Column:
-  additions = []
+    base_column = base_car.get_column(column, STAR_ICON, FOOTNOTE_TAG)
+    new_column = new_car.get_column(column, STAR_ICON, FOOTNOTE_TAG)
 
-  # Changes (tier + stars)
+    if base_column != new_column:
-  for base_car_model, base_car in base_car_info.items():
+      row_builder.append(f"{base_column} {ARROW_SYMBOL} {new_column}")
-    if base_car_model not in new_car_info:
+    else:
-      continue
+      row_builder.append(new_column)
 
-    new_car = new_car_info[base_car_model]
+  return format_row(row_builder)
 
-    # Tier changes
-    if base_car.tier != new_car.tier:
-      tier_changes.append(f"- Tier for {base_car.make} {base_car.model} changed! ({base_car.tier.name.title()} {ARROW_SYMBOL} {new_car.tier.name.title()})")
 
-    # Star changes
+def format_row(builder):
-    diff = get_star_diff(base_car, new_car)
+  return "|" + "|".join(builder) + "|"
-    if not len(diff):
-      continue
-
-    row_builder = []
-    for column in list(Column):
-      if column not in diff:
-        row_builder.append(new_car.get_column(column, STAR_ICON, "{}"))
-      else:
-        row_builder.append(base_car.get_column(column, STAR_ICON, "{}") + ARROW_SYMBOL + new_car.get_column(column, STAR_ICON, "{}"))
-
-    star_changes.append(format_row(row_builder))
 
-  # Removals
-  for model in set(base_car_info) - set(new_car_info):
-    car_info = base_car_info[model]
-    removals.append(format_row([car_info.get_column(column, STAR_ICON, "{}") for column in Column]))
 
-  # Additions
+def print_car_info_diff(path):
-  for model in set(new_car_info) - set(base_car_info):
+  base_car_info = defaultdict(list)
-    car_info = new_car_info[model]
+  new_car_info = defaultdict(list)
-    additions.append(format_row([car_info.get_column(column, STAR_ICON, "{}") for column in Column]))
+
+  for car in load_base_car_info(path):
+    base_car_info[car.car_fingerprint].append(car)
+  for car in get_all_car_info():
+    new_car_info[car.car_fingerprint].append(car)
+
+  changes = defaultdict(list)
+  for base_car_model, base_cars in base_car_info.items():
+    # Match car info changes, and get additions and removals
+    new_cars = new_car_info[base_car_model]
+    car_changes, car_additions, car_removals = match_cars(base_cars, new_cars)
+
+    # Removals
+    for car_info in car_removals:
+      changes["removals"].append(format_row([car_info.get_column(column, STAR_ICON, FOOTNOTE_TAG) for column in Column]))
+
+    # Additions
+    for car_info in car_additions:
+      changes["additions"].append(format_row([car_info.get_column(column, STAR_ICON, FOOTNOTE_TAG) for column in Column]))
+
+    for new_car, base_car in car_changes:
+      # Tier changes
+      if base_car.tier != new_car.tier:
+        changes["tier"].append(f"- Tier for {base_car.make} {base_car.model} changed! ({base_car.tier.name.title()} {ARROW_SYMBOL} {new_car.tier.name.title()})")
+
+      # Column changes
+      row_diff = build_column_diff(base_car, new_car)
+      if ARROW_SYMBOL in row_diff:
+        changes["column"].append(row_diff)
 
   # Print diff
-  if len(star_changes) or len(tier_changes) or len(removals) or len(additions):
+  if any(len(c) for c in changes.values()):
     markdown_builder = ["### ⚠️ This PR makes changes to [CARS.md](../blob/master/docs/CARS.md) ⚠️"]
 
-    for title, category in (("## 🏅 Tier Changes", tier_changes), ("## 🔀 Star Changes", star_changes), ("## ❌ Removed", removals), ("## ➕ Added", additions)):
+    for title, category in (("## 🏅 Tier Changes", "tier"), ("## 🔀 Column Changes", "column"), ("## ❌ Removed", "removals"), ("## ➕ Added", "additions")):
-      if len(category):
+      if len(changes[category]):
         markdown_builder.append(title)
         if "Tier" not in title:
           markdown_builder.append(COLUMNS)
           markdown_builder.append(COLUMN_HEADER)
-        markdown_builder.extend(category)
+        markdown_builder.extend(changes[category])
 
     print("\n".join(markdown_builder))