diff --git a/poetry.lock b/poetry.lock
index 5120cd0620..70a9e5b576 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:305c1ec645193b0916dd8254b13a5bd6582eac513dcb59e357cf9ba003ac8183
-size 445997
+oid sha256:d5af206fbe7c5b1e572047626db4eaab9c470b77334efc543e94d6b15c2f49ca
+size 447702
diff --git a/pyproject.toml b/pyproject.toml
index 38818cc62a..63bcd46de5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -124,7 +124,6 @@ av = "*"
 azure-identity = "*"
 azure-storage-blob = "*"
 breathe = "*"
-carla = { url = "https://github.com/commaai/carla/releases/download/3.11.4/carla-0.9.14-cp311-cp311-linux_x86_64.whl", platform = "linux", markers = "platform_machine == 'x86_64'" }
 coverage = "*"
 dictdiffer = "*"
 ft4222 = "*"
@@ -167,6 +166,11 @@ types-requests = "*"
 types-tabulate = "*"
 pyqt5 = { version = "*", markers = "platform_machine == 'x86_64'" } # no aarch64 wheels for macOS/linux
 
+[tool.poetry.group.carla]
+optional = true
+
+[tool.poetry.group.carla.dependencies]
+carla = { url = "https://github.com/commaai/carla/releases/download/3.11.4/carla-0.9.14-cp311-cp311-linux_x86_64.whl", platform = "linux", markers = "platform_machine == 'x86_64'" }
 
 [build-system]
 requires = ["poetry-core"]
diff --git a/tools/sim/README.md b/tools/sim/README.md
index dd248cd996..0892e77736 100644
--- a/tools/sim/README.md
+++ b/tools/sim/README.md
@@ -57,7 +57,13 @@ Start bridge processes located in tools/sim:
 
 ## Carla
 
-CARLA is also partially supported, though the performance is not great. openpilot doesn't have any extreme hardware requirements, however CARLA requires an NVIDIA graphics card and is very resource-intensive and may not run smoothly on your system.
+CARLA is also partially supported, though the performance is not great. It needs to be manually installed with:
+
+```bash
+poetry install --with=carla
+```
+
+openpilot doesn't have any extreme hardware requirements, however CARLA requires an NVIDIA graphics card and is very resource-intensive and may not run smoothly on your system.
 For this case, we have the simulator in low quality by default.
 
 You can also check out the [CARLA python documentation](https://carla.readthedocs.io/en/latest/python_api/) to find more parameters to tune that might increase performance on your system.
diff --git a/tools/sim/bridge/carla/carla_bridge.py b/tools/sim/bridge/carla/carla_bridge.py
index 949f3ed89e..dc2377df30 100644
--- a/tools/sim/bridge/carla/carla_bridge.py
+++ b/tools/sim/bridge/carla/carla_bridge.py
@@ -1,5 +1,3 @@
-import carla
-
 from openpilot.tools.sim.bridge.common import SimulatorBridge
 from openpilot.tools.sim.bridge.carla.carla_world import CarlaWorld
 
@@ -15,6 +13,8 @@ class CarlaBridge(SimulatorBridge):
     self.num_selected_spawn_point = arguments.num_selected_spawn_point
 
   def spawn_world(self):
+    import carla
+
     client = carla.Client(self.host, self.port)
     client.set_timeout(5)
 
diff --git a/tools/sim/bridge/carla/carla_world.py b/tools/sim/bridge/carla/carla_world.py
index 7886a865c9..14b300ed7b 100644
--- a/tools/sim/bridge/carla/carla_world.py
+++ b/tools/sim/bridge/carla/carla_world.py
@@ -1,4 +1,3 @@
-import carla
 import numpy as np
 
 from openpilot.common.params import Params
@@ -10,6 +9,8 @@ from openpilot.tools.sim.lib.camerad import W, H
 class CarlaWorld(World):
   def __init__(self, client, high_quality, dual_camera, num_selected_spawn_point, town):
     super().__init__(dual_camera)
+    import carla
+
     low_quality_layers = carla.MapLayer(carla.MapLayer.Ground | carla.MapLayer.Walls | carla.MapLayer.Decals)
 
     layers = carla.MapLayer.All if high_quality else low_quality_layers
@@ -139,5 +140,6 @@ class CarlaWorld(World):
     self.world.tick()
 
   def reset(self):
+    import carla
     self.vehicle.set_transform(self.spawn_point)
     self.vehicle.set_target_velocity(carla.Vector3D())