sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

commit

Browse Source
pull/36071/head
workinright 4 days ago
parent 64ec967711
commit 1e31ec7ab0
2 changed files with 135 additions and 101 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 123
      selfdrive/test/build.sh
  2. 113
      selfdrive/test/build_common.sh

123
selfdrive/test/build.sh
Unescape View File

@ -1,114 +1,38 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
trap restore_root ERR
ORG_PWD="$PWD" ORG_PWD="$PWD"
SELF_PATH="$(realpath $0)"
SCRIPT_DIR="$(dirname "$SELF_PATH")"
REPO="$HOME/work/openpilot/openpilot" REPO="$HOME/work/openpilot/openpilot"
CACHE_ROOTFS_TARBALL_PATH="/tmp/rootfs_cache.tar" CACHE_ROOTFS_TARBALL_PATH="/tmp/rootfs_cache.tar"
unpack_rootfs_tarball() { source "$SCRIPT_DIR/build_common.sh"
cd /
sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
cd
}
commit_root() {
sudo mkdir -p /base /newroot /upper /work
sudo unshare -f --kill-child -m $ORG_PWD/selfdrive/test/build.sh build_inside_namespace
ec=$?
echo "end of ns"
sudo rm -rf /base /newroot /work
# finally, create the rootfs diff tarball (to be pushed into the CI native cache)
mkdir -p /tmp/rootfs_cache
sudo rm -f "$CACHE_ROOTFS_TARBALL_PATH" # remove the old tarball from previous run, if exists
cd /upper
sudo tar -cf "$CACHE_ROOTFS_TARBALL_PATH" .
cd
sudo rm -rf /upper
unpack_rootfs_tarball
# before the next tasks are run, finalize the environment for them
prepare_mounts
exit $ec
}
prepare_mounts() {
# create and mount the required volumes where they're expected
mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
sudo mount --bind "$REPO" /tmp/openpilot
sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
# needed for the unit tests not to fail
sudo chmod 755 /sys/fs/pstore
}
restore_root() {
echo failed at ${BASH_LINENO[0]}
}
build_inside_namespace() {
mount --bind / /base
mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
rm -f /newroot/etc/resolv.conf
touch /newroot/etc/resolv.conf
cat /etc/resolv.conf > /newroot/etc/resolv.conf
mkdir -p /newroot/old
cd /newroot
pivot_root . old
mount -t proc proc /proc
mount -t devtmpfs devtmpfs /dev
mkdir -p /dev/pts
mount -t devpts devpts /dev/pts
mount -t proc proc /proc
mount -t sysfs sysfs /sys
touch /root_committed
sudo -u runner /home/runner/work/openpilot/openpilot/selfdrive/test/build.sh
ec=$?
exit $ec
}
if [ "$1" = "build_inside_namespace" ]
then
build_inside_namespace
exit
fi
# if the rootfs diff tarball (also created by this script) got restored from the CI native cache
if [ -f "$CACHE_ROOTFS_TARBALL_PATH" ] if [ -f "$CACHE_ROOTFS_TARBALL_PATH" ]
then then
# if the rootfs diff tarball (also created by this script) got restored from the CI native cache, unpack it, upgrading the rootfs # apply it, upgrading the rootfs
echo "restoring rootfs from the native build cache" echo "restoring rootfs from the native build cache"
unpack_rootfs_tarball apply_rootfs_diff
rm "$CACHE_ROOTFS_TARBALL_PATH" rm "$CACHE_ROOTFS_TARBALL_PATH"
# before the next tasks are run, finalize the environment for them # before the next tasks are run, finalize the environment for them
prepare_mounts prepare_build
# EXITS HERE - if the rootfs could been prepared entirely from the cache, there's no need for any further action like re-building # EXITS HERE - if the rootfs could been prepared entirely from the cache, there's no need for any further action like re-building
exit 0 exit 0
else else
# otherwise, we'll have to install everything from scratch and build the tarball to be available for the next run # otherwise, we'll have to install everything from scratch and build the tarball to be available for the next run
echo "no native build cache entry restored, rebuilding" if ! [ -f /root_committed ]
then
echo "no native build cache entry restored, rebuilding"
fi
fi fi
# in order to be able to build a diff rootfs tarball, we need to commit its initial state by moving it on-the-fly to overlayfs; # in order to be able to build a diff rootfs tarball, we need to commit its initial state
# below, we prepare the system and the new rootfs itself # by moving it on-the-fly to overlayfs; below, we prepare the system and the new rootfs itself
if ! [ -e /root_committed ]
then
commit_root commit_root
fi
# -------- at this point, the original rootfs was committed and all the changes to it done below will be saved to the newly created rootfs diff tarball -------- # -------- at this point, the original rootfs was committed and all the changes to it done below will be saved to the newly created rootfs diff tarball --------
@ -118,11 +42,11 @@ DEBIAN_FRONTEND=noninteractive
mkdir -p /tmp/tools mkdir -p /tmp/tools
cp "$REPO/tools/install_ubuntu_dependencies.sh" /tmp/tools/ cp "$REPO/tools/install_ubuntu_dependencies.sh" /tmp/tools/
sudo /tmp/tools/install_ubuntu_dependencies.sh &>/dev/null sudo /tmp/tools/install_ubuntu_dependencies.sh
sudo apt-get install -y --no-install-recommends \ sudo apt-get install -y --no-install-recommends \
sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev \ sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev \
apt-utils alien unzip tar curl xz-utils dbus gcc-arm-none-eabi tmux vim libx11-6 wget &>/dev/null apt-utils alien unzip tar curl xz-utils dbus gcc-arm-none-eabi tmux vim libx11-6 wget
sudo rm -rf /var/lib/apt/lists/* sudo rm -rf /var/lib/apt/lists/*
sudo apt-get clean sudo apt-get clean
@ -155,7 +79,6 @@ sudo ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2
sudo mkdir -p /etc/ld.so.conf.d sudo mkdir -p /etc/ld.so.conf.d
sudo bash -c "echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf" sudo bash -c "echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf"
sudo ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf sudo ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf
cd /
rm -rf /tmp/opencl-driver-intel rm -rf /tmp/opencl-driver-intel
cd cd
@ -166,18 +89,16 @@ NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
QTWEBENGINE_DISABLE_SANDBOX=1 QTWEBENGINE_DISABLE_SANDBOX=1
# install and set up the Python dependencies needed # install and set up the Python dependencies needed
sudo cp "/home/runner/work/openpilot/openpilot/pyproject.toml" "/home/runner/work/openpilot/openpilot/uv.lock" "/home/runner/work/openpilot/openpilot/tools/install_python_dependencies.sh" \ cp "$REPO/pyproject.toml" "$REPO/uv.lock" "$HOME/"
/home/runner/ mkdir "$HOME/tools"
cp "$REPO/tools/install_python_dependencies.sh" "$HOME/tools/"
cd VIRTUAL_ENV=/home/$USER/.venv
rm -rf .venv PATH="$VIRTUAL_ENV/bin:$PATH"
mkdir aaa
cd aaa
../install_python_dependencies.sh
cd cd
rm pyproject.toml uv.lock install_python_dependencies.sh tools/install_python_dependencies.sh
rm -rf tools/ pyproject.toml uv.lock .cache
# add a git safe directory for compiling openpilot # add a git safe directory for compiling openpilot
sudo git config --global --add safe.directory /tmp/openpilot sudo git config --global --add safe.directory /tmp/openpilot

113
selfdrive/test/build_common.sh
Unescape View File

@ -0,0 +1,113 @@
error_handler() {
echo failed at ${BASH_LINENO[0]}
}
trap error_handler ERR
create_rootfs_diff() {
sudo rm -f "$CACHE_ROOTFS_TARBALL_PATH" # remove the old diff tarball from previous run, if exists
cd /upper
sudo tar -cf "$CACHE_ROOTFS_TARBALL_PATH" .
cd
}
apply_rootfs_diff() {
cd /
sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
cd
}
prepare_build() {
# create and mount the required volumes where they're expected
mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
sudo mount --bind "$REPO" /tmp/openpilot
sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
# needed for the unit tests not to fail
sudo chmod 755 /sys/fs/pstore
}
post_commit_root() {
# we have the diff tarball, now let's remove the folder too
sudo rm -rf /upper
# now we apply it straight away
apply_rootfs_diff
# before the next tasks are run, finalize the environment for them
prepare_build
}
# warning: this function initiates a somewhat complicated program flow, follow carefully
# (even despite this part was made sure to not be too relevant for the rest of the job)
commit_root() {
# if that's a first execution
if ! [ -e /root_committed ]
then
# prepare directories
sudo mkdir -p /base /newroot /upper /work
# re-execute the main script (causing it to go straight to `build_inside_namespace`), but
# inside the newly created namespace, in a way which would cause all mounts
# created to automatically umount before it exits
sudo unshare -f --kill-child -m "$SELF_PATH" build_inside_namespace
ec=$?
# after it exited, remove the created directories (except the one containing created diff)
sudo rm -rf /base /newroot /work
# finally, create the rootfs diff tarball (to be pushed into the CI native cache)
create_rootfs_diff
# after creating the rootfs diff, bring the system into a state as if it was restored from cache
post_commit_root
exit $ec
fi
}
reexecute() {
touch /root_committed
sudo -u runner "$SELF_PATH"
ec=$?
exit $ec
}
# that's where the script goes after being re-executed for the first time
build_inside_namespace() {
# initialize the mounts namespace on overlayfs to be able to prepare the rootfs diff
mount --bind / /base
mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
# apply the current DNS config (beware: systemd often symlinks /etc/resolv.conf, that's why it's needed)
rm -f /newroot/etc/resolv.conf
touch /newroot/etc/resolv.conf
cat /etc/resolv.conf > /newroot/etc/resolv.conf
# switch the namespace's root mount to the newly created one
mkdir -p /newroot/old
cd /newroot
pivot_root . old
# initialize basic required POSIX-standard additional mounts
mount -t proc proc /proc
mount -t devtmpfs devtmpfs /dev
mkdir -p /dev/pts
mount -t devpts devpts /dev/pts
mount -t proc proc /proc
mount -t sysfs sysfs /sys
# re-execute the main script for the 2nd time, causing it to go back to the main flow
# (but this time already inside the newly created namespace)
reexecute
# after the main flow terminates and the namespace exist, post_commit_root is executed - be sure to look at it
}
if [ "$1" = "build_inside_namespace" ]
then
build_inside_namespace
exit
fi
Write Preview
Loading…
Cancel
Save