Jenkins pipeline to create master-ci (#1019)
* Added Jenkinsfile
* Added Jenkinsfile
* Added Jenkinsfile
* change order
* sudo
* whoami?
* Added Jenkinsfile
* install git
* Untested build scripts
* Add lockable resource
* Fix syntax
* Only one stage
* fix target dir
* Use deploy key
* noqa on test_openpilot
* Fix version.h path
* Cleanup release files
* Add linter scripts to release
* Update jenkinsfile
* Fix path
* this should work
* Use python3 docker container
* Run in correct directory
* Setup /data/pythonpath
Co-authored-by: commaci-public <60409688+commaci-public@users.noreply.github.com>
old-commit-hash: 0319861700
commatwo_master
Willem Melching
5 years ago
committed by
GitHub
parent
e58c9f6586
commit
6a6a1fbbab
7 changed files with 335 additions and 157 deletions
@ -0,0 +1,23 @@ |
|||||||
|
pipeline { |
||||||
|
agent { |
||||||
|
docker { |
||||||
|
image 'python:3.7.3' |
||||||
|
args '--user=root' |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
stages { |
||||||
|
stage('EON Build/Test') { |
||||||
|
steps { |
||||||
|
lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){ |
||||||
|
timeout(time: 30, unit: 'MINUTES') { |
||||||
|
dir(path: 'release') { |
||||||
|
sh 'pip install paramiko' |
||||||
|
sh 'python remote_build.py' |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -1,9 +0,0 @@ |
|||||||
|
|
||||||
code_dir := $(shell pwd)
|
|
||||||
|
|
||||||
# TODO: Add a global build system
|
|
||||||
|
|
||||||
.PHONY: all |
|
||||||
all: |
|
||||||
cd selfdrive && PYTHONPATH=$(code_dir) PREPAREONLY=1 ./manager.py
|
|
||||||
|
|
||||||
@ -0,0 +1,120 @@ |
|||||||
|
#!/data/data/com.termux/files/usr/bin/bash -e |
||||||
|
|
||||||
|
mkdir -p /dev/shm |
||||||
|
chmod 777 /dev/shm |
||||||
|
|
||||||
|
|
||||||
|
add_subtree() { |
||||||
|
echo "[-] adding $2 subtree T=$SECONDS" |
||||||
|
if [ -d "$2" ]; then |
||||||
|
if git subtree pull --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash -m "Merge $2 subtree"; then |
||||||
|
echo "git subtree pull succeeds" |
||||||
|
else |
||||||
|
echo "git subtree pull failed, fixing" |
||||||
|
git merge --abort || true |
||||||
|
git rm -r $2 |
||||||
|
git commit -m "Remove old $2 subtree" |
||||||
|
git subtree add --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash |
||||||
|
fi |
||||||
|
else |
||||||
|
git subtree add --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash |
||||||
|
fi |
||||||
|
} |
||||||
|
|
||||||
|
SOURCE_DIR=/data/openpilot_source |
||||||
|
TARGET_DIR=/data/openpilot |
||||||
|
|
||||||
|
ln -sf $TARGET_DIR /data/pythonpath |
||||||
|
|
||||||
|
export GIT_COMMITTER_NAME="Vehicle Researcher" |
||||||
|
export GIT_COMMITTER_EMAIL="user@comma.ai" |
||||||
|
export GIT_AUTHOR_NAME="Vehicle Researcher" |
||||||
|
export GIT_AUTHOR_EMAIL="user@comma.ai" |
||||||
|
export GIT_SSH_COMMAND="ssh -i /tmp/deploy_key" |
||||||
|
|
||||||
|
echo "[-] Setting up repo T=$SECONDS" |
||||||
|
if [ ! -d "$TARGET_DIR" ]; then |
||||||
|
mkdir -p $TARGET_DIR |
||||||
|
cd $TARGET_DIR |
||||||
|
git init |
||||||
|
git remote add origin git@github.com:commaai/openpilot.git |
||||||
|
fi |
||||||
|
|
||||||
|
|
||||||
|
echo "[-] fetching public T=$SECONDS" |
||||||
|
cd $TARGET_DIR |
||||||
|
git prune || true |
||||||
|
git remote prune origin || true |
||||||
|
|
||||||
|
echo "[-] bringing master-ci and devel in sync T=$SECONDS" |
||||||
|
git fetch origin master-ci |
||||||
|
git fetch origin devel |
||||||
|
|
||||||
|
git checkout --track origin/master-ci || true |
||||||
|
git reset --hard master-ci |
||||||
|
git checkout master-ci |
||||||
|
git reset --hard origin/devel |
||||||
|
git clean -xdf |
||||||
|
|
||||||
|
# subtrees to make updates more reliable. updating them needs a clean tree |
||||||
|
add_subtree "cereal" "cereal" master |
||||||
|
add_subtree "panda" "panda" master |
||||||
|
add_subtree "opendbc" "opendbc" master |
||||||
|
add_subtree "openpilot-pyextra" "pyextra" master |
||||||
|
|
||||||
|
# leave .git alone |
||||||
|
echo "[-] erasing old openpilot T=$SECONDS" |
||||||
|
rm -rf $TARGET_DIR/* $TARGET_DIR/.gitmodules |
||||||
|
|
||||||
|
# dont delete our subtrees |
||||||
|
git checkout -- cereal panda opendbc pyextra |
||||||
|
|
||||||
|
# reset tree and get version |
||||||
|
cd $SOURCE_DIR |
||||||
|
git clean -xdf |
||||||
|
git checkout -- selfdrive/common/version.h |
||||||
|
|
||||||
|
VERSION=$(cat selfdrive/common/version.h | awk -F\" '{print $2}') |
||||||
|
echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h |
||||||
|
|
||||||
|
# do the files copy |
||||||
|
echo "[-] copying files T=$SECONDS" |
||||||
|
cd $SOURCE_DIR |
||||||
|
cp -pR --parents $(cat release/files_common) $TARGET_DIR/ |
||||||
|
|
||||||
|
# in the directory |
||||||
|
cd $TARGET_DIR |
||||||
|
|
||||||
|
echo "[-] committing version $VERSION T=$SECONDS" |
||||||
|
git add -f . |
||||||
|
git status |
||||||
|
git commit -a -m "openpilot v$VERSION release" |
||||||
|
|
||||||
|
# Run build |
||||||
|
SCONS_CACHE=1 scons -j3 |
||||||
|
|
||||||
|
echo "[-] testing openpilot T=$SECONDS" |
||||||
|
PYTHONPATH="$SOURCE_DIR:$SOURCE_DIR/pyextra" nosetests -s selfdrive/test/test_openpilot.py |
||||||
|
|
||||||
|
echo "[-] testing panda build T=$SECONDS" |
||||||
|
pushd panda/board/ |
||||||
|
make bin |
||||||
|
popd |
||||||
|
|
||||||
|
echo "[-] testing pedal build T=$SECONDS" |
||||||
|
pushd panda/board/pedal |
||||||
|
make obj/comma.bin |
||||||
|
popd |
||||||
|
|
||||||
|
if [ ! -z "$PUSH" ]; then |
||||||
|
echo "[-] Pushing to $PUSH T=$SECONDS" |
||||||
|
git push -f origin master:$PUSH |
||||||
|
fi |
||||||
|
|
||||||
|
echo "[-] done pushing T=$SECONDS" |
||||||
|
|
||||||
|
# reset version |
||||||
|
cd $SOURCE_DIR |
||||||
|
git checkout -- selfdrive/common/version.h |
||||||
|
|
||||||
|
echo "[-] done T=$SECONDS" |
||||||
@ -0,0 +1,73 @@ |
|||||||
|
#!/usr/bin/env python2 |
||||||
|
import paramiko |
||||||
|
import os |
||||||
|
import sys |
||||||
|
import re |
||||||
|
import time |
||||||
|
import socket |
||||||
|
|
||||||
|
|
||||||
|
def start_build(name): |
||||||
|
ssh = paramiko.SSHClient() |
||||||
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
||||||
|
|
||||||
|
key_file = open(os.path.join(os.path.dirname(__file__), "id_rsa_public")) |
||||||
|
key = paramiko.RSAKey.from_private_key(key_file) |
||||||
|
|
||||||
|
print("SSH to phone {}".format(name)) |
||||||
|
|
||||||
|
# Try connecting for one minute |
||||||
|
t_start = time.time() |
||||||
|
while True: |
||||||
|
try: |
||||||
|
ssh.connect(hostname=name, port=8022, pkey=key, timeout=10) |
||||||
|
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError): |
||||||
|
print("Connection failed") |
||||||
|
if time.time() - t_start > 60: |
||||||
|
raise |
||||||
|
else: |
||||||
|
break |
||||||
|
time.sleep(1) |
||||||
|
|
||||||
|
conn = ssh.invoke_shell() |
||||||
|
branch = os.environ['GIT_BRANCH'] |
||||||
|
commit = os.environ.get('GIT_COMMIT', branch) |
||||||
|
|
||||||
|
conn.send('uname -a\n') |
||||||
|
|
||||||
|
conn.send('cd /data/openpilot_source\n') |
||||||
|
conn.send("git reset --hard\n") |
||||||
|
conn.send("git fetch origin\n") |
||||||
|
conn.send("git checkout %s\n" % commit) |
||||||
|
conn.send("git clean -xdf\n") |
||||||
|
conn.send("git submodule update --init\n") |
||||||
|
conn.send("git submodule foreach --recursive git reset --hard\n") |
||||||
|
conn.send("git submodule foreach --recursive git clean -xdf\n") |
||||||
|
conn.send("echo \"git took $SECONDS seconds\"\n") |
||||||
|
|
||||||
|
push = "PUSH=one-master" if branch == "master" else "" |
||||||
|
|
||||||
|
conn.send("%s /data/openpilot_source/release/go.sh\n" % push) |
||||||
|
conn.send('echo "RESULT:" $?\n') |
||||||
|
conn.send("exit\n") |
||||||
|
return conn |
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
eon_name = os.environ.get('eon_name', None) |
||||||
|
|
||||||
|
conn = start_build(eon_name) |
||||||
|
|
||||||
|
dat = b"" |
||||||
|
|
||||||
|
while True: |
||||||
|
recvd = conn.recv(4096) |
||||||
|
if len(recvd) == 0: |
||||||
|
break |
||||||
|
|
||||||
|
dat += recvd |
||||||
|
sys.stdout.buffer.write(recvd) |
||||||
|
sys.stdout.flush() |
||||||
|
|
||||||
|
returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE) |
||||||
|
sys.exit(int(returns[0])) |
||||||
Loading…
Reference in new issue