From 44afc4ae3cf2cf1e11193aaaf6dd799216c5c909 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 7 Jul 2020 14:17:09 -0700 Subject: [PATCH] Build release2 and dashcam in CI (#1834) * start building release2 in CI * build script * bring over changes from test branch * fix build release test * remove references to test branches * ucnomment push * fix duplicate remove * two commits for dashcam * need the key old-commit-hash: c7156616568ce2fa780db3eafc5f912ab2ead14e --- .github/workflows/test.yaml | 2 +- Jenkinsfile | 30 ++++++++++++++++++++- release/build_release2.sh | 54 ++++++++++++++++++++++--------------- release/files_common | 5 +++- selfdrive/test/id_rsa | 28 +++++++++++++++++++ selfdrive/test/phone_ci.py | 4 ++- tools/ssh/key/id_rsa | 29 +------------------- 7 files changed, 99 insertions(+), 53 deletions(-) create mode 100644 selfdrive/test/id_rsa mode change 100644 => 120000 tools/ssh/key/id_rsa diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fc851e62ac..55fc9e6e47 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,7 +39,7 @@ jobs: # need these so docker copy won't fail cp Pipfile Pipfile.lock .pylintrc .coveragerc-app .pre-commit-config.yaml $TEST_DIR cd $TEST_DIR - mkdir laika laika_repo tools release + mkdir laika laika_repo tools - name: Build Docker image run: cd $TEST_DIR && eval "$BUILD" - name: Build openpilot and run quick check diff --git a/Jenkinsfile b/Jenkinsfile index 6afac3a236..2029379229 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,14 +7,42 @@ pipeline { } environment { COMMA_JWT = credentials('athena-test-jwt') - CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}" } stages { + + stage('Release Build') { + when { + branch 'devel-staging' + } + steps { + lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){ + timeout(time: 60, unit: 'MINUTES') { + dir(path: 'selfdrive/test') { + sh 'pip install paramiko' + sh 'python phone_ci.py "cd release && PUSH=1 ./build_release2.sh"' + } + } + } + } + } + stage('On-device Tests') { + when { + not { + anyOf { + branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging' + } + } + } + parallel { stage('Build') { + environment { + CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}" + } + steps { lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){ timeout(time: 30, unit: 'MINUTES') { diff --git a/release/build_release2.sh b/release/build_release2.sh index 08f727b1b2..9378bb723b 100755 --- a/release/build_release2.sh +++ b/release/build_release2.sh @@ -8,24 +8,32 @@ export GIT_AUTHOR_EMAIL="user@comma.ai" export GIT_SSH_COMMAND="ssh -i /data/gitkey" -# Create folders -rm -rf /data/openpilot -mkdir -p /data/openpilot -cd /data/openpilot - -# Create git repo -git init -git remote add origin git@github.com:commaai/openpilot.git -git fetch origin devel +# set CLEAN to build outside of CI +if [ ! -z "$CLEAN" ]; then + # Create folders + rm -rf /data/openpilot + mkdir -p /data/openpilot + cd /data/openpilot + + # Create git repo + git init + git remote add origin git@github.com:commaai/openpilot.git + git fetch origin devel-staging +else + cd /data/openpilot + git clean -xdf + git branch -D release2-staging || true +fi + git fetch origin release2-staging git fetch origin dashcam-staging -# Checkout devel -#git checkout origin/devel -#git clean -xdf - # Create release2 with no history -git checkout --orphan release2-staging origin/devel +if [ ! -z "$CLEAN" ]; then + git checkout --orphan release2-staging origin/devel-staging +else + git checkout --orphan release2-staging +fi VERSION=$(cat selfdrive/common/version.h | awk -F\" '{print $2}') git commit -m "openpilot v$VERSION" @@ -55,7 +63,7 @@ find . -name '*.o' -delete find . -name '*.os' -delete find . -name '*.pyc' -delete find . -name '__pycache__' -delete -rm .sconsign.dblite +rm -rf .sconsign.dblite Jenkinsfile release/ # Restore phonelibs git checkout phonelibs/ @@ -70,11 +78,15 @@ git commit --amend -m "openpilot v$VERSION" # Print committed files that are normally gitignored #git status --ignored -# Push to release2-staging -git push -f origin release2-staging +if [ ! -z "$PUSH" ]; then + git remote set-url origin git@github.com:commaai/openpilot.git + + # Push to release2-staging + git push -f origin release2-staging -# Create dashcam release -git rm selfdrive/car/*/carcontroller.py + # Create dashcam release + git rm selfdrive/car/*/carcontroller.py -git commit -m "create dashcam release from release2" -git push -f origin release2-staging:dashcam-staging + git commit -m "create dashcam release from release2" + git push -f origin release2-staging:dashcam-staging +fi diff --git a/release/files_common b/release/files_common index 745026f6c5..5d7a9f28aa 100644 --- a/release/files_common +++ b/release/files_common @@ -58,6 +58,8 @@ common/api/__init__.py models/supercombo.dlc models/dmonitoring_model_q.dlc +release/build_release2.sh + selfdrive/version.py selfdrive/__init__.py @@ -314,10 +316,11 @@ selfdrive/thermald/thermald.py selfdrive/thermald/power_monitoring.py selfdrive/test/__init__.py +selfdrive/test/id_rsa selfdrive/test/helpers.py +selfdrive/test/phone_ci.py selfdrive/test/test_openpilot.py selfdrive/test/test_fingerprints.py -selfdrive/test/test_car_models.py selfdrive/test/test_cpu_usage.py selfdrive/ui/SConscript diff --git a/selfdrive/test/id_rsa b/selfdrive/test/id_rsa new file mode 100644 index 0000000000..3f269afe22 --- /dev/null +++ b/selfdrive/test/id_rsa @@ -0,0 +1,28 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N +Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB +t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt +Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ +acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6 +1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ +rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q +x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC +gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix +Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1 +LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL +s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb +Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj +6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO +uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh +0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q +FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo +ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi +4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA +nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it +VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl +s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l +jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7 +b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I +RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY +6co17RFaAZHwGfCFFjO76Q== +-----END RSA PRIVATE KEY----- diff --git a/selfdrive/test/phone_ci.py b/selfdrive/test/phone_ci.py index 3d2e6f0672..b002547196 100755 --- a/selfdrive/test/phone_ci.py +++ b/selfdrive/test/phone_ci.py @@ -19,7 +19,7 @@ def run_on_phone(test_cmd): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa")) + key_file = open(os.path.join(os.path.dirname(__file__), "id_rsa")) key = paramiko.RSAKey.from_private_key(key_file) print("SSH to phone at {}".format(eon_ip)) @@ -52,6 +52,8 @@ def run_on_phone(test_cmd): conn.send(f"cd {SOURCE_DIR}\n") conn.send("git reset --hard\n") conn.send("git fetch origin\n") + conn.send("find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \\;\n") + conn.send(f"git reset --hard {commit}\n") conn.send(f"git checkout {commit}\n") conn.send("git clean -xdf\n") conn.send("git submodule update --init\n") diff --git a/tools/ssh/key/id_rsa b/tools/ssh/key/id_rsa deleted file mode 100644 index 3f269afe22..0000000000 --- a/tools/ssh/key/id_rsa +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N -Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB -t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt -Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ -acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6 -1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ -rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q -x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC -gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix -Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1 -LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL -s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb -Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj -6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO -uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh -0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q -FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo -ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi -4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA -nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it -VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl -s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l -jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7 -b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I -RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY -6co17RFaAZHwGfCFFjO76Q== ------END RSA PRIVATE KEY----- diff --git a/tools/ssh/key/id_rsa b/tools/ssh/key/id_rsa new file mode 120000 index 0000000000..d4e6dd344a --- /dev/null +++ b/tools/ssh/key/id_rsa @@ -0,0 +1 @@ +../../../selfdrive/test/id_rsa \ No newline at end of file