From cf9adbc099680af95b0c2f0d2a216df5653b9d77 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Aug 22 2018 10:19:13 +0000 Subject: CI/CD: Split integration tests Typically, integration tests should be rerun - when a new container image of WaiverDB is built - when any service that WaiverDB depends on is updated to a new version - to ensure an image is mature enough for promotion Hence, splitting the functional test stage makes it possible to run integration tests individually. It can also be a step of a larger pipeline. Instructions can be found from `openshift/README.md`. --- diff --git a/openshift/README.md b/openshift/README.md index 2b275c0..4dfa093 100644 --- a/openshift/README.md +++ b/openshift/README.md @@ -116,6 +116,65 @@ test against: ``` You can go to the OpenShift Web console to check the details of the pipeline build. +### Integration Test Pipeline +Typically, integration tests should be rerun +- when a new container image of WaiverDB is built +- when any service that WaiverDB depends on is updated to a new version +- to ensure an image is mature enough for promotion + +Hence, splitting the functional test stage makes it possible to run integration +tests individually. It can also be a step of a larger pipeline. + +#### Installation +To install this OpenShift pipeline: +```bash +oc process --local -f pipelines/templates/waiverdb-integration-test-template.yaml \ + -p NAME=waiverdb-integration-test \ + | oc apply -f - +``` + +Additional installations with default parameters for dev, stage, and prod environments: +```bash +# for dev +oc process --local -f pipelines/templates/waiverdb-integration-test-template.yaml \ + -p NAME=waiverdb-dev-integration-test \ + -p IMAGE="quay.io/factory2/waiverdb:latest" \ + | oc apply -f - +# for stage +oc process --local -f pipelines/templates/waiverdb-integration-test-template.yaml \ + -p NAME=waiverdb-stage-integration-test \ + -p IMAGE="quay.io/factory2/waiverdb:stage" \ + | oc apply -f - +# for prod +oc process --local -f pipelines/templates/waiverdb-integration-test-template.yaml \ + -p NAME=waiverdb-prod-integration-test \ + -p IMAGE="quay.io/factory2/waiverdb:prod" \ + | oc apply -f - +``` + +#### Usage +To trigger a pipeline build for each environment, run: +```bash +# for dev +oc start-build waiverdb-dev-integration-test +# for stage +oc start-build waiverdb-stage-integration-test +# for prod +oc start-build waiverdb-prod-integration-test +``` + +To trigger a custom integration test, start a new pipeline build with the image reference you want to test against and the Git repository and commit ID/branch name +where the functional test suite is used: +```bash +oc start-build waiverdb-integration-test \ + -e IMAGE="quay.io/factory2/waiverdb:test" \ + -e WAIVERDB_GIT_REPO=https://pagure.io/forks//waiverdb.git \ + -e WAIVERDB_GIT_REF=my-branch # master branch is default +``` + +#### NOTE +The stage of reporting test results to ResultsDB has not been implemented. + [OpenShift Pipeline]: https://docs.okd.io/3.9/dev_guide/openshift_pipeline.html [Jenkins Pipeline Build Strategy]: https://docs.openshift.com/container-platform/3.9/dev_guide/dev_tutorials/openshift_pipeline.html [Jenkinsfiles]: https://jenkins.io/doc/book/pipeline/jenkinsfile/ diff --git a/openshift/pipelines/jobs/waiverdb-dev-integration-test.env b/openshift/pipelines/jobs/waiverdb-dev-integration-test.env new file mode 100644 index 0000000..cc605f7 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-dev-integration-test.env @@ -0,0 +1,2 @@ +NAME=waiverdb-dev-integration-test +IMAGE=quay.io/factory2/waiverdb:latest diff --git a/openshift/pipelines/jobs/waiverdb-dev-integration-test.tmpl b/openshift/pipelines/jobs/waiverdb-dev-integration-test.tmpl new file mode 100644 index 0000000..f02d245 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-dev-integration-test.tmpl @@ -0,0 +1 @@ +waiverdb-integration-test-template.yaml diff --git a/openshift/pipelines/jobs/waiverdb-integration-test.env b/openshift/pipelines/jobs/waiverdb-integration-test.env new file mode 100644 index 0000000..723e655 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-integration-test.env @@ -0,0 +1,2 @@ +NAME=waiverdb-integration-test +IMAGE=quay.io/factory2/waiverdb:latest diff --git a/openshift/pipelines/jobs/waiverdb-integration-test.tmpl b/openshift/pipelines/jobs/waiverdb-integration-test.tmpl new file mode 100644 index 0000000..f02d245 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-integration-test.tmpl @@ -0,0 +1 @@ +waiverdb-integration-test-template.yaml diff --git a/openshift/pipelines/jobs/waiverdb-prod-integration-test.env b/openshift/pipelines/jobs/waiverdb-prod-integration-test.env new file mode 100644 index 0000000..35d6dcb --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-prod-integration-test.env @@ -0,0 +1,2 @@ +NAME=waiverdb-prod-integration-test +IMAGE=quay.io/factory2/waiverdb:prod diff --git a/openshift/pipelines/jobs/waiverdb-prod-integration-test.tmpl b/openshift/pipelines/jobs/waiverdb-prod-integration-test.tmpl new file mode 100644 index 0000000..f02d245 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-prod-integration-test.tmpl @@ -0,0 +1 @@ +waiverdb-integration-test-template.yaml diff --git a/openshift/pipelines/jobs/waiverdb-stage-integration-test.env b/openshift/pipelines/jobs/waiverdb-stage-integration-test.env new file mode 100644 index 0000000..3f5d060 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-stage-integration-test.env @@ -0,0 +1,2 @@ +NAME=waiverdb-stage-integration-test +IMAGE=quay.io/factory2/waiverdb:stage diff --git a/openshift/pipelines/jobs/waiverdb-stage-integration-test.tmpl b/openshift/pipelines/jobs/waiverdb-stage-integration-test.tmpl new file mode 100644 index 0000000..f02d245 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-stage-integration-test.tmpl @@ -0,0 +1 @@ +waiverdb-integration-test-template.yaml diff --git a/openshift/pipelines/templates/waiverdb-integration-test-template.yaml b/openshift/pipelines/templates/waiverdb-integration-test-template.yaml new file mode 100644 index 0000000..5e8486a --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-integration-test-template.yaml @@ -0,0 +1,93 @@ +# Template to produce a new OpenShift pipeline for running integration tests +# +--- +apiVersion: v1 +kind: Template +metadata: + name: waiverdb-integration-test +labels: + template: waiverdb-integration-test +parameters: +- name: NAME + displayName: Short unique identifier for the templated instances + description: This field is used to deploy multiple pipelines to one OpenShift project from this template. + required: true + value: waiverdb-integration-test +- name: IMAGE + displayName: The container image to be tested + description: This field must be in repo:tag or repo@sha256 format + value: quay.io/factory2/waiverdb:latest +- name: WAIVERDB_GIT_REPO + displayName: WaiverDB Git repo URL + description: Default WaiverDB Git repo URL in which to run functional tests against + required: true + value: "https://pagure.io/waiverdb.git" +- name: WAIVERDB_GIT_REF + displayName: WaiverDB Git repo ref + description: Default WaiverDB Git repo ref in which to run functional tests against + required: true + value: master +- name: JENKINS_AGENT_IMAGE + displayName: Container image for Jenkins slave pods + required: true + value: docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest +- name: CONTAINER_REGISTRY_CREDENTIALS + displayName: Secret name of container registries used for pulling and pushing images + value: factory2-pipeline-registry-credentials + required: false +- name: JENKINS_AGENT_CLOUD_NAME + displayName: Name of OpenShift cloud in Jenkins master configuration + required: true + value: openshift +objects: +- kind: ServiceAccount + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave" + labels: + app: "${NAME}" +- kind: RoleBinding + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave_edit" + labels: + app: "${NAME}" + subjects: + - kind: ServiceAccount + name: "${NAME}-jenkins-slave" + roleRef: + name: edit +- kind: "BuildConfig" + apiVersion: "v1" + metadata: + name: "${NAME}" + labels: + app: "${NAME}" + spec: + runPolicy: "Serial" # FIXME: Parallel is supported, but we have limited quota in UpShift. + completionDeadlineSeconds: 1800 + source: + git: + uri: "${WAIVERDB_GIT_REPO}" + ref: "${WAIVERDB_GIT_REF}" + strategy: + type: JenkinsPipeline + jenkinsPipelineStrategy: + env: + - name: "WAIVERDB_GIT_REPO" + value: "${WAIVERDB_GIT_REPO}" + - name: "WAIVERDB_GIT_REF" + value: "${WAIVERDB_GIT_REF}" + - name: "IMAGE" + value: "${IMAGE}" + - name: "CONTAINER_REGISTRY_CREDENTIALS" + value: "${CONTAINER_REGISTRY_CREDENTIALS}" + - name: "TEST_ID" + value: "" + - name: JENKINS_AGENT_IMAGE + value: "${JENKINS_AGENT_IMAGE}" + - name: JENKINS_AGENT_CLOUD_NAME + value: "${JENKINS_AGENT_CLOUD_NAME}" + - name: JENKINS_AGENT_SERVICE_ACCOUNT + value: "${NAME}-jenkins-slave" + jenkinsfilePath: openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile diff --git a/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile b/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile new file mode 100644 index 0000000..d14529c --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile @@ -0,0 +1,131 @@ +pipeline { + agent { + kubernetes { + cloud "${params.JENKINS_AGENT_CLOUD_NAME}" + label "jenkins-slave-${UUID.randomUUID().toString()}" + serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}" + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: "${env.JOB_BASE_NAME}" + factory2-pipeline-kind: "waiverdb-integration-test-pipeline" + factory2-pipeline-build-number: "${env.BUILD_NUMBER}" + spec: + containers: + - name: jnlp + image: "${params.JENKINS_AGENT_IMAGE}" + imagePullPolicy: Always + tty: true + env: + - name: REGISTRY_CREDENTIALS + valueFrom: + secretKeyRef: + name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" + key: '.dockerconfigjson' + resources: + requests: + memory: 384Mi + cpu: 200m + limits: + memory: 512Mi + cpu: 300m + """ + } + } + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + } + stages { + stage('Prepare') { + steps { + checkout([$class: 'GitSCM', + branches: [[name: params.WAIVERDB_GIT_REF]], + userRemoteConfigs: [[url: params.WAIVERDB_GIT_REPO]], + ]) + } + } + stage('Run functional tests') { + environment { + // Jenkins BUILD_TAG could be too long (> 63 characters) for OpenShift to consume + TEST_ID = "${params.TEST_ID ?: 'jenkins-' + currentBuild.id}" + ENVIRONMENT_LABEL = "test-${env.TEST_ID}" + } + steps { + echo "Container image ${params.IMAGE} will be tested." + script { + openshift.withCluster() { + def imageTag = (params.IMAGE =~ /(?::(\w[\w.-]{0,127}))?$/)[0][1] + def imageRepo = imageTag ? params.IMAGE.substring(0, params.IMAGE.length() - imageTag.length() - 1) : params.IMAGE + def template = readYaml file: 'openshift/waiverdb-test-template.yaml' + def webPodReplicas = 1 // The current quota in UpShift is agressively limited + def models = openshift.process(template, + '-p', "TEST_ID=${env.TEST_ID}", + '-p', "WAIVERDB_APP_IMAGE_REPO=${imageRepo}", + '-p', "WAIVERDB_APP_VERSION=${imageTag ?: 'latest'}", + '-p', "WAIVERDB_REPLICAS=${webPodReplicas}", + ) + def objects = openshift.apply(models) + echo "Waiting for test pods with label environment=${env.ENVIRONMENT_LABEL} to become Ready" + //def rm = dcSelector.rollout() + def dcs = openshift.selector('dc', ['environment': env.ENVIRONMENT_LABEL]) + def rm = dcs.rollout() + def pods = openshift.selector('pods', ['environment': env.ENVIRONMENT_LABEL]) + timeout(15) { + pods.untilEach(webPodReplicas + 1) { + def pod = it.object() + if (pod.status.phase in ["New", "Pending", "Unknown"]) { + return false + } + if (pod.status.phase == "Running") { + for (cond in pod.status.conditions) { + if (cond.type == 'Ready' && cond.status == 'True') { + return true + } + } + return false + } + error("Test pod ${pod.metadata.name} is not running. Current phase is ${pod.status.phase}.") + } + } + // Run functional tests + def route_hostname = objects.narrow('route').object().spec.host + echo "Running tests against https://${route_hostname}/" + withEnv(["WAIVERDB_TEST_URL=https://${route_hostname}/"]) { + sh 'py.test-3 -v --junitxml=junit-functional-tests.xml functional-tests/' + } + } + } + } + post { + always { + script { + junit 'junit-functional-tests.xml' + openshift.withCluster() { + /* Extract logs for debugging purposes */ + openshift.selector('deploy,pods', ['environment': env.ENVIRONMENT_LABEL]).logs() + } + } + } + cleanup { + script { + openshift.withCluster() { + /* Tear down everything we just created */ + echo "Tearing down test resources..." + openshift.selector('dc,deploy,configmap,secret,svc,route', + ['environment': env.ENVIRONMENT_LABEL]).delete() + } + } + } + } + } + stage('Report to ResultsDB') { + steps { + echo 'This is a placeholder.' + } + } + } +}