Summary Table

Categories	Total Count
PII	0
URL	2
DNS	0
EKL	0
IP	0
PORT	2
VsID	0
CF	0
AI	0
VPD	0
PL	0
Other	0

File Content

properties([
parameters([
booleanParam(name: 'CLEAN', defaultValue: true, description: 'cleans workspace on start up'),
choice(name: 'FAIL_WAIT', choices: "0\n1\n5\n10\n15\n20", description: 'On failure, amount of time to wait before failing and terminating the job. This allows for troubleshooting in the job environment.')
])
])

/************************************
* Application-specific values
************************************/
def appCheckoutDir = "my-va-images"
def environmentFileList = []


def APIGATEWAY_ADDR = 'localhost: PORT '
def STASH_URL = env.STASH_URL ?: "https:// URL "
def buildUserID

node('build-docker-large'){
def repos = [:]
def executionError

/**********
* These will be used throughout the build process as the application workspace and the stack workspace.
* Both will be set to the directory into which their respective source code is pulled/checked out
**********/
def app_workspace = appCheckoutDir

buildUserID = sh (
script: "id -u",
returnStdout: true
).trim()

stage('initialization') {
sh "docker stop \$(docker ps -a -q) || true ; docker rm \$(docker ps -a -q) || true"
notifySlack()
}

stage('clean workspace') {
if (params.CLEAN) {
cleanWs notFailBuild: true
}
}

stage('checkout'){
//Can't obtain this directory name from nextgenConfig.yml because that file isn't yet checked out
dir(appCheckoutDir){
checkout scm
repos = readYaml file: 'nextgenConfig.yml'
}
/**********
* Intent is to use existing artifacts in Jenkins to speed the build process; however,
* acceptance tests exist within the individual repos. Perform shallow checkouts of all
* source repos
**********/
parallel repos.collectEntries { key, value ->
value.collectEntries { subkey, subvalue ->
["$subkey-checkout" : sourceCheckout(subvalue)]
}
}

app_workspace = pwd() + "/" + repos["core"]["application"]["checkoutDir"]
}
stage('registry login'){
withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'innovations_gitlab', passwordVariable: 'DTR_PASS', usernameVariable: 'DTR_USER']]){
sh "docker login https:// URL:PORT -u \"${env.DTR_USER}\" -p \"${env.DTR_PASS}\""
}
sh "aws ecr get-login --region us-east-1 --no-include-email | bash"
}
stage('prep'){
parallel 'permissions':{
def chmods = ""
repos.each { key, value ->
value.each { subkey, subvalue ->
def dirPerm = dirPermissions(subvalue)
chmods += dirPerm ? dirPerm + " && " : ""
}
}
chmods = chmods.endsWith("&& ") ? chmods[0..-4] : chmods
sh chmods
}

}

try {
/* Environment settings that will be used throughout the build/run/test process */
withEnv(["NEXTGEN_APP_HOME=${app_workspace}", "COMPOSE_HTTP_TIMEOUT=480"]) {

/* Workaround - performing actions that use docker-compose before running run-all.sh */
stage('create docker network'){
sh "docker network inspect apigateway-network || docker network create apigateway-network"
}

/* The builder will be used in subsequent steps, and thus should be pulled "up front" */
stage('pull builder'){
dir(app_workspace){
sh "./gradlew pullBuildImages"
}
}

stage('pull and build dependencies'){
def buildSteps = repos["services"].collectEntries { subkey, subvalue ->
["$subkey-build" : applicationBuild(subkey, subvalue, buildUserID, repos["core"]["application"]["checkoutDir"])]
}

/* Don't perform more than maxParallel build actions simultaneously:
* Transform: [k1:v1,k2:v2,k3:v3,k4:v4,k5:v5]
* Into: [[k1:v1,k2:v2,k3:v3],[k4:v4,k5:v5]]
*/
def maxParallel = 3
def buildSubSteps = []
/* Initialize buildSubSteps as a collection of maps */
(buildSteps.size()/maxParallel).setScale(0, BigDecimal.ROUND_CEILING).times { buildSubSteps[it] = [:] }
/* Iterate over the map, putting the values into the buildSubsteps[floor(idx/maxParallel)] */
buildSteps.eachWithIndex { key, val, idx ->
buildSubSteps[((idx / maxParallel).setScale(0, BigDecimal.ROUND_FLOOR)).intValue()][key] = val
}
parallel 'nextgen image pull':{
dir(app_workspace){
sh "./gradlew pullStack"
}
}, 'dev image pull':{
dir(app_workspace){
sh "./gradlew pullDev"
}
}, 'build dependencies':{
buildSubSteps.each { subSteps ->
parallel subSteps
}
}
}

stage('build application'){
script applicationBuild("application", repos["core"]["application"], buildUserID, repos["core"]["application"]["checkoutDir"])
}

stage('build dev images'){
def imageBuildSteps = repos["services"].collectEntries { subkey, subvalue ->
["$subkey-image-build" : applicationImageBuild(subkey, subvalue, buildUserID, repos["core"]["application"]["checkoutDir"])]
}
imageBuildSteps += ["application-image-build" : applicationBuild("application", repos["core"]["application"], buildUserID, repos["core"]["application"]["checkoutDir"])]

def maxParallel = 3
def imageBuildSubSteps = []
/* Initialize buildSubSteps as a collection of maps */
(imageBuildSteps.size()/maxParallel).setScale(0, BigDecimal.ROUND_CEILING).times { imageBuildSubSteps[it] = [:] }
/* Iterate over the map, putting the values into the imageBuildSubSteps[floor(idx/maxParallel)] */
imageBuildSteps.eachWithIndex { key, val, idx ->
imageBuildSubSteps[((idx / maxParallel).setScale(0, BigDecimal.ROUND_FLOOR)).intValue()][key] = val
}
imageBuildSubSteps.each { subSteps ->
parallel subSteps
}
}

stage('start nextgen-stack'){
sh "docker network prune -f"
dir (app_workspace) {
sh "./gradlew runStack"
}
}

stage('start application') {
dir (app_workspace) {
sh "./gradlew runDev"
}
}

stage('verify stack up') {
sh "docker ps -a --format 'table {{.Image}}\t{{.Names}}\t{{.Status}}' ; docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Digest}}'"

def firstRun = true

retry(10) {
if(!firstRun) {
sleep time: 1, unit: 'MINUTES'
dir(repos["core"]["stack"]["checkoutDir"]){
sh "docker-compose -f core/docker-compose.core.yml restart registrator"
}
}
firstRun = false

verifyUrl("http://${APIGATEWAY_ADDR}/users/v1/session", 200)
verifyUrl("http://${APIGATEWAY_ADDR}/eula/v1/agreement", 200)
verifyUrl("http://${APIGATEWAY_ADDR}/wayf/v1/", 200)
verifyUrl("http://${APIGATEWAY_ADDR}/eula-web/", 403)
}
}
stage('test'){
sh "docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Digest}}'"
/* Bring up the selenium_hub and selenium_node */
sh "docker-compose -f $app_workspace/nextgen/docker-compose.build.yml -f $app_workspace/nextgen/docker-compose.test.yml up -d"
script componentTest("application", repos["core"]["application"], buildUserID, "${app_workspace}/nextgen")
}
}
}
catch(err) {
currentBuild.result = 'FAILURE'
echo "EXECUTION ERROR: ${err}"
executionError = err
sleep time: params.FAIL_WAIT.toInteger(), unit: 'MINUTES'

}
finally{
stage('stop') {
sh "docker ps -a"
sh "mkdir -p logs && for container in \$(docker ps -a -q) ; do docker logs \$container &> logs/\$(docker ps -af \"ID=\$container\" --format \"{{.Names}}\").log ; done"
sh "docker stop \$(docker ps -a -q) && docker rm \$(docker ps -a -q)" // Stop all containers
}
stage('archive'){
archiveArtifacts 'logs/*.log,**/screenshots/*.png'
junit allowEmptyResults: true, testResults: '**/*_test.xml'
}
if (executionError) {
error "${executionError}"
}
}
}

/********************************************************************************************
* SUPPORT METHODS
*
* The following methods are used by the pipeline to perform repetitive operations or
* return closures (anonymous code blocks) that represent common build steps populated
* with values from the nextgenConfig yaml
********************************************************************************************/

/**********
* Name: sourceCheckout
* Description: Returns a closure representing a Jenkinsfile checkout step for a git repository;
* allows variable setting of:
* - git branch (`branch`)
* - submodules (`hasSubmodules` - If `true`, disableSubmodules will be set to `false`)
* - repository project/repo (`repo`)
* Parameters: repo: A <Map> representing a repository construct
* Return: A closure representing a Jenkinsfile checkout step (or a descriptive no-op)
**********/
def sourceCheckout(repo){
/* If there is a repo for the entry, perform checkout */
if (repo.repo){
return {
checkout(
changelog: false,
poll: false,
scm: [
$class: 'GitSCM',
branches: [
[ name: "*/${repo.branch}" ]
],
doGenerateSubmoduleConfigurations: false,
extensions: [
[ $class: 'SubmoduleOption', disableSubmodules: !repo.hasSubmodules, parentCredentials: true, recursiveSubmodules: false, reference: '', trackingSubmodules: false ],
[ $class: 'CheckoutOption', timeout: 20 ],
[ $class: 'CloneOption', noTags: false, reference: '', shallow: true ],
[ $class: 'RelativeTargetDirectory', relativeTargetDir: repo.checkoutDir ]
], submoduleCfg: [],
userRemoteConfigs: [
[ credentialsId: 'STASH_USER', url: "${STASH_URL}/scm/${repo.repo}.git" ]
]
]
)
}
}
/* Otherwise, return a no-op */
else{
return {
echo "No repo specified"
}
}
}

/**********
* Name: dirPermissions
* Description: Generate strings representing shell chmod commands
* Parameters: repo: A <Map> representing a repository construct
* Output: A string representing a shell chmod command
**********/
def dirPermissions(repo){
return repo.dirPermission ? "sudo chmod -R ${repo.dirPermission} ${repo.checkoutDir}" : ""
}

/**********
* Name: verifyUrl
* Description: Uses `curl` to verify if a URL response code is as expected
* Parameters: url: A <String> URL to test
* expectedResponse: A <String> or <Integer> HTTP response code
* Output: None (executes a Jenkinsfile build step that may succeed or fail)
**********/
def verifyUrl(url, expectedResponse) {
sh "test \$(curl -sL -w '%{http_code}' -o /dev/null '${url}') = '${expectedResponse}'"
}

/**********
* Name: componentTest
* Description: Returns a closure representing the acceptance test step(s) for a component (application, applet, etc);
* this may be no-op; one-off test (like a curl), or an rspec acceptance test
* Parameters: name: A <String> name of the component
* repo: A <Map> representing a repository construct
* buildUserID (optional): A <String> user ID
* builderDir (optional): The directory in which the build/test compose files are located
* * NOTE: It is assumed that docker-compose.build.yml AND docker-compose.test.yml will exist
* in this or the workspace root directory
* Output: A closure representing Jenkinsfile test step(s)
**********/
def componentTest(name, repo, buildUserID, builderDir){
/* Default no-op build action */
def testClosure = { echo "No tests to run for ${name}" }
/* If there is an artifact specified, build step should copy and explode that artifact */
if(repo.testScript) {
testClosure = {
if(!buildUserID){
buildUserID = sh (
script: "id -u",
returnStdout: true
).trim()
}
dir(builderDir ?: "."){
sh "docker-compose -f docker-compose.build.yml -f docker-compose.test.yml run --name ${name}-test --rm -u $buildUserID builder /bin/bash -c \"cd ${repo.checkoutDir ?: "."}/${repo.buildDir ?: "."} && bundle install && gem list && cd ${repo.testDir ?: "."} && \\\$GEM_HOME/bin/rspec --format RspecJunitFormatter --out ${name}_test.xml ${repo.testScript}\""
}
}
}
return testClosure
}


/**********
* Name: applicationBuild
* Description: Returns a closure representing the build step(s) for an application;
* will be skipped if the yaml entry sets skipCIBuild=true
* Parameters: name: A <String> name of the container
* repo: A <Map> representing a repository construct
* buildUserID (optional): A <String> user ID
* baseDir: The directory in which to initiate the build, if not repo.checkoutDir
* Output: A closure representing Jenkinsfile build step(s)
**********/
def applicationBuild(name, repo, buildUserID, baseDir=null){
def closure = { echo "Skipping build" }
if(repo.skipCIBuild){
closure = { echo "Skipping build per `skipCIBuild` flag" }
}
else if(!repo.buildCommand){
closure = { echo "Skipping build; no buildCommand defined"}
}
else {
closure = {
if(!buildUserID){
buildUserID = sh (
script: "id -u",
returnStdout: true
).trim()
}
buildDir = baseDir ?: repo.checkoutDir ?: "."
dir(buildDir){
sh "COMPOSE_HTTP_TIMEOUT=300 docker-compose -f nextgen/docker-compose.build.yml run --name ${name}-build --rm -u ${buildUserID} builder /bin/bash -c \"cd ${buildDir} && ./gradlew --no-daemon --project-cache-dir=/tmp/project_cache ${name}Build -PskipCheckout=true --stacktrace\""
}
}
}
return closure
}

/**********
* Name: applicationImageBuild
* Description: Returns a closure representing the docker image build step(s) for an application;
* will be skipped if the yaml entry sets skipCIBuild=true
* Parameters: name: A <String> name of the container
* repo: A <Map> representing a repository construct
* buildUserID (optional): A <String> user ID
* baseDir: The directory in which to initiate the build, if not repo.checkoutDir
* Output: A closure representing Jenkinsfile build step(s)
**********/
def applicationImageBuild(name, repo, buildUserID, baseDir=null){
def closure = { echo "Skipping build" }
if(repo.skipCIBuild){
closure = { echo "Skipping image build per `skipCIBuild` flag" }
}
else if(!repo.buildCommand){
closure = { echo "Skipping image build; no buildCommand defined"}
}
else {
closure ={
if(!buildUserID){
buildUserID = sh (
script: "id -u",
returnStdout: true
).trim()
}
buildDir = baseDir ?: repo.checkoutDir ?: "."
dir(buildDir){
sh "./gradlew ${name}ImageBuild --stacktrace\""
}
}
}
return closure
}


def notifySlack(String buildStatus = 'STARTED') {
// Build status of null means success.
buildStatus = buildStatus ?: 'SUCCESS'

def color

if (buildStatus == 'STARTED') {
color = '#D4DADF'
} else if (buildStatus == 'SUCCESS') {
color = '#BDFFC3'
} else if (buildStatus == 'UNSTABLE') {
color = '#FFFE89'
} else {
color = '#FF9FA1'
}

def msg = "${buildStatus}: `${env.JOB_NAME}` #${env.BUILD_NUMBER}:\n${env.BUILD_URL}"

slackSend(color: color, message: msg)
}