From 5ce993ada9c79e6a3b3833c7296260886cdefbcc Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Mon, 22 Aug 2016 16:12:18 -0400 Subject: [PATCH 01/19] AWS builder, does not run --- shibboleth-appliance-aws.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 shibboleth-appliance-aws.json diff --git a/shibboleth-appliance-aws.json b/shibboleth-appliance-aws.json new file mode 100644 index 0000000..8851f07 --- /dev/null +++ b/shibboleth-appliance-aws.json @@ -0,0 +1,29 @@ +{ + "min_packer_version": "0.8.6", + "description": "Shibboleth appliance ", + "variables": { + "region": "us-west-1", + "timestamper": "{{timestamp}}" + }, + "provisioners": [ + { + "type": "ansible", + "playbook_file": "ansible-playbooks/shibboleth-appliance-vbox.yml", + "sftp_command": "/usr/libexec/sftp-server -e", + "ansible_env_vars": [ "ANSIBLE_HOST_KEY_CHECKING=False" ], + "user": "centos" + } + ], + "builders": [ + { + "type": "amazon-ebs", + "access_key": "AKIAJGTQSCEQTDCIUU6A", + "secret_key": "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", + "region": "us-west-1", + "source_ami": "ami-af4333cf", + "instance_type": "t2.large", + "ssh_username": "centos", + "ami_name": "shibboleth-idp {{timestamp}}" + } + ] +} From e83b39f8c2c2fe30b5a802309cb685d8db39df0a Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Fri, 26 Aug 2016 13:11:19 -0400 Subject: [PATCH 02/19] This can and does drive builds --- shibboleth-appliance.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shibboleth-appliance.json b/shibboleth-appliance.json index b937963..873f19c 100644 --- a/shibboleth-appliance.json +++ b/shibboleth-appliance.json @@ -6,6 +6,7 @@ "disk_size": "8192", "memory": "6122", "cpus": "2", + "guest_os_type": "RedHat_64", "ssh_username": "centos", "ssh_password": "centos", "region": "us-west-1", @@ -39,7 +40,8 @@ "disk_size": "{{user `disk_size`}}", "vboxmanage": [ ["modifyvm", "{{.Name}}", "--memory", "{{user `memory`}}"], - ["modifyvm", "{{.Name}}", "--cpus", "{{user `cpus`}}"] + ["modifyvm", "{{.Name}}", "--cpus", "{{user `cpus`}}"], + ["modifyvm", "{{.Name}}", "--ostype", "{{user `guest_os_type`}}"] ], "iso_url": "http://mirror.vcu.edu/pub/gnu+linux/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1511.iso", "iso_checksum": "f90e4d28fa377669b2db16cbcb451fcb9a89d2460e3645993e30e137ac37d284", From c4515b0c8de7fa1d36492160b4c8021550226200 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 10:31:16 -0400 Subject: [PATCH 03/19] Lowering requirements/suggestions for VM --- shibboleth-appliance.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shibboleth-appliance.json b/shibboleth-appliance.json index 873f19c..13777c1 100644 --- a/shibboleth-appliance.json +++ b/shibboleth-appliance.json @@ -4,8 +4,8 @@ "variables": { "profile": "xfs", "disk_size": "8192", - "memory": "6122", - "cpus": "2", + "memory": "2048", + "cpus": "1", "guest_os_type": "RedHat_64", "ssh_username": "centos", "ssh_password": "centos", From d94425efcf29f34466869159b019bf998f8d0f08 Mon Sep 17 00:00:00 2001 From: Mason Packard Date: Tue, 6 Sep 2016 12:02:17 -0400 Subject: [PATCH 04/19] Initial setup for grouper appliance build. --- Jenkinsfile | 61 +++++++++++++++++++++++++++++++ grouper-appliance.json | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 grouper-appliance.json diff --git a/Jenkinsfile b/Jenkinsfile index 664acbd..9cd8786 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -119,6 +119,67 @@ if (BRANCH_TEST=="shibboleth-production"){ } + catch (err) { + currentBuild.result = "FAILURE" + + slackSend color: 'danger', message: "BUILD ERROR: There was a problem with Comanage Appliance. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>) \n\n ${err}" + + throw err + } + } +} else if (BRANCH_TEST=="grouper-production") { + node('packer') { + currentBuild.result = "SUCCESS" + + try { + stage 'Cleanup & Setup ' + sh 'rm -rf output-grouper-*' + sh 'rm -rf "/Users/levvel/VirtualBox VMs/grouper-appliance-*"' + sh 'rm -rf builds' + sh 'mkdir -p builds' + + withCredentials([[$class: 'FileBinding', credentialsId: 'our-aws-config', variable: 'FILE']]) { + ws { + sh 'mkdir -p ~/.aws' + sh 'cp $FILE ~/.aws/config' + } + } + + withCredentials([[$class: 'FileBinding', credentialsId: 'our-aws-credentials', variable: 'FILE']]) { + ws { + sh 'mkdir -p ~/.aws' + sh 'cp $FILE ~/.aws/credentials' + } + } + + stage 'Checkout' + + checkout scm + + stage 'Acquire ansible-playbooks' + + sh 'mkdir -p ansible-playbooks' + dir('ansible-playbooks'){ + git([ url: "https://github.internet2.edu/docker/ansible-playbooks.git", + branch: "grouper-production", credentialsId: "jenkins-github-access-token" ]) + } + + stage 'Validate' + + sh '/usr/local/bin/packer validate grouper-appliance.json' + + stage 'Build Appliance - Grouper' + + sh '/usr/local/bin/packer build grouper-appliance.json' + + stage 'Sync appliances to S3 bucket' + + sh 'aws s3 sync builds s3://internet2-appliance-us-west-1' + + slackSend color: 'good', message: "BUILD SUCCESS: Comanage Appliance build success. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>)" + + } + catch (err) { currentBuild.result = "FAILURE" diff --git a/grouper-appliance.json b/grouper-appliance.json new file mode 100644 index 0000000..c72ea94 --- /dev/null +++ b/grouper-appliance.json @@ -0,0 +1,82 @@ +{ + "min_packer_version": "0.8.6", + "description": "Grouper appliance ", + "variables": { + "profile": "xfs", + "disk_size": "8192", + "memory": "6122", + "cpus": "2", + "ssh_username": "centos", + "ssh_password": "centos", + "region": "us-west-1", + "timestamper": "{{timestamp}}", + "outputfile_name": "grouper-appliance-centos7" + }, + "provisioners": [ + { + "type": "shell", + "execute_command": "echo 'centos' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/ansible.sh" + }, + { + "type": "ansible-local", + "playbook_dir": "ansible-playbooks", + "playbook_file": "ansible-playbooks/grouper-appliance-vbox.yml" + }, + { + "type": "shell", + "execute_command": "echo 'centos' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/cleanup.sh" + } + ], + "builders": [ + { + "type": "virtualbox-iso", + "name": "grouper-appliance-centos7", + "vm_name": "grouper-appliance-centos7-{{user `profile`}}", + "headless": true, + "guest_os_type": "RedHat_64", + "disk_size": "{{user `disk_size`}}", + "vboxmanage": [ + ["modifyvm", "{{.Name}}", "--memory", "{{user `memory`}}"], + ["modifyvm", "{{.Name}}", "--cpus", "{{user `cpus`}}"] + ], + "iso_url": "http://mirror.vcu.edu/pub/gnu+linux/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1511.iso", + "iso_checksum": "f90e4d28fa377669b2db16cbcb451fcb9a89d2460e3645993e30e137ac37d284", + "iso_checksum_type": "sha256", + "http_directory": "http", + "boot_command": [" text ks=http://{{.HTTPIP}}:{{.HTTPPort}}/ks-{{user `profile`}}.cfg"], + "boot_wait": "10s", + "ssh_username": "{{user `ssh_username`}}", + "ssh_password": "{{user `ssh_password`}}", + "ssh_port": 22, + "ssh_wait_timeout": "10000s", + "guest_additions_mode": "disable", + "shutdown_command": "sudo -S shutdown -P now", + "output_directory": "output-grouper-{{user `profile`}}", + "format": "ova" + } + ], + "post-processors": [ + { + "type": "compress", + "compression_level": "9", + "output": "builds/{{user `outputfile_name`}}-ova-{{user `timestamper`}}.tar" + }, + { + "type": "vagrant", + "compression_level": "9", + "output": "builds/{{user `outputfile_name`}}-vagrant-{{user `timestamper`}}.tar" + }, + { + "type": "amazon-import", + "access_key": "{{user `aws_access_key`}}", + "secret_key": "{{user `aws_secret_key`}}", + "region": "{{user `region`}}", + "s3_bucket_name": "internet2-ami-import-us-west-1", + "tags": { + "Description": "packer amazon-import-grouper-appliance {{user `timestamper`}}" + } + } + ] +} From b99759b9012f7addb21c530908cc994a6e667e19 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 14:18:18 -0400 Subject: [PATCH 05/19] Adding ability to build grouper, and to share results --- bin/perms.sh | 17 +++++++++++++++++ grouper-appliance.json | 10 ++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100755 bin/perms.sh diff --git a/bin/perms.sh b/bin/perms.sh new file mode 100755 index 0000000..71d8e1b --- /dev/null +++ b/bin/perms.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +aws_acct_id="823003027569" +bucket="internet2-tier-appliance-us-west-1" +vagrant_key=$(aws s3api list-objects-v2 --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `vagrant`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) +ova_key=$(aws s3api list-objects-v2 --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `ova`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) + +echo "Found $vagrant_key" +aws s3api put-object-acl --bucket $bucket --key $vagrant_key --acl public-read + +echo "Found $ova_key" +aws s3api put-object-acl --bucket $bucket --key $ova_key --acl public-read + +ami_id=$(aws ec2 describe-images --owners $aws_acct_id --query 'reverse(sort_by(Images[?starts_with(Name, `import-ami`)], &CreationDate))[0].[ImageId]' --output text | tr -d '\n' ) +echo "Found $ami_id" + +aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id\"}]}" \ No newline at end of file diff --git a/grouper-appliance.json b/grouper-appliance.json index c72ea94..bd84099 100644 --- a/grouper-appliance.json +++ b/grouper-appliance.json @@ -4,8 +4,9 @@ "variables": { "profile": "xfs", "disk_size": "8192", - "memory": "6122", - "cpus": "2", + "memory": "2048", + "cpus": "1", + "guest_os_type": "RedHat_64", "ssh_username": "centos", "ssh_password": "centos", "region": "us-west-1", @@ -39,7 +40,8 @@ "disk_size": "{{user `disk_size`}}", "vboxmanage": [ ["modifyvm", "{{.Name}}", "--memory", "{{user `memory`}}"], - ["modifyvm", "{{.Name}}", "--cpus", "{{user `cpus`}}"] + ["modifyvm", "{{.Name}}", "--cpus", "{{user `cpus`}}"], + ["modifyvm", "{{.Name}}", "--ostype", "{{user `guest_os_type`}}"] ], "iso_url": "http://mirror.vcu.edu/pub/gnu+linux/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1511.iso", "iso_checksum": "f90e4d28fa377669b2db16cbcb451fcb9a89d2460e3645993e30e137ac37d284", @@ -73,7 +75,7 @@ "access_key": "{{user `aws_access_key`}}", "secret_key": "{{user `aws_secret_key`}}", "region": "{{user `region`}}", - "s3_bucket_name": "internet2-ami-import-us-west-1", + "s3_bucket_name": "internet2-tier-ami-import-us-west-1", "tags": { "Description": "packer amazon-import-grouper-appliance {{user `timestamper`}}" } From 954a24c2581aa7a66581db7b125deb044c8c9867 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 14:35:30 -0400 Subject: [PATCH 06/19] Refactored Jenkinsfile --- Jenkinsfile | 180 ++++++++++++++++++++-------------------------------- 1 file changed, 69 insertions(+), 111 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9cd8786..b76244d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,131 +1,81 @@ -#!groovy -// pipeline file for building appliances - -echo "My Branch is: ${env.BRANCH_NAME}" - -BRANCH_TEST = "${env.BRANCH_NAME}" - -if (BRANCH_TEST=="shibboleth-production"){ - node('packer') { - currentBuild.result = "SUCCESS" - - try { - stage 'Cleanup & Setup' - sh 'rm -rf output-shibboleth-*' - sh 'rm -rf "/Users/levvel/VirtualBox VMs/shibboleth-appliance-*"' - sh 'rm -rf builds' - sh 'mkdir -p builds' - - withCredentials([[$class: 'FileBinding', credentialsId: 'tier-aws-config', variable: 'FILE']]) { - ws { - sh 'mkdir -p ~/.aws' - sh 'cp $FILE ~/.aws/config' - } - } - - withCredentials([[$class: 'FileBinding', credentialsId: 'tier-aws-credentials', variable: 'FILE']]) { - ws { - sh 'mkdir -p ~/.aws' - sh 'cp $FILE ~/.aws/credentials' - } - } - - stage 'Checkout' - - checkout scm - - stage 'Acquire ansible-playbooks' - - sh 'mkdir -p ansible-playbooks' - dir('ansible-playbooks'){ - git([ url: "https://github.internet2.edu/docker/ansible-playbooks.git", - branch: "shibboleth-production", credentialsId: "jenkins-github-access-token" ]) - } - - stage 'Validate' - - sh '/usr/local/bin/packer validate shibboleth-appliance.json' - - stage 'Build Appliance - Shibboleth' - - sh '/usr/local/bin/packer build shibboleth-appliance.json' - - stage 'Sync appliances to S3 bucket' - - sh 'aws s3 sync builds s3://internet2-tier-appliance-us-west-1' - - slackSend color: 'good', message: "BUILD SUCCESS: Shibboleth Appliance build success. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>)" +node('packer') { + currentBuild.result = "SUCCESS" + + stage 'Appliance build' + echo "My Branch is: ${env.BRANCH_NAME}" + + BRANCH_TEST = "${env.BRANCH_NAME}" + + if (BRANCH_TEST=="shibboleth-production"){ + performBuild("shibboleth") + } else if (BRANCH_TEST=="grouper-production"){ + performBuild("grouper") + } else if (BRANCH_TEST=="comanage-production"){ + performBuild("comanage") + } + +} +def performBuild(String project){ + try { + stage 'Cleanup & Setup' + sh "rm -rf output-${project}-*" + sh "rm -rf \"/Users/levvel/VirtualBox VMs/${project}-appliance-*\"" + sh 'rm -rf builds' + sh 'mkdir -p builds' + + withCredentials([[$class: 'FileBinding', credentialsId: 'tier-aws-config', variable: 'FILE']]) { + ws { + sh 'mkdir -p ~/.aws' + sh 'cp $FILE ~/.aws/config' + } } - catch (err) { - currentBuild.result = "FAILURE" - - slackSend color: 'danger', message: "BUILD ERROR: There was a problem with Shibboleth Appliance. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>) \n\n ${err}" - - throw err + withCredentials([[$class: 'FileBinding', credentialsId: 'tier-aws-credentials', variable: 'FILE']]) { + ws { + sh 'mkdir -p ~/.aws' + sh 'cp $FILE ~/.aws/credentials' + } } - } -} else if (BRANCH_TEST=="comanage-production"){ - node('packer') { - currentBuild.result = "SUCCESS" - try { - stage 'Cleanup & Setup ' - sh 'rm -rf output-comanage-*' - sh 'rm -rf "/Users/levvel/VirtualBox VMs/comange-appliance-*"' - sh 'rm -rf builds' - sh 'mkdir -p builds' - - withCredentials([[$class: 'FileBinding', credentialsId: 'tier-aws-config', variable: 'FILE']]) { - ws { - sh 'mkdir -p ~/.aws' - sh 'cp $FILE ~/.aws/config' - } - } + stage 'Checkout' - withCredentials([[$class: 'FileBinding', credentialsId: 'tier-aws-credentials', variable: 'FILE']]) { - ws { - sh 'mkdir -p ~/.aws' - sh 'cp $FILE ~/.aws/credentials' - } - } + checkout scm - stage 'Checkout' + stage 'Acquire ansible-playbooks' - checkout scm - - stage 'Acquire ansible-playbooks' - - sh 'mkdir -p ansible-playbooks' - dir('ansible-playbooks'){ - git([ url: "https://github.internet2.edu/docker/ansible-playbooks.git", - branch: "comanage-production", credentialsId: "jenkins-github-access-token" ]) - } + sh 'mkdir -p ansible-playbooks' + dir('ansible-playbooks'){ + git([ url: "https://github.internet2.edu/docker/ansible-playbooks.git", + branch: "{{project}}-production", credentialsId: "jenkins-github-access-token" ]) + } - stage 'Validate' + stage 'Validate' - sh '/usr/local/bin/packer validate comange-appliance.json' + sh "/usr/local/bin/packer validate {{project}}-appliance.json" - stage 'Build Appliance - Comanage' + stage 'Build Appliance' - sh '/usr/local/bin/packer build comanage-appliance.json' + sh "/usr/local/bin/packer build {{project}}-appliance.json" - stage 'Sync appliances to S3 bucket' + stage 'Sync appliances to S3 bucket' - sh 'aws s3 sync builds s3://internet2-tier-appliance-us-west-1' + sh "aws s3 sync builds s3://internet2-tier-appliance-us-west-1" + + stage 'Publish results' + + sh "bin/perms.sh > results.txt" - slackSend color: 'good', message: "BUILD SUCCESS: Comanage Appliance build success. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>)" + slackSend color: 'good', message: "BUILD SUCCESS: Shibboleth Appliance build success. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>)" + + slackSend color: 'good', message: readFile('./results.txt') + + sh "rm results.txt" - } - - catch (err) { - currentBuild.result = "FAILURE" - - slackSend color: 'danger', message: "BUILD ERROR: There was a problem with Comanage Appliance. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>) \n\n ${err}" + } - throw err - } + catch (err) { + handleError("BUILD ERROR: There was a problem with ${project} Appliance. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>) \n\n ${err}") } } else if (BRANCH_TEST=="grouper-production") { node('packer') { @@ -189,3 +139,11 @@ if (BRANCH_TEST=="shibboleth-production"){ } } } + +def handleError(String message){ + echo "${message}" + currentBuild.setResult("FAILED") + slackSend color: 'danger', message: "${message}" + //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'chris.bynum@levvel.io', sendToIndividuals: true]) + sh 'exit 1' +} \ No newline at end of file From c45fced9443f2a10a899afb2144ecb34b61771fd Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 14:41:02 -0400 Subject: [PATCH 07/19] Adding even more info --- bin/perms.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/perms.sh b/bin/perms.sh index 71d8e1b..ff580aa 100755 --- a/bin/perms.sh +++ b/bin/perms.sh @@ -8,10 +8,19 @@ ova_key=$(aws s3api list-objects-v2 --bucket $bucket --query 'reverse(sort_by(Co echo "Found $vagrant_key" aws s3api put-object-acl --bucket $bucket --key $vagrant_key --acl public-read +echo "(https://s3-us-west-1.amazonaws.com/$bucket/$vagrant_key)" + +echo $(md5 builds/$vagrant_key) + echo "Found $ova_key" aws s3api put-object-acl --bucket $bucket --key $ova_key --acl public-read +echo "(https://s3-us-west-1.amazonaws.com/$bucket/$ova_key)" + +echo $(md5 builds/$ova_key) + ami_id=$(aws ec2 describe-images --owners $aws_acct_id --query 'reverse(sort_by(Images[?starts_with(Name, `import-ami`)], &CreationDate))[0].[ImageId]' --output text | tr -d '\n' ) -echo "Found $ami_id" +echo "Publishing $ami_id to $aws_acct_id" + +aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id\"}]}" -aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id\"}]}" \ No newline at end of file From af9eb3cc2ec64cdaaa38d16273e833830019d090 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 14:51:39 -0400 Subject: [PATCH 08/19] Improving syntax --- Jenkinsfile | 75 +++++------------------------------------------------ 1 file changed, 7 insertions(+), 68 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b76244d..a878a18 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,10 @@ node('packer') { currentBuild.result = "SUCCESS" + stage 'Checkout' + + checkout scm + stage 'Appliance build' echo "My Branch is: ${env.BRANCH_NAME}" @@ -38,25 +42,21 @@ def performBuild(String project){ } } - stage 'Checkout' - - checkout scm - stage 'Acquire ansible-playbooks' sh 'mkdir -p ansible-playbooks' dir('ansible-playbooks'){ git([ url: "https://github.internet2.edu/docker/ansible-playbooks.git", - branch: "{{project}}-production", credentialsId: "jenkins-github-access-token" ]) + branch: "${project}-production", credentialsId: "jenkins-github-access-token" ]) } stage 'Validate' - sh "/usr/local/bin/packer validate {{project}}-appliance.json" + sh "/usr/local/bin/packer validate ${project}-appliance.json" stage 'Build Appliance' - sh "/usr/local/bin/packer build {{project}}-appliance.json" + sh "/usr/local/bin/packer build ${project}-appliance.json" stage 'Sync appliances to S3 bucket' @@ -77,67 +77,6 @@ def performBuild(String project){ catch (err) { handleError("BUILD ERROR: There was a problem with ${project} Appliance. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>) \n\n ${err}") } -} else if (BRANCH_TEST=="grouper-production") { - node('packer') { - currentBuild.result = "SUCCESS" - - try { - stage 'Cleanup & Setup ' - sh 'rm -rf output-grouper-*' - sh 'rm -rf "/Users/levvel/VirtualBox VMs/grouper-appliance-*"' - sh 'rm -rf builds' - sh 'mkdir -p builds' - - withCredentials([[$class: 'FileBinding', credentialsId: 'our-aws-config', variable: 'FILE']]) { - ws { - sh 'mkdir -p ~/.aws' - sh 'cp $FILE ~/.aws/config' - } - } - - withCredentials([[$class: 'FileBinding', credentialsId: 'our-aws-credentials', variable: 'FILE']]) { - ws { - sh 'mkdir -p ~/.aws' - sh 'cp $FILE ~/.aws/credentials' - } - } - - stage 'Checkout' - - checkout scm - - stage 'Acquire ansible-playbooks' - - sh 'mkdir -p ansible-playbooks' - dir('ansible-playbooks'){ - git([ url: "https://github.internet2.edu/docker/ansible-playbooks.git", - branch: "grouper-production", credentialsId: "jenkins-github-access-token" ]) - } - - stage 'Validate' - - sh '/usr/local/bin/packer validate grouper-appliance.json' - - stage 'Build Appliance - Grouper' - - sh '/usr/local/bin/packer build grouper-appliance.json' - - stage 'Sync appliances to S3 bucket' - - sh 'aws s3 sync builds s3://internet2-appliance-us-west-1' - - slackSend color: 'good', message: "BUILD SUCCESS: Comanage Appliance build success. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>)" - - } - - catch (err) { - currentBuild.result = "FAILURE" - - slackSend color: 'danger', message: "BUILD ERROR: There was a problem with Comanage Appliance. ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL} |Open>) \n\n ${err}" - - throw err - } - } } def handleError(String message){ From 415e8f802537d92f8b5b400f025c22cc12014bf7 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 15:51:21 -0400 Subject: [PATCH 09/19] awscli aging differently cross platform --- bin/perms.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/perms.sh b/bin/perms.sh index ff580aa..0b378a6 100755 --- a/bin/perms.sh +++ b/bin/perms.sh @@ -2,8 +2,8 @@ aws_acct_id="823003027569" bucket="internet2-tier-appliance-us-west-1" -vagrant_key=$(aws s3api list-objects-v2 --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `vagrant`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) -ova_key=$(aws s3api list-objects-v2 --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `ova`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) +vagrant_key=$(aws s3api list-objects --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `vagrant`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) +ova_key=$(aws s3api list-objects --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `ova`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) echo "Found $vagrant_key" aws s3api put-object-acl --bucket $bucket --key $vagrant_key --acl public-read From 407b06c67819759feb40a68a20d9f4d16eac1c18 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 6 Sep 2016 15:57:01 -0400 Subject: [PATCH 10/19] The post-processor is working as far as AWS access goes --- shibboleth-appliance-aws.json | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 shibboleth-appliance-aws.json diff --git a/shibboleth-appliance-aws.json b/shibboleth-appliance-aws.json deleted file mode 100644 index 8851f07..0000000 --- a/shibboleth-appliance-aws.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "min_packer_version": "0.8.6", - "description": "Shibboleth appliance ", - "variables": { - "region": "us-west-1", - "timestamper": "{{timestamp}}" - }, - "provisioners": [ - { - "type": "ansible", - "playbook_file": "ansible-playbooks/shibboleth-appliance-vbox.yml", - "sftp_command": "/usr/libexec/sftp-server -e", - "ansible_env_vars": [ "ANSIBLE_HOST_KEY_CHECKING=False" ], - "user": "centos" - } - ], - "builders": [ - { - "type": "amazon-ebs", - "access_key": "AKIAJGTQSCEQTDCIUU6A", - "secret_key": "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", - "region": "us-west-1", - "source_ami": "ami-af4333cf", - "instance_type": "t2.large", - "ssh_username": "centos", - "ami_name": "shibboleth-idp {{timestamp}}" - } - ] -} From 1503ce5434f98eff93aaf03d9cd4c52b4c11458a Mon Sep 17 00:00:00 2001 From: Mason Packard Date: Mon, 12 Sep 2016 10:48:43 -0400 Subject: [PATCH 11/19] Increase VM size, per Jim Jokl feedback. --- shibboleth-appliance.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shibboleth-appliance.json b/shibboleth-appliance.json index b937963..c7f5164 100644 --- a/shibboleth-appliance.json +++ b/shibboleth-appliance.json @@ -3,7 +3,7 @@ "description": "Shibboleth appliance ", "variables": { "profile": "xfs", - "disk_size": "8192", + "disk_size": "20000", "memory": "6122", "cpus": "2", "ssh_username": "centos", From 91588f08edb46a895f323d8f6470f878fc876d1b Mon Sep 17 00:00:00 2001 From: Mason Packard Date: Wed, 14 Sep 2016 13:03:50 -0400 Subject: [PATCH 12/19] Help ensure that the cleanup stage for Jenkins pipeline is more flexible regarding user directory cleanup. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3eb29a1..fc9c6c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ def performBuild(String project){ try { stage 'Cleanup & Setup' sh "rm -rf output-${project}-*" - sh "rm -rf \"/Users/levvel/VirtualBox VMs/${project}-appliance-*\"" + sh "rm -rf \"~/VirtualBox VMs/${project}-appliance-*\"" sh 'rm -rf builds' sh 'mkdir -p builds' From fe9dce9abfacf1d096f3a4e905c5508253309d42 Mon Sep 17 00:00:00 2001 From: villadalmine Date: Tue, 15 Nov 2016 18:01:09 +0100 Subject: [PATCH 13/19] Add perms.sh change to remove vagrant creation --- bin/perms.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bin/perms.sh b/bin/perms.sh index cf3b5d4..50e2631 100755 --- a/bin/perms.sh +++ b/bin/perms.sh @@ -2,15 +2,8 @@ aws_acct_id="823003027569" bucket="internet2-tier-appliance-us-west-1" -vagrant_key=$(aws s3api list-objects --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `vagrant`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) -ova_key=$(aws s3api list-objects --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `ova`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) - -echo "Found $vagrant_key" -aws s3api put-object-acl --bucket $bucket --key $vagrant_key --acl public-read - -echo "(https://s3-us-west-1.amazonaws.com/$bucket/$vagrant_key)" -echo $(md5sum builds/$vagrant_key) +ova_key=$(aws s3api list-objects --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `ova`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) echo "Found $ova_key" aws s3api put-object-acl --bucket $bucket --key $ova_key --acl public-read @@ -22,8 +15,5 @@ echo $(md5sum builds/$ova_key) ami_id=$(aws ec2 describe-images --owners $aws_acct_id --query 'reverse(sort_by(Images[?starts_with(Name, `import-ami`)], &CreationDate))[0].[ImageId]' --output text | tr -d '\n' ) echo "Publishing $ami_id to $aws_acct_id" -ami_image_description=$(aws ec2 describe-images --image-ids $ami_id --query 'Images[0].{Description:Tags[0].Value}') -echo $ami_image_description - aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id\"}]}" From e6ba9c808d5ae581fcb8d16c62824eab3d9dd248 Mon Sep 17 00:00:00 2001 From: villadalmine Date: Tue, 15 Nov 2016 18:06:14 +0100 Subject: [PATCH 14/19] Removing vagrant processing --- bin/perms.sh | 3 +++ comanage-appliance.json | 5 ----- grouper-appliance.json | 5 ----- shibboleth-appliance.json | 5 ----- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/bin/perms.sh b/bin/perms.sh index 50e2631..4044f28 100755 --- a/bin/perms.sh +++ b/bin/perms.sh @@ -15,5 +15,8 @@ echo $(md5sum builds/$ova_key) ami_id=$(aws ec2 describe-images --owners $aws_acct_id --query 'reverse(sort_by(Images[?starts_with(Name, `import-ami`)], &CreationDate))[0].[ImageId]' --output text | tr -d '\n' ) echo "Publishing $ami_id to $aws_acct_id" +ami_image_description=$(aws ec2 describe-images --image-ids $ami_id --query 'Images[0].{Description:Tags[0].Value}') +echo $ami_image_description + aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id\"}]}" diff --git a/comanage-appliance.json b/comanage-appliance.json index 28a30cc..84f02c3 100644 --- a/comanage-appliance.json +++ b/comanage-appliance.json @@ -63,11 +63,6 @@ "compression_level": "9", "output": "builds/{{user `outputfile_name`}}-ova-{{user `timestamper`}}.tar" }, - { - "type": "vagrant", - "compression_level": "9", - "output": "builds/{{user `outputfile_name`}}-vagrant-{{user `timestamper`}}.box" - }, { "type": "amazon-import", "access_key": "{{user `aws_access_key`}}", diff --git a/grouper-appliance.json b/grouper-appliance.json index e4418e4..77859ca 100644 --- a/grouper-appliance.json +++ b/grouper-appliance.json @@ -65,11 +65,6 @@ "compression_level": "9", "output": "builds/{{user `outputfile_name`}}-ova-{{user `timestamper`}}.tar" }, - { - "type": "vagrant", - "compression_level": "9", - "output": "builds/{{user `outputfile_name`}}-vagrant-{{user `timestamper`}}.tar" - }, { "type": "amazon-import", "access_key": "{{user `aws_access_key`}}", diff --git a/shibboleth-appliance.json b/shibboleth-appliance.json index e3cccf9..a58007e 100644 --- a/shibboleth-appliance.json +++ b/shibboleth-appliance.json @@ -65,11 +65,6 @@ "compression_level": "9", "output": "builds/{{user `outputfile_name`}}-ova-{{user `timestamper`}}.tar" }, - { - "type": "vagrant", - "compression_level": "9", - "output": "builds/{{user `outputfile_name`}}-vagrant-{{user `timestamper`}}.tar" - }, { "type": "amazon-import", "access_key": "{{user `aws_access_key`}}", From 1797d4f62ad5478c0346405609395b6e8f089bca Mon Sep 17 00:00:00 2001 From: villadalmine Date: Tue, 15 Nov 2016 18:12:41 +0100 Subject: [PATCH 15/19] Added the new account --- bin/perms.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/perms.sh b/bin/perms.sh index 4044f28..ebe273d 100755 --- a/bin/perms.sh +++ b/bin/perms.sh @@ -1,6 +1,7 @@ #!/bin/bash aws_acct_id="823003027569" +aws_acct_id_plus="886593122405" bucket="internet2-tier-appliance-us-west-1" ova_key=$(aws s3api list-objects --bucket $bucket --query 'reverse(sort_by(Contents[?contains(Key, `ova`)], &LastModified))[0].[Key]' --output text | tr -d '\n' ) @@ -13,10 +14,11 @@ echo "(https://s3-us-west-1.amazonaws.com/$bucket/$ova_key)" echo $(md5sum builds/$ova_key) ami_id=$(aws ec2 describe-images --owners $aws_acct_id --query 'reverse(sort_by(Images[?starts_with(Name, `import-ami`)], &CreationDate))[0].[ImageId]' --output text | tr -d '\n' ) -echo "Publishing $ami_id to $aws_acct_id" +echo "Publishing $ami_id to $aws_acct_id $aws_acct_id_plus" ami_image_description=$(aws ec2 describe-images --image-ids $ami_id --query 'Images[0].{Description:Tags[0].Value}') echo $ami_image_description aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id\"}]}" +aws ec2 modify-image-attribute --image-id $ami_id --launch-permission "{\"Add\":[{\"UserId\":\"$aws_acct_id_plus\"}]}" From a08b12e50a64f538126131812c287bb986225730 Mon Sep 17 00:00:00 2001 From: villadalmine Date: Tue, 15 Nov 2016 21:03:27 +0100 Subject: [PATCH 16/19] Update perms.sh --- bin/perms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/perms.sh b/bin/perms.sh index ebe273d..c9ab1ab 100755 --- a/bin/perms.sh +++ b/bin/perms.sh @@ -14,7 +14,7 @@ echo "(https://s3-us-west-1.amazonaws.com/$bucket/$ova_key)" echo $(md5sum builds/$ova_key) ami_id=$(aws ec2 describe-images --owners $aws_acct_id --query 'reverse(sort_by(Images[?starts_with(Name, `import-ami`)], &CreationDate))[0].[ImageId]' --output text | tr -d '\n' ) -echo "Publishing $ami_id to $aws_acct_id $aws_acct_id_plus" +echo "Publishing $ami_id to all accounts" ami_image_description=$(aws ec2 describe-images --image-ids $ami_id --query 'Images[0].{Description:Tags[0].Value}') echo $ami_image_description From 3fcc7f89de6d5312d82501d4c9541566d0bbf07d Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Mon, 9 Jan 2017 11:44:55 -0500 Subject: [PATCH 17/19] Updating CentOS minimal ISO location and checksum --- shibboleth-appliance.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shibboleth-appliance.json b/shibboleth-appliance.json index a58007e..11ca4f6 100644 --- a/shibboleth-appliance.json +++ b/shibboleth-appliance.json @@ -43,8 +43,8 @@ ["modifyvm", "{{.Name}}", "--cpus", "{{user `cpus`}}"], ["modifyvm", "{{.Name}}", "--ostype", "{{user `guest_os_type`}}"] ], - "iso_url": "http://mirror.vcu.edu/pub/gnu+linux/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1511.iso", - "iso_checksum": "f90e4d28fa377669b2db16cbcb451fcb9a89d2460e3645993e30e137ac37d284", + "iso_url": "http://mirrors.mit.edu/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1611.iso", + "iso_checksum": "89e9b4587e2f9e942e857034bb9aee0085e07d094a94057f8f16c7841f218db2", "iso_checksum_type": "sha256", "http_directory": "http", "boot_command": [" text ks=http://{{.HTTPIP}}:{{.HTTPPort}}/ks-{{user `profile`}}.cfg"], From 250ffd92ba05c10113040e3993daed9fcabc5842 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Mon, 9 Jan 2017 12:58:57 -0500 Subject: [PATCH 18/19] Updating checksum --- README.md | 26 +++++++++++++------------- shibboleth-appliance.json | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 775062d..b085209 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,6 @@ The OS versions are: For all operating systems we generate images for -- centos7.json will build: - - VirtualBox (user: packer/changeme1234) - -- aws-centos7-base.json will build: - - AMI for AWS (user: centos/centos) - - sibboleth-appliance.json will build: - VirtualBox (user: centos/centos) @@ -72,12 +66,8 @@ This template only is tested against 64 bit systems. With the following sizing r ``` "profile": "xfs", "disk_size": "20000", - "memory": "6122", - "cpus": "2", -``` - -``` - "disk_size": 81920 + "memory": "2048", + "cpus": "1", ``` ## Post Processors @@ -88,9 +78,19 @@ The final VM appliance formats will be uploaded into AWS infrastructure. The two - Compressed Vagrant format - Imported AWS AMI +## Updating CentOS release + +When CentOS distributes a new Minimal ISO build to mirrors, the ISO checksum will fail. + +To update the repo, download the file in question, and perform this command on it: + +`shasum -a 256 CentOS-7-x86_64-Minimal-1611.iso` + +You can then update the JSON config with the output. + # License -TBD +Apache2 diff --git a/shibboleth-appliance.json b/shibboleth-appliance.json index 11ca4f6..22c2d8c 100644 --- a/shibboleth-appliance.json +++ b/shibboleth-appliance.json @@ -44,7 +44,7 @@ ["modifyvm", "{{.Name}}", "--ostype", "{{user `guest_os_type`}}"] ], "iso_url": "http://mirrors.mit.edu/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1611.iso", - "iso_checksum": "89e9b4587e2f9e942e857034bb9aee0085e07d094a94057f8f16c7841f218db2", + "iso_checksum": "27bd866242ee058b7a5754e83d8ee8403e216b93d130d800852a96f41c34d86a", "iso_checksum_type": "sha256", "http_directory": "http", "boot_command": [" text ks=http://{{.HTTPIP}}:{{.HTTPPort}}/ks-{{user `profile`}}.cfg"], From 45df8b595de8f564355a12bcd7e2a6b7ea8dbffb Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Thu, 19 Jan 2017 15:18:16 -0500 Subject: [PATCH 19/19] Delete shibboleth-appliance-aws.json --- shibboleth-appliance-aws.json | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 shibboleth-appliance-aws.json diff --git a/shibboleth-appliance-aws.json b/shibboleth-appliance-aws.json deleted file mode 100644 index 8851f07..0000000 --- a/shibboleth-appliance-aws.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "min_packer_version": "0.8.6", - "description": "Shibboleth appliance ", - "variables": { - "region": "us-west-1", - "timestamper": "{{timestamp}}" - }, - "provisioners": [ - { - "type": "ansible", - "playbook_file": "ansible-playbooks/shibboleth-appliance-vbox.yml", - "sftp_command": "/usr/libexec/sftp-server -e", - "ansible_env_vars": [ "ANSIBLE_HOST_KEY_CHECKING=False" ], - "user": "centos" - } - ], - "builders": [ - { - "type": "amazon-ebs", - "access_key": "AKIAJGTQSCEQTDCIUU6A", - "secret_key": "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", - "region": "us-west-1", - "source_ami": "ami-af4333cf", - "instance_type": "t2.large", - "ssh_username": "centos", - "ami_name": "shibboleth-idp {{timestamp}}" - } - ] -}