From 4f3fbcb3c358afd5b544990d55e3c54dedd6760f Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 31 Aug 2016 10:28:08 -0400 Subject: [PATCH 1/4] Getting started with new repo --- .gitignore | 9 ++++++ BINSCRIPTS.md | 41 ++++++++++++++++++++++++ Jenkinsfile | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ common.bash | 2 ++ 4 files changed, 140 insertions(+) create mode 100644 .gitignore create mode 100644 BINSCRIPTS.md create mode 100644 Jenkinsfile create mode 100644 common.bash diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..811f8fa --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +bin/build.sh +bin/destroy.sh +bin/install.sh +bin/rebuild.sh +bin/rerun.sh +bin/run.sh +bin/start.sh +bin/stop.sh +bin/test.sh diff --git a/BINSCRIPTS.md b/BINSCRIPTS.md new file mode 100644 index 0000000..7818108 --- /dev/null +++ b/BINSCRIPTS.md @@ -0,0 +1,41 @@ +# util + +This repository is intended to be downloaded into a container repository during development for convenience purposes. Since the major mechanisms of developing to the Docker container construction lifecycle are identical across container images, this repository allows for consistency and additional ease of use across all container images. + +## Install + +If you are reading this file in BINSCRIPTS.md, your container project is likely to be util-enabled. + +To use these scripts yourself, issue this command: + +``` +curl "https://github.internet2.edu/raw/docker/util/master/bin/install.sh?token=AAAAE4VRBLPB8VExPHSR5nCe791IAYqaks5Xzug5wA%3D%3D" | bash +``` + +### common.bash + +The installation process will create a common.bash file. This file should be the central, canonical authority for management of environment variables. While a subprocess may override them, the files in common.bash should be treated as authoritative defaults. Processes (e.g. `docker build`, `bats`, inside `Jenkinsfile`) can read this file and process the results therein. + +You should edit this file to change the image name, and add any other helpful environment variables. + +### Jenkinsfile + +This will also install a Jenkinsfile to your repository, if it doesn't have one. This will ensure that your Jenkins pipeline can leverage these scripts in the way intended. Ensuring the commands that you issue on your laptop match the commands issued by the build pipeline is critical to ensure predictable, reliable results. + +## Use + + +### Building + +#### build.sh +`bin/build.sh ` +#### destroy.sh +#### rebuild.sh + +### Running +### rerun.sh +### run.sh + + +### Testing +#### test.sh diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..c4b3931 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,88 @@ +node { + + stage 'Checkout' + + checkout scm + + stage 'Acquire util' + + sh 'mkdir -p tmp && mkdir -p bin' + dir('tmp'){ + git([ url: "https://github.internet2.edu/docker/util.git", + credentialsId: "jenkins-github-access-token" ]) + sh 'mv ./bin/* ../bin/.' + } + sh 'rm -rf tmp' + + stage 'Setting build context' + + def maintainer = maintainer() + def imagename = imagename() + def tag + + // Tag images created on master branch with 'latest' + if(env.BRANCH_NAME == "master"){ + tag = "latest" + }else{ + tag = env.BRANCH_NAME + } + + if(!imagename){ + echo "You must define an imagename in common.bash" + currentBuild.result = 'FAILURE' + } + if(maintainer){ + echo "Building ${imagename}:${tag} for ${maintainer}" + } + + stage 'Build' + try{ + sh 'bin/build.sh &> debug' + } catch(error) { + def error_details = readFile('./debug'); + def message = "BUILD ERROR: There was a problem building ${imagename}:${tag}. \n\n ${error_details}" + sh "rm -f ./debug" + handleError(message) + } + + stage 'Tests' + + try{ + sh 'bin/test.sh &> debug' + } catch(error) { + def error_details = readFile('./debug'); + def message = "BUILD ERROR: There was a problem building ${imagename}:${tag}. \n\n ${error_details}" + sh "rm -f ./debug" + handleError(message) + } + + stage 'Push' + + docker.withRegistry('https://registry.hub.docker.com/', "dockerhub-$maintainer") { + def baseImg = docker.build("$maintainer/$imagename") + baseImg.push("$tag") + } + + stage 'Notify' + + slackSend color: 'good', message: "$maintainer/$imagename:$tag pushed to DockerHub" + +} + +def maintainer() { + def matcher = readFile('common.bash') =~ 'maintainer="(.+)"' + matcher ? matcher[0][1] : 'tier' +} + +def imagename() { + def matcher = readFile('common.bash') =~ 'imagename="(.+)"' + matcher ? matcher[0][1] : null +} + +def handleError(String message){ + echo "${message}" + currentBuild.setResult("FAILED") + slackSend color: 'danger', message: "${message}" + //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'chris.bynum@levvel.io', sendToIndividuals: true]) + sh 'exit 1' +} \ No newline at end of file diff --git a/common.bash b/common.bash new file mode 100644 index 0000000..96e0143 --- /dev/null +++ b/common.bash @@ -0,0 +1,2 @@ +maintainer="tier" +imagename="imagename-replaceme-in-common-bash" \ No newline at end of file From 79351048cd9e375326494b00e7e6e33d4f257ddb Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 31 Aug 2016 10:48:56 -0400 Subject: [PATCH 2/4] Image builds, testable --- .gitignore | 1 - Dockerfile | 42 +++++++++++++++ bin/start.sh | 8 +++ common.bash | 3 +- container_files/container_start.sh | 84 ++++++++++++++++++++++++++++++ container_files/fix-permissions.sh | 16 ++++++ tests/image.bats | 11 ++++ 7 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100755 bin/start.sh create mode 100755 container_files/container_start.sh create mode 100755 container_files/fix-permissions.sh create mode 100644 tests/image.bats diff --git a/.gitignore b/.gitignore index 811f8fa..0e47596 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,5 @@ bin/install.sh bin/rebuild.sh bin/rerun.sh bin/run.sh -bin/start.sh bin/stop.sh bin/test.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..74a9ea5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +FROM tier/centos7base + +# Define args and set a default value +ARG registry=docker.io +ARG maintainer=tier +ARG imagename=mariadb +ARG version=5.5 + +MAINTAINER $maintainer +LABEL Vendor="Internet2" +LABEL ImageType="Database" +LABEL ImageName=$imagename +LABEL ImageOS=centos7 +LABEL Version=$version + +LABEL Build docker build --rm --tag $registry/$maintainer/$imagename . + +# Install base deps +RUN yum -y install --setopt=tsflags=nodocs mariadb-server bind-utils pwgen psmisc hostname vim + +# Add starters and installers +ADD ./container_files /root + +# Add Volumes and Set permissions +RUN mkdir /opt/shared && chmod 777 /opt/shared && chmod 777 /root/*.sh + +# Place VOLUME statement below all changes to /var/lib/mysql +VOLUME /var/lib/mysql + +# Environment variables +ENV CREATE_NEW_DATABASE "1" +ENV MYSQL_ROOT_PASSWORD "123321" +ENV MYSQL_DATABASE "registry" +ENV MYSQL_USER "registry_user" +ENV MYSQL_PASSWORD "WJzesbe3poNZ91qIbmR7" +ENV MYSQL_DATADIR "/var/lib/mysqlmounted" +ENV TERM "testterm" + +# Port +EXPOSE 3306 + +CMD ["/root/container_start.sh"] diff --git a/bin/start.sh b/bin/start.sh new file mode 100755 index 0000000..9b071e9 --- /dev/null +++ b/bin/start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +source common.bash . + +echo "Running Docker image($registry/$maintainer/$imagename)" +docker run --name=$imagename -d -p 3306:3306 $maintainer/$imagename + +exit 0 \ No newline at end of file diff --git a/common.bash b/common.bash index 96e0143..89fe631 100644 --- a/common.bash +++ b/common.bash @@ -1,2 +1,3 @@ maintainer="tier" -imagename="imagename-replaceme-in-common-bash" \ No newline at end of file +imagename="mariadb" +version="5.5" \ No newline at end of file diff --git a/container_files/container_start.sh b/container_files/container_start.sh new file mode 100755 index 0000000..87e9515 --- /dev/null +++ b/container_files/container_start.sh @@ -0,0 +1,84 @@ +#!/bin/bash -x + +log="/tmp/start.log" + +echo "Starting Container: " > $log +date >> $log +echo "" >> $log + +if [ -e "/tmp/firsttimerunning" ]; then + + set -e + + echo "Checking args" >> $log + if [ "${1:0:1}" = '-' ]; then + set -- mysqld_safe "$@" >> $log + fi + + echo "Setting DataDir: $MYSQL_DATADIR" >> $log + + if [ "$CREATE_NEW_DATABASE" == "1" ]; then + + echo "Installing MariaDB" >> $log + + if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" ]; then + echo >&2 'error: database is uninitialized and MYSQL_ROOT_PASSWORD not set' >> $log + echo >&2 ' Did you forget to add -e MYSQL_ROOT_PASSWORD=... ?' >> $log + exit 1 + fi + + echo 'Running mysql_install_db ...' >> $log + mysql_install_db --datadir="$MYSQL_DATADIR" >> $log + echo 'Finished mysql_install_db' >> $log + + # These statements _must_ be on individual lines, and _must_ end with + # semicolons (no line breaks or comments are permitted). + # TODO proper SQL escaping on ALL the things D: + + tempSqlFile='/tmp/mysql-first-time.sql' + echo "DELETE FROM mysql.user ;" > $tempSqlFile + echo "CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;" >> $tempSqlFile + echo "GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ;" >> $tempSqlFile + echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}';" >> $tempSqlFile + echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' WITH GRANT OPTION;" >> $tempSqlFile + echo "DROP DATABASE IF EXISTS test ;" >> $tempSqlFile + + + if [ "$MYSQL_DATABASE" != "" ]; then + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "GRANT ALL ON $MYSQL_DATABASE.* TO '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "GRANT ALL ON $MYSQL_DATABASE.* TO '$MYSQL_USER'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "GRANT ALL ON $MYSQL_DATABASE.* TO '$MYSQL_USER'@'comanage.compose_i2network' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "GRANT ALL ON $MYSQL_DATABASE.* TO '$MYSQL_USER'@'comanage.comanage_i2network' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "GRANT ALL ON $MYSQL_DATABASE.* TO '$MYSQL_USER'@'comanage.config_i2network' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + fi + + echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" + + echo "character-set-server = utf8mb4" >> /etc/my.cnf + echo "collation-server = utf8mb4_unicode_ci" >> /etc/my.cnf + echo "" >> /etc/my.cnf + + echo "Fixing Permissions" >> $log + chown -R mysql:mysql $MYSQL_DATADIR + /root/fix-permissions.sh $MYSQL_DATADIR >> $log + /root/fix-permissions.sh /var/log/mariadb/ >> $log + /root/fix-permissions.sh /var/run/ >> $log + echo "Done Fixing Permissions" >> $log + + /usr/bin/mysqld_safe --init-file="$tempSqlFile" --datadir="$MYSQL_DATADIR" + else + echo "Not Creating a MariaDB - Using Existing from DataDir: $MYSQL_DATADIR" >> $log + /usr/bin/mysqld_safe --datadir="$MYSQL_DATADIR" + fi + + rm -f /tmp/firsttimerunning +else + echo "Using Existing MariaDB from DataDir: $MYSQL_DATADIR" >> $log + /usr/bin/mysqld_safe --datadir="$MYSQL_DATADIR" +fi + +tail -f $log + +exit 0 diff --git a/container_files/fix-permissions.sh b/container_files/fix-permissions.sh new file mode 100755 index 0000000..9b9615a --- /dev/null +++ b/container_files/fix-permissions.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +log="/tmp/permissions.log" + +echo "Starting Container: " > $log +date >> $log +echo "" >> $log + +# Taken from https://raw.githubusercontent.com/openshift/sti-base/master/bin/fix-permissions +# Fix permissions on the given directory to allow group read/write of +# regular files and execute of directories. +chgrp -R 0 $1 >> $log +chmod -R g+rw $1 >> $log +find $1 -type d -exec chmod g+x {} + >> $log + +exit 0 diff --git a/tests/image.bats b/tests/image.bats new file mode 100644 index 0000000..db6dfb8 --- /dev/null +++ b/tests/image.bats @@ -0,0 +1,11 @@ +#!/usr/bin/env bats + +load ../common + +@test "Data directory created" { + docker run -i $maintainer/$imagename find /var/lib/mysql +} + +@test "MariaDB service available" { + docker run -i $maintainer/$imagename find /usr/lib/systemd/system/mariadb.service +} \ No newline at end of file From 055a3882f0f7436d611c36da49812bde1295cb46 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 31 Aug 2016 11:08:15 -0400 Subject: [PATCH 3/4] Updating tests --- Jenkinsfile | 15 ++++++++++++++- tests/running.bats | 13 +++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/running.bats diff --git a/Jenkinsfile b/Jenkinsfile index c4b3931..810b50d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,6 @@ node { credentialsId: "jenkins-github-access-token" ]) sh 'mv ./bin/* ../bin/.' } - sh 'rm -rf tmp' stage 'Setting build context' @@ -44,6 +43,10 @@ node { sh "rm -f ./debug" handleError(message) } + + stage 'Start' + + sh 'bin/start.sh' stage 'Tests' @@ -54,8 +57,14 @@ node { def message = "BUILD ERROR: There was a problem building ${imagename}:${tag}. \n\n ${error_details}" sh "rm -f ./debug" handleError(message) + } finally { + sh 'bin/stop.sh' } + stage 'Stop' + + sh 'bin/stop.sh' + stage 'Push' docker.withRegistry('https://registry.hub.docker.com/', "dockerhub-$maintainer") { @@ -66,6 +75,10 @@ node { stage 'Notify' slackSend color: 'good', message: "$maintainer/$imagename:$tag pushed to DockerHub" + + stage 'Cleanup' + + sh 'rm -rf tmp' } diff --git a/tests/running.bats b/tests/running.bats new file mode 100644 index 0000000..0263d88 --- /dev/null +++ b/tests/running.bats @@ -0,0 +1,13 @@ +#!/usr/bin/env bats + +load ../common + +# Assumes process is running + +@test "MySQL user running process" { + docker exec -i $imagename ps -u mysql +} + +@test "Creates root account and database" { + docker exec -i $imagename mysql -u root --password="123321" -e 'show tables' registry +} \ No newline at end of file From 61602d70c2ba4ce0e83e3e75e261ee6e61b5cc49 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 31 Aug 2016 11:13:50 -0400 Subject: [PATCH 4/4] Adding user account test --- tests/running.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/running.bats b/tests/running.bats index 0263d88..78075be 100644 --- a/tests/running.bats +++ b/tests/running.bats @@ -10,4 +10,8 @@ load ../common @test "Creates root account and database" { docker exec -i $imagename mysql -u root --password="123321" -e 'show tables' registry +} + +@test "Creates user account with password" { + docker exec -i $imagename mysql -u registry_user --password="WJzesbe3poNZ91qIbmR7" -e 'show tables' registry } \ No newline at end of file