update jenkins

main
Yik Teng Hie 5 years ago
parent f1fd68d348
commit 9c256505e1

@ -70,3 +70,14 @@ public class StarterApplication {
[Build Application Guide](https://spring.io/guides/gs/spring-boot/)
## Run Application
* From CLI
```shell
// spring.profiles.active to load .properties file according to profile setting
// below load [resources/application-prod.properties]
$ java -Dspring.profiles.active=prod -jar app.jar
```
* dd

@ -23,7 +23,7 @@
// build docker image. image name cannot use camel case
$ docker build -t <target-image-name> .
// store image in tart file
// store image in tar file
$ docker save <generated-image-name> -o ./store/<generated-image-name>.tar
// upload tar image to server
@ -34,6 +34,9 @@
$ docker-compose stop
$ docker load -i <image-name>.tar
$ docker-compose up -d --build
// skip docker entrypoint
$ docker run -it --entrypoint=/bin/bash $IMAGE -i
```
@ -88,4 +91,76 @@
* s
* docker image versioning
* Semantic Versioning ([Semver](https://medium.com/@mccode/using-semantic-versioning-for-docker-image-tags-dfde8be06699))
* [blog](https://stevelasker.blog/2018/03/01/docker-tagging-best-practices-for-tagging-and-versioning-docker-images/)
* Tag and push separately
```shell
$ docker build -t registry:latest .
$ docker push registry:latest
$ docker tag registry:latest registry:2
$ docker push registry:2
$ docker tag registry:latest registry:2.6
$ docker push registry:2.6
$ docker tag registry:latest registry 2.6.3
$ docker push registry:2.6.3
```
* ```shell
// check image size
$ docker run --entrypoint=/bin/sh $IMAGE:$VERSION -c 'du -s / 2 > /dev/null | cut -f1'
```
* [Best practice](https://cloud.google.com/architecture/best-practices-for-building-containers)
* Copy docker image to another machine
```shell
// build docker image on source machine
$ docker build -t razer-pay-direct-fast-middleware:latest .
// save Docker image as tar file on source machine
$ docker save -o <path for generated tar file> <image name>
// eg.
$ docker save -o c:/myfile.tar centos:16
$ docker save razer-pay-direct-fast-middleware:latest -o ./razer-pay-direct-fast-middleware.tar
// copy to target move via scp / rsync
// scp copy tar to aws ec2
$ scp -i ssh-identity.pem -o StrictHostKeyChecking=no ./razer-pay-direct-fast-middleware.tar $RPS_PRD_SSH_USER@$RPS_PRD_HOST:~/apps/razer-pay-direct-fast-middleware/
// Then, load the tar file into Docker
$ docker load -i <path to image tar file>
// using pipe to run all on the fly
// bzipping the on the fly
// pv - to show progress indicator
$ docker save <image> | bzip2 | pv | ssh user@host 'bunzip2 | docker load'
// run docker image
$ ssh ubuntu@172.88.1.101
$ cd ~/apps/razer-pay-direct-fast-middleware
$ pwd
echo "Stopping Container..."
$ docker-compose stop
$ echo "Loading new Docker image..."
$ docker load -i razer-pay-direct-fast-middleware.tar
$ echo "Restarting Container..."
// --build build image before starting container
$ docker-compose up -d --build
$ echo "Container restarted."
```

@ -0,0 +1,20 @@
# Git command for jenkins build
* command
```shell
// latest commit SHA
$ git rev-parse refs/remotes/origin/dev
// checkout SHA
$ git checkout -f 0760d8575d5e21774f1c571a0f7d41271afb640a
$ git init /var/lib/jenkins/workspace/razer-pay-admin-portal
$ git fetch --tags --progress git@bitbucket.org:razersw/razer-pay-admin-portal.git +refs/heads/*:refs/remotes/admin-portal/*
$ git config remote.admin-portal.url git@bitbucket.org:razersw/razer-pay-admin-portal.git
$ git config --add remote.admin-portal.fetch +refs/heads/*:refs/remotes/admin-portal/*
$ git rev-parse refs/remotes/admin-portal/master
$ git config core.sparsecheckout
$ git checkout -f f5d0db852cfb15c6ee14ca8fc2944862fdd8a521
```

@ -0,0 +1,565 @@
# Sample Jenkins Script
* Jenkins builtin variables
```groovy
// build name . eg: #241
${BUILD_DISPLAY_NAME}
// build id . eg : 241
${BUILD_ID}
${BUILD_NUMBER}
// present working dir
${PWD}
```
* test_infrastructure_aws_eu-west-1
```groovy
pipeline {
agent any
stages {
stage('Checkout') {
steps {
checkout([
$class: 'GitSCM',
branches: [[name: '*/serverless']],
doGenerateSubmoduleConfigurations: false,
extensions: [],
gitTool: 'Default',
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: 'a19fa3f5-7075-4129-82b7-0b6f49c24dd0',
url: 'https://elenapistol@bitbucket.org/razersw/razer-pay-infrastructure.git'
]]
])
}
}
stage('Build terraform image') {
steps {
//withCredentials([usernamePassword(credentialsId: 'terraform_aws_key', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) {
withCredentials([[ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'terraform_aws_key', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh 'echo $AWS_ACCESS_KEY_ID'
sh 'cd serverless/aws && docker build --build-arg AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID --build-arg AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -t terraform_aws .'
}
}
}
stage('Validate terraform') {
steps {
sh 'echo "Validate tf"'
sh 'docker run -t terraform_aws bash -c "cd environments/dev2 && terraform validate"'
sh "CONTAINER_ID=\$(docker ps -a | grep 'terraform' | awk '{ print \$1 }') && CONT=\$(echo \${CONTAINER_ID} | cut -d' ' -f1) && docker rm \${CONT}"
//sh "CONTAINER_ID=\$(docker ps -n=1 -a | grep 'terraform_aws' | awk '{ print \$1 }')"
//sh "CONTAINER_ID=\$(docker ps -a | grep 'terraform' | awk '{ print $1 }') && CONT=\$(echo \${CONTAINER_ID} | cut -d" " -f1) && sudo docker rm \$CONT"
//sh "CONTAINER_ID=\$(docker ps -n=1 -a | grep 'terraform_aws' | awk '{ print \$1 }') && docker rm $CONTAINER_ID"
}
}
stage('Generate plan') {
steps {
sh 'echo "tf plan output"'
sh 'docker run -t terraform_aws bash -c "cd environments/dev2 && terraform plan -out tfplan"'
sh "CONTAINER_ID=\$(docker ps -a | grep 'terraform' | awk '{ print \$1 }') && CONT=\$(echo \${CONTAINER_ID} | cut -d' ' -f1) && docker rm \${CONT}"
//sh 'docker run -t terraform_aws bash -c "cd environments/dev2 && terraform show -no-color tfplan > tfplan.txt"'
}
}
stage('Security testing') {
steps {
sh 'echo "execute security testing"'
sh 'docker run -t terraform_aws bash -c "cd environments/dev2 && terraform plan -out tfplan && terraform show -json tfplan > tfplan.json && terraform-compliance -p tfplan.json -f ../../security_testing/ > security_results.txt ; cat security_results.txt ; terraform-compliance -p tfplan.json -f ../../security_testing/ > security_results.txt"'
sh "CONTAINER_ID=\$(docker ps -a | grep 'terraform' | awk '{ print \$1 }') && CONT=\$(echo \${CONTAINER_ID} | cut -d' ' -f1) && docker rm \${CONT}"
//sh 'docker run -t terraform_aws bash -c "cd environments/dev2 && terraform show -no-color tfplan > tfplan.txt"'
}
}
stage('Approval step') {
steps {
script {
env.APPROVED = input message: 'Approve this infrastructure deploy?',
ok: 'Deploy!',
parameters: [choice(name: 'Approving', choices: "YES\nNO\n", description: 'Proceed with deployment?')]
}
}
}
stage('Deploy infrastructure') {
steps {
script {
sh 'echo "Approved?"'
sh "echo ${APPROVED}"
sh 'echo "Deploying..."'
sh 'docker run -t terraform_aws bash -c "cd environments/dev2 && terraform apply -auto-approve"'
sh "CONTAINER_ID=\$(docker ps -a | grep 'terraform' | awk '{ print \$1 }') && CONT=\$(echo \${CONTAINER_ID} | cut -d' ' -f1) && docker rm \${CONT}"
}
}
}
}
}
```
* wallet-aws-ecs-dev
```groovy
pipeline {
agent any
stages {
stage('Build json config file') {
steps {
sh 'echo "Build config files..."'
checkout([
$class: 'GitSCM',
branches: [[name: '*/master']],
doGenerateSubmoduleConfigurations: false,
extensions: [],
gitTool: 'Default',
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: 'a19fa3f5-7075-4129-82b7-0b6f49c24dd0',
url: 'https://elenapistol@bitbucket.org/razersw/razer-pay-deployment-scripts.git'
]]
])
withCredentials([[ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'terraform_aws_key', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh 'cd wallet && python3 wallet.py --aws-access-key-id=$AWS_ACCESS_KEY_ID --aws-secret-access-key=$AWS_SECRET_ACCESS_KEY --aws-region=eu-west-1 --environment-name=dev2 --json-input-filename=default.json.dev2-sg'
sh 'mv wallet/default.json /tmp/default.json.dev2.wallet-${BUILD_NUMBER}'
}
sh 'mv wallet/default.json.local /tmp/default.json.local.wallet-${BUILD_NUMBER}'
}
}
stage('Checkout') {
steps {
//checkout([
// $class: 'GitSCM',
// branches: [[name: '*/containers-pipeline']],
// doGenerateSubmoduleConfigurations: false,
// extensions: [],
// gitTool: 'Default',
// submoduleCfg: [],
// userRemoteConfigs: [[
// credentialsId: 'a19fa3f5-7075-4129-82b7-0b6f49c24dd0',
// url: 'https://elenapistol@bitbucket.org/razersw/razer-pay-docker-tools.git'
// ]]
//])
//sh 'mkdir /tmp/razer-pay-docker-tools-${BUILD_NUMBER}'
//sh 'mv * /tmp/razer-pay-docker-tools-${BUILD_NUMBER}'
//sh 'ls -l /tmp/razer-pay-docker-tools-${BUILD_NUMBER}'
checkout([
$class: 'GitSCM',
branches: [[name: '*/containers-pipeline']],
doGenerateSubmoduleConfigurations: false,
extensions: [],
gitTool: 'Default',
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: 'a19fa3f5-7075-4129-82b7-0b6f49c24dd0',
url: 'https://elenapistol@bitbucket.org/razersw/razer-pay-wallet-api.git'
]]
])
sh 'ls -l'
}
}
// stage('Execute component tests') {
// steps {
// sh 'echo "Testing..."'
// sh 'ls -l'
// }
// }
stage('Add config files') {
steps {
sh 'mv /tmp/default.json.dev2.wallet-${BUILD_NUMBER} config/default.json'
withCredentials([[ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'terraform_aws_key', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh 'aws ssm get-parameter --name /dev2/wallet/ec_public --with-decryption --query Parameter.Value --output text --region eu-west-1 > common/jwt/ec_public.pem'
sh 'aws ssm get-parameter --name /dev2/wallet/ec_private --with-decryption --query Parameter.Value --output text --region eu-west-1 > common/jwt/ec_private.pem'
sh 'aws ssm get-parameter --name /dev2/wallet/ec_public_visa_middleware --with-decryption --query Parameter.Value --output text --region eu-west-1 > common/jwt/ec_public_visa_middleware.pem'
}
sh 'ls -l common/jwt'
}
}
stage('Build docker image') {
steps {
sh 'echo "Build docker image"'
sh 'docker build -t dev/wallet-api .'
}
}
stage('Execute component tests') {
steps {
sh 'echo "Testing..."'
sh 'mv /tmp/default.json.local.wallet-${BUILD_NUMBER} config/default.json'
sh 'docker build -t local/wallet-api .'
checkout([
$class: 'GitSCM',
branches: [[name: '*/containers-pipeline']],
doGenerateSubmoduleConfigurations: false,
extensions: [],
gitTool: 'Default',
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: 'a19fa3f5-7075-4129-82b7-0b6f49c24dd0',
url: 'https://elenapistol@bitbucket.org/razersw/razer-pay-docker-tools.git'
]]
])
//sh 'ls -l /tmp/razer-pay-docker-tools-${BUILD_NUMBER}'
//sh 'cd /tmp/razer-pay-docker-tools-${BUILD_NUMBER} && docker-compose up -d'
//sh 'npm install'
// sh 'npm run component-test'
// sh 'docker ps -a'
sh 'docker-compose up -d'
script {
try {
sh 'docker exec -t wallet npm run component-test'
} catch (Exception e) {
currentBuild.result = 'UNSTABLE'
}
}
//sh 'cd /tmp/razer-pay-docker-tools-${BUILD_NUMBER} && docker-compose down'
sh 'docker-compose down'
}
}
stage('Push docker image') {
steps {
sh 'echo "push to ecr"'
withCredentials([[ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'terraform_aws_key', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh 'DOCKER_LOGIN=$(AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY aws ecr get-login --no-include-email --region eu-west-1) && \${DOCKER_LOGIN}'
sh 'docker tag dev/wallet-api:latest 877859673258.dkr.ecr.eu-west-1.amazonaws.com/wallet-dev2:${BUILD_NUMBER}'
sh 'docker push 877859673258.dkr.ecr.eu-west-1.amazonaws.com/wallet-dev2:${BUILD_NUMBER}'
}
}
}
stage('Deploy to AWS ECS') {
steps {
sh 'echo "Deploy"'
withCredentials([[ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'terraform_aws_key', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
//sh ''
sh 'ecs deploy dev2-cluster wallet --region eu-west-1 --image wallet 877859673258.dkr.ecr.eu-west-1.amazonaws.com/wallet-dev2:${BUILD_NUMBER} --timeout 600'
}
}
}
}
post {
always {
//sh 'cd /tmp/razer-pay-docker-tools-${BUILD_NUMBER} && docker-compose down'
//sh 'rm -r /tmp/razer-pay-docker-tools-${BUILD_NUMBER}'
sh 'docker-compose down'
}
}
}
```
* 7-eleven-admin-frontend
```groovy
pipeline {
agent
{
docker {
image 'node:10-alpine'
args '-p 20001-20100:3000'
}
}
environment {
HOME = '.'
npm_config_cache = 'npm-cache'
dev_bucket_region = 'ap-northeast-1'
dev_AWS_Jenkins_Credential_ID = '2739896c-8292-4061-98a3-dc178bb2abe2'
dev_bucket_name = '7e-adminportal'
staging_bucket_region = 'ap-northeast-1'
staging_AWS_Staging_Jenkins_Credential_ID = '2739896c-8292-4061-98a3-dc178bb2abe2'
staging_bucket_name = '7e-adminportal'
production_bucket_region = 'ap-northeast-1'
production_AWS_Staging_Jenkins_Credential_ID = '2739896c-8292-4061-98a3-dc178bb2abe2'
production_bucket_name = '7e-adminportal'
}
stages {
stage('Install Packages') {
steps {
sh 'printenv'
sh 'npm install'
}
}
stage('Test and Build') {
parallel {
// stage('Run Tests') {
// steps {
// sh 'npm run test'
// }
// }
stage('Create Build Artifacts') {
steps {
sh 'CI=false npm run build:dev'
}
}
}
}
stage('Deployment') {
parallel {
stage('Dev') {
when {
expression {env.GIT_BRANCH == 'origin/dev'}
}
steps {
withAWS(region:"${dev_bucket_region}",credentials:"${dev_AWS_Jenkins_Credential_ID}") {
s3Delete(bucket: "${dev_bucket_name}", path:'**/*')
s3Upload(bucket: "${dev_bucket_name}", workingDir:'build', includePathPattern:'**/*');
}
// mail(subject: 'Dev Build', body: 'New Deployment to Staging', to: 'jenkins-mailing-list@mail.com')
}
}
stage('Staging') {
when {
expression {env.GIT_BRANCH == 'origin/staging'}
}
steps {
withAWS(region:"${staging_bucket_region}",credentials:"${staging_AWS_Staging_Jenkins_Credential_ID}") {
s3Delete(bucket: "${staging_bucket_name}", path:'**/*')
s3Upload(bucket: "${staging_bucket_name}", workingDir:'build', includePathPattern:'**/*');
}
// mail(subject: 'Staging Build', body: 'New Deployment to Staging', to: 'jenkins-mailing-list@mail.com')
}
}
stage('Production') {
when {
expression {env.GIT_BRANCH == 'origin/master'}
}
steps {
withAWS(region:"${production_bucket_region}",credentials:"${production_AWS_Staging_Jenkins_Credential_ID}") {
s3Delete(bucket: "${production_bucket_name}", path:'**/*')
s3Upload(bucket: "${production_bucket_name}", workingDir:'build', includePathPattern:'**/*');
}
// mail(subject: 'Production Build', body: 'New Deployment to Production', to: 'jenkins-mailing-list@mail.com')
}
}
}
}
}
}
```
* admin-portal
```groovy
pipeline {
agent any
tools {
maven 'Maven 3.5.4'
jdk 'jdk8'
}
stages {
stage ('Initialize') {
steps {
sh '''
echo "PATH = ${PATH}"
echo "WORKSPACE = ${WORKSPACE}"
'''
}
}
stage ('Build - Admin') {
steps {
sh 'mvn clean -Dmaven.test.failure.ignore=true install'
}
}
stage ('Deploy - Admin') {
steps {
sh '''#!/bin/bash
set -a
source /home/jenkins/config/${DEPLOY_ENV}_${DEPLOY_COUNTRY}.properties
set +a
ls -l "${WORKSPACE}/target"
echo "Deployment Environment: $DEPLOY_ENV"
echo "Deployment Country: $DEPLOY_COUNTRY"
echo "Target Host: $ADMIN_IP"
if [[ $USE_KEY == *"false"* ]]
then
sshpass -p $SSH_PASSWORD scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $WORKSPACE/target/razer-pay-admin-portal.war jenkins@$ADMIN_IP:/var/lib/tomcat8/webapps/ROOT.war
else
cd /home/jenkins/keys
CLEANED_KEY=${SSH_KEY_PATH//[$'\t\r\n ']}
scp -i $CLEANED_KEY -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $WORKSPACE/target/razer-pay-admin-portal.war jenkins@$ADMIN_IP:/var/lib/tomcat8/webapps/ROOT.war
fi
'''
}
}
stage ('Build AIM Image') {
steps {
sh '''#!/bin/bash
set -a
source /home/jenkins/config/${DEPLOY_ENV}_${DEPLOY_COUNTRY}.properties
set +a
case $DEPLOY_COUNTRY in
SG)
case $BRANCH_NAME in
uat)
aws ec2 create-image --instance-id "${INSTANCE_ID//[$'\t\r\n']}" --no-reboot --name "BackendAPIDeployment_"${BRANCH_NAME}"_"$(date +"%Y%m%d-%H%M%S") --description "Jenkins Built Image"
echo "AWS image build-"${DEPLOY_COUNTRY}"-"${BRANCH_NAME}
;;
master)
aws ec2 create-image --instance-id "${INSTANCE_ID//[$'\t\r\n']}" --no-reboot --name "BackendAPIDeployment_"${BRANCH_NAME}"_"$(date +"%Y%m%d-%H%M%S") --description "Jenkins Built Image"
echo "AWS image build-"${DEPLOY_COUNTRY}"-"${BRANCH_NAME}
;;
*)
echo "Sorry, invalid input values"
;;
esac
;;
MY)
case $BRANCH_NAME in
uat)
aliyun ecs CreateImage --InstanceId "${INSTANCE_ID//[$'\t\r\n']}" --ImageName "BackendAPIDeployment_${DEPLOY_ENV}_"$(date +"%Y%m%d-%H%M%S") --RegionId "${REGION_ID//[$'\t\r\n']}" --Description "Jenkins Built Image"
echo "Alibaba Cloud image build-"${DEPLOY_COUNTRY}"-"${BRANCH_NAME}
;;
master)
aliyun ecs CreateImage --InstanceId "${INSTANCE_ID//[$'\t\r\n']}" --ImageName "BackendAPIDeployment_${DEPLOY_ENV}_"$(date +"%Y%m%d-%H%M%S") --RegionId "${REGION_ID//[$'\t\r\n']}" --Description "Jenkins Built Image"
echo "Alibaba Cloud image build-"${DEPLOY_COUNTRY}"-"${BRANCH_NAME}
;;
*)
echo "Sorry, invalid input values"
;;
esac
;;
esac
'''
}
}
}
}
```
* razerpay-dashboard (analytics-portal)
```groovy
pipeline {
agent any
tools {
maven 'Maven 3.5.4'
jdk 'jdk8'
}
stages {
stage ('Initialize') {
steps {
sh '''
echo "PATH = ${PATH}"
echo "M2_HOME = ${M2_HOME}"
'''
}
}
stage ('Build') {
steps {
dir ('ui') {
sh 'mvn clean install -DskipTests'
}
dir ('resource') {
sh 'mvn clean install -DskipTests'
}
dir ('authserver') {
sh 'mvn clean install -DskipTests'
}
}
}
stage('Build image') {
steps {
script {
dir ('ui') {
docker.withRegistry('https://registry-intl-vpc.ap-southeast-3.aliyuncs.com', 'dockerrepo') {
def app = docker.build "razerpay-report-malaysia/ui:latest"
app.push()
}
}
dir ('resource') {
docker.withRegistry('https://registry-intl-vpc.ap-southeast-3.aliyuncs.com', 'dockerrepo') {
def app = docker.build "razerpay-report-malaysia/resource:latest"
app.push()
}
}
dir ('authserver') {
docker.withRegistry('https://registry-intl-vpc.ap-southeast-3.aliyuncs.com', 'dockerrepo') {
def app = docker.build "razerpay-report-malaysia/authserver:latest"
app.push()
}
}
}
}
}
stage('Dangling Images') {
steps {
sh 'docker images -q -f dangling=true | xargs --no-run-if-empty docker rmi'
}
}
}
}
```
Loading…
Cancel
Save