diff --git a/elasticsearch/.env b/elasticsearch/.env new file mode 100644 index 0000000..05c6757 --- /dev/null +++ b/elasticsearch/.env @@ -0,0 +1,29 @@ +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD=yth123456789 + +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD=yth123456789 + +# Version of Elastic products +STACK_VERSION=8.2.3 + +# Set the cluster name +CLUSTER_NAME=docker-cluster + +# Set to 'basic' or 'trial' to automatically start the 30-day trial +LICENSE=basic +#LICENSE=trial + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 +#ES_PORT=127.0.0.1:9200 + +# Port to expose Kibana to the host +KIBANA_PORT=5601 +#KIBANA_PORT=80 + +# Increase or decrease based on the available host memory (in bytes) +MEM_LIMIT=1073741824 + +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject \ No newline at end of file diff --git a/elasticsearch/README.md b/elasticsearch/README.md new file mode 100644 index 0000000..6a03d7f --- /dev/null +++ b/elasticsearch/README.md @@ -0,0 +1,37 @@ +# Setup Elasticsearch + + + +- For single node setup +- [Reference](https://www.elastic.co/guide/en/kibana/8.2/docker.html) +```shell +# create docker elastic network +$ docker network create elastic + +# Start elasticsearch +$ docker pull docker.elastic.co/elasticsearch/elasticsearch:8.2.3 +$ docker run --name es01 --net elastic -p 9200:9200 -p 9300:9300 -it docker.elastic.co/elasticsearch/elasticsearch:8.2.3 + +# start kibana +$ docker pull docker.elastic.co/kibana/kibana:8.2.3 +$ docker run --name kib-01 --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:8.2.3 + +# remove +$ docker network rm elastic +$ docker rm es-node01 +$ docker rm kib-01 +``` + +- For multinode elasticsearch setup +- [Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) + +```shell +# start multi-node elasticsearch +$ docker-compose up -d + +# stop elasticsearch +$ docker-compose down + +# Remove all volume and network +$ docker-compose down -v +``` \ No newline at end of file diff --git a/elasticsearch/docker-compose.yml b/elasticsearch/docker-compose.yml new file mode 100644 index 0000000..0d26df7 --- /dev/null +++ b/elasticsearch/docker-compose.yml @@ -0,0 +1,231 @@ +# Please setup password inside .env +version: "2.2" + +services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: es02\n"\ + " dns:\n"\ + " - es02\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: es03\n"\ + " dns:\n"\ + " - es03\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: [ "CMD-SHELL", "[ -f config/certs/es01/es01.crt ]" ] + interval: 1s + timeout: 5s + retries: 120 + + es01: + depends_on: + setup: + condition: service_healthy + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es02,es03 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'" + ] + interval: 10s + timeout: 10s + retries: 120 + + es02: + depends_on: + - es01 + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata02:/usr/share/elasticsearch/data + environment: + - node.name=es02 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es01,es03 + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es02/es02.key + - xpack.security.http.ssl.certificate=certs/es02/es02.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es02/es02.key + - xpack.security.transport.ssl.certificate=certs/es02/es02.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'" + ] + interval: 10s + timeout: 10s + retries: 120 + + es03: + depends_on: + - es02 + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata03:/usr/share/elasticsearch/data + environment: + - node.name=es03 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es01,es02 + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es03/es03.key + - xpack.security.http.ssl.certificate=certs/es03/es03.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es03/es03.key + - xpack.security.transport.ssl.certificate=certs/es03/es03.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'" + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + es01: + condition: service_healthy + es02: + condition: service_healthy + es03: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + volumes: + - certs:/usr/share/kibana/config/certs + - kibanadata:/usr/share/kibana/data + ports: + - ${KIBANA_PORT}:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + mem_limit: ${MEM_LIMIT} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'" + ] + interval: 10s + timeout: 10s + retries: 120 + +volumes: + certs: + driver: local + esdata01: + driver: local + esdata02: + driver: local + esdata03: + driver: local + kibanadata: + driver: local diff --git a/razer-sql5-uat/docker-compose.yml b/razer-sql5-uat/docker-compose.yml index a647b2f..18036ed 100644 --- a/razer-sql5-uat/docker-compose.yml +++ b/razer-sql5-uat/docker-compose.yml @@ -12,7 +12,8 @@ services: MYSQL_ROOT_PASSWORD: root volumes: - ./persistent_data:/var/lib/mysql - # - ./etc/conf.d:/etc/mysql/conf.d + - ./etc/conf.d/my-custom.cnf:/etc/mysql/conf.d/my-custom.cnf + - ./etc/conf.d/mysql.cnf:/etc/mysql/conf.d/mysql.cnf #adminer: # image: adminer diff --git a/razer-sql8/docker-compose.yml b/razer-sql8/docker-compose.yml index 23897b2..df1664d 100644 --- a/razer-sql8/docker-compose.yml +++ b/razer-sql8/docker-compose.yml @@ -13,7 +13,7 @@ services: MYSQL_ROOT_PASSWORD: root volumes: - ./persistent_data:/var/lib/mysql - - ./etc/conf.d:/etc/mysql/conf.d + # - ./etc/conf.d:/etc/mysql/conf.d #adminer: # image: adminer