From d15ea1ff51c1b32ebbfd3d3f2862d2a0e7d8be01 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Thu, 10 Oct 2019 15:53:24 +0200 Subject: [PATCH] Elasticserach 7 - Template mangement (#262) Former-commit-id: 2113b5b3d50bd9ba6225a7cf5aca024accd6c220 --- elasticsearch/Dockerfile | 8 +- .../config/35-entrypoint_load_settings.sh | 39 ++++++--- .../config/35-load_settings_aliases.sh | 86 +++++++++++++++++++ .../config/35-load_settings_policies.sh | 86 +++++++++++++++++++ .../config/35-load_settings_templates.sh | 81 +++++++++++++++++ .../config/20-entrypoint_kibana_settings.sh | 17 +++- logstash/config/01-wazuh.conf | 2 +- 7 files changed, 305 insertions(+), 14 deletions(-) create mode 100644 elasticsearch/config/35-load_settings_aliases.sh create mode 100644 elasticsearch/config/35-load_settings_policies.sh create mode 100644 elasticsearch/config/35-load_settings_templates.sh diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index cabac130..4ab5f86e 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -74,6 +74,9 @@ COPY --chown=elasticsearch:elasticsearch ./config/35-entrypoint.sh /entrypoint-s COPY --chown=elasticsearch:elasticsearch ./config/35-entrypoint_load_settings.sh ./ COPY config/35-load_settings_configure_s3.sh ./config/35-load_settings_configure_s3.sh COPY --chown=elasticsearch:elasticsearch ./config/35-load_settings_users_management.sh ./ +COPY --chown=elasticsearch:elasticsearch ./config/35-load_settings_policies.sh ./ +COPY --chown=elasticsearch:elasticsearch ./config/35-load_settings_templates.sh ./ +COPY --chown=elasticsearch:elasticsearch ./config/35-load_settings_aliases.sh ./ RUN chmod +x /entrypoint-scripts/10-config_cluster.sh && \ chmod +x /entrypoint-scripts/15-get_CA_key.sh && \ chmod +x /entrypoint-scripts/20-security_instances.sh && \ @@ -84,7 +87,10 @@ RUN chmod +x /entrypoint-scripts/10-config_cluster.sh && \ chmod +x /entrypoint-scripts/35-entrypoint.sh && \ chmod +x ./35-entrypoint_load_settings.sh && \ chmod 755 ./config/35-load_settings_configure_s3.sh && \ - chmod +x ./35-load_settings_users_management.sh + chmod +x ./35-load_settings_users_management.sh && \ + chmod +x ./35-load_settings_policies.sh && \ + chmod +x ./35-load_settings_templates.sh && \ + chmod +x ./35-load_settings_aliases.sh ENTRYPOINT ["/entrypoint.sh"] CMD ["elasticsearch"] diff --git a/elasticsearch/config/35-entrypoint_load_settings.sh b/elasticsearch/config/35-entrypoint_load_settings.sh index 3c24c068..dc6b7756 100644 --- a/elasticsearch/config/35-entrypoint_load_settings.sh +++ b/elasticsearch/config/35-entrypoint_load_settings.sh @@ -106,17 +106,13 @@ if [ $ENABLE_CONFIGURE_S3 ]; then fi + ############################################################################## -# Elastic Stack users creation. -# Only security main node can manage users. +# Load custom policies. ############################################################################## -echo "LOAD SETTINGS - Run users_management.sh." -MY_HOSTNAME=`hostname` -echo "LOAD SETTINGS - Hostname: $MY_HOSTNAME" -if [[ $SECURITY_MAIN_NODE == $MY_HOSTNAME ]]; then - bash /usr/share/elasticsearch/35-load_settings_users_management.sh & -fi +echo "LOAD SETTINGS - Loading custom Elasticsearch policies." +bash /usr/share/elasticsearch/35-load_settings_policies.sh ############################################################################## @@ -132,9 +128,30 @@ sed -i 's:"index.number_of_replicas"\: "0":"index.number_of_replicas"\: "'$WAZUH # Load default templates ############################################################################## -echo "LOAD SETTINGS - Loading wazuh-alerts template." -cat /usr/share/elasticsearch/config/wazuh-template.json | curl -XPUT "$el_url/_template/wazuh" ${auth} -H 'Content-Type: application/json' -d @- -sleep 5 +echo "LOAD SETTINGS - Loading wazuh-alerts template" +bash /usr/share/elasticsearch/35-load_settings_templates.sh + + +############################################################################## +# Load custom aliases. +############################################################################## + +echo "LOAD SETTINGS - Loading custom Elasticsearch aliases." +bash /usr/share/elasticsearch/35-load_settings_aliases.sh + + +############################################################################## +# Elastic Stack users creation. +# Only security main node can manage users. +############################################################################## + +echo "LOAD SETTINGS - Run users_management.sh." +MY_HOSTNAME=`hostname` +echo "LOAD SETTINGS - Hostname: $MY_HOSTNAME" +if [[ $SECURITY_MAIN_NODE == $MY_HOSTNAME ]]; then + bash /usr/share/elasticsearch/35-load_settings_users_management.sh & +fi + ############################################################################## # Prepare Wazuh API credentials diff --git a/elasticsearch/config/35-load_settings_aliases.sh b/elasticsearch/config/35-load_settings_aliases.sh new file mode 100644 index 00000000..cf67d7b1 --- /dev/null +++ b/elasticsearch/config/35-load_settings_aliases.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Wazuh Docker Copyright (C) 2019 Wazuh Inc. (License GPLv2) + +set -e + + +############################################################################## +# Set Elasticsearch API url +############################################################################## + +if [[ "x${ELASTICSEARCH_PROTOCOL}" = "x" || "x${ELASTICSEARCH_IP}" = "x" || "x${ELASTICSEARCH_PORT}" = "x" ]]; then + el_url="http://elasticsearch:9200" +else + el_url="${ELASTICSEARCH_PROTOCOL}://${ELASTICSEARCH_IP}:${ELASTICSEARCH_PORT}" +fi + +echo "ALIASES - Elasticsearch url: $el_url" + + +############################################################################## +# If Elasticsearch security is enabled get the elastic user password. +############################################################################## + +ELASTIC_PASS="" + +if [[ "x${SECURITY_CREDENTIALS_FILE}" == "x" ]]; then + ELASTIC_PASS=${SECURITY_ELASTIC_PASSWORD} +else + input=${SECURITY_CREDENTIALS_FILE} + while IFS= read -r line + do + if [[ $line == *"ELASTIC_PASSWORD"* ]]; then + arrIN=(${line//:/ }) + ELASTIC_PASS=${arrIN[1]} + fi + done < "$input" + +fi + + +############################################################################## +# If Elasticsearch security is enabled get the users credentials. +############################################################################## + +# The user must get the credentials of the users. +# TO DO. + +############################################################################## +# Set authentication for curl if Elasticsearch security is enabled. +############################################################################## + +if [ ${SECURITY_ENABLED} != "no" ]; then + auth="-uelastic:${ELASTIC_PASS} -k" + echo "ALIASES - authentication for curl established." +elif [[ ${ENABLED_XPACK} != "true" || "x${ELASTICSEARCH_USERNAME}" = "x" || "x${ELASTICSEARCH_PASSWORD}" = "x" ]]; then + auth="" + echo "ALIASES - authentication for curl not established." +else + auth="--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" + echo "ALIASES - authentication for curl established." +fi + + +############################################################################## +# Wait until Elasticsearch is active. +############################################################################## + +until curl ${auth} -XGET $el_url; do + >&2 echo "ALIASES - Elastic is unavailable - sleeping" + sleep 5 +done + +>&2 echo "ALIASES - Elastic is up - executing command" + + +############################################################################## +# Add custom aliases. +############################################################################## + +# The user must add the credentials of the users. +# TO DO. +# Example +# echo "ALIASES - Add custom_user password and role:" +# curl ${auth} -k -XPOST -H 'Content-Type: application/json' 'https://localhost:9200/_ilm/policy/my_policy?pretty' -d' +# { "policy": { "phases": { "hot": { "actions": { "rollover": {"max_size": "50GB", "max_age": "5m"}}}}}}' + diff --git a/elasticsearch/config/35-load_settings_policies.sh b/elasticsearch/config/35-load_settings_policies.sh new file mode 100644 index 00000000..3e89e4c5 --- /dev/null +++ b/elasticsearch/config/35-load_settings_policies.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Wazuh Docker Copyright (C) 2019 Wazuh Inc. (License GPLv2) + +set -e + + +############################################################################## +# Set Elasticsearch API url +############################################################################## + +if [[ "x${ELASTICSEARCH_PROTOCOL}" = "x" || "x${ELASTICSEARCH_IP}" = "x" || "x${ELASTICSEARCH_PORT}" = "x" ]]; then + el_url="http://elasticsearch:9200" +else + el_url="${ELASTICSEARCH_PROTOCOL}://${ELASTICSEARCH_IP}:${ELASTICSEARCH_PORT}" +fi + +echo "POLICIES - Elasticsearch url: $el_url" + + +############################################################################## +# If Elasticsearch security is enabled get the elastic user password. +############################################################################## + +ELASTIC_PASS="" + +if [[ "x${SECURITY_CREDENTIALS_FILE}" == "x" ]]; then + ELASTIC_PASS=${SECURITY_ELASTIC_PASSWORD} +else + input=${SECURITY_CREDENTIALS_FILE} + while IFS= read -r line + do + if [[ $line == *"ELASTIC_PASSWORD"* ]]; then + arrIN=(${line//:/ }) + ELASTIC_PASS=${arrIN[1]} + fi + done < "$input" + +fi + + +############################################################################## +# If Elasticsearch security is enabled get the users credentials. +############################################################################## + +# The user must get the credentials of the users. +# TO DO. + +############################################################################## +# Set authentication for curl if Elasticsearch security is enabled. +############################################################################## + +if [ ${SECURITY_ENABLED} != "no" ]; then + auth="-uelastic:${ELASTIC_PASS} -k" + echo "POLICIES - authentication for curl established." +elif [[ ${ENABLED_XPACK} != "true" || "x${ELASTICSEARCH_USERNAME}" = "x" || "x${ELASTICSEARCH_PASSWORD}" = "x" ]]; then + auth="" + echo "POLICIES - authentication for curl not established." +else + auth="--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" + echo "POLICIES - authentication for curl established." +fi + + +############################################################################## +# Wait until Elasticsearch is active. +############################################################################## + +until curl ${auth} -XGET $el_url; do + >&2 echo "POLICIES - Elastic is unavailable - sleeping" + sleep 5 +done + +>&2 echo "POLICIES - Elastic is up - executing command" + + +############################################################################## +# Add custom policies. +############################################################################## + +# The user must add the credentials of the users. +# TO DO. +# Example +# echo "POLICIES - Add custom_user password and role:" +# curl ${auth} -k -XPOST -H 'Content-Type: application/json' 'https://localhost:9200/_ilm/policy/my_policy?pretty' -d' +# { "policy": { "phases": { "hot": { "actions": { "rollover": {"max_size": "50GB", "max_age": "5m"}}}}}}' + diff --git a/elasticsearch/config/35-load_settings_templates.sh b/elasticsearch/config/35-load_settings_templates.sh new file mode 100644 index 00000000..dcee8ef8 --- /dev/null +++ b/elasticsearch/config/35-load_settings_templates.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Wazuh Docker Copyright (C) 2019 Wazuh Inc. (License GPLv2) + +set -e + + +############################################################################## +# Set Elasticsearch API url +############################################################################## + +if [[ "x${ELASTICSEARCH_PROTOCOL}" = "x" || "x${ELASTICSEARCH_IP}" = "x" || "x${ELASTICSEARCH_PORT}" = "x" ]]; then + el_url="http://elasticsearch:9200" +else + el_url="${ELASTICSEARCH_PROTOCOL}://${ELASTICSEARCH_IP}:${ELASTICSEARCH_PORT}" +fi + +echo "TEMPLATES - Elasticsearch url: $el_url" + + +############################################################################## +# If Elasticsearch security is enabled get the elastic user password. +############################################################################## + +ELASTIC_PASS="" + +if [[ "x${SECURITY_CREDENTIALS_FILE}" == "x" ]]; then + ELASTIC_PASS=${SECURITY_ELASTIC_PASSWORD} +else + input=${SECURITY_CREDENTIALS_FILE} + while IFS= read -r line + do + if [[ $line == *"ELASTIC_PASSWORD"* ]]; then + arrIN=(${line//:/ }) + ELASTIC_PASS=${arrIN[1]} + fi + done < "$input" + +fi + + +############################################################################## +# If Elasticsearch security is enabled get the users credentials. +############################################################################## + +# The user must get the credentials of the users. +# TO DO. + +############################################################################## +# Set authentication for curl if Elasticsearch security is enabled. +############################################################################## + +if [ ${SECURITY_ENABLED} != "no" ]; then + auth="-uelastic:${ELASTIC_PASS} -k" + echo "TEMPLATES - authentication for curl established." +elif [[ ${ENABLED_XPACK} != "true" || "x${ELASTICSEARCH_USERNAME}" = "x" || "x${ELASTICSEARCH_PASSWORD}" = "x" ]]; then + auth="" + echo "TEMPLATES - authentication for curl not established." +else + auth="--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" + echo "TEMPLATES - authentication for curl established." +fi + + +############################################################################## +# Wait until Elasticsearch is active. +############################################################################## + +until curl ${auth} -XGET $el_url; do + >&2 echo "TEMPLATES - Elastic is unavailable - sleeping" + sleep 5 +done + +>&2 echo "TEMPLATES - Elastic is up - executing command" + + +############################################################################## +# Add wazuh-alerts templates. +############################################################################## + +echo "TEMPLATES - Loading default wazuh-alerts template." +cat /usr/share/elasticsearch/config/wazuh-template.json | curl -XPUT "$el_url/_template/wazuh" ${auth} -H 'Content-Type: application/json' -d @- diff --git a/kibana/config/20-entrypoint_kibana_settings.sh b/kibana/config/20-entrypoint_kibana_settings.sh index e3c062be..8abe2947 100644 --- a/kibana/config/20-entrypoint_kibana_settings.sh +++ b/kibana/config/20-entrypoint_kibana_settings.sh @@ -109,7 +109,9 @@ echo "SETTINGS - Prepare index selection." default_index="/tmp/default_index.json" -cat > ${default_index} << EOF +if [[ $PATTERN == "" ]]; then + + cat > ${default_index} << EOF { "changes": { "defaultIndex": "wazuh-alerts-${WAZUH_MAJOR}.x-*" @@ -117,6 +119,19 @@ cat > ${default_index} << EOF } EOF +else + + cat > ${default_index} << EOF +{ + "changes": { + "defaultIndex": "$PATTERN" + } +} +EOF + +fi + + sleep 5 diff --git a/logstash/config/01-wazuh.conf b/logstash/config/01-wazuh.conf index 57a6dba5..f51d38ab 100644 --- a/logstash/config/01-wazuh.conf +++ b/logstash/config/01-wazuh.conf @@ -34,7 +34,7 @@ filter { } date { match => ["timestamp", "ISO8601"] - target => "timestamp" + target => "@timestamp" } mutate { remove_field => [ "beat", "input_type", "tags", "count", "@version", "log", "offset", "type", "@src_ip", "host"]