Terraform Elasticsearch in Amazon Web Services

The elasticsearch cluster needs to know the seed ip addresses, which makes it a little bit tricker to do in terraform. We actually need to do have two separate ec2 declarations. The second batch will grab the first batches ip addresses. The userdata automatically will install and set up everything. This also allows us to dynamically change the size of the cluster if we need to scale up or down depending on the job. Any new ec2 instances that come up will automatically join the cluster. I specify the version of elasticsearch in order to make sure new virtual machines don’t automatically grab an incompatible version. 

Files

ec2.tf

variable "ami_id_arm64" {
  description = "Amazon Linux 2 ARM64 AMI"
  default = "ami-07acebf185d439fa0"
}

#arm64 sizes
variable "instance_type_arm64" {
  type = "map"
  default = {
    L3 = "a1.large"
    L2 = "a1.4xlarge"
    L1 = "a1.4xlarge"
  }
}

variable "instance_type_arm64_metal" {
  type = "map"
  default = {
    L3 = "a1.large"
    L2 = "a1.metal"
    L1 = "a1.metal"
  }
}

variable "instance_count_reserved_arm64" {
  type = "map"
  default = {
    L3 = "2"
    L2 = "2"
    L1 = "20"
  }
}

variable "instance_count_reserved_2_arm64" {
  type = "map"
  default = {
    L3 = "0"
    L2 = "0"
    L1 = "160"
  }
}

variable "instance_count_reserved_failover_arm64" {
  type = "map"
  default = {
    L3 = "0"
    L2 = "0"
    L1 = "0"
  }
}

variable "instance_count_spot_arm64" {
  type = "map"
  default = {
    L3 = "0"
    L2 = "0"
    L1 = "0"
  }
}

data "template_cloudinit_config" "ondemand"  {
  gzip          = true
  base64_encode = true

  part {
    content      = "${file("userdata_arm64.yml")}"
  }

  part {
    merge_type    = "list(append)+dict(recurse_array)+str()"
    content_type = "text/cloud-config"
    content = <<EOF
#cloud-config
---
write_files:
- path: /etc/elasticsearch/elasticsearch.yml
  permissions: 0660
  content: |
    cluster.name: "BPI"
    network.host: 0.0.0.0
    xpack.ml.enabled: false
    xpack.monitoring.enabled: false
    bootstrap.memory_lock: true
    path.data: /var/lib/elasticsearch
    path.logs: /var/log/elasticsearch
    discovery.zen.minimum_master_nodes: 1
    http.cors.enabled: true
    http.cors.allow-origin: /http?://localhost(:[0-9]+)?/
    #cluster.routing.allocation.total_shards_per_node: 1
EOF
  }
}


data "template_cloudinit_config" "spot"  {
  gzip          = true
  base64_encode = true

  part {
    content      = "${file("userdata_arm64.yml")}"
  }

  part {
    merge_type    = "list(append)+dict(recurse_array)+str()"
    content_type = "text/cloud-config"
    content = <<EOF
#cloud-config
---
write_files:
  - path: /etc/elasticsearch/elasticsearch.yml
    permissions: 0660
    content: |
      cluster.name: "BPI"
      network.host: 0.0.0.0
      xpack.ml.enabled: false
      xpack.monitoring.enabled: false
      bootstrap.memory_lock: true
      path.data: /var/lib/elasticsearch
      path.logs: /var/log/elasticsearch
      discovery.zen.minimum_master_nodes: 1
      #cluster.routing.allocation.total_shards_per_node: 1
      http.cors.enabled: true
      http.cors.allow-origin: /http?://localhost(:[0-9]+)?/
      discovery.zen.ping.unicast.hosts: ["${element(aws_instance.bgt-bpi-arm64.*.private_ip, 0)}:9300", "${element(aws_instance.bgt-bpi-arm64.*.private_ip, 1)}:9300", "${element(aws_instance.bgt-bpi-arm64.*.private_ip, 2)}:9300", "${element(aws_instance.bgt-bpi-arm64.*.private_ip, 3)}:9300"]

EOF
  }
}

resource "aws_instance" "bgt-bpi-arm64" {
  ami = "${var.ami_id_arm64}"
  instance_type = "${var.instance_type_arm64["${data.terraform_remote_state.global.aws_environment}"]}"
  iam_instance_profile    = "${lower(var.stack_id)}"

  subnet_id               = "${data.terraform_remote_state.global.default_vpc_server_subnet_ids_list[1]}"
  user_data               = "${data.template_cloudinit_config.ondemand.rendered}"
  vpc_security_group_ids  = ["${data.terraform_remote_state.global.base_security_group_ids_default_vpc_list[0]}"]

  count                   = "${var.instance_count_reserved_arm64["${data.terraform_remote_state.global.aws_environment}"]}"
  placement_group         = "${aws_placement_group.bgt_bpi_arm64_pg.name}"
  ebs_optimized           = "${var.ebs_optimized["${data.terraform_remote_state.global.aws_environment}"]}"

  lifecycle {
    ignore_changes = ["user_data", "ami", "ebs_optimized"]
  }

  root_block_device {
    volume_type           = "gp2"
    volume_size           = "165"
    delete_on_termination = true
  }

  tags {
    Name                = "bgt-bpi-arm64-p1-z1"
    Team                = "Bigtree Services"
    Tool                = "Terraform"
    StackId             = "${var.stack_id}"
    Deploy              = "arm64"
  }
}

resource "aws_elb" "elb" {
  name               = "bgt-bpi-elb"
  subnets = ["${data.terraform_remote_state.global.default_vpc_server_subnet_ids_list}"]
  security_groups = ["${data.terraform_remote_state.global.base_security_group_ids_default_vpc_list[0]}"]
  internal = true

  listener {
    instance_port = 9200
    instance_protocol = "http"
    lb_port = 80
    lb_protocol = "http"
  }

  health_check {
    healthy_threshold = 2
    interval = 10
    target = "http:9200/"
    timeout = 5
    unhealthy_threshold = 3
  }

  tags {
    Name                = "bgt-bpi-elb"
    Team                = "Bigtree Services"
    Tool                = "Terraform"
    StackId             = "${var.stack_id}"
    Deploy              = "arm64"
  }
}

resource "aws_elb_attachment" "elb" {
  count = "${var.instance_count_reserved_arm64["${data.terraform_remote_state.global.aws_environment}"]}"
  elb = "${aws_elb.elb.id}"
  instance = "${element(aws_instance.bgt-bpi-arm64.*.id, count.index)}"
}




resource "aws_instance" "bgt-bpi-arm64-part2" {
  ami = "${var.ami_id_arm64}"
  instance_type = "${var.instance_type_arm64["${data.terraform_remote_state.global.aws_environment}"]}"
  iam_instance_profile    = "${lower(var.stack_id)}"

  subnet_id               = "${data.terraform_remote_state.global.default_vpc_server_subnet_ids_list[1]}"
  user_data               = "${data.template_cloudinit_config.spot.rendered}"
  vpc_security_group_ids  = ["${data.terraform_remote_state.global.base_security_group_ids_default_vpc_list[0]}"]

  count                   = "${var.instance_count_reserved_2_arm64["${data.terraform_remote_state.global.aws_environment}"]}"
  ebs_optimized           = "${var.ebs_optimized["${data.terraform_remote_state.global.aws_environment}"]}"

  lifecycle {
    ignore_changes = ["user_data", "ami", "ebs_optimized", "placement_group"]
  }

  root_block_device {
    volume_type           = "gp2"
    volume_size           = "165"
    delete_on_termination = true
  }

  tags {
    Name                = "bgt-bpi-arm64-a2-z1"
    Team                = "Bigtree Services"
    Tool                = "Terraform"
    StackId             = "${var.stack_id}"
    Deploy              = "arm64"
  }
}

resource "aws_elb_attachment" "elb-part2" {
  count = "${var.instance_count_reserved_2_arm64["${data.terraform_remote_state.global.aws_environment}"]}"
  elb = "${aws_elb.elb.id}"
  instance = "${element(aws_instance.bgt-bpi-arm64-part2.*.id, count.index)}"
}

In this example L3/L2/L1 are the same as having a dev/stage/prod environment. 

userdata.yml

#cloud-config
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCTSRtWzW/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
yum_repos:
  elasticsearch.repo:
    name: Elasticsearch repository for 6.x packages
    baseurl: https://artifacts.elastic.co/packages/6.x/yum
    gpgcheck: 1
    gpgkey: https://artifacts.elastic.co/GPG-KEY-elasticsearch
    enabled: 1
    autorefresh: 1
    type: rpm-md

package_upgrade: true
packages:
- vim
- htop
- wget
- gcc

write_files:
- path: /etc/systemd/timesyncd.conf
  permissions: 0644
  owner: root
  content: |
    [Time]
    NTP=0.amazon.pool.ntp.org 1.amazon.pool.ntp.org 2.amazon.pool.ntp.org 3.amazon.pool.ntp.org

- path: /etc/sysctl.d/net.ipv4.neigh.default.conf
  content: |
    net.ipv4.neigh.default.gc_thresh1=4096
    net.ipv4.neigh.default.gc_thresh2=8192
    net.ipv4.neigh.default.gc_thresh3=16384

- path: /etc/sysctl.d/fs.inotify.max_user_instances.conf
  content: |
    fs.inotify.max_user_instances=4096

- path: /etc/sysctl.d/net.conf
  content: |
    net.core.somaxconn = 1000
    net.core.netdev_max_backlog = 5000
    net.core.rmem_default = 524280
    net.core.wmem_default = 524280
    net.core.rmem_max = 16777216
    net.core.wmem_max = 16777216
    net.ipv4.udp_rmem_min = 10240
    net.nf_conntrack_max = 1048576

- path: /etc/sysctl.d/vm.conf
  content: |
    vm.max_map_count=262144

- path: /etc/security/limits.conf
  content: |
    * soft memlock unlimited
    * hard memlock unlimited
    *  -  nofile  65536

- path: /etc/systemd/system/elasticsearch.service.d/override.conf
  permissions: 666
  content: |
    [Service]
    LimitMEMLOCK=infinity

- path: /etc/elasticsearch/jvm.options
  permissions: 0666
  content: |
    ## JVM configuration
    -Xms20g
    -Xmx20g
    -XX:+UseConcMarkSweepGC
    -XX:CMSInitiatingOccupancyFraction=75
    -XX:+UseCMSInitiatingOccupancyOnly
    -Des.networkaddress.cache.ttl=60
    -Des.networkaddress.cache.negative.ttl=10
    -XX:+AlwaysPreTouch
    -Xss1m
    -Djava.awt.headless=true
    -Dfile.encoding=UTF-8
    -Djna.nosys=true
    -XX:-OmitStackTraceInFastThrow
    -Dio.netty.noUnsafe=true
    -Dio.netty.noKeySetOptimization=true
    -Dio.netty.recycler.maxCapacityPerThread=0
    -Dlog4j.shutdownHookEnabled=false
    -Dlog4j2.disable.jmx=true
    -Djava.io.tmpdir=${ES_TMPDIR}
    -XX:+HeapDumpOnOutOfMemoryError
    -XX:HeapDumpPath=/var/lib/elasticsearch
    -XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
    8:-XX:+PrintGCDetails
    8:-XX:+PrintGCDateStamps
    8:-XX:+PrintTenuringDistribution
    8:-XX:+PrintGCApplicationStoppedTime
    8:-Xloggc:/var/log/elasticsearch/gc.log
    8:-XX:+UseGCLogFileRotation
    8:-XX:NumberOfGCLogFiles=32
    8:-XX:GCLogFileSize=64m
    9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
    9-:-Djava.locale.providers=COMPAT
    10-:-XX:UseAVX=2


runcmd:
- [ amazon-linux-extras, install, corretto8, -y ]
- [ yum, install, elasticsearch-6.4.3-1, -y ]
- [ systemctl, daemon-reload ]
- [ systemctl, enable, elasticsearch.service ]
- [ systemctl, start, elasticsearch.service ]