Added Bastion Host

bastion/.gitignore

@@ -0,0 +1,5 @@
+terraform.tfvars
+terraform.tfstate
+terraform.tfstate.backup
+*.tfplan
+files/*/cloud-init.txt

bastion/Makefile

@@ -0,0 +1,30 @@
+HOSTS     := bastion consul
+HOST_DIRS := $(addprefix files/,$(HOSTS))
+TARGETS   := $(addsuffix /cloud-init.txt,$(HOST_DIRS))
+
+all: $(TARGETS)
+
+define make-goal
+$1/cloud-init.txt: $(wildcard files/common/*.sh) $(wildcard $1/*.yaml) $(wildcard $1/*.sh)
+endef
+
+$(foreach hdir,$(HOST_DIRS),$(eval $(call make-goal,$(hdir))))
+
+$(TARGETS):
+	$(CURDIR)/bin/write-mime-multipart --output=$@ $^
+
+plan: $(TARGETS)
+	terraform $@
+
+apply: $(TARGETS)
+	terraform $@
+
+destroy:
+	terraform plan -destroy -out=destroy.tfplan
+	terraform apply destroy.tfplan
+
+clean:
+	rm $(TARGETS)
+
+.PHONY: all plan apply clean destroy
+

bastion/bin/write-mime-multipart

@@ -0,0 +1,118 @@
+#!/usr/bin/python2.6
+# largely taken from python examples
+# http://docs.python.org/library/email-examples.html
+
+import os
+import sys
+import smtplib
+# For guessing MIME type based on file name extension
+import mimetypes
+
+from email import encoders
+from email.message import Message
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from optparse import OptionParser
+import gzip
+
+from base64 import b64encode
+
+COMMASPACE = ', '
+
+starts_with_mappings={
+    '#include' : 'text/x-include-url',
+    '#!' : 'text/x-shellscript',
+    '#cloud-config' : 'text/cloud-config',
+    '#upstart-job'  : 'text/upstart-job',
+    '#part-handler' : 'text/part-handler',
+    '#cloud-boothook' : 'text/cloud-boothook'
+}
+
+def get_type(fname,deftype):
+    f = file(fname,"rb")
+    line = f.readline()
+    f.close()
+    rtype = deftype
+    for str,mtype in starts_with_mappings.items():
+        if line.startswith(str):
+            rtype = mtype
+            break
+    return(rtype)
+
+def main():
+    outer = MIMEMultipart()
+    #outer['Subject'] = 'Contents of directory %s' % os.path.abspath(directory)
+    #outer['To'] = COMMASPACE.join(opts.recipients)
+    #outer['From'] = opts.sender
+    #outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
+
+    parser = OptionParser()
+    
+    parser.add_option("-o", "--output", dest="output",
+        help="write output to FILE [default %default]", metavar="FILE", 
+        default="-")
+    parser.add_option("-z", "--gzip", dest="compress", action="store_true",
+        help="compress output", default=False)
+    parser.add_option("-d", "--default", dest="deftype",
+        help="default mime type [default %default]", default="text/plain")
+    parser.add_option("--delim", dest="delim",
+        help="delimiter [default %default]", default=":")
+    parser.add_option("-b", "--base64", dest="base64", action="store_true",
+        help="encode content base64", default=False)
+
+    (options, args) = parser.parse_args()
+
+    if (len(args)) < 1:
+        parser.error("Must give file list see '--help'")
+
+    for arg in args:
+        t = arg.split(options.delim, 1)
+        path=t[0]
+        if len(t) > 1:
+            mtype = t[1]
+        else:
+            mtype = get_type(path,options.deftype)
+
+        maintype, subtype = mtype.split('/', 1)
+        if maintype == 'text':
+            fp = open(path)
+            # Note: we should handle calculating the charset
+            msg = MIMEText(fp.read(), _subtype=subtype)
+            fp.close()
+        else:
+            fp = open(path, 'rb')
+            msg = MIMEBase(maintype, subtype)
+            msg.set_payload(fp.read())
+            fp.close()
+            # Encode the payload using Base64
+            encoders.encode_base64(msg)
+
+        # Set the filename parameter
+        msg.add_header('Content-Disposition', 'attachment',
+            filename=os.path.basename(path))
+
+        outer.attach(msg)
+
+    if options.output is "-":
+        ofile = sys.stdout
+    else:
+        ofile = file(options.output,"wb")
+    
+    if options.base64:
+        output = b64encode(outer.as_string())
+    else:
+        output = outer.as_string()
+        
+    if options.compress:
+        gfile = gzip.GzipFile(fileobj=ofile, filename = options.output )
+        gfile.write(output)
+        gfile.close()
+    else:
+        ofile.write(output)
+
+    ofile.close()
+
+if __name__ == '__main__':
+    main()
+

bastion/files/bastion/cloud-config.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+write_files:
+    - path: /etc/terraform_environment
+      content: |
+          ROLE="bastion consul-ui"
+

bastion/files/common/configure-dhclient.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+echo "Using Consul in dhclient..."
+cat >/etc/dhcp/dhclient.conf << EOF
+timeout 300;
+supersede domain-name "node.dc1.consul";
+supersede domain-search "service.dc1.consul", "node.dc1.consul";
+supersede domain-name-servers 127.0.0.1, 10.0.0.2;
+EOF
+chmod 0644 /etc/dhcp/dhclient.conf
+
+service network reload
+

bastion/files/common/install-consul.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+set -e
+
+source /etc/terraform_environment
+
+SERVER_ARGS=""
+UI_DIR="null"
+HTTP_CLIENT_ADDR="127.0.0.1"
+
+echo "Installing Consul..."
+pushd /tmp
+wget https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip -O consul.zip
+unzip consul.zip >/dev/null
+chmod +x consul
+mv consul /usr/local/bin/consul
+mkdir -p /etc/consul.d
+mkdir -p /mnt/consul/data
+mkdir -p /etc/service
+rm /tmp/consul.zip
+popd
+
+if [[ "${ROLE}" == *consul-server* ]]; then
+    echo "Configure as Consul Server..."
+
+    SERVER_ARGS="-server -bootstrap-expect=3"
+else
+    echo "Configure as Consul Client..."
+fi
+
+if [[ "${ROLE}" == *consul-ui* ]]; then
+    echo "Installing Consul UI..."
+    pushd /tmp
+    wget https://dl.bintray.com/mitchellh/consul/0.4.1_web_ui.zip -O consul-ui.zip
+    unzip consul-ui.zip >/dev/null
+    mkdir -p /mnt/consul/ui
+    mv dist/* /mnt/consul/ui/
+    rm /tmp/consul-ui.zip
+    popd
+
+    HTTP_CLIENT_ADDR="0.0.0.0"
+    UI_DIR="\"/mnt/consul/ui\""
+fi
+
+# Configuration file
+echo "Creating configuration..."
+cat >/etc/consul.d/config.json << EOF
+{
+    "addresses"                   : {
+        "http" : "${HTTP_CLIENT_ADDR}"
+    },
+    "ports"                       : {
+        "dns" : 53
+    },
+    "recursor"                    : "10.0.0.2",
+    "disable_anonymous_signature" : true,
+    "disable_update_check"        : true,
+    "data_dir"                    : "/mnt/consul/data",
+    "ui_dir"                      : $UI_DIR
+}
+EOF
+chmod 0644 /etc/consul.d/config.json
+
+# Setup the join address
+echo "Configure IPs..."
+cat >/etc/service/consul-join << EOF
+export CONSUL_JOIN="10.0.1.10 10.0.1.11 10.0.1.12"
+EOF
+chmod 0644 /etc/service/consul-join
+
+# Configure the server
+echo "Configure server..."
+cat >/etc/service/consul << EOF
+export CONSUL_FLAGS="${SERVER_ARGS}"
+EOF
+chmod 0644 /etc/service/consul
+
+# Add "first start" join service
+echo "Creating 'join' service..."
+cat >/etc/init/consul-join.conf <<"EOF"
+description "Join the consul cluster"
+
+start on started consul
+stop on stopped consul
+
+task
+
+script
+  if [ -f "/etc/service/consul-join" ]; then
+    . /etc/service/consul-join
+  fi
+
+  # Keep trying to join until it succeeds
+  set +e
+  while :; do
+    logger -t "consul-join" "Attempting join: ${CONSUL_JOIN}"
+    /usr/local/bin/consul join \
+      ${CONSUL_JOIN} \
+      >>/var/log/consul-join.log 2>&1
+    [ $? -eq 0 ] && break
+    sleep 5
+  done
+
+  logger -t "consul-join" "Join success!"
+end script
+EOF
+chmod 0644 /etc/init/consul-join.conf
+
+# Add actual service
+echo "Creating service..."
+cat >/etc/init/consul.conf <<"EOF"
+description "Consul agent"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+script
+  if [ -f "/etc/service/consul" ]; then
+    . /etc/service/consul
+  fi
+
+  # Make sure to use all our CPUs, because Consul can block a scheduler thread
+  export GOMAXPROCS=`nproc`
+
+  exec /usr/local/bin/consul agent \
+    -config-dir="/etc/consul.d" \
+    ${CONSUL_FLAGS} \
+    >>/var/log/consul.log 2>&1
+end script
+EOF
+chmod 0644 /etc/init/consul.conf
+
+# Start service
+echo "Starting service..."
+initctl start consul
+

bastion/files/consul/cloud-config.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+write_files:
+    - path: /etc/terraform_environment
+      content: |
+          ROLE="consul-server"
+

bastion/main.tf

@@ -0,0 +1,84 @@
+##
+# Create a bastion host to allow SSH in to the test network.
+# Connections are only allowed from ${var.allowed_network}
+# This box also acts as a NAT for the private network
+##
+
+resource "aws_security_group" "bastion" {
+    name = "bastion"
+    description = "Allow access from allowed_network to SSH/Consul, and NAT internal traffic"
+    vpc_id = "${var.vpc_id}"
+
+    # SSH
+    ingress = {
+        from_port = 22
+        to_port = 22
+        protocol = "tcp"
+        cidr_blocks = [ "${var.allowed_network}" ]
+        self = false
+    }
+
+    # Consul
+    ingress = {
+        from_port = 8500
+        to_port = 8500
+        protocol = "tcp"
+        cidr_blocks = [ "${var.allowed_network}" ]
+        self = false
+    }
+
+    # NAT
+    # ingress {
+    #     from_port = 0
+    #     to_port = 65535
+    #     protocol = "tcp"
+    #     cidr_blocks = [
+    #         "${var.subnet_public.cidr_block}",
+    #         "${var.subnet_private.cidr_block}"
+    #     ]
+    #     self = false
+    # }
+}
+
+resource "aws_security_group" "allow_bastion" {
+    name = "allow_bastion_ssh"
+    description = "Allow access from bastion host"
+    vpc_id = "${var.vpc_id}"
+    ingress {
+        from_port = 0
+        to_port = 65535
+        protocol = "tcp"
+        security_groups = ["${aws_security_group.bastion.id}"]
+        self = false
+    }
+}
+
+resource "aws_instance" "bastion" {
+    connection {
+        user = "ec2-user"
+        key_file = "${var.key_path}"
+    }
+    ami = "${lookup(var.amazon_nat_amis, var.region)}"
+    instance_type = "t2.micro"
+    key_name = "${var.key_name}"
+    security_groups = [
+        "${aws_security_group.allow_bastion.id}"
+    ]
+    subnet_id = "${var.subnet_public}"
+    associate_public_ip_address = true
+    source_dest_check = false
+    user_data = "${file(format("%s/files/bastion/cloud-config.yaml", path.module))}"
+    tags = {
+        Name                = "bastion"
+        subnet              = "dmz"
+        environment         = "devops"
+        billing-category    = "internal"
+        billing-subcategory = "devops"
+        role                = "devops.bastion"
+        managed_by          = "terraform"
+    }
+}
+
+output "bastion" {
+    value = "${aws_instance.bastion.public_ip}"
+}

bastion/terraform-aws-consul

@@ -0,0 +1 @@
+Subproject commit 4f026e71e1e2bee0420140d1ffd1dd2615075729

bastion/tmp/bastion.tf

@@ -0,0 +1,82 @@
+##
+# Create a bastion host to allow SSH in to the test network.
+# Connections are only allowed from ${var.allowed_network}
+# This box also acts as a NAT for the private network
+##
+
+resource "aws_security_group" "bastion" {
+    name = "bastion"
+    description = "Allow access from allowed_network to SSH/Consul, and NAT internal traffic"
+    vpc_id = "${aws_vpc.test.id}"
+
+    # SSH
+    ingress = {
+        from_port = 22
+        to_port = 22
+        protocol = "tcp"
+        cidr_blocks = [ "${var.allowed_network}" ]
+        self = false
+    }
+
+    # Consul
+    ingress = {
+        from_port = 8500
+        to_port = 8500
+        protocol = "tcp"
+        cidr_blocks = [ "${var.allowed_network}" ]
+        self = false
+    }
+
+    # NAT
+    ingress {
+        from_port = 0
+        to_port = 65535
+        protocol = "tcp"
+        cidr_blocks = [
+            "${aws_subnet.public.cidr_block}",
+            "${aws_subnet.private.cidr_block}"
+        ]
+        self = false
+    }
+}
+
+resource "aws_security_group" "allow_bastion" {
+    name = "allow_bastion_ssh"
+    description = "Allow access from bastion host"
+    vpc_id = "${aws_vpc.test.id}"
+    ingress {
+        from_port = 0
+        to_port = 65535
+        protocol = "tcp"
+        security_groups = ["${aws_security_group.bastion.id}"]
+        self = false
+    }
+}
+
+resource "aws_instance" "bastion" {
+    connection {
+        user = "ec2-user"
+        key_file = "${var.key_path}"
+    }
+    ami = "${lookup(var.amazon_nat_amis, var.region)}"
+    instance_type = "t2.micro"
+    key_name = "${var.key_name}"
+    security_groups = [
+        "${aws_security_group.bastion.id}"
+    ]
+    subnet_id = "${aws_subnet.dmz.id}"
+    associate_public_ip_address = true
+    source_dest_check = false
+    user_data = "${file("files/bastion/cloud-init.txt")}"
+    tags = {
+        Name = "bastion"
+        subnet = "dmz"
+        role = "bastion"
+        environment = "test"
+    }
+}
+
+output "bastion" {
+    value = "${aws_instance.bastion.public_ip}"
+}
+

bastion/tmp/consul.tf

@@ -0,0 +1,72 @@
+##
+# Consul cluster setup
+##
+
+resource "aws_security_group" "consul" {
+    name = "consul"
+    description = "Consul internal traffic + maintenance."
+    vpc_id = "${aws_vpc.test.id}"
+
+    ingress {
+        from_port = 53
+        to_port = 53
+        protocol = "tcp"
+        self = true
+    }
+    ingress {
+        from_port = 53
+        to_port = 53
+        protocol = "udp"
+        self = true
+    }
+    ingress {
+        from_port = 8300
+        to_port = 8302
+        protocol = "tcp"
+        self = true
+    }
+    ingress {
+        from_port = 8301
+        to_port = 8302
+        protocol = "udp"
+        self = true
+    }
+    ingress {
+        from_port = 8400
+        to_port = 8400
+        protocol = "tcp"
+        self = true
+    }
+    ingress {
+        from_port = 8500
+        to_port = 8500
+        protocol = "tcp"
+        self = true
+    }
+}
+
+resource "aws_instance" "consul" {
+    depends_on = [ "aws_instance.bastion" ]
+    connection {
+        user = "ec2-user"
+        key_file = "${var.key_path}"
+    }
+    ami = "${lookup(var.amazon_amis, var.region)}"
+    instance_type = "t2.micro"
+    count = 3
+    key_name = "${var.key_name}"
+    security_groups = [
+        "${aws_security_group.allow_bastion.id}",
+        "${aws_security_group.consul.id}"
+    ]
+    subnet_id = "${aws_subnet.private.id}"
+    private_ip = "10.0.1.1${count.index}"
+    tags = {
+        Name = "consul${count.index}"
+        subnet = "private"
+        role = "dns"
+        environment = "test"
+    }
+    user_data = "${file("files/consul/cloud-init.txt")}"
+}
+

bastion/tmp/network.tf

@@ -0,0 +1,77 @@
+##
+# VPC
+##
+resource "aws_vpc" "test" {
+    cidr_block = "10.0.0.0/16"
+}
+
+resource "aws_internet_gateway" "gateway" {
+    vpc_id = "${aws_vpc.test.id}"
+}
+
+##
+# DMZ
+##
+
+resource "aws_subnet" "dmz" {
+    vpc_id = "${aws_vpc.test.id}"
+    cidr_block = "10.0.201.0/24"
+}
+
+resource "aws_route_table" "dmz" {
+    vpc_id = "${aws_vpc.test.id}"
+    route {
+        cidr_block = "0.0.0.0/0"
+        gateway_id = "${aws_internet_gateway.gateway.id}"
+    }
+}
+
+resource "aws_route_table_association" "dmz" {
+    subnet_id = "${aws_subnet.dmz.id}"
+    route_table_id = "${aws_route_table.dmz.id}"
+}
+
+##
+# Public
+##
+
+resource "aws_subnet" "public" {
+    vpc_id = "${aws_vpc.test.id}"
+    cidr_block = "10.0.0.0/24"
+}
+
+resource "aws_route_table" "public" {
+    vpc_id = "${aws_vpc.test.id}"
+    route {
+        cidr_block = "0.0.0.0/0"
+        instance_id = "${aws_instance.bastion.id}"
+    }
+}
+
+resource "aws_route_table_association" "public" {
+    subnet_id = "${aws_subnet.public.id}"
+    route_table_id = "${aws_route_table.public.id}"
+}
+
+##
+# Private
+##
+
+resource "aws_subnet" "private" {
+    vpc_id = "${aws_vpc.test.id}"
+    cidr_block = "10.0.1.0/24"
+}
+
+resource "aws_route_table" "private" {
+    vpc_id = "${aws_vpc.test.id}"
+    route {
+        cidr_block = "0.0.0.0/0"
+        instance_id = "${aws_instance.bastion.id}"
+    }
+}
+
+resource "aws_route_table_association" "private" {
+    subnet_id = "${aws_subnet.private.id}"
+    route_table_id = "${aws_route_table.private.id}"
+}
+

bastion/variables.tf

@@ -0,0 +1,53 @@
+variable "vpc_id" {
+}
+
+variable "subnet_public" {
+}
+
+variable "allowed_network" {
+    description = "The CIDR of network that is allowed to access the bastion host"
+}
+
+variable "region" {
+    description = "The AWS region to create things in."
+    default = "us-west-2"
+}
+
+variable "key_name" {
+    description = "Name of the keypair to use in EC2."
+    default = "terraform"
+}
+
+variable "key_path" {
+    descriptoin = "Path to your private key."
+    default = "~/.ssh/id_rsa"
+}
+
+variable "amazon_amis" {
+    description = "Amazon Linux AMIs"
+    default = {
+        us-west-2 = "ami-b5a7ea85"
+    }
+}
+
+variable "amazon_nat_amis" {
+    description = "Amazon Linux NAT AMIs"
+    default = {
+        us-west-2 = "ami-bb69128b"
+    }
+}
+
+variable "centos7_amis" {
+    description = "CentOS 7 AMIs"
+    default = {
+        us-east-1 = "ami-96a818fe"
+        us-west-2 = "ami-c7d092f7"
+        us-west-1 = "ami-6bcfc42e"
+        eu-west-1 =  "ami-e4ff5c93"
+        ap-southeast-1 = "ami-aea582fc"
+        ap-southeast-2 = "ami-bd523087"
+        ap-northeast-1 = "ami-89634988"
+        sa-east-1 = "ami-bf9520a2"
+    }
+}
+

main.tf

@@ -54,7 +54,8 @@ resource "aws_route53_record" "dns" {
 module "s3" {
   source = "./s3/"
   stack_name = "${var.stack_name}"
-  cust_id = "${random_id.customer.b64}"
+  #cust_id = "${random_id.customer.b64}"
+  cust_id = "${uuid()}"
 }
 
 
@@ -103,4 +104,12 @@ module "nuxeo" {
   subnet_id="${element(module.net.private_subnets, 0)}"
 }
 
+module "bastion" {
+  source = "bastion/"
+
+  vpc_id = "${var.vpc_id}"
+  allowed_network="10.0.0.0/16"
+  subnet_public="${module.net.public_subnets}"
+}
+
 

net/variables.tf

@@ -47,4 +47,4 @@ variable "ssl_certificate_id" {
   default = "ASCAI627UM4G2NSLWDTMM"
 }
 
-data "aws_availability_zones" "all" {}
+data "aws_availability_zones" "available" {}