update k8s version and implement tls

- use coreos-config yaml to configure instances - create aws auto-scaling group - tls for workers is generated by workes themselves using the ca and ca-key uploaded via cloud-config (should probably be made more secure)

Author: mettjus

Makefile

@@ -1,16 +1,25 @@
-plan: etcd_discovery_url.txt kube_token.txt
+plan: etcd_discovery_url.txt kube_token.txt tls-assets/ca-key.pem tls-assets/ca.pem
 terraform plan
 
 etcd_discovery_url.txt:
-curl -s https://discovery.etcd.io/new?size=3 > etcd_discovery_url.txt
+curl -s https://discovery.etcd.io/new?size=2 > etcd_discovery_url.txt
 
 destroy:
 terraform destroy
 rm etcd_discovery_url.txt kube_token.txt
+rm -rf tls-assets
 
-apply: etcd_discovery_url.txt kube_token.txt
+apply: etcd_discovery_url.txt kube_token.txt tls-assets/ca-key.pem tls-assets/ca.pem
 terraform apply
 
 kube_token.txt:
 openssl rand -base64 8 |md5 |head -c8 > kube_token.txt
 echo >> kube_token.txt
+
+tls-assets/ca-key.pem:
+mkdir -p tls-assets
+openssl genrsa -out tls-assets/ca-key.pem 2048
+
+tls-assets/ca.pem:
+mkdir -p tls-assets
+openssl req -x509 -new -nodes -key tls-assets/ca-key.pem -days 10000 -out tls-assets/ca.pem -subj "/CN=kube-ca"

kubernetes.tf

@@ -65,8 +65,8 @@ resource "aws_security_group_rule" "allow_ssh" {
 
 resource "aws_security_group_rule" "allow_kube_api" {
  type = "ingress"
- from_port = 6443
- to_port = 6443
+ from_port = "${var.api_secure_port}"
+ to_port = "${var.api_secure_port}"
  protocol = "tcp"
  cidr_blocks = ["0.0.0.0/0"]
  security_group_id = "${aws_security_group.kubernetes.id}"
@@ -90,29 +90,9 @@ resource "aws_security_group_rule" "allow_all_egress" {
  security_group_id = "${aws_security_group.kubernetes.id}"
 }
 
-resource "template_file" "kubernetes" {
- template = "templates/kubernetes.sh"
-
- vars = {
- etcd_dicovery_url = "${replace(file("etcd_discovery_url.txt"), "/\n/", "")}"
- containers_cidr = "${var.containers_cidr}"
- kubernetes_version = "${var.kubernetes_version}"
- portal_net = "${var.portal_net}"
- etcd_version = "${var.etcd_version}"
- }
-}
-
-resource "template_file" "tokens" {
- template = "templates/tokens.csv"
-
- vars = {
- token = "${replace(file("kube_token.txt"), "/\n/", "")}"
- }
-}
-
 resource "aws_instance" "etcd" {
  ami = "${var.ami}"
- instance_type = "t2.medium"
+ instance_type = "${var.etcd_instance_type}"
  count = "${var.num_etcd}"
  security_groups = [ "${aws_security_group.kubernetes.id}" ]
  subnet_id = "${aws_subnet.kubernetes.id}"
@@ -125,32 +105,74 @@ resource "aws_instance" "etcd" {
  }
 
  tags {
- Name = "kubernetes-${var.cluster_name}-etcd"
+ Name = "${var.cluster_name}-etcd"
  Cluster = "${var.cluster_name}"
  Role = "etcd"
  }
 
+ user_data = "${template_file.etcd-user-data.rendered}"
+}
 
- provisioner "file" {
-  source = "scripts"
-  destination = "/tmp/scripts"
- }
+resource "aws_iam_instance_profile" "master" {
+ name = "k8s-master"
+ roles = ["${aws_iam_role.master.name}"]
+}
 
- provisioner "remote-exec" {
- inline = [
- "echo 'PRIVATE_IP=${self.private_ip}' > /tmp/network.env",
- "echo 'PUBLIC_IP=${self.public_ip}' >> /tmp/network.env",
- "sudo mv /tmp/network.env /etc/network.env",
- "cat <<'EOF' > /tmp/kubernetes.env\n${template_file.kubernetes.rendered}\nEOF",
- "sudo mv /tmp/kubernetes.env /etc/kubernetes.env",
- "sudo bash /tmp/scripts/${self.tags.Role}.sh"
- ]
+resource "aws_iam_role_policy" "master" {
+ name = "k8s-master"
+ role = "${aws_iam_role.master.id}"
+ policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ec2:*"
+ ],
+ "Resource": [
+ "*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "elasticloadbalancing:*"
+ ],
+ "Resource": [
+ "*"
+ ]
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_role" "master" {
+ name = "k8s-master"
+ path = "/"
+ assume_role_policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
  }
+ ]
+}
+EOF
 }
 
 resource "aws_instance" "master" {
  ami = "${var.ami}"
- instance_type = "t2.medium"
+ instance_type = "${var.master_instance_type}"
+ count = "${var.num_master}"
+ iam_instance_profile = "${aws_iam_instance_profile.master.name}"
  security_groups = [ "${aws_security_group.kubernetes.id}" ]
  subnet_id = "${aws_subnet.kubernetes.id}"
  associate_public_ip_address = true
@@ -162,7 +184,7 @@ resource "aws_instance" "master" {
  }
 
  tags {
- Name = "kubernetes-${var.cluster_name}-master"
+ Name = "${var.cluster_name}-master"
  Cluster = "${var.cluster_name}"
  Role = "master"
  }
@@ -174,67 +196,178 @@ resource "aws_instance" "master" {
 
  provisioner "remote-exec" {
  inline = [
- "cat <<'EOF' > /tmp/tokens.csv\n${template_file.tokens.rendered}\nEOF",
- "sudo mkdir -p mkdir /etc/kubernetes",
- "sudo mv /tmp/tokens.csv /etc/kubernetes/tokens.csv",
- "echo 'PRIVATE_IP=${self.private_ip}' > /tmp/network.env",
- "echo 'PUBLIC_IP=${self.public_ip}' >> /tmp/network.env",
- "sudo mv /tmp/network.env /etc/network.env",
- "cat <<'EOF' > /tmp/kubernetes.env\n${template_file.kubernetes.rendered}\nEOF",
+ "cat <<'EOF' > /tmp/kubernetes.env\n${template_file.master-env.rendered}\nEOF",
+ "echo 'PRIVATE_IP=${self.private_ip}' >> /tmp/kubernetes.env",
+ "echo 'PUBLIC_IP=${self.public_ip}' >> /tmp/kubernetes.env",
  "sudo mv /tmp/kubernetes.env /etc/kubernetes.env",
- "sudo bash /tmp/scripts/${self.tags.Role}.sh"
+ "sudo bash /tmp/scripts/master.sh",
+ # "cat << 'EOF' > /tmp/kubernetes/tokens.csv\n${template_file.tokens.rendered}\nEOF",
+ "sudo mkdir -p /etc/kubernetes/ssl"
+ # "sudo mv /tmp/kubernetes/tokens.csv /etc/kubernetes/ssl/tokens.csv",
  ]
  }
+
+ user_data = "${template_file.master-user-data.rendered}"
 }
 
-resource "aws_instance" "worker" {
- ami = "${var.ami}"
- instance_type = "t2.medium"
- count = "${var.num_worker}"
+resource "null_resource" "master" {
+ count = "${var.num_master}"
+ triggers {
+ cluster_instance_ids = "${join(",", aws_instance.master.*.id)}"
+ }
+ connection {
+ host = "${element(aws_instance.master.*.public_ip, count.index)}"
+ user = "core"
+ agent = true
+ }
+ provisioner "remote-exec" {
+ inline = [
+ "sudo rm -rf /tmp/ssl",
+ "sudo rm -rf /etc/kubernetes/ssl",
+ "mkdir /tmp/ssl",
+ "sudo mkdir -p /etc/kubernetes"
+ ]
+ }
+ provisioner "local-exec" {
+ command = "cat << 'EOF' > tls-assets/openssl.cnf\n${template_file.openssl.rendered}\nEOF\n${template_file.create-master-tls.rendered} && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q tls-assets/ca.pem tls-assets/apiserver.pem tls-assets/apiserver-key.pem core@${element(aws_instance.master.*.public_ip, count.index)}:/tmp/ssl/"
+ }
+ provisioner "remote-exec" {
+ inline = [
+ "sudo mv /tmp/ssl /etc/kubernetes/ssl",
+ "sudo chmod 600 /etc/kubernetes/ssl/*-key.pem",
+ "sudo chown root:root /etc/kubernetes/ssl/*-key.pem"
+ ]
+ }
+}
+
+resource "aws_iam_role_policy" "worker" {
+ name = "k8s-worker"
+ role = "${aws_iam_role.worker.id}"
+ policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "ec2:Describe*",
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": "ec2:AttachVolume",
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": "ec2:DetachVolume",
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ecr:GetAuthorizationToken",
+ "ecr:BatchCheckLayerAvailability",
+ "ecr:GetDownloadUrlForLayer",
+ "ecr:GetRepositoryPolicy",
+ "ecr:DescribeRepositories",
+ "ecr:ListImages",
+ "ecr:BatchGetImage"
+ ],
+ "Resource": "*"
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_instance_profile" "worker" {
+ name = "k8s-worker"
+ roles = ["${aws_iam_role.worker.name}"]
+}
+
+resource "aws_iam_role" "worker" {
+ name = "k8s-worker"
+ path = "/"
+ assume_role_policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_launch_configuration" "worker" {
+ image_id = "${var.ami}"
+ instance_type = "${var.worker_instance_type}"
+ iam_instance_profile = "${aws_iam_instance_profile.worker.name}"
  security_groups = [ "${aws_security_group.kubernetes.id}" ]
- subnet_id = "${aws_subnet.kubernetes.id}"
  associate_public_ip_address = true
  key_name = "${var.ssh_key_name}"
 
- connection {
- user = "core"
- agent = true
+
+ user_data = "${template_file.worker-user-data.rendered}"
+}
+
+resource "aws_autoscaling_group" "worker" {
+ name = "${var.cluster_name}-k8s-worker"
+ launch_configuration = "${aws_launch_configuration.worker.name}"
+ max_size = "${var.num_worker}"
+ min_size = "${var.num_worker}"
+ desired_capacity = "${var.num_worker}"
+ vpc_zone_identifier = [ "${aws_subnet.kubernetes.id}" ]
+
+
+ tag {
+ key = "Name"
+ value = "${var.cluster_name}-worker"
+ propagate_at_launch = true
  }
 
- tags {
- Name = "kubernetes-${var.cluster_name}-worker"
- Cluster = "${var.cluster_name}"
- Role = "worker"
+ tag {
+ key = "Cluster"
+ value = "${var.cluster_name}"
+ propagate_at_launch = true
  }
 
- provisioner "file" {
- source = "scripts"
- destination = "/tmp/scripts"
+ tag {
+ key = "Role"
+ value = "worker"
+ propagate_at_launch = true
  }
+}
 
+resource "null_resource" "init-kubernetes" {
+ triggers {
+ id = "${element(aws_instance.master.*.id,0)}"
+ }
+ connection {
+ host = "${element(aws_instance.master.*.public_ip, 0)}"
+ user = "core"
+ agent = true
+ }
  provisioner "remote-exec" {
  inline = [
- "echo 'PRIVATE_IP=${self.private_ip}' > /tmp/network.env",
- "echo 'PUBLIC_IP=${self.public_ip}' >> /tmp/network.env",
- "sudo mv /tmp/network.env /etc/network.env",
- "cat <<'EOF' > /tmp/kubernetes.env\n${template_file.kubernetes.rendered}\nEOF",
- "echo 'KUBERNETES_MASTER=http://${aws_instance.master.private_ip}:8080' >> /tmp/kubernetes.env",
- "sudo mv /tmp/kubernetes.env /etc/kubernetes.env",
- "sudo bash /tmp/scripts/${self.tags.Role}.sh"
+ "${template_file.init-kubernetes.rendered}"
  ]
  }
-}
+ provisioner "local-exec" {
+ command = "${template_file.create-admin-tls.rendered}"
+ }
 
-resource "template_file" "kubectl-config" {
- template = "templates/kubectl-config.sh"
- vars = {
- cluster_name = "${var.cluster_name}"
- token = "${replace(file("kube_token.txt"), "/\n/", "")}"
- server = "${aws_instance.master.public_ip}"
+ provisioner "local-exec" {
+ command = "cat <<EOF > kubeconfig\n${template_file.kubectl-config-file.rendered}\nEOF"
  }
 }
 
 output "kubernetes-api-server" {
  value = "${template_file.kubectl-config.rendered}"
-}
-
+}