Nomad CSI Sample
약 8 분
Nomad CSI Sample
- AWS에 EFS를 Nomad CSI로 활용
- full code는 아래 github를 참고
- 참고 github: https://github.com/Great-Stone/nomad-demo-with-ecs
- branches: ung
- 참고 github: https://github.com/Great-Stone/nomad-demo-with-ecs
ec2(nomad client node)에 efs의 volume관련 권한이 필요합니다.
- ec2와 efs는 같은 subnet이여야 합니다.
- vpc에는 dns관련 권한설정이 필요합니다.
ec2 iam policy
resource "aws_vpc" "nomad_demo" {
cidr_block = var.vpc_cidr_block
#dns 권한설정이 필요함
enable_dns_support = true
enable_dns_hostnames = true
tags = {
env = "nomad"
}
}
resource "aws_subnet" "nomad_demo" {
cidr_block = var.vpc_cidr_block
vpc_id = aws_vpc.nomad_demo.id
#efs와 az가 같아야함
availability_zone = "ap-northeast-2a"
}
############
# Policy
data "aws_iam_policy_document" "instance_role" {
statement {
effect = "Allow"
actions = [
"sts:AssumeRole",
]
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
resource "aws_iam_role" "instance_role" {
name_prefix = "${var.prefix}-nomad"
assume_role_policy = data.aws_iam_policy_document.instance_role.json
}
resource "aws_iam_role" "instance_role" {
name_prefix = "${var.prefix}-nomad"
assume_role_policy = data.aws_iam_policy_document.instance_role.json
}
resource "aws_iam_instance_profile" "test_profile" {
name = "test_profile"
role = aws_iam_role.instance_role.name
}
resource "aws_iam_role_policy" "cluster_discovery" {
name = "${var.prefix}-nomad-cluster_discovery"
role = aws_iam_role.instance_role.id
policy = data.aws_iam_policy_document.cluster_discovery.json
}
data "aws_iam_policy_document" "cluster_discovery" {
# allow role with this policy to do the following: list instances, list tags, autoscale
statement {
effect = "Allow"
actions = [
"ec2:DescribeInstances",
"autoscaling:CompleteLifecycleAction",
"ec2:DescribeTags",
"ecs:ListClusters",
"ecs:DescribeClusters",
"ecs:DeregisterContainerInstance",
"ecs:ListContainerInstances",
"ecs:RegisterContainerInstance",
"ecs:SubmitContainerStateChange",
"ecs:SubmitTaskStateChange",
"ecs:DescribeContainerInstances",
"ecs:DescribeTasks",
"ecs:ListTasks",
"ecs:UpdateContainerAgent",
"ecs:StartTask",
"ecs:StopTask",
"ecs:RunTask",
"elasticfilesystem:ClientMount",
"elasticfilesystem:ClientWrite",
"elasticfilesystem:ClientRootAccess"
]
resources = ["*"]
}
}
efs volume and iam policy
############
#EFS
resource "aws_iam_role_policy" "mount_efs_volumes" {
name = "mount-efs-volumes"
role = aws_iam_role.instance_role.id
policy = data.aws_iam_policy_document.mount_efs_volumes.json
}
data "aws_iam_policy_document" "mount_efs_volumes" {
statement {
effect = "Allow"
actions = [
"ec2:DescribeInstances",
"ec2:DescribeTags",
"ec2:DescribeVolumes",
"ec2:AttachVolume",
"ec2:DetachVolume",
]
resources = ["*"]
}
}
# csi efs volume
resource "aws_efs_file_system" "nomad_csi" {
creation_token = "nomad-csi"
performance_mode = "generalPurpose"
throughput_mode = "bursting"
tags = {
Name = "nomad-csi"
}
#az가 ec2와 동일해야함
availability_zone_name = "ap-northeast-2a"
}
#ec2와 subnet이 같아야함
resource "aws_efs_mount_target" "nomad_efs" {
file_system_id = aws_efs_file_system.nomad_csi.id
subnet_id = aws_subnet.nomad_demo.id
security_groups = [ aws_security_group.efs.id ]
}
resource "aws_security_group" "efs" {
name = "allow_efs"
description = "Allow EFS inbound traffic"
vpc_id = aws_vpc.nomad_demo.id
ingress {
description = "TLS from VPC"
from_port = 443
to_port = 443
protocol = "tcp"
cidr_blocks = [ "0.0.0.0/0" ]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = {
Name = "allow_tls"
}
}
nomad csi create
- nomad csi job을 배포합니다.
- plugins을 생성합니다.
nomad csi create
#efs csi job을 생성
resource "nomad_job" "nomad_csi_node_job" {
jobspec = file("./job_file/csi-node.tpl")
}
resource "time_sleep" "wait_30_seconds" {
depends_on = [nomad_job.nomad_csi_node_job]
create_duration = "30s"
}
#생성된 plugin을 기다림
data "nomad_plugin" "efs" {
depends_on = [time_sleep.wait_30_seconds]
plugin_id = "aws-efs0"
wait_for_registration = true
}
#efs volume을 nomad에서 사용할 수 있게 plugins을 생성
resource "nomad_volume" "efs_csi_volume" {
depends_on = [data.nomad_plugin.efs]
type = "csi"
plugin_id = "aws-efs0"
volume_id = "efs_csi_volume"
name = "efs_csi_volume"
external_id = data.terraform_remote_state.net.outputs.nomad_efs_name.id
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
mount_options {
fs_type = "ext4"
}
}
efs test job 배포
job "efs_csi_job" {
datacenters = ["dc1"]
type = "system"
group "cache" {
count = 1
network {
port "db" {
to = 6379
}
}
# 생성한 volume id 값을 명시한 volume을 선언
volume "cache" {
type = "csi"
source = "efs_csi_volume"
attachment_mode = "file-system"
access_mode = "single-node-writer"
read_only = false
}
task "redis" {
driver = "docker"
config {
image = "redis:6.2.6-alpine3.15"
ports = ["db"]
}
resources {
cpu = 500
memory = 511
}
#선언한 volume을 사용할 위치에 mount
volume_mount {
volume = "cache"
destination = "/data"
read_only = false
}
}
}
}