Thank you for reading this post, don't forget to subscribe!
есть модуль karpenter
infra/terraform/modules/karpenter/karpenter.tf
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 |
locals { karpenter = merge(var.infra_apps[index(var.infra_apps.*.chart, "karpenter")], var.karpenter) karpenter_crd = merge(var.infra_apps[index(var.infra_apps.*.chart, "karpenter-crd")], var.karpenter_crd) karpernter_values = <<VALUES webhook: # -- Whether to enable the webhooks and webhook permissions. enabled: false serviceAccount: create: true name: "karpenter" annotations: eks.amazonaws.com/role-arn: "${aws_iam_role.karpenter_controller_iam_trust_role.arn}" serviceMonitor: enabled: true nodeSelector: ${indent(2, yamlencode(local.karpenter.node_selector))} tolerations: ${indent(2, yamlencode(local.karpenter.tolerations))} controller: env: - name: AWS_REGION value: "${var.region}" resources: ${indent(4, yamlencode(local.karpenter.resources))} logConfig: enabled: true outputPaths: - stdout errorOutputPaths: - stderr - stdout logEncoding: console logLevel: global: debug controller: debug webhook: error settings: clusterName: "${var.eks_cluster_id}" clusterEndpoint: "${var.eks_cluster_endpoint}" VALUES #-------------------------------------------------------------- # Karpenter EC2NodeClass #-------------------------------------------------------------- ec2nodeclass_values = <<VALUES apiVersion: karpenter.k8s.aws/v1beta1 kind: EC2NodeClass metadata: name: services-private spec: amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${var.eks_cluster_id}" subnetSelectorTerms: - tags: karpenter.sh/discovery: ${local.karpenter.subnets_tags_discovery} securityGroupSelectorTerms: - id: ${var.eks_node_security_group_id} tags: karpenter.sh/discovery: ${var.eks_cluster_id} blockDeviceMappings: - deviceName: /dev/xvda ebs: volumeSize: 100Gi volumeType: gp3 iops: 1000 encrypted: false deleteOnTermination: true throughput: 125 amiSelectorTerms: - tags: karpenter.sh/discovery: ${var.eks_cluster_id} - id: "${local.karpenter.instance_ami}" VALUES } resource "helm_release" "karpenter" { count = local.karpenter.enabled ? 1 : 0 name = local.karpenter.name chart = local.karpenter.chart repository = local.karpenter.repository version = local.karpenter.chart_version namespace = local.karpenter.namespace max_history = local.karpenter.max_history values = [local.karpernter_values] } resource "helm_release" "karpenter_crd" { count = local.karpenter_crd.enabled ? 1 : 0 name = local.karpenter_crd.name chart = local.karpenter_crd.chart repository = local.karpenter_crd.repository version = local.karpenter_crd.chart_version namespace = local.karpenter_crd.namespace max_history = local.karpenter_crd.max_history } resource "kubectl_manifest" "karpenter_ec2nodeclass_private" { yaml_body = local.ec2nodeclass_values wait_for_rollout = false apply_only = true depends_on = [helm_release.karpenter, helm_release.karpenter_crd] } resource "kubectl_manifest" "karpenter_nodepool" { for_each = var.nodepools wait_for_rollout = false apply_only = true yaml_body = yamlencode({ apiVersion : "karpenter.sh/v1beta1" kind : "NodePool" metadata = { name = each.key } spec = { template = { metadata = { labels = each.value.labels } spec = { nodeClassRef = { name = "services-private" } taints = each.value.taints requirements = each.value.requirements kubelet = each.value.kubelet } } disruption = each.value.disruption } }) depends_on = [helm_release.karpenter, helm_release.karpenter_crd] } #------------------------------------------------------------------ # IAM role and policies for Karpenter #------------------------------------------------------------------ resource "aws_iam_service_linked_role" "karpenter" { count = local.karpenter.service_role_enabled ? 1 : 0 description = "Default EC2 Spot Service Linked Role" aws_service_name = "spot.amazonaws.com" } resource "aws_iam_role" "karpenter_instance_profile_trust_role" { name = "KarpenterNodeRole-${var.eks_cluster_id}" description = "Policy for Karpenter-provisioned nodes" assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ { Effect = "Allow", Principal = { Service = "ec2.amazonaws.com" }, Action = "sts:AssumeRole" } ] }) } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_worker_node_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_eks_sni_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_ecr_read_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_ssm_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_instance_profile" "karpenter_instance_profile" { name = "KarpenterNodeInstanceProfile-${var.eks_cluster_id}" role = aws_iam_role.karpenter_instance_profile_trust_role.name tags = { Name = "KarpenterNodeInstanceProfile-${var.eks_cluster_id}" } } resource "aws_iam_role" "karpenter_controller_iam_trust_role" { name = "KarpenterControllerRole-${var.eks_cluster_id}" description = "Policy for Karpenter controller" assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ { Effect = "Allow", Principal = { Federated = var.oidc_provider_arn }, Action = "sts:AssumeRoleWithWebIdentity", Condition = { StringEquals = { "${var.oidc_provider}:aud" = "sts.amazonaws.com" "${var.oidc_provider}:sub" = "system:serviceaccount:karpenter:karpenter" } } } ] }) } resource "aws_iam_policy" "karpenter_controller_iam_policy" { name = "KarpenterControllerPolicy-${var.eks_cluster_id}" description = "IAM policy for Karpenter Controller" policy = jsonencode({ "Statement" : [ { "Action" : [ "ssm:GetParameter", "ec2:DescribeImages", "ec2:RunInstances", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", "ec2:DescribeLaunchTemplates", "ec2:DescribeInstances", "ec2:DescribeInstanceTypes", "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeAvailabilityZones", "ec2:DeleteLaunchTemplate", "ec2:CreateTags", "ec2:CreateLaunchTemplate", "ec2:CreateFleet", "ec2:DescribeSpotPriceHistory", "pricing:GetProducts", "iam:GetInstanceProfile", "iam:CreateInstanceProfile", "iam:TagInstanceProfile", "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", "iam:DeleteInstanceProfile", "iam:PassRole" ], "Effect" : "Allow", "Resource" : "*", "Sid" : "Karpenter" }, { "Action" : "ec2:TerminateInstances", "Condition" : { "StringLike" : { "ec2:ResourceTag/karpenter.sh/nodepool" : "*" } }, "Effect" : "Allow", "Resource" : "*", "Sid" : "ConditionalEC2Termination" }, { "Effect" : "Allow", "Action" : "iam:PassRole", "Resource" : "arn:aws:iam::${var.account_id}:role/KarpenterNodeRole-${var.eks_cluster_id}", "Sid" : "PassNodeIAMRole" }, { "Effect" : "Allow", "Action" : "eks:DescribeCluster", "Resource" : "arn:aws:eks:${var.region}:${var.account_id}:cluster/${var.eks_cluster_id}", "Sid" : "EKSClusterEndpointLookup" } ], "Version" : "2012-10-17" }) } resource "aws_iam_role_policy_attachment" "karpenter_controller_policy_attach" { policy_arn = aws_iam_policy.karpenter_controller_iam_policy.arn role = aws_iam_role.karpenter_controller_iam_trust_role.name } |
infra/terraform/modules/karpenter/main.tf
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
provider "kubernetes" { host = data.aws_eks_cluster.main.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority.0.data) token = data.aws_eks_cluster_auth.main.token } provider "kubectl" { host = data.aws_eks_cluster.main.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority.0.data) token = data.aws_eks_cluster_auth.main.token } provider "helm" { kubernetes { host = data.aws_eks_cluster.main.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority.0.data) token = data.aws_eks_cluster_auth.main.token } experiments { manifest = false } } data "aws_eks_cluster" "main" { name = var.eks_cluster_id } data "aws_eks_cluster_auth" "main" { name = var.eks_cluster_id } data "aws_caller_identity" "current" {} |
infra/terraform/modules/karpenter/variables.tf
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
variable "region" { description = "Target region for all infrastructure resources" default = "" } variable "account_id" { type = string description = "Account ID" } variable "eks_cluster_id" { type = string description = "ID of the created EKS cluster." } variable "eks_node_security_group_id" { type = string description = "Security group ids attached to the nodes." } variable "eks_cluster_endpoint" { type = string description = "Endpoint for EKS control plane." } variable "oidc_provider_arn" { description = "The ARN of the OIDC Provider" default = "" } variable "oidc_provider" { description = "The OIDC Provider" default = "" } variable "vpc_cidr" { type = string default = "10.0.0.0/16" description = "CIDR block of the VPC" } variable "public_subnets" { type = string default = "" description = "Public subnets of the VPC" } variable "private_subnets" { type = string default = "" description = "Private subnets of the VPC" } variable "private_subnets_map" { type = map(string) default = {} description = "Private subnets map" } variable "helm_release_history_size" { description = "How much helm releases to store" default = 5 } variable "infra_apps" { description = "Base infra apps list" type = any } variable "karpenter" { type = any default = {} description = "Karpenter configuration" } variable "karpenter_crd" { type = any default = {} description = "Karpenter CRDs" } variable "nodepools" { description = "Nodepools" type = any } |
и вот мой terragrunt
infra/terragrunt/test/eu-central-1/pre/infra_apps.hcl
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 |
locals { #---------------------------------------------------------------------------------- # Infra apps locals #---------------------------------------------------------------------------------- infra_apps = [ #-------------------------------------------------------------------------------- # External secrets #-------------------------------------------------------------------------------- { name = "eso" enabled = true chart = "external-secrets" repository = "https://charts.external-secrets.io" chart_version = "0.9.19" namespace = "external-secrets" node_selector = { dedication = "infra" provisioning = "karpenter" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] resources = { requests = { cpu = "100m" memory = "128Mi" } limits = { cpu = "200m" memory = "128Mi" } } webhook_resources = { requests = { cpu = "100m" memory = "128Mi" } limits = { cpu = "200m" memory = "128Mi" } } cert_controller_resources = { requests = { cpu = "100m" memory = "128Mi" } limits = { cpu = "200m" memory = "128Mi" } } }, #-------------------------------------------------------------------------------- # Cert manager #-------------------------------------------------------------------------------- { name = "cert-manager" enabled = true chart = "cert-manager" repository = "https://charts.jetstack.io" chart_version = "v1.14.5" namespace = "cert-manager" node_selector = { dedication = "infra" provisioning = "karpenter" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] resources = { requests = { cpu = "100m" memory = "64Mi" } limits = { cpu = "200m" memory = "128Mi" } } webhook_resources = { requests = { cpu = "100m" memory = "64Mi" } limits = { cpu = "200m" memory = "128Mi" } } cainjector_resources = { requests = { cpu = "100m" memory = "128Mi" } limits = { cpu = "200m" memory = "256Mi" } } }, #-------------------------------------------------------------------------------- # Keda #-------------------------------------------------------------------------------- { name = "keda" enabled = true chart = "keda" repository = "https://kedacore.github.io/charts" chart_version = "2.12.0" namespace = "keda" node_selector = { dedication = "infra" provisioning = "karpenter" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] }, #-------------------------------------------------------------------------------- # Reloader #-------------------------------------------------------------------------------- { name = "reloader" enabled = true chart = "reloader" repository = "https://stakater.github.io/stakater-charts" chart_version = "1.0.104" namespace = "reloader" node_selector = { dedication = "infra" provisioning = "karpenter" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] resources = { requests = { cpu = "10m" memory = "128Mi" } limits = { cpu = "100m" memory = "512Mi" } } }, #-------------------------------------------------------------------------------- # Load balancer controller #-------------------------------------------------------------------------------- { name = "aws-lb-controller" enabled = true chart = "aws-load-balancer-controller" repository = "https://aws.github.io/eks-charts" chart_version = "1.6.2" namespace = "kube-system" service_account_name = "aws-load-balancer-controller" tolerations = [ { key = "dedicated" operator = "Equal" value = "service" effect = "NoSchedule" }, { key = "dedicated" operator = "Equal" value = "infra" effect = "NoExecute" } ] resources = { requests = { cpu = "100m" memory = "128Mi" } limits = { cpu = "200m" memory = "256Mi" } } }, #-------------------------------------------------------------------------------- # Ingress Nginx external #-------------------------------------------------------------------------------- { name = "ingress-nginx-ext" enabled = true chart = "ingress-nginx" repository = "https://kubernetes.github.io/ingress-nginx" chart_version = "4.11.3" namespace = "web-ingress-ext" ext_memory_requests = "256Mi" ext_cpu_requests = "100m" ext_memory_limit = "2Gi" ext_cpu_limit = "2000m" node_selector = { dedication = "infra" provisioning = "nodegroup" } topology_spread_constraints = [ { maxSkew = 1 topologyKey = "kubernetes.io/hostname" whenUnsatisfiable = "DoNotSchedule" labelSelector = { matchLabels = { "app.kubernetes.io/name" = "ingress-nginx" } } } ] tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] }, #-------------------------------------------------------------------------------- # Ingress Nginx internal #-------------------------------------------------------------------------------- { name = "ingress-nginx-int" enabled = true chart = "ingress-nginx" repository = "https://kubernetes.github.io/ingress-nginx" chart_version = "4.11.3" namespace = "web-ingress-int" int_memory_requests = "256Mi" int_cpu_requests = "100m" int_memory_limit = "2Gi" int_cpu_limit = "2000m" node_selector = { dedication = "infra" provisioning = "nodegroup" } topology_spread_constraints = [ { maxSkew = 1 topologyKey = "kubernetes.io/hostname" whenUnsatisfiable = "DoNotSchedule" labelSelector = { matchLabels = { "app.kubernetes.io/name" = "ingress-nginx" } } } ] tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] }, #-------------------------------------------------------------------------------- # Karpenter #-------------------------------------------------------------------------------- { name = "karpenter" enabled = true chart = "karpenter" repository = "oci://public.ecr.aws/karpenter" chart_version = "0.37.0" namespace = "karpenter" max_history = 3 service_role_enabled = false subnets_tags_discovery = "private" instance_ami = "ami-04f90211b3fc5e253" node_selector = { "kubernetes.io/os" = "linux" dedication = "infra" provisioning = "nodegroup" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] resources = { requests = { cpu = "200m" memory = "256Mi" } limits = { cpu = "400m" memory = "512Mi" } } }, #-------------------------------------------------------------------------------- # Karpeneter CRD #-------------------------------------------------------------------------------- { name = "karpenter-crd" enabled = true chart = "karpenter-crd" repository = "oci://public.ecr.aws/karpenter" chart_version = "0.37.0" namespace = "karpenter" max_history = 3 } ] } |
nodepool
infra/terragrunt/test/eu-central-1/pre/nodepools.hcl
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 |
locals { nodepools = { #-------------------------------------------------------------- # Karpenter NodePool Services #-------------------------------------------------------------- services = { labels = { dedication = "service" provisioning = "karpenter" } taints = [ { key = "dedicated" value = "service" effect = "NoSchedule" } ] requirements = [ { key = "kubernetes.io/arch" operator = "In" values = ["amd64"] }, { key = "karpenter.sh/capacity-type" operator = "In" values = ["spot"] }, { key = "karpenter.k8s.aws/instance-family" operator = "In" values = ["t3a","m5","m5a","m6a","m6in","m7a", "m7i", "c6a", "c7a"] }, { key = "karpenter.k8s.aws/instance-size" operator = "In" values = ["large","xlarge","medium"] }, { key = "topology.kubernetes.io/zone" operator = "In" values: ["eu-central-1b"] } ] kubelet = { systemReserved = { cpu = "100m" memory = "100Mi" "ephemeral-storage" = "1Gi" } kubeReserved = { cpu = "200m" memory = "100Mi" "ephemeral-storage" = "3Gi" } evictionMaxPodGracePeriod = 60 imageGCHighThresholdPercent = 85 imageGCLowThresholdPercent = 80 cpuCFSQuota = true maxPods = 110 } disruption = { consolidationPolicy = "WhenUnderutilized" budgets = [ { nodes = "10%" } ] } }, #-------------------------------------------------------------- # Karpenter NodePool Infra #-------------------------------------------------------------- infra = { labels = { dedication = "infra" provisioning = "karpenter" } taints = [ { key = "dedicated" value = "infra" effect = "NoSchedule" } ] requirements = [ { key = "kubernetes.io/arch" operator = "In" values = ["amd64"] }, { key = "karpenter.sh/capacity-type" operator = "In" values = ["spot"] }, { key = "karpenter.k8s.aws/instance-family" operator = "In" values = ["t3a","m5","m5a","m6a","m6in","m7a", "m7i", "c6a", "c7a"] }, { key = "karpenter.k8s.aws/instance-size" operator = "In" values = ["large","xlarge","medium"] }, { key = "topology.kubernetes.io/zone" operator = "In" values: ["eu-central-1b"] } ] kubelet = { systemReserved = { cpu = "100m" memory = "100Mi" "ephemeral-storage" = "1Gi" } kubeReserved = { cpu = "200m" memory = "100Mi" "ephemeral-storage" = "3Gi" } evictionMaxPodGracePeriod = 60 imageGCHighThresholdPercent = 85 imageGCLowThresholdPercent = 80 cpuCFSQuota = true maxPods = 110 } disruption = { consolidationPolicy = "WhenUnderutilized" budgets = [ { nodes = "10%" } ] } } #-------------------------------------------------------------- # Karpenter NodePool Infra-ondemand #-------------------------------------------------------------- infra-ondemand = { labels = { dedication = "infra-ondemand" provisioning = "karpenter" } taints = [ { key = "dedicated" value = "infra-ondemand" effect = "NoSchedule" } ] requirements = [ { key = "kubernetes.io/arch" operator = "In" values = ["amd64"] }, { key = "karpenter.sh/capacity-type" operator = "In" values = ["on-demand"] }, { key = "karpenter.k8s.aws/instance-family" operator = "In" values = ["t3a","t3"] }, { key = "karpenter.k8s.aws/instance-size" operator = "In" values = ["large","xlarge","medium", "small"] }, { key = "topology.kubernetes.io/zone" operator = "In" values: ["eu-central-1b"] } ] kubelet = { systemReserved = { cpu = "100m" memory = "100Mi" "ephemeral-storage" = "1Gi" } kubeReserved = { cpu = "200m" memory = "100Mi" "ephemeral-storage" = "3Gi" } evictionMaxPodGracePeriod = 60 imageGCHighThresholdPercent = 85 imageGCLowThresholdPercent = 80 cpuCFSQuota = true maxPods = 110 } disruption = { consolidationPolicy = "WhenUnderutilized" budgets = [ { nodes = "10%" } ] } } } } |
директория откуда запускаем apply
infra/terragrunt/test/eu-central-1/pre/common/karpenter/terragrunt.hcl
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 |
include "root" { path = find_in_parent_folders() expose = true merge_strategy = "deep" } include "env" { path = find_in_parent_folders("env.hcl") expose = true merge_strategy = "deep" } include "infra_apps" { path = find_in_parent_folders("infra_apps.hcl") expose = true merge_strategy = "deep" } include "nodepools" { path = find_in_parent_folders("nodepools.hcl") expose = true merge_strategy = "deep" } terraform { source = "${get_path_to_repo_root()}/terraform/modules//karpenter" } dependency "eks" { config_path = "../aws-eks" mock_outputs_allowed_terraform_commands = ["init", "validate", "plan", "destroy"] mock_outputs = { eks_cluster_id = "test" eks_oidc_provider_arn = "arn:aws:iam::11111111:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/D55EEBDFE5510B81EEE2381B88888888" node_group_default_iam_role_arn = "arn::" node_group_default_iam_role_name = "test" } } dependency "vpc" { config_path = "../aws-vpc" mock_outputs_allowed_terraform_commands = ["init", "validate", "plan", "destroy"] mock_outputs = { vpc_id = "vpc-0f5b1f788888888" vpc_cidr = "10.0.0.0/16" private_subnets = ["10.0.0.0/16"] public_subnets = ["10.0.0.0/16"] } } generate "providers_versions" { path = "versions.tf" if_exists = "overwrite" contents = <<EOF terraform { required_version = ">= 1.7.0" required_providers { aws = { source = "hashicorp/aws" version = "${include.root.locals.tf_providers.aws}" } kubernetes = { source = "hashicorp/kubernetes" version = "${include.root.locals.tf_providers.kubernetes}" } kubectl = { source = "gavinbunney/kubectl" version = "${include.root.locals.tf_providers.kubectl}" } helm = { source = "hashicorp/helm" version = "${include.root.locals.tf_providers.helm}" } http = { source = "hashicorp/http" version = "${include.root.locals.tf_providers.http}" } } } EOF } inputs = { region = include.env.locals.region environment = include.env.locals.environment name = include.env.locals.name infra_apps = include.infra_apps.locals.infra_apps account_id = get_aws_account_id() eks_cluster_id = dependency.eks.outputs.eks_cluster_id eks_cluster_endpoint = dependency.eks.outputs.eks_cluster_endpoint eks_node_security_group_id = dependency.eks.outputs.eks_node_security_group_id oidc_provider = dependency.eks.outputs.eks_oidc_provider oidc_provider_arn = dependency.eks.outputs.eks_oidc_provider_arn vpc_id = dependency.vpc.outputs.vpc_id private_subnets = dependency.vpc.outputs.private_subnets public_subnets = dependency.vpc.outputs.public_subnets private_subnets_map = dependency.vpc.outputs.private_subnets_map security_groups = [dependency.eks.outputs.eks_node_security_group_id] project_name = include.env.locals.project_name namespaces = include.env.locals.namespaces nodepools = include.nodepools.locals.nodepools } |
теперь нужно сделать обновление до версии 1,3,3 для этого правим модуль
infra/terraform/modules/karpenter/karpenter.tf
включаем webhook меняем api на v1 и добавляем
group = "karpenter.k8s.aws"
kind = "EC2NodeClass"
к
nodeClassRef
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
webhook: # -- Whether to enable the webhooks and webhook permissions. enabled: true ----------- #-------------------------------------------------------------- # Karpenter EC2NodeClass #-------------------------------------------------------------- ec2nodeclass_values = <<VALUES apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass ---------------- resource "kubectl_manifest" "karpenter_nodepool" { for_each = var.nodepools wait_for_rollout = false apply_only = true yaml_body = yamlencode({ apiVersion : "karpenter.sh/v1" kind : "NodePool" metadata = { name = each.key } spec = { template = { metadata = { labels = each.value.labels } spec = { nodeClassRef = { name = "services-private" group = "karpenter.k8s.aws" kind = "EC2NodeClass" } taints = each.value.taints requirements = each.value.requirements kubelet = each.value.kubelet } } disruption = each.value.disruption } }) depends_on = [helm_release.karpenter, helm_release.karpenter_crd] } |
в файле
infra/terragrunt/test/eu-central-1/pre/nodepools.hcl
меняем
consolidationPolicy = "WhenUnderutilized"
на
consolidationPolicy = "WhenEmptyOrUnderutilized"
в файле
infra/terragrunt/test/eu-central-1/pre/infra_apps.hcl
меняем версии сначала на 1,0,6
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
#-------------------------------------------------------------------------------- # Karpenter #-------------------------------------------------------------------------------- { name = "karpenter" enabled = true chart = "karpenter" repository = "oci://public.ecr.aws/karpenter" chart_version = "1.0.6" namespace = "karpenter" max_history = 3 service_role_enabled = false subnets_tags_discovery = "private" instance_ami = "ami-04f90211b3fc5e253" node_selector = { "kubernetes.io/os" = "linux" dedication = "infra" provisioning = "nodegroup" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] resources = { requests = { cpu = "200m" memory = "256Mi" } limits = { cpu = "400m" memory = "512Mi" } } }, #-------------------------------------------------------------------------------- # Karpeneter CRD #-------------------------------------------------------------------------------- { name = "karpenter-crd" enabled = true chart = "karpenter-crd" repository = "oci://public.ecr.aws/karpenter" chart_version = "1.0.6" namespace = "karpenter" max_history = 3 } ] } |
аплаим всё норм.
потом нам нужно переехать дальше на 1,3,3 для этого правим nodepool убираем из него kubelet и добавляем в ec2nodeclass
меняем версии в файле
infra/terragrunt/test/eu-central-1/pre/infra_apps.hcl
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
#-------------------------------------------------------------------------------- # Karpenter #-------------------------------------------------------------------------------- { name = "karpenter" enabled = true chart = "karpenter" repository = "oci://public.ecr.aws/karpenter" chart_version = "1.3.3" namespace = "karpenter" max_history = 3 service_role_enabled = false subnets_tags_discovery = "private" instance_ami = "ami-04f90211b3fc5e253" node_selector = { "kubernetes.io/os" = "linux" dedication = "infra" provisioning = "nodegroup" } tolerations = [ { key = "dedicated" operator = "Equal" value = "infra" effect = "NoSchedule" } ] resources = { requests = { cpu = "200m" memory = "256Mi" } limits = { cpu = "400m" memory = "512Mi" } } }, #-------------------------------------------------------------------------------- # Karpeneter CRD #-------------------------------------------------------------------------------- { name = "karpenter-crd" enabled = true chart = "karpenter-crd" repository = "oci://public.ecr.aws/karpenter" chart_version = "1.3.3" namespace = "karpenter" max_history = 3 } ] } |
вот nodepool
/infra/terragrunt/test/eu-central-1/pre/nodepools.hcl
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 |
locals { nodepools = { #-------------------------------------------------------------- # Karpenter NodePool Services #-------------------------------------------------------------- services = { labels = { dedication = "service" provisioning = "karpenter" } taints = [ { key = "dedicated" value = "service" effect = "NoSchedule" } ] requirements = [ { key = "kubernetes.io/arch" operator = "In" values = ["amd64"] }, { key = "karpenter.sh/capacity-type" operator = "In" values = ["spot"] }, { key = "karpenter.k8s.aws/instance-family" operator = "In" values = ["t3a","m5","m5a","m6a","m6in","m7a", "m7i", "c6a", "c7a"] }, { key = "karpenter.k8s.aws/instance-size" operator = "In" values = ["large","xlarge","medium"] }, { key = "topology.kubernetes.io/zone" operator = "In" values: ["eu-central-1c"] } ] disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" budgets = [ { nodes = "10%" } ] } }, #-------------------------------------------------------------- # Karpenter NodePool Infra #-------------------------------------------------------------- infra = { labels = { dedication = "infra" provisioning = "karpenter" } taints = [ { key = "dedicated" value = "infra" effect = "NoSchedule" } ] requirements = [ { key = "kubernetes.io/arch" operator = "In" values = ["amd64"] }, { key = "karpenter.sh/capacity-type" operator = "In" values = ["spot"] }, { key = "karpenter.k8s.aws/instance-family" operator = "In" values = ["t3a","m5","m5a","m6a","m6in","m7a", "m7i", "c6a", "c7a"] }, { key = "karpenter.k8s.aws/instance-size" operator = "In" values = ["large","xlarge","medium"] }, { key = "topology.kubernetes.io/zone" operator = "In" values: ["eu-central-1c"] } ] disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" budgets = [ { nodes = "10%" } ] } }, #-------------------------------------------------------------- # Karpenter NodePool Infra-ondemand #-------------------------------------------------------------- infra-ondemand = { labels = { dedication = "infra-ondemand" provisioning = "karpenter" } taints = [ { key = "dedicated" value = "infra-ondemand" effect = "NoSchedule" } ] requirements = [ { key = "kubernetes.io/arch" operator = "In" values = ["amd64"] }, { key = "karpenter.sh/capacity-type" operator = "In" values = ["on-demand"] }, { key = "karpenter.k8s.aws/instance-family" operator = "In" values = ["t3a", "t3"] }, { key = "karpenter.k8s.aws/instance-size" operator = "In" values = ["large","xlarge","medium"] }, { key = "topology.kubernetes.io/zone" operator = "In" values: ["eu-central-1c"] } ] disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" budgets = [ { nodes = "10%" } ] } } } } |
а вот модуль:
infra/terraform/modules/karpenter/karpenter.tf
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 |
locals { karpenter = merge(var.infra_apps[index(var.infra_apps.*.chart, "karpenter")], var.karpenter) karpenter_crd = merge(var.infra_apps[index(var.infra_apps.*.chart, "karpenter-crd")], var.karpenter_crd) karpernter_values = <<VALUES webhook: # -- Whether to enable the webhooks and webhook permissions. enabled: true serviceAccount: create: true name: "karpenter" annotations: eks.amazonaws.com/role-arn: "${aws_iam_role.karpenter_controller_iam_trust_role.arn}" serviceMonitor: enabled: true nodeSelector: ${indent(2, yamlencode(local.karpenter.node_selector))} tolerations: ${indent(2, yamlencode(local.karpenter.tolerations))} controller: env: - name: AWS_REGION value: "${var.region}" resources: ${indent(4, yamlencode(local.karpenter.resources))} logConfig: enabled: true outputPaths: - stdout errorOutputPaths: - stderr - stdout logEncoding: console logLevel: global: debug controller: debug webhook: error settings: clusterName: "${var.eks_cluster_id}" clusterEndpoint: "${var.eks_cluster_endpoint}" VALUES #-------------------------------------------------------------- # Karpenter EC2NodeClass #-------------------------------------------------------------- ec2nodeclass_values = <<VALUES apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: services-private spec: kubelet: systemReserved: cpu: "100m" memory: "100Mi" ephemeral-storage: "1Gi" kubeReserved: cpu: "200m" memory: "100Mi" ephemeral-storage: "3Gi" evictionMaxPodGracePeriod: 60 imageGCHighThresholdPercent: 85 imageGCLowThresholdPercent: 80 cpuCFSQuota: true maxPods: 110 amiFamily: AL2 # Amazon Linux 2 role: "KarpenterNodeRole-${var.eks_cluster_id}" subnetSelectorTerms: - tags: karpenter.sh/discovery: ${local.karpenter.subnets_tags_discovery} securityGroupSelectorTerms: - id: ${var.eks_node_security_group_id} tags: karpenter.sh/discovery: ${var.eks_cluster_id} blockDeviceMappings: - deviceName: /dev/xvda ebs: volumeSize: 100Gi volumeType: gp3 iops: 1000 encrypted: false deleteOnTermination: true throughput: 125 amiSelectorTerms: - tags: karpenter.sh/discovery: ${var.eks_cluster_id} - id: "${local.karpenter.instance_ami}" VALUES } resource "helm_release" "karpenter" { count = local.karpenter.enabled ? 1 : 0 name = local.karpenter.name chart = local.karpenter.chart repository = local.karpenter.repository version = local.karpenter.chart_version namespace = local.karpenter.namespace max_history = local.karpenter.max_history values = [local.karpernter_values] } resource "helm_release" "karpenter_crd" { count = local.karpenter_crd.enabled ? 1 : 0 name = local.karpenter_crd.name chart = local.karpenter_crd.chart repository = local.karpenter_crd.repository version = local.karpenter_crd.chart_version namespace = local.karpenter_crd.namespace max_history = local.karpenter_crd.max_history } resource "kubectl_manifest" "karpenter_ec2nodeclass_private" { yaml_body = local.ec2nodeclass_values wait_for_rollout = false apply_only = true depends_on = [helm_release.karpenter, helm_release.karpenter_crd] } resource "kubectl_manifest" "karpenter_nodepool" { for_each = var.nodepools wait_for_rollout = false apply_only = true yaml_body = yamlencode({ apiVersion : "karpenter.sh/v1" kind : "NodePool" metadata = { name = each.key } spec = { template = { metadata = { labels = each.value.labels } spec = { expireAfter = "Never" nodeClassRef = { name = "services-private" group = "karpenter.k8s.aws" kind = "EC2NodeClass" } taints = each.value.taints requirements = each.value.requirements } } disruption = each.value.disruption } }) depends_on = [helm_release.karpenter, helm_release.karpenter_crd] } #------------------------------------------------------------------ # IAM role and policies for Karpenter #------------------------------------------------------------------ resource "aws_iam_service_linked_role" "karpenter" { count = local.karpenter.service_role_enabled ? 1 : 0 description = "Default EC2 Spot Service Linked Role" aws_service_name = "spot.amazonaws.com" } resource "aws_iam_role" "karpenter_instance_profile_trust_role" { name = "KarpenterNodeRole-${var.eks_cluster_id}" description = "Policy for Karpenter-provisioned nodes" assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ { Effect = "Allow", Principal = { Service = "ec2.amazonaws.com" }, Action = "sts:AssumeRole" } ] }) } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_worker_node_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_eks_sni_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_ecr_read_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_role_policy_attachment" "karpenter_instance_profile_ssm_policy_attach" { policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" role = aws_iam_role.karpenter_instance_profile_trust_role.name } resource "aws_iam_instance_profile" "karpenter_instance_profile" { name = "KarpenterNodeInstanceProfile-${var.eks_cluster_id}" role = aws_iam_role.karpenter_instance_profile_trust_role.name tags = { Name = "KarpenterNodeInstanceProfile-${var.eks_cluster_id}" } } resource "aws_iam_role" "karpenter_controller_iam_trust_role" { name = "KarpenterControllerRole-${var.eks_cluster_id}" description = "Policy for Karpenter controller" assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ { Effect = "Allow", Principal = { Federated = var.oidc_provider_arn }, Action = "sts:AssumeRoleWithWebIdentity", Condition = { StringEquals = { "${var.oidc_provider}:aud" = "sts.amazonaws.com" "${var.oidc_provider}:sub" = "system:serviceaccount:karpenter:karpenter" } } } ] }) } resource "aws_iam_policy" "karpenter_controller_iam_policy" { name = "KarpenterControllerPolicy-${var.eks_cluster_id}" description = "IAM policy for Karpenter Controller" policy = jsonencode({ "Statement" : [ { "Action" : [ "ssm:GetParameter", "ec2:DescribeImages", "ec2:RunInstances", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", "ec2:DescribeLaunchTemplates", "ec2:DescribeInstances", "ec2:DescribeInstanceTypes", "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeAvailabilityZones", "ec2:DeleteLaunchTemplate", "ec2:CreateTags", "ec2:CreateLaunchTemplate", "ec2:CreateFleet", "ec2:DescribeSpotPriceHistory", "pricing:GetProducts", "iam:GetInstanceProfile", "iam:CreateInstanceProfile", "iam:TagInstanceProfile", "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", "iam:DeleteInstanceProfile", "iam:PassRole" ], "Effect" : "Allow", "Resource" : "*", "Sid" : "Karpenter" }, { "Action" : "ec2:TerminateInstances", "Condition" : { "StringLike" : { "ec2:ResourceTag/karpenter.sh/nodepool" : "*" } }, "Effect" : "Allow", "Resource" : "*", "Sid" : "ConditionalEC2Termination" }, { "Effect" : "Allow", "Action" : "iam:PassRole", "Resource" : "arn:aws:iam::${var.account_id}:role/KarpenterNodeRole-${var.eks_cluster_id}", "Sid" : "PassNodeIAMRole" }, { "Effect" : "Allow", "Action" : "eks:DescribeCluster", "Resource" : "arn:aws:eks:${var.region}:${var.account_id}:cluster/${var.eks_cluster_id}", "Sid" : "EKSClusterEndpointLookup" } ], "Version" : "2012-10-17" }) } resource "aws_iam_role_policy_attachment" "karpenter_controller_policy_attach" { policy_arn = aws_iam_policy.karpenter_controller_iam_policy.arn role = aws_iam_role.karpenter_controller_iam_trust_role.name } |
как видим kubelet перенесён в EC2NodeClass
можем аплаить, но при аплае возникнет следующая ошибка
|
1 2 3 |
kubectl logs -n karpenter karpenter-5f979dd8dd-w9jhg 2025/04/03 06:01:49 The kubelet compatibility annotation, compatibility.karpenter.sh/v1beta1-kubelet-conversion, is not supported on Karpenter v1.1+. Please refer to the upgrade guide in the docs. The following NodePools still have the compatibility annotation: infra, infra-ondemand, services |
она возникает из-за того что в nodepool в annotation есть v1beta
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
kubectl get nodepools.karpenter.sh infra -o yaml apiVersion: karpenter.sh/v1 kind: NodePool metadata: annotations: compatibility.karpenter.sh/v1beta1-kubelet-conversion: '{"maxPods":110,"systemReserved":{"cpu":"100m","ephemeral-storage":"1Gi","memory":"100Mi"},"kubeReserved":{"cpu":"200m","ephemeral-storage":"3Gi","memory":"100Mi"},"evictionMaxPodGracePeriod":60,"imageGCHighThresholdPercent":85,"imageGCLowThresholdPercent":80,"cpuCFSQuota":true}' compatibility.karpenter.sh/v1beta1-nodeclass-reference: '{"name":"services-private"}' karpenter.sh/nodepool-hash: "2820547658529482212" karpenter.sh/nodepool-hash-version: v3 karpenter.sh/stored-version-migrated: "true" kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"karpenter.sh/v1","kind":"NodePool","metadata":{"annotations":{},"name":"infra"},"spec":{"disruption":{"budgets":[{"nodes":"10%"}],"consolidationPolicy":"WhenEmptyOrUnderutilized"},"template":{"metadata":{"labels":{"dedication":"infra","provisioning":"karpenter"}},"spec":{"kubelet":{"cpuCFSQuota":true,"evictionMaxPodGracePeriod":60,"imageGCHighThresholdPercent":85,"imageGCLowThresholdPercent":80,"kubeReserved":{"cpu":"200m","ephemeral-storage":"3Gi","memory":"100Mi"},"maxPods":110,"systemReserved":{"cpu":"100m","ephemeral-storage":"1Gi","memory":"100Mi"}},"nodeClassRef":{"group":"karpenter.k8s.aws","kind":"EC2NodeClass","name":"services-private"},"requirements":[{"key":"kubernetes.io/arch","operator":"In","values":["amd64"]},{"key":"karpenter.sh/capacity-type","operator":"In","values":["spot"]},{"key":"karpenter.k8s.aws/instance-family","operator":"In","values":["t3a","m5","m5a","m6a","m6in","m7a","m7i","c6a","c7a"]},{"key":"karpenter.k8s.aws/instance-size","operator":"In","values":["large","xlarge","medium"]},{"key":"topology.kubernetes.io/zone","operator":"In","values":["eu-central-1b"]}],"taints":[{"effect":"NoSchedule","key":"dedicated","value":"infra"}]}}}} creationTimestamp: "2025-03-06T10:20:07Z" generation: 1 name: infra resourceVersion: "285709123" uid: eb337814-e3b5-4d02-a485-09cc3ebc6d10 |
можно поправить руками а можно использовать следующие команды
|
1 2 3 4 |
kubectl annotate nodepool infra compatibility.karpenter.sh/v1beta1-kubelet-conversion- compatibility.karpenter.sh/v1beta1-nodeclass-reference- karpenter.sh/stored-version-migrated- kubectl annotate nodepool infra-ondemand compatibility.karpenter.sh/v1beta1-kubelet-conversion- compatibility.karpenter.sh/v1beta1-nodeclass-reference- karpenter.sh/stored-version-migrated- kubectl annotate nodepool services compatibility.karpenter.sh/v1beta1-kubelet-conversion- compatibility.karpenter.sh/v1beta1-nodeclass-reference- karpenter.sh/stored-version-migrated- |
у меня 3 нод пула infra infra-ondemand services для всех них чистим аннотации и можем аплаить.
переехать сразу с 0.37 на 1.3.3 у меня не получилось, поймал следующую ошибку:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
kubectl logs -n karpenter karpenter-5f979dd8dd-74bpl {"level":"ERROR","time":"2025-04-03T05:47:54.674Z","logger":"controller","message":"failed to setup nodeclaim provider id indexer","commit":"ad71530","error":"no matches for kind \"NodeClaim\" in version \"karpenter.sh/v1\""} {"level":"ERROR","time":"2025-04-03T05:47:54.677Z","logger":"controller","message":"failed to setup nodeclaim nodeclassref apiversion indexer","commit":"ad71530","error":"no matches for kind \"NodeClaim\" in version \"karpenter.sh/v1\""} {"level":"ERROR","time":"2025-04-03T05:47:54.681Z","logger":"controller","message":"failed to setup nodeclaim nodeclassref kind indexer","commit":"ad71530","error":"no matches for kind \"NodeClaim\" in version \"karpenter.sh/v1\""} {"level":"ERROR","time":"2025-04-03T05:47:54.738Z","logger":"controller","message":"failed to setup nodeclaim nodeclassref name indexer","commit":"ad71530","error":"no matches for kind \"NodeClaim\" in version \"karpenter.sh/v1\""} {"level":"ERROR","time":"2025-04-03T05:47:54.741Z","logger":"controller","message":"failed to setup nodepool nodeclassref apiversion indexer","commit":"ad71530","error":"no matches for kind \"NodePool\" in version \"karpenter.sh/v1\""} {"level":"ERROR","time":"2025-04-03T05:47:54.745Z","logger":"controller","message":"failed to setup nodepool nodeclassref kind indexer","commit":"ad71530","error":"no matches for kind \"NodePool\" in version \"karpenter.sh/v1\""} {"level":"ERROR","time":"2025-04-03T05:47:54.750Z","logger":"controller","message":"failed to setup nodepool nodeclassref name indexer","commit":"ad71530","error":"no matches for kind \"NodePool\" in version \"karpenter.sh/v1\""} panic: unable to retrieve the complete list of server APIs: karpenter.sh/v1: no matches for karpenter.sh/v1, Resource= goroutine 1 [running]: github.com/samber/lo.must({0x3c72700, 0xc00009ea98}, {0x0, 0x0, 0x0}) github.com/samber/lo@v1.49.1/errors.go:53 +0x1df github.com/samber/lo.Must0(...) github.com/samber/lo@v1.49.1/errors.go:72 github.com/aws/karpenter-provider-aws/pkg/operator.NewOperator({0x46bf4b0, 0xc00056d9b0}, 0xc000b836c0) github.com/aws/karpenter-provider-aws/pkg/operator/operator.go:107 +0x147 main.main() github.com/aws/karpenter-provider-aws/cmd/controller/main.go:31 +0x2a |