Thank you for reading this post, don't forget to subscribe!
we have vault with backend = postgres and efs volume for plugin
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
resource "kubernetes_manifest" "vault-efs-vol-test" { manifest = yamldecode(<<EOF apiVersion: v1 kind: PersistentVolumeClaim metadata: name: vault-plugins-volume namespace: vault-test spec: accessModes: - ReadWriteMany storageClassName: efs-sc resources: requests: storage: 5Gi EOF ) provider = kubernetes } resource "helm_release" "vault_test" { name = "vault-test" repository = "https://helm.releases.hashicorp.com" chart = "vault" namespace = "vault-test" create_namespace = false version = local.settings.eks.vault_helm_version values = [<<EOF global: enabled: true injector: enabled: false server: image: repository: "1111111.dkr.ecr.eu-central-1.amazonaws.com/vault" tag: "1.12.3-bd428767" pullPolicy: Always updateStrategyType: "RollingUpdate" nodeSelector: dedication: "infra" tolerations: - key: "dedicated" operator: "Equal" value: "infra" effect: "NoSchedule" volumes: - name: plugins persistentVolumeClaim: claimName: vault-plugins-volume volumeMounts: - name: plugins mountPath: /usr/local/libexec/vault resources: requests: memory: 512Mi cpu: 500m limits: memory: 1024Mi cpu: 1000m ingress: enabled: true ingressClassName: "nginx-int" activeService: true pathType: Prefix hosts: - host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}" paths: - / tls: - secretName: "${local.vault_tls_secret_name}" hosts: - "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}" ha: enabled: true replicas: 1 config: | ui = true plugin_directory = "/usr/local/libexec/vault" listener "tcp" { tls_disable = 1 address = "[::]:8200" cluster_address = "[::]:8201" } storage "postgresql" { connection_url = "postgres://vault:fdgsdfkgh34098fsd@dev-db.chfnpsvsvxhn.eu-central-1.rds.amazonaws.com:5432/vault" ha_enabled = "true" table = "vault_kv_store" ha_table = "vault_ha_locks" } service_registration "kubernetes" {} seal "awskms" { region = "${local.settings.vpc.region}" kms_key_id = "${aws_kms_key.vault_test.id}" } serviceAccount: name: "vault-test-sa" annotations: eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}" ui: enabled: true EOF ] # disable_webhooks = "true" provider = helm } |
kubectl exec -ti -n vault-test vault-test-0 sh
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
/ $ vault status Key Value --- ----- Recovery Seal Type shamir Initialized true Sealed false Total Recovery Shares 5 Threshold 3 Version 1.12.3 Build Date 2023-02-02T09:07:27Z Storage Type postgresql Cluster Name vault-cluster-489a2d88 Cluster ID eb2d635d-d10f-7b7c-2465-2ca0927aaa13 HA Enabled true HA Cluster https://vault-test-0.vault-test-internal:8201 HA Mode active Active Since 2024-05-08T06:10:27.536091234Z |
lets check secrets in vault UI
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
resource "helm_release" "vault_test" { name = "vault-test" repository = "https://helm.releases.hashicorp.com" chart = "vault" version = local.settings.eks.vault_helm_version namespace = "vault-test" create_namespace = false values = [<<EOF global: enabled: true serverTelemetry: prometheusOperator: true injector: enabled: false server: image: repository: "1111111111.dkr.ecr.eu-central-1.amazonaws.com/vault" tag: "1.12.3-bd428767" pullPolicy: Always updateStrategyType: "RollingUpdate" nodeSelector: dedication: "infra" tolerations: - key: "dedicated" operator: "Equal" value: "infra" effect: "NoSchedule" resources: requests: memory: 512Mi cpu: 500m limits: memory: 1024Mi cpu: 1000m ingress: enabled: true ingressClassName: "nginx-int" pathType: Prefix activeService: true hosts: - host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}" paths: - / tls: - secretName: "${local.vault_tls_secret_name}" hosts: - "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}" volumes: - name: plugins persistentVolumeClaim: claimName: vault-plugins-volume volumeMounts: - name: plugins mountPath: /usr/local/libexec/vault dataStorage: size: 10Gi storageClass: gp3 auditStorage: enabled: true size: 10Gi storageClass: gp3 standalone: enabled: false ha: enabled: true replicas: 3 raft: enabled: true setNodeId: true config: | ui = true plugin_directory = "/usr/local/libexec/vault" telemetry { unauthenticated_metrics_access = "true" prometheus_retention_time = "24h" } listener "tcp" { tls_disable = 1 address = "[::]:8200" cluster_address = "[::]:8201" # Enable unauthenticated metrics access (necessary for Prometheus Operator) telemetry { unauthenticated_metrics_access = "true" } } storage "raft" { path = "/vault/data" retry_join { leader_api_addr = "http://vault-test-0.vault-test-internal:8200" } retry_join { leader_api_addr = "http://vault-test-1.vault-test-internal:8200" } retry_join { leader_api_addr = "http://vault-test-2.vault-test-internal:8200" } } service_registration "kubernetes" {} seal "awskms" { region = "${local.settings.vpc.region}" kms_key_id = "${aws_kms_key.vault_test.id}" } serviceAccount: name: "vault-test-sa" annotations: eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}" serverTelemetry: serviceMonitor: enabled: true interval: 30s selectors: release: kube-prometheus-stack ui: enabled: true EOF ] provider = helm } |
delete helm chart
helm delete -n vault-test vault-test
apply new chart with raft
terraform apply --target helm_release.vault_test
Wait until Terraform apply does not timeout.
1 2 3 4 5 6 7 |
helm_release.vault_test: Still creating… [5m30s elapsed] │ Error: context deadline exceeded │ │ with helm_release.vault_test, │ on vault.tf line 409, in resource "helm_release" "vault_test": │ 409: resource "helm_release" "vault_test" { │ |
check and we will see that vault pod run but not ready
1 2 3 4 5 |
kubectl get pod -n vault-test NAME READY STATUS RESTARTS AGE vault-test-0 0/1 Running 0 4m48s vault-test-1 0/1 Running 0 4m48s vault-test-2 0/1 Running 0 4m48s |
go to the zero pod
1 2 3 4 |
kubectl exec -ti -n vault-test vault-test-0 sh / $ cd /vault/data /vault/data $ |
create migration file
cat > migrate.hcl
1 2 3 4 5 6 7 8 9 10 11 |
storage_source "postgresql" { connection_url = "postgres://vault:fdgsdfkgh34098fsd@dev-db.chfnpsvsvxhn.eu-central-1.rds.amazonaws.com:5432/vault" } storage_destination "raft" { path = "/vault/data/" node_id = "vault-test-0" } cluster_addr = "https://vault-test-0.vault-test-internal:8201" |
delete directory raft and file vault.db
1 2 3 4 5 6 7 8 9 10 11 12 |
/vault/data $ ls -lah total 48K drwxrwsr-x 4 root vault 4.0K May 8 07:11 . drwxr-xr-x 1 vault vault 31 May 8 06:35 .. drwxrws--- 2 root vault 16.0K May 8 06:35 lost+found -rw-r--r-- 1 vault vault 299 May 8 07:11 migrate.hcl drwx--S--- 3 vault vault 4.0K May 8 06:35 raft -rw------- 1 vault vault 20.0K May 8 07:11 vault.db /vault/data $ rm -rf raft/ vault.db |
run migration
vault operator migrate -config=migrate.hcl
if we see message like this:
1 2 3 4 5 6 7 8 9 |
2024-05-08T07:13:55.697Z [INFO] copied key: path=sys/policy/control-group 2024-05-08T07:13:55.705Z [INFO] copied key: path=sys/policy/default 2024-05-08T07:13:55.712Z [INFO] copied key: path=sys/policy/response-wrapping 2024-05-08T07:13:55.718Z [INFO] copied key: path=sys/policy/test-policy 2024-05-08T07:13:55.729Z [INFO] copied key: path=sys/token/accessor/b4e0e965a8be25e557588afa3445db8de8adaad7 2024-05-08T07:13:55.737Z [INFO] copied key: path=sys/token/id/h0bd87b407b9f953455fe304257cf522e9184311cd5733f4732739ffb282606c0 2024-05-08T07:13:55.742Z [INFO] copied key: path=sys/token/salt Success! All of the keys have been migrated. |
all ok.
now we will run terraform apply again
terraform apply --target helm_release.vault_test
1 2 |
Plan: 1 to add, 0 to change, 1 to destroy. |
say yes
helm chart will be delete and after create again
1 |
Apply complete! Resources: 1 added, 0 changed, 1 destroyed. |
check vault with raft
1 2 3 4 5 |
kubectl get pod -n vault-test NAME READY STATUS RESTARTS AGE vault-test-0 1/1 Running 0 2m22s vault-test-1 1/1 Running 0 2m22s vault-test-2 1/1 Running 0 2m22s |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
kubectl exec -ti -n vault-test vault-test-0 sh / $ vault status Key Value --- ----- Recovery Seal Type shamir Initialized true Sealed false Total Recovery Shares 5 Threshold 3 Version 1.12.3 Build Date 2023-02-02T09:07:27Z Storage Type raft Cluster Name vault-cluster-489a2d88 Cluster ID eb2d635d-d10f-7b7c-2465-2ca0927aaa13 HA Enabled true HA Cluster https://vault-test-0.vault-test-internal:8201 HA Mode active Active Since 2024-05-08T07:18:04.343169233Z Raft Committed Index 58 Raft Applied Index 58 / $ / $ vault login / $ vault operator raft list-peers Node Address State Voter ---- ------- ----- ----- vault-test-0 vault-test-0.vault-test-internal:8201 leader true vault-test-1 vault-test-1.vault-test-internal:8201 follower true vault-test-2 vault-test-2.vault-test-internal:8201 follower true / $ vault operator members Host Name API Address Cluster Address Active Node Version Upgrade Version Redundancy Zone Last Echo --------- ----------- --------------- ----------- ------- --------------- --------------- --------- vault-test-0 http://10.117.1.147:8200 https://vault-test-0.vault-test-internal:8201 true 1.12.3 1.12.3 n/a n/a vault-test-1 http://10.117.47.234:8200 https://vault-test-1.vault-test-internal:8201 false 1.12.3 1.12.3 n/a 2024-05-08T07:23:06Z vault-test-2 http://10.117.6.211:8200 https://vault-test-2.vault-test-internal:8201 false 1.12.3 1.12.3 n/a 2024-05-08T07:23:07Z / $ |
block zero pod
1 2 3 |
/ $ vault operator seal Success! Vault is sealed. |
check from another pod
kubectl exec -ti -n vault-test vault-test-2 sh
/ $ vault login
as we see vault-0 disable and leader bacome vault-1
1 2 3 4 5 6 7 8 9 10 11 12 13 |
/ $ vault operator raft list-peers Node Address State Voter ---- ------- ----- ----- vault-test-0 vault-test-0.vault-test-internal:8201 follower true vault-test-1 vault-test-1.vault-test-internal:8201 leader true vault-test-2 vault-test-2.vault-test-internal:8201 follower true / $ / $ vault operator members Host Name API Address Cluster Address Active Node Version Upgrade Version Redundancy Zone Last Echo --------- ----------- --------------- ----------- ------- --------------- --------------- --------- vault-test-1 http://10.117.47.234:8200 https://vault-test-1.vault-test-internal:8201 true 1.12.3 1.12.3 n/a n/a vault-test-2 http://10.117.6.211:8200 https://vault-test-2.vault-test-internal:8201 false 1.12.3 1.12.3 n/a 2024-05-08T08:04:45Z |
now check vault UI
ok now we will return vault-0
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
kubectl get pod -n vault-test NAME READY STATUS RESTARTS AGE vault-test-0 0/1 Running 0 51m vault-test-1 1/1 Running 0 51m vault-test-2 1/1 Running 0 51m kubectl delete pod -n vault-test vault-test-0 pod "vault-test-0" deleted kubectl get pod -n vault-test NAME READY STATUS RESTARTS AGE vault-test-0 1/1 Running 0 38s vault-test-1 1/1 Running 0 52m vault-test-2 1/1 Running 0 52m |