HOW TO migrate vault backend from file to raft

Thank you for reading this post, don't forget to subscribe!

есть задача перетащить vault с бэкенда где используется file на бекенд где используется raft

we have vault with some data:

secrets,

access

policies

this vault install with backend file

storage "file" {
        path = "/vault/data"
      }

storage "file" {

path = "/vault/data"

}

full resource

resource "helm_release" "vault_test" {
  name       = "vault-test"
  repository = "https://helm.releases.hashicorp.com"
  chart      = "vault"
  version    = local.settings.eks.vault_helm_version

  namespace        = "vault-test"
  create_namespace = false

  values = [<<EOF
global:
  enabled: true
  serverTelemetry:
    prometheusOperator: true

injector:
  enabled: false

server:
  resources:
    requests:
      memory: 512Mi
      cpu: 500m
    limits:
      memory: 1024Mi
      cpu: 1000m

  nodeSelector:
    dedication: "infra"
  tolerations:
    - key: "dedicated"
      operator: "Equal"
      value: "infra"
      effect: "NoSchedule"

  ingress:
    enabled: true
    ingressClassName: "nginx-int"
    pathType: Prefix
    activeService: true
    hosts:
      - host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"
        paths:
          - /
    tls:
      - secretName: "${local.vault_tls_secret_name}"
        hosts:
          - "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

  dataStorage:
    size: 10Gi
    storageClass: gp3

  auditStorage:
    enabled: true
    size: 10Gi
    storageClass: gp3

  standalone:
    enabled: true

    config: |
      ui = true

      telemetry {
        unauthenticated_metrics_access = "true"
        prometheus_retention_time = "24h"
      }

      listener "tcp" {
        tls_disable = 1
        address = "[::]:8200"
        cluster_address = "[::]:8201"
        # Enable unauthenticated metrics access (necessary for Prometheus Operator)
        telemetry {
         unauthenticated_metrics_access = "true"
        }
      }
      storage "file" {
        path = "/vault/data"
      }

      seal "awskms" {
        region     = "${local.settings.vpc.region}"
        kms_key_id = "${aws_kms_key.vault_test.id}"
      }

  serviceAccount:
    name: "vault-test-sa"
    annotations:
      eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

serverTelemetry:
  serviceMonitor:
    enabled: true
    interval: 30s
    selectors:
      release: kube-prometheus-stack    

ui:
  enabled: true

  EOF
  ]

  provider = helm
}

100

101

102

103

104

105

106

107

resource "helm_release" "vault_test" {

name = "vault-test"

repository = "https://helm.releases.hashicorp.com"

chart = "vault"

version = local.settings.eks.vault_helm_version

namespace = "vault-test"

create_namespace = false

values = [<<EOF

global:

enabled: true

serverTelemetry:

prometheusOperator: true

injector:

enabled: false

server:

resources:

requests:

memory: 512Mi

cpu: 500m

limits:

memory: 1024Mi

cpu: 1000m

nodeSelector:

dedication: "infra"

tolerations:

- key: "dedicated"

operator: "Equal"

value: "infra"

effect: "NoSchedule"

ingress:

enabled: true

ingressClassName: "nginx-int"

pathType: Prefix

activeService: true

hosts:

- host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

paths:

- /

tls:

- secretName: "${local.vault_tls_secret_name}"

hosts:

- "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

dataStorage:

size: 10Gi

storageClass: gp3

auditStorage:

enabled: true

size: 10Gi

storageClass: gp3

standalone:

enabled: true

config: |

ui = true

telemetry {

unauthenticated_metrics_access = "true"

prometheus_retention_time = "24h"

}

listener "tcp" {

tls_disable = 1

address = "[::]:8200"

cluster_address = "[::]:8201"

# Enable unauthenticated metrics access (necessary for Prometheus Operator)

telemetry {

unauthenticated_metrics_access = "true"

}

storage "file" {

path = "/vault/data"

}

seal "awskms" {

region = "${local.settings.vpc.region}"

kms_key_id = "${aws_kms_key.vault_test.id}"

}

serviceAccount:

name: "vault-test-sa"

annotations:

eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

serverTelemetry:

serviceMonitor:

enabled: true

interval: 30s

selectors:

release: kube-prometheus-stack

ui:

enabled: true

EOF

]

provider = helm

}

we will migrate it to raft

kubectl exec -ti -n vault-test vault-test-0 sh

check hostname

hostname
vault-test-0

check name of svc

cat /etc/hosts
# Kubernetes-managed hosts file.
127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
fe00::0 ip6-mcastprefix
fe00::1 ip6-allnodes
fe00::2 ip6-allrouters
10.117.45.80    vault-test-0.vault-test-internal.vault-test.svc.cluster.local   vault-test-0

cat /etc/hosts

# Kubernetes-managed hosts file.

127.0.0.1 localhost

::1 localhost ip6-localhost ip6-loopback

fe00::0 ip6-localnet

fe00::0 ip6-mcastprefix

fe00::1 ip6-allnodes

fe00::2 ip6-allrouters

10.117.45.80 vault-test-0.vault-test-internal.vault-test.svc.cluster.local vault-test-0

we need this vault-test-0.vault-test-internal

go to PV

cd /vault/data
/vault/data $

create file for migration

/vault/data $ cat > migrate.hcl

storage_source "file" {
path = "/vault/data/"
}

storage_destination "raft" {
path = "/vault/data/"
node_id = "vault-test-0"
}
cluster_addr = "https://vault-test-0.vault-test-internal:8201"

/vault/data $ cat > migrate.hcl

storage_source "file" {

path = "/vault/data/"

}

storage_destination "raft" {

path = "/vault/data/"

node_id = "vault-test-0"

}

cluster_addr = "https://vault-test-0.vault-test-internal:8201"

node_id - it is our hostname
cluster_addr - it is our svc

run migration

/vault/data $ vault operator migrate -config=migrate.hcl

if all messages look like INFO and in the end we see:
Success! All of the keys have been migrated.

it means that all OK and we can continue

NOW we apply helm chart with raft backend

        storage "raft" {
          path = "/vault/data"
          retry_join {
            leader_api_addr = "http://vault-test-0.vault-test-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-test-1.vault-test-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-test-2.vault-test-internal:8200"
          }
        }

storage "raft" {

path = "/vault/data"

retry_join {

leader_api_addr = "http://vault-test-0.vault-test-internal:8200"

}

retry_join {

leader_api_addr = "http://vault-test-1.vault-test-internal:8200"

}

retry_join {

leader_api_addr = "http://vault-test-2.vault-test-internal:8200"

}

full resource

resource "helm_release" "vault_test" {
  name       = "vault-test"
  repository = "https://helm.releases.hashicorp.com"
  chart      = "vault"
  version    = local.settings.eks.vault_helm_version

  namespace        = "vault-test"
  create_namespace = false

  values = [<<EOF
global:
  enabled: true
  serverTelemetry:
    prometheusOperator: true

injector:
  enabled: false

server:
  resources:
    requests:
      memory: 512Mi
      cpu: 500m
    limits:
      memory: 1024Mi
      cpu: 1000m

  nodeSelector:
    dedication: "infra"
  tolerations:
    - key: "dedicated"
      operator: "Equal"
      value: "infra"
      effect: "NoSchedule"

  ingress:
    enabled: true
    ingressClassName: "nginx-int"
    pathType: Prefix
    activeService: true
    hosts:
      - host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"
        paths:
          - /
    tls:
      - secretName: "${local.vault_tls_secret_name}"
        hosts:
          - "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

  dataStorage:
    size: 10Gi
    storageClass: gp3

  auditStorage:
    enabled: true
    size: 10Gi
    storageClass: gp3

  logLevel: "debug"

  standalone:
    enabled: false

  ha:
    enabled: true
    replicas: 3

    raft:
      enabled: true
      setNodeId: true

      config: |
        ui = true
        plugin_directory = "/usr/local/libexec/vault"

        telemetry {
          # unauthenticated_metrics_access = "true"
          prometheus_retention_time = "24h"
        }

        listener "tcp" {
          tls_disable = "true"
          address = "[::]:8200"
          cluster_address = "[::]:8201"
          # Enable unauthenticated metrics access (necessary for Prometheus Operator)
          telemetry {
            unauthenticated_metrics_access = "true"
          }
        }

        storage "raft" {
          path = "/vault/data"
          retry_join {
            leader_api_addr = "http://vault-test-0.vault-test-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-test-1.vault-test-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-test-2.vault-test-internal:8200"
          }
        }

        service_registration "kubernetes" {}

        seal "awskms" {
          region     = "${local.settings.vpc.region}"
          kms_key_id = "${aws_kms_key.vault_test.id}"
        }

  serviceAccount:
    name: "vault-test-sa"
    annotations:
      eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

serverTelemetry:
  serviceMonitor:
    enabled: true
    interval: 30s
    selectors:
      release: kube-prometheus-stack    

ui:
  enabled: true

  EOF
  ]

  provider = helm
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

resource "helm_release" "vault_test" {

name = "vault-test"

repository = "https://helm.releases.hashicorp.com"

chart = "vault"

version = local.settings.eks.vault_helm_version

namespace = "vault-test"

create_namespace = false

values = [<<EOF

global:

enabled: true

serverTelemetry:

prometheusOperator: true

injector:

enabled: false

server:

resources:

requests:

memory: 512Mi

cpu: 500m

limits:

memory: 1024Mi

cpu: 1000m

nodeSelector:

dedication: "infra"

tolerations:

- key: "dedicated"

operator: "Equal"

value: "infra"

effect: "NoSchedule"

ingress:

enabled: true

ingressClassName: "nginx-int"

pathType: Prefix

activeService: true

hosts:

- host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

paths:

- /

tls:

- secretName: "${local.vault_tls_secret_name}"

hosts:

- "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

dataStorage:

size: 10Gi

storageClass: gp3

auditStorage:

enabled: true

size: 10Gi

storageClass: gp3

logLevel: "debug"

standalone:

enabled: false

ha:

enabled: true

replicas: 3

raft:

enabled: true

setNodeId: true

config: |

ui = true

plugin_directory = "/usr/local/libexec/vault"

telemetry {

# unauthenticated_metrics_access = "true"

prometheus_retention_time = "24h"

}

listener "tcp" {

tls_disable = "true"

address = "[::]:8200"

cluster_address = "[::]:8201"

# Enable unauthenticated metrics access (necessary for Prometheus Operator)

telemetry {

unauthenticated_metrics_access = "true"

}

storage "raft" {

path = "/vault/data"

retry_join {

leader_api_addr = "http://vault-test-0.vault-test-internal:8200"

}

retry_join {

leader_api_addr = "http://vault-test-1.vault-test-internal:8200"

}

retry_join {

leader_api_addr = "http://vault-test-2.vault-test-internal:8200"

}

service_registration "kubernetes" {}

seal "awskms" {

region = "${local.settings.vpc.region}"

kms_key_id = "${aws_kms_key.vault_test.id}"

}

serviceAccount:

name: "vault-test-sa"

annotations:

eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

serverTelemetry:

serviceMonitor:

enabled: true

interval: 30s

selectors:

release: kube-prometheus-stack

ui:

enabled: true

EOF

]

provider = helm

}

terraform apply --target helm_release.vault_test

check installation

kubectl get pod -n vault-test
NAME           READY   STATUS    RESTARTS   AGE
vault-test-0   1/1     Running   0          30m
vault-test-1   0/1     Running   0          22s
vault-test-2   0/1     Running   0          22s

kubectl get pod -n vault-test

NAME READY STATUS RESTARTS AGE

vault-test-0 1/1 Running 0 30m

vault-test-1 0/1 Running 0 22s

vault-test-2 0/1 Running 0 22s

as we see pods vault-test-1 vault-test-2 not ready, check it:

kubectl logs -n vault-test vault-test-2
2024-05-03T06:46:22.415Z [INFO]  core: security barrier not initialized
2024-05-03T06:46:22.418Z [INFO]  core: attempting to join possible raft leader node: leader_addr=http://vault-test-0.vault-test-internal:8200
2024-05-03T06:46:22.418Z [INFO]  core: attempting to join possible raft leader node: leader_addr=http://vault-test-1.vault-test-internal:8200
2024-05-03T06:46:22.418Z [INFO]  core: attempting to join possible raft leader node: leader_addr=http://vault-test-2.vault-test-internal:8200
2024-05-03T06:46:22.421Z [ERROR] core: failed to get raft challenge: leader_addr=http://vault-test-1.vault-test-internal:8200
  error=
  | error during raft bootstrap init call: Error making API request.
  | 
  | URL: PUT http://vault-test-1.vault-test-internal:8200/v1/sys/storage/raft/bootstrap/challenge
  | Code: 503. Errors:
  | 
  | * Vault is sealed
  
2024-05-03T06:46:22.423Z [ERROR] core: failed to get raft challenge: leader_addr=http://vault-test-0.vault-test-internal:8200
  error=
  | error during raft bootstrap init call: Error making API request.
  | 
  | URL: PUT http://vault-test-0.vault-test-internal:8200/v1/sys/storage/raft/bootstrap/challenge
  | Code: 404. Errors:
  | 
  | * unsupported path

kubectl logs -n vault-test vault-test-2

2024-05-03T06:46:22.415Z [INFO] core: security barrier not initialized

2024-05-03T06:46:22.418Z [INFO] core: attempting to join possible raft leader node: leader_addr=http://vault-test-0.vault-test-internal:8200

2024-05-03T06:46:22.418Z [INFO] core: attempting to join possible raft leader node: leader_addr=http://vault-test-1.vault-test-internal:8200

2024-05-03T06:46:22.418Z [INFO] core: attempting to join possible raft leader node: leader_addr=http://vault-test-2.vault-test-internal:8200

2024-05-03T06:46:22.421Z [ERROR] core: failed to get raft challenge: leader_addr=http://vault-test-1.vault-test-internal:8200

error=

| error during raft bootstrap init call: Error making API request.

| URL: PUT http://vault-test-1.vault-test-internal:8200/v1/sys/storage/raft/bootstrap/challenge

| Code: 503. Errors:

| * Vault is sealed

2024-05-03T06:46:22.423Z [ERROR] core: failed to get raft challenge: leader_addr=http://vault-test-0.vault-test-internal:8200

error=

| error during raft bootstrap init call: Error making API request.

| URL: PUT http://vault-test-0.vault-test-internal:8200/v1/sys/storage/raft/bootstrap/challenge

| Code: 404. Errors:

| * unsupported path

it is normal - now check vault vault-test-0

kubectl exec -ti -n vault-test vault-test-0 sh
vault status
Key                      Value
---                      -----
Recovery Seal Type       shamir
Initialized              true
Sealed                   false
Total Recovery Shares    5
Threshold                3
Version                  1.13.1
Build Date               2023-03-23T12:51:35Z
Storage Type             file
Cluster Name             vault-cluster-6884ef67
Cluster ID               830fd2b2-12ef-6c31-29f7-7ca33a0eb9db
HA Enabled               false

kubectl exec -ti -n vault-test vault-test-0 sh

vault status

Key Value

--- -----

Recovery Seal Type shamir

Initialized true

Sealed false

Total Recovery Shares 5

Threshold 3

Version 1.13.1

Build Date 2023-03-23T12:51:35Z

Storage Type file

Cluster Name vault-cluster-6884ef67

Cluster ID 830fd2b2-12ef-6c31-29f7-7ca33a0eb9db

HA Enabled false

As we see this vault still uses a file backend.

Storage Type file

we need to update this pod

kubectl delete pod -n vault-test vault-test-0
pod "vault-test-0" deleted

wait several seconds and check

kubectl get pod -n vault-test
NAME           READY   STATUS    RESTARTS   AGE
vault-test-0   1/1     Running   0          61s
vault-test-1   1/1     Running   0          7m13s
vault-test-2   1/1     Running   0          7m13s

kubectl get pod -n vault-test

NAME READY STATUS RESTARTS AGE

vault-test-0 1/1 Running 0 61s

vault-test-1 1/1 Running 0 7m13s

vault-test-2 1/1 Running 0 7m13s

kubectl exec -ti -n vault-test vault-test-0 sh
/ $ vault status
Key                      Value
---                      -----
Recovery Seal Type       shamir
Initialized              true
Sealed                   false
Total Recovery Shares    5
Threshold                3
Version                  1.13.1
Build Date               2023-03-23T12:51:35Z
Storage Type             raft
Cluster Name             vault-cluster-6884ef67
Cluster ID               830fd2b2-12ef-6c31-29f7-7ca33a0eb9db
HA Enabled               true
HA Cluster               https://vault-test-0.vault-test-internal:8201
HA Mode                  active
Active Since             2024-05-03T06:52:26.731944925Z
Raft Committed Index     50
Raft Applied Index       50

kubectl exec -ti -n vault-test vault-test-0 sh

/ $ vault status

Key Value

--- -----

Recovery Seal Type shamir

Initialized true

Sealed false

Total Recovery Shares 5

Threshold 3

Version 1.13.1

Build Date 2023-03-23T12:51:35Z

Storage Type raft

Cluster Name vault-cluster-6884ef67

Cluster ID 830fd2b2-12ef-6c31-29f7-7ca33a0eb9db

HA Enabled true

HA Cluster https://vault-test-0.vault-test-internal:8201

HA Mode active

Active Since 2024-05-03T06:52:26.731944925Z

Raft Committed Index 50

Raft Applied Index 50

/ $ vault login
Token (will be hidden): 
Success! You are now authenticated. The token information displayed below
is already stored in the token helper. You do NOT need to run "vault login"
again. Future Vault requests will automatically use this token.
Key                  Value
---                  -----
token                hvs.I7QlNzu30v8sMYRqniFtrELt
token_accessor       xe8EJ4bW4CExvjO6vTMa8m5z
token_duration       ∞
token_renewable      false
token_policies       ["root"]
identity_policies    []
policies             ["root"]

/ $ vault login

Token (will be hidden):

Success! You are now authenticated. The token information displayed below

is already stored in the token helper. You do NOT need to run "vault login"

again. Future Vault requests will automatically use this token.

Key Value

--- -----

token hvs.I7QlNzu30v8sMYRqniFtrELt

token_accessor xe8EJ4bW4CExvjO6vTMa8m5z

token_duration ∞

token_renewable false

token_policies ["root"]

identity_policies []

policies ["root"]

/ $ vault operator raft list-peers
Node            Address                                  State       Voter
----            -------                                  -----       -----
vault-test-0    vault-test-0.vault-test-internal:8201    leader      true
vault-test-1    vault-test-1.vault-test-internal:8201    follower    true
vault-test-2    vault-test-2.vault-test-internal:8201    follower    true

/ $ vault operator raft list-peers

Node Address State Voter

---- ------- ----- -----

vault-test-0 vault-test-0.vault-test-internal:8201 leader true

vault-test-1 vault-test-1.vault-test-internal:8201 follower true

vault-test-2 vault-test-2.vault-test-internal:8201 follower true

as we see now uses raft
Storage Type raft

and all PODS in cluster vault-test-0 = leader

Let's check what will happen if we seal vault-test-0.

/ $ vault operator seal
Success! Vault is sealed.


/ $ vault operator raft list-peers
Error reading the raft cluster configuration: Error making API request.

URL: GET http://127.0.0.1:8200/v1/sys/storage/raft/configuration
Code: 503. Errors:

* Vault is sealed

/ $ vault operator seal

Success! Vault is sealed.

/ $ vault operator raft list-peers

Error reading the raft cluster configuration: Error making API request.

URL: GET http://127.0.0.1:8200/v1/sys/storage/raft/configuration

Code: 503. Errors:

* Vault is sealed

enter to another pod

kubectl get pod -n vault-test
NAME           READY   STATUS    RESTARTS   AGE
vault-test-0   0/1     Running   0          9m17s
vault-test-1   1/1     Running   0          15m
vault-test-2   1/1     Running   0          15m


kubectl exec -ti -n vault-test vault-test-1 sh
/ $ vault login
/ $ vault operator raft list-peers
Node            Address                                  State       Voter
----            -------                                  -----       -----
vault-test-0    vault-test-0.vault-test-internal:8201    follower    true
vault-test-1    vault-test-1.vault-test-internal:8201    leader      true
vault-test-2    vault-test-2.vault-test-internal:8201    follower    true

kubectl get pod -n vault-test

NAME READY STATUS RESTARTS AGE

vault-test-0 0/1 Running 0 9m17s

vault-test-1 1/1 Running 0 15m

vault-test-2 1/1 Running 0 15m

kubectl exec -ti -n vault-test vault-test-1 sh

/ $ vault login

/ $ vault operator raft list-peers

Node Address State Voter

---- ------- ----- -----

vault-test-0 vault-test-0.vault-test-internal:8201 follower true

vault-test-1 vault-test-1.vault-test-internal:8201 leader true

vault-test-2 vault-test-2.vault-test-internal:8201 follower true

now check data in vault UI

as we see all OK, vault-test-0 sealed and all data accessable from vault-test-1 vault-test-2

After our check, we can unseal vault-test-0, or delete this pod, and it will start and unseal automatically.

Centos

HOW TO migrate vault backend from file to raft

NOW we apply helm chart with raft backend

https://github.com/midnight47/

Май 2024
Пн	Вт	Ср	Чт	Пт	Сб	Вс
		1	2	3	4	5
6	7	8	9	10	11	12
13	14	15	16	17	18	19
20	21	22	23	24	25	26
27	28	29	30	31