HOW TO migrate vault backend file to postgres

Thank you for reading this post, don't forget to subscribe!

we have vault with backend file

resource "helm_release" "vault_test" {
  name             = "vault-test"
  repository       = "https://helm.releases.hashicorp.com"
  chart            = "vault"
  version          = local.settings.eks.vault_helm_version
  namespace        = "vault-test"
  create_namespace = false

  values = [<<EOF
global:
  enabled: true
  serverTelemetry:
    prometheusOperator: true

injector:
  enabled: false

server:
  resources:
    requests:
      memory: 512Mi
      cpu: 500m
    limits:
      memory: 1024Mi
      cpu: 1000m

  ingress:
    enabled: true
    ingressClassName: "nginx-int"
    pathType: Prefix
    activeService: true
    hosts:
      - host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"
        paths:
          - /
    tls:
      - secretName: "${local.vault_tls_secret_name}"
        hosts:
          - "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

  dataStorage:
    size: 10Gi
    storageClass: gp3

  auditStorage:
    enabled: true
    size: 10Gi
    storageClass: gp3

  standalone:
    enabled: true

    config: |
      ui = true

      telemetry {
        unauthenticated_metrics_access = "true"
        prometheus_retention_time = "24h"
      }

      listener "tcp" {
        tls_disable = 1
        address = "[::]:8200"
        cluster_address = "[::]:8201"
        # Enable unauthenticated metrics access (necessary for Prometheus Operator)
        telemetry {
         unauthenticated_metrics_access = "true"
        }
      }
      storage "file" {
        path = "/vault/data"
      }

      seal "awskms" {
        region     = "${local.settings.vpc.region}"
        kms_key_id = "${aws_kms_key.vault_test.id}"
      }

  serviceAccount:
    name: "vault-test-sa"
    annotations:
      eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"


serverTelemetry:
  serviceMonitor:
    enabled: true
    interval: 30s
    selectors:
      release: kube-prometheus-stack    

ui:
  enabled: true
  EOF
  ]

  provider = helm
}

resource "helm_release" "vault_test" {

name = "vault-test"

repository = "https://helm.releases.hashicorp.com"

chart = "vault"

version = local.settings.eks.vault_helm_version

namespace = "vault-test"

create_namespace = false

values = [<<EOF

global:

enabled: true

serverTelemetry:

prometheusOperator: true

injector:

enabled: false

server:

resources:

requests:

memory: 512Mi

cpu: 500m

limits:

memory: 1024Mi

cpu: 1000m

ingress:

enabled: true

ingressClassName: "nginx-int"

pathType: Prefix

activeService: true

hosts:

- host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

paths:

- /

tls:

- secretName: "${local.vault_tls_secret_name}"

hosts:

- "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

dataStorage:

size: 10Gi

storageClass: gp3

auditStorage:

enabled: true

size: 10Gi

storageClass: gp3

standalone:

enabled: true

config: |

ui = true

telemetry {

unauthenticated_metrics_access = "true"

prometheus_retention_time = "24h"

}

listener "tcp" {

tls_disable = 1

address = "[::]:8200"

cluster_address = "[::]:8201"

# Enable unauthenticated metrics access (necessary for Prometheus Operator)

telemetry {

unauthenticated_metrics_access = "true"

}

storage "file" {

path = "/vault/data"

}

seal "awskms" {

region = "${local.settings.vpc.region}"

kms_key_id = "${aws_kms_key.vault_test.id}"

}

serviceAccount:

name: "vault-test-sa"

annotations:

eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

serverTelemetry:

serviceMonitor:

enabled: true

interval: 30s

selectors:

release: kube-prometheus-stack

ui:

enabled: true

EOF

]

provider = helm

}

kubectl exec -ti -n vault-test vault-test-0  sh

/ $ vault status
Key                      Value
---                      -----
Recovery Seal Type       shamir
Initialized              true
Sealed                   false
Total Recovery Shares    5
Threshold                3
Version                  1.13.1
Build Date               2023-03-23T12:51:35Z
Storage Type             file
Cluster Name             vault-cluster-3bc45695
Cluster ID               5e0ccb87-18c5-4042-8763-c82512a80339
HA Enabled               false

kubectl exec -ti -n vault-test vault-test-0 sh

/ $ vault status

Key Value

--- -----

Recovery Seal Type shamir

Initialized true

Sealed false

Total Recovery Shares 5

Threshold 3

Version 1.13.1

Build Date 2023-03-23T12:51:35Z

Storage Type file

Cluster Name vault-cluster-3bc45695

Cluster ID 5e0ccb87-18c5-4042-8763-c82512a80339

HA Enabled false

check data in vault

Now we need to create the database.

create database vault;
CREATE USER vault WITH PASSWORD 'fdgsdfkgh34098fsd';
GRANT ALL PRIVILEGES ON DATABASE vault TO vault;
ALTER DATABASE vault OWNER TO vault;

switch to database vault:
\c vault

CREATE TABLE vault_kv_store (
	parent_path TEXT COLLATE "C" NOT NULL,
	path        TEXT COLLATE "C",
	key         TEXT COLLATE "C",
	value       BYTEA,
	CONSTRAINT pkey PRIMARY KEY (path, key)
);



CREATE INDEX parent_path_idx ON vault_kv_store (parent_path);


CREATE TABLE vault_ha_locks (
	ha_key                                      TEXT COLLATE "C" NOT NULL,
	ha_identity                                 TEXT COLLATE "C" NOT NULL,
	ha_value                                    TEXT COLLATE "C",
	valid_until                                 TIMESTAMP WITH TIME ZONE NOT NULL,
	CONSTRAINT ha_key PRIMARY KEY (ha_key)
);

GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE vault_kv_store TO vault;
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE vault_ha_locks TO vault;

CREATE TABLE vault_kv_store (

parent_path TEXT COLLATE "C" NOT NULL,

path TEXT COLLATE "C",

key TEXT COLLATE "C",

value BYTEA,

CONSTRAINT pkey PRIMARY KEY (path, key)

);

CREATE INDEX parent_path_idx ON vault_kv_store (parent_path);

CREATE TABLE vault_ha_locks (

ha_key TEXT COLLATE "C" NOT NULL,

ha_identity TEXT COLLATE "C" NOT NULL,

ha_value TEXT COLLATE "C",

valid_until TIMESTAMP WITH TIME ZONE NOT NULL,

CONSTRAINT ha_key PRIMARY KEY (ha_key)

);

GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE vault_kv_store TO vault;

GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE vault_ha_locks TO vault;

now create migration file

kubectl exec -ti -n vault-test vault-test-0 sh
cd /vault/data/

cat > migrate.hcl

storage_source "file" {
	path = "/vault/data"
}
storage_destination "postgresql" {
  connection_url = "postgres://vault:fdgsdfkgh34098fsd@dev-db.chfnpsvsvxhn.eu-central-1.rds.amazonaws.com:5432/vault"
}
cluster_addr = "https://vault-test-0.vault-test-internal:8201"

storage_source "file" {

path = "/vault/data"

}

storage_destination "postgresql" {

connection_url = "postgres://vault:fdgsdfkgh34098fsd@dev-db.chfnpsvsvxhn.eu-central-1.rds.amazonaws.com:5432/vault"

}

cluster_addr = "https://vault-test-0.vault-test-internal:8201"

run migration

vault operator migrate -config=migrate.hcl

if we see

2024-05-10T08:15:46.668Z [INFO]  copied key: path=core/keyring
2024-05-10T08:15:46.668Z [INFO]  copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_provider/assignment/allow_all
2024-05-10T08:15:46.669Z [INFO]  copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_tokens/public_keys/e47839d3-55ab-7654-3c19-8617dc37d66b
2024-05-10T08:15:46.667Z [INFO]  copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_provider/provider/default
2024-05-10T08:15:46.673Z [INFO]  copied key: path=sys/policy/control-group
2024-05-10T08:15:46.675Z [INFO]  copied key: path=sys/policy/default
2024-05-10T08:15:46.675Z [INFO]  copied key: path=sys/policy/response-wrapping
2024-05-10T08:15:46.676Z [INFO]  copied key: path=sys/token/id/h69a08b5531ff96121a8a7e62813a0a461483cacd6698744be942aef2f6f1020f
2024-05-10T08:15:46.676Z [INFO]  copied key: path=core/local-auth
2024-05-10T08:15:46.690Z [INFO]  copied key: path=core/auth
2024-05-10T08:15:46.696Z [INFO]  copied key: path=sys/token/salt
2024-05-10T08:15:46.704Z [INFO]  copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/casesensitivity
2024-05-10T08:15:46.725Z [INFO]  copied key: path=sys/policy/test-policy
2024-05-10T08:15:46.725Z [INFO]  copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_tokens/named_keys/default
2024-05-10T08:15:46.726Z [INFO]  copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_tokens/public_keys/52a9d801-b680-433e-1ef6-48d47e32794a
2024-05-10T08:15:46.726Z [INFO]  copied key: path=sys/token/accessor/a00f03820e3563d472e5f55123b7640953ef24ec
Success! All of the keys have been migrated.

2024-05-10T08:15:46.668Z [INFO] copied key: path=core/keyring

2024-05-10T08:15:46.668Z [INFO] copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_provider/assignment/allow_all

2024-05-10T08:15:46.669Z [INFO] copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_tokens/public_keys/e47839d3-55ab-7654-3c19-8617dc37d66b

2024-05-10T08:15:46.667Z [INFO] copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_provider/provider/default

2024-05-10T08:15:46.673Z [INFO] copied key: path=sys/policy/control-group

2024-05-10T08:15:46.675Z [INFO] copied key: path=sys/policy/default

2024-05-10T08:15:46.675Z [INFO] copied key: path=sys/policy/response-wrapping

2024-05-10T08:15:46.676Z [INFO] copied key: path=sys/token/id/h69a08b5531ff96121a8a7e62813a0a461483cacd6698744be942aef2f6f1020f

2024-05-10T08:15:46.676Z [INFO] copied key: path=core/local-auth

2024-05-10T08:15:46.690Z [INFO] copied key: path=core/auth

2024-05-10T08:15:46.696Z [INFO] copied key: path=sys/token/salt

2024-05-10T08:15:46.704Z [INFO] copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/casesensitivity

2024-05-10T08:15:46.725Z [INFO] copied key: path=sys/policy/test-policy

2024-05-10T08:15:46.725Z [INFO] copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_tokens/named_keys/default

2024-05-10T08:15:46.726Z [INFO] copied key: path=logical/43cc4dcf-38d6-e6d2-d769-b86cdaee757b/oidc_tokens/public_keys/52a9d801-b680-433e-1ef6-48d47e32794a

2024-05-10T08:15:46.726Z [INFO] copied key: path=sys/token/accessor/a00f03820e3563d472e5f55123b7640953ef24ec

Success! All of the keys have been migrated.

all ok.

Now we can delete this helm chart, and pvc

terraform destroy --target helm_release.vault_test

kubectl get pvc -n vault-test
NAME                   STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
audit-vault-test-0     Bound    pvc-9e4df335-9000-4fdc-9490-4fd540dab4a1   10Gi       RWO            gp3            33m
data-vault-test-0      Bound    pvc-e9218391-94c8-4c19-a1ac-6a2f9ee5ebde   10Gi       RWO            gp3            33m
vault-plugins-volume   Bound    pvc-8f4499f4-f3cf-4444-8fe4-76b8d2965cde   5Gi        RWX            efs-sc         3h9m

kubectl get pvc -n vault-test

NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE

audit-vault-test-0 Bound pvc-9e4df335-9000-4fdc-9490-4fd540dab4a1 10Gi RWO gp3 33m

data-vault-test-0 Bound pvc-e9218391-94c8-4c19-a1ac-6a2f9ee5ebde 10Gi RWO gp3 33m

vault-plugins-volume Bound pvc-8f4499f4-f3cf-4444-8fe4-76b8d2965cde 5Gi RWX efs-sc 3h9m

kubectl delete pvc -n vault-test audit-vault-test-0 data-vault-test-0

apply helm chart with backend postgresql

resource "helm_release" "vault_test" {
  name             = "vault-test"
  repository       = "https://helm.releases.hashicorp.com"
  chart            = "vault"
  namespace        = "vault-test"
  create_namespace = false
  version          = local.settings.eks.vault_helm_version

  values = [<<EOF
global:
  enabled: true
injector:
  enabled: false
server:
  image:
    repository: "11111111111.dkr.ecr.eu-central-1.amazonaws.com/vault"
    tag: "1.12.3-bd428767"
    pullPolicy: Always

  updateStrategyType: "RollingUpdate"

  nodeSelector:
    dedication: "infra"
  tolerations:
    - key: "dedicated"
      operator: "Equal"
      value: "infra"
      effect: "NoSchedule"

  volumes:
    - name: plugins
      persistentVolumeClaim:
        claimName: vault-plugins-volume
    # - name: vault-data
    #   persistentVolumeClaim:
    #     claimName: vault-data-volume

  volumeMounts:
    - name: plugins
      mountPath: /usr/local/libexec/vault
    # - name: vault-data
    #   mountPath: /vault/data

  resources:
    requests:
      memory: 512Mi
      cpu: 500m
    limits:
      memory: 1024Mi
      cpu: 1000m

  ingress:
    enabled: true
    ingressClassName: "nginx-int"
    activeService: true
    pathType: Prefix
    hosts:
      - host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"
        paths:
          - /
    tls: 
      - secretName: "${local.vault_tls_secret_name}"
        hosts:
          - "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

  ha:
    enabled: true
    replicas: 1

    config: |
      ui = true
      plugin_directory = "/usr/local/libexec/vault"

      listener "tcp" {
        tls_disable = 1
        address = "[::]:8200"
        cluster_address = "[::]:8201"
      }

      storage "postgresql" {
        connection_url = "postgres://vault:fdgsdfkgh34098fsd@dev-db.chfnpsvsvxhn.eu-central-1.rds.amazonaws.com:5432/vault"
        ha_enabled = "true"
        table = "vault_kv_store"
        ha_table = "vault_ha_locks"
      }

      service_registration "kubernetes" {}

      seal "awskms" {
        region     = "${local.settings.vpc.region}"
        kms_key_id = "${aws_kms_key.vault_test.id}"
      }

  serviceAccount:
    name: "vault-test-sa"
    annotations:
      eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

ui:
  enabled: true
  EOF
  ]

  # disable_webhooks = "true"

  provider = helm
}

100

101

102

103

104

105

106

107

resource "helm_release" "vault_test" {

name = "vault-test"

repository = "https://helm.releases.hashicorp.com"

chart = "vault"

namespace = "vault-test"

create_namespace = false

version = local.settings.eks.vault_helm_version

values = [<<EOF

global:

enabled: true

injector:

enabled: false

server:

image:

repository: "11111111111.dkr.ecr.eu-central-1.amazonaws.com/vault"

tag: "1.12.3-bd428767"

pullPolicy: Always

updateStrategyType: "RollingUpdate"

nodeSelector:

dedication: "infra"

tolerations:

- key: "dedicated"

operator: "Equal"

value: "infra"

effect: "NoSchedule"

volumes:

- name: plugins

persistentVolumeClaim:

claimName: vault-plugins-volume

# - name: vault-data

# persistentVolumeClaim:

# claimName: vault-data-volume

volumeMounts:

- name: plugins

mountPath: /usr/local/libexec/vault

# - name: vault-data

# mountPath: /vault/data

resources:

requests:

memory: 512Mi

cpu: 500m

limits:

memory: 1024Mi

cpu: 1000m

ingress:

enabled: true

ingressClassName: "nginx-int"

activeService: true

pathType: Prefix

hosts:

- host: "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

paths:

- /

tls:

- secretName: "${local.vault_tls_secret_name}"

hosts:

- "vault-test.${local.settings.dns.base_domain}.${local.settings.dns.tech_domain}"

ha:

enabled: true

replicas: 1

config: |

ui = true

plugin_directory = "/usr/local/libexec/vault"

listener "tcp" {

tls_disable = 1

address = "[::]:8200"

cluster_address = "[::]:8201"

}

storage "postgresql" {

connection_url = "postgres://vault:fdgsdfkgh34098fsd@dev-db.chfnpsvsvxhn.eu-central-1.rds.amazonaws.com:5432/vault"

ha_enabled = "true"

table = "vault_kv_store"

ha_table = "vault_ha_locks"

}

service_registration "kubernetes" {}

seal "awskms" {

region = "${local.settings.vpc.region}"

kms_key_id = "${aws_kms_key.vault_test.id}"

}

serviceAccount:

name: "vault-test-sa"

annotations:

eks.amazonaws.com/role-arn: "${module.iam_assumable_role_vault_test.iam_role_arn}"

ui:

enabled: true

EOF

]

# disable_webhooks = "true"

provider = helm

}

terraform apply --target helm_release.vault_test

lets check

kubectl get pod -n vault-test 
NAME           READY   STATUS    RESTARTS   AGE
vault-test-0   1/1     Running   0          5m23s

kubectl get pod -n vault-test

NAME READY STATUS RESTARTS AGE

vault-test-0 1/1 Running 0 5m23s

kubectl exec -ti -n vault-test vault-test-0 sh

/ $ vault status
Key                      Value
---                      -----
Recovery Seal Type       shamir
Initialized              true
Sealed                   false
Total Recovery Shares    5
Threshold                3
Version                  1.12.3
Build Date               2023-02-02T09:07:27Z
Storage Type             postgresql
Cluster Name             vault-cluster-3bc45695
Cluster ID               5e0ccb87-18c5-4042-8763-c82512a80339
HA Enabled               true
HA Cluster               https://vault-test-0.vault-test-internal:8201
HA Mode                  active
Active Since             2024-05-10T08:32:39.198808996Z

/ $ vault status

Key Value

--- -----

Recovery Seal Type shamir

Initialized true

Sealed false

Total Recovery Shares 5

Threshold 3

Version 1.12.3

Build Date 2023-02-02T09:07:27Z

Storage Type postgresql

Cluster Name vault-cluster-3bc45695

Cluster ID 5e0ccb87-18c5-4042-8763-c82512a80339

HA Enabled true

HA Cluster https://vault-test-0.vault-test-internal:8201

HA Mode active

Active Since 2024-05-10T08:32:39.198808996Z

all ok

Centos

HOW TO migrate vault backend file to postgres

https://github.com/midnight47/

Сентябрь 2024
Пн	Вт	Ср	Чт	Пт	Сб	Вс
						1
2	3	4	5	6	7	8
9	10	11	12	13	14	15
16	17	18	19	20	21	22
23	24	25	26	27	28	29
30