k8s - force delete namespace

Thank you for reading this post, don't forget to subscribe!

бывает что подвисло удаление namespace в статусе terminating

kubectl get ns
NAME                                     STATUS        AGE
cattle-fleet-clusters-system             Active        35d
cattle-fleet-local-system                Active        35d
cattle-fleet-system                      Active        35d
cattle-global-data                       Active        35d
cattle-global-nt                         Active        35d
cattle-impersonation-system              Active        35d
cattle-provisioning-capi-system          Terminating   35d
cattle-system                            Active        13d
cattle-ui-plugin-system                  Active        35d
cert-manager                             Active        293d
cluster-fleet-local-local-1a3d67d0a899   Active        41d
default                                  Active        300d
dev                                      Active        155d
dex                                      Active        139d
elk                                      Active        288d
fleet-default                            Active        293d
fleet-local                              Active        293d
ingress-nginx                            Active        13d
keycloak                                 Active        112d
kube-node-lease                          Active        300d
kube-public                              Active        300d
kube-system                              Active        300d
local                                    Active        293d
loki                                     Active        160d
metallb-system                           Active        294d
monitoring                               Active        293d
openunison                               Terminating   83d
p-l9xbk                                  Terminating   41d
p-ztvwf                                  Terminating   41d
prod                                     Active        156d
teleport                                 Active        13d
test                                     Active        157d
user-cl57d                               Terminating   40d
vault                                    Active        159d

kubectl get ns

NAME STATUS AGE

cattle-fleet-clusters-system Active 35d

cattle-fleet-local-system Active 35d

cattle-fleet-system Active 35d

cattle-global-data Active 35d

cattle-global-nt Active 35d

cattle-impersonation-system Active 35d

cattle-provisioning-capi-system Terminating 35d

cattle-system Active 13d

cattle-ui-plugin-system Active 35d

cert-manager Active 293d

cluster-fleet-local-local-1a3d67d0a899 Active 41d

default Active 300d

dev Active 155d

dex Active 139d

elk Active 288d

fleet-default Active 293d

fleet-local Active 293d

ingress-nginx Active 13d

keycloak Active 112d

kube-node-lease Active 300d

kube-public Active 300d

kube-system Active 300d

local Active 293d

loki Active 160d

metallb-system Active 294d

monitoring Active 293d

openunison Terminating 83d

p-l9xbk Terminating 41d

p-ztvwf Terminating 41d

prod Active 156d

teleport Active 13d

test Active 157d

user-cl57d Terminating 40d

vault Active 159d

вот такой describe

 kubectl describe ns cattle-provisioning-capi-system
Name:         cattle-provisioning-capi-system
Labels:       kubernetes.io/metadata.name=cattle-provisioning-capi-system
Annotations:  cattle.io/status:
                {"Conditions":[{"Type":"ResourceQuotaInit","Status":"True","Message":"","LastUpdateTime":"2025-03-22T07:11:59Z"},{"Type":"InitialRolesPopu...
              lifecycle.cattle.io/create.namespace-auth: true
Status:       Terminating
Conditions:
  Type                                         Status  LastTransitionTime               Reason                Message
  ----                                         ------  ------------------               ------                -------
  NamespaceDeletionDiscoveryFailure            False   Sun, 27 Apr 2025 10:48:35 +0600  ResourcesDiscovered   All resources successfully discovered
  NamespaceDeletionGroupVersionParsingFailure  False   Sun, 13 Apr 2025 15:47:11 +0600  ParsedGroupVersions   All legacy kube types successfully parsed
  NamespaceDeletionContentFailure              False   Sun, 13 Apr 2025 15:47:11 +0600  ContentDeleted        All content successfully deleted, may be waiting on finalization
  NamespaceContentRemaining                    True    Sun, 13 Apr 2025 15:47:11 +0600  SomeResourcesRemain   Some resources are remaining: rolebindings.rbac.authorization.k8s.io has 2 resource instances, roles.rbac.authorization.k8s.io has 2 resource instances
  NamespaceFinalizersRemaining                 True    Sun, 13 Apr 2025 15:47:11 +0600  SomeFinalizersRemain  Some content in the namespace has finalizers remaining: wrangler.cattle.io/auth-prov-v2-rb in 2 resource instances, wrangler.cattle.io/auth-prov-v2-role in 2 resource instances

kubectl describe ns cattle-provisioning-capi-system

Name: cattle-provisioning-capi-system

Labels: kubernetes.io/metadata.name=cattle-provisioning-capi-system

Annotations: cattle.io/status:

{"Conditions":[{"Type":"ResourceQuotaInit","Status":"True","Message":"","LastUpdateTime":"2025-03-22T07:11:59Z"},{"Type":"InitialRolesPopu…

lifecycle.cattle.io/create.namespace-auth: true

Status: Terminating

Conditions:

Type Status LastTransitionTime Reason Message

---- ------ ------------------ ------ -------

NamespaceDeletionDiscoveryFailure False Sun, 27 Apr 2025 10:48:35 +0600 ResourcesDiscovered All resources successfully discovered

NamespaceDeletionGroupVersionParsingFailure False Sun, 13 Apr 2025 15:47:11 +0600 ParsedGroupVersions All legacy kube types successfully parsed

NamespaceDeletionContentFailure False Sun, 13 Apr 2025 15:47:11 +0600 ContentDeleted All content successfully deleted, may be waiting on finalization

NamespaceContentRemaining True Sun, 13 Apr 2025 15:47:11 +0600 SomeResourcesRemain Some resources are remaining: rolebindings.rbac.authorization.k8s.io has 2 resource instances, roles.rbac.authorization.k8s.io has 2 resource instances

NamespaceFinalizersRemaining True Sun, 13 Apr 2025 15:47:11 +0600 SomeFinalizersRemain Some content in the namespace has finalizers remaining: wrangler.cattle.io/auth-prov-v2-rb in 2 resource instances, wrangler.cattle.io/auth-prov-v2-role in 2 resource instances

проверяем какие ресурсы есть в этом ns

kubectl get rolebindings,roles -n cattle-provisioning-capi-system
NAME                                                                                    ROLE                                            AGE
rolebinding.rbac.authorization.k8s.io/capi-leader-election-rolebinding                  Role/capi-leader-election-role                  35d
rolebinding.rbac.authorization.k8s.io/helm-operation-sff2l-helm-installer-rolebinding   Role/helm-operation-sff2l-helm-installer-role   35d

NAME                                                                      CREATED AT
role.rbac.authorization.k8s.io/capi-leader-election-role                  2025-03-22T07:12:06Z
role.rbac.authorization.k8s.io/helm-operation-sff2l-helm-installer-role   2025-03-22T07:12:01Z

kubectl get rolebindings,roles -n cattle-provisioning-capi-system

NAME ROLE AGE

rolebinding.rbac.authorization.k8s.io/capi-leader-election-rolebinding Role/capi-leader-election-role 35d

rolebinding.rbac.authorization.k8s.io/helm-operation-sff2l-helm-installer-rolebinding Role/helm-operation-sff2l-helm-installer-role 35d

NAME CREATED AT

role.rbac.authorization.k8s.io/capi-leader-election-role 2025-03-22T07:12:06Z

role.rbac.authorization.k8s.io/helm-operation-sff2l-helm-installer-role 2025-03-22T07:12:01Z

чтобы удалить все ресурсы из namespace используем скриптец:

NAMESPACE="cattle-provisioning-capi-system"


for resource in $(kubectl api-resources --namespaced=true --verbs=list -o name); do
  for item in $(kubectl get "$resource" -n "$NAMESPACE" --no-headers --ignore-not-found -o custom-columns=":metadata.name"); do
    echo "Patching $resource/$item to remove finalizers..."
    kubectl get "$resource" "$item" -n "$NAMESPACE" -o json | jq 'del(.metadata.finalizers)' | kubectl apply -f -
  done
done

NAMESPACE="cattle-provisioning-capi-system"

for resource in $(kubectl api-resources --namespaced=true --verbs=list -o name); do

for item in $(kubectl get "$resource" -n "$NAMESPACE" --no-headers --ignore-not-found -o custom-columns=":metadata.name"); do

echo "Patching $resource/$item to remove finalizers…"

kubectl get "$resource" "$item" -n "$NAMESPACE" -o json | jq 'del(.metadata.finalizers)' | kubectl apply -f -

done

мы сначала определяем namespace а уже потом проходимся по всем ресурсам и удаляем их

рассмотрим ещё вариант

NAMESPACE="openunison"

  for item in $(kubectl get "$resource" -n "$NAMESPACE" --no-headers --ignore-not-found -o custom-columns=":metadata.name"); do
    echo "Patching $resource/$item to remove finalizers..."
    kubectl get "$resource" "$item" -n "$NAMESPACE" -o json | jq 'del(.metadata.finalizers)' | kubectl apply -f -
  done
done
Warning: metallb.io v1beta1 AddressPool is deprecated, consider using IPAddressPool

for item in $(kubectl get "$resource" -n "$NAMESPACE" --no-headers --ignore-not-found -o custom-columns=":metadata.name"); do

echo "Patching $resource/$item to remove finalizers…"

kubectl get "$resource" "$item" -n "$NAMESPACE" -o json | jq 'del(.metadata.finalizers)' | kubectl apply -f -

done

Warning: metallb.io v1beta1 AddressPool is deprecated, consider using IPAddressPool

kubectl describe ns openunison
Name:         openunison
Labels:       kubernetes.io/metadata.name=openunison
Annotations:  cattle.io/status:
                {"Conditions":[{"Type":"ResourceQuotaInit","Status":"True","Message":"","LastUpdateTime":"2025-03-16T10:30:04Z"},{"Type":"InitialRolesPopu...
              lifecycle.cattle.io/create.namespace-auth: true
Status:       Terminating
Conditions:
  Type                                         Status  LastTransitionTime               Reason                  Message
  ----                                         ------  ------------------               ------                  -------
  NamespaceDeletionDiscoveryFailure            False   Sun, 27 Apr 2025 11:30:13 +0600  ResourcesDiscovered     All resources successfully discovered
  NamespaceDeletionGroupVersionParsingFailure  False   Sun, 27 Apr 2025 11:30:13 +0600  ParsedGroupVersions     All legacy kube types successfully parsed
  NamespaceDeletionContentFailure              False   Sun, 27 Apr 2025 11:30:13 +0600  ContentDeleted          All content successfully deleted, may be waiting on finalization
  NamespaceContentRemaining                    False   Sun, 27 Apr 2025 11:30:13 +0600  ContentRemoved          All content successfully removed
  NamespaceFinalizersRemaining                 False   Sun, 27 Apr 2025 11:30:13 +0600  ContentHasNoFinalizers  All content-preserving finalizers finished

No resource quota.

No LimitRange resource.

kubectl describe ns openunison

Name: openunison

Labels: kubernetes.io/metadata.name=openunison

Annotations: cattle.io/status:

{"Conditions":[{"Type":"ResourceQuotaInit","Status":"True","Message":"","LastUpdateTime":"2025-03-16T10:30:04Z"},{"Type":"InitialRolesPopu…

lifecycle.cattle.io/create.namespace-auth: true

Status: Terminating

Conditions:

Type Status LastTransitionTime Reason Message

---- ------ ------------------ ------ -------

NamespaceDeletionDiscoveryFailure False Sun, 27 Apr 2025 11:30:13 +0600 ResourcesDiscovered All resources successfully discovered

NamespaceDeletionGroupVersionParsingFailure False Sun, 27 Apr 2025 11:30:13 +0600 ParsedGroupVersions All legacy kube types successfully parsed

NamespaceDeletionContentFailure False Sun, 27 Apr 2025 11:30:13 +0600 ContentDeleted All content successfully deleted, may be waiting on finalization

NamespaceContentRemaining False Sun, 27 Apr 2025 11:30:13 +0600 ContentRemoved All content successfully removed

NamespaceFinalizersRemaining False Sun, 27 Apr 2025 11:30:13 +0600 ContentHasNoFinalizers All content-preserving finalizers finished

No resource quota.

No LimitRange resource.

удаление через force так же не помогло

kubectl delete ns openunison --grace-period=0 --force

несмотря на force delete, namespace openunison всё ещё висит в статусе Terminating.

Причина в том, что API-сервер Kubernetes уже считает, что namespace удалён, но в etcd запись о нём не исчезла (она "залипла").

Это бывает когда:

Было много force deletion
Либо kube-apiserver/etcd багует на удалении старых namespaces (особенно в Rancher кластерах)

что в таком случае можно сделать,

запускаем

kubectl proxy &

и выполняем жёсткое удаление через etcd

curl -k -H "Content-Type: application/json" -X PUT http://127.0.0.1:8001/api/v1/namespaces/openunison/finalize -d '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"openunison"}}'

1 2	curl -k -H "Content-Type: application/json" -X PUT http://127.0.0.1:8001/api/v1/namespaces/openunison/finalize -d '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"openunison"}}'

вывод будет примерно такой:

 curl -k -H "Content-Type: application/json" -X PUT http://127.0.0.1:8001/api/v1/namespaces/openunison/finalize -d '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"openunison"}}'
{
  "kind": "Namespace",
  "apiVersion": "v1",
  "metadata": {
    "name": "openunison",
    "uid": "2525f5b4-b396-4868-ac5c-fa19987a628d",
    "resourceVersion": "4324710",
    "creationTimestamp": "2025-02-02T11:03:40Z",
    "deletionTimestamp": "2025-04-27T05:30:08Z",
    "deletionGracePeriodSeconds": 0,
    "labels": {
      "kubernetes.io/metadata.name": "openunison"
    },
    "managedFields": [
      {
        "manager": "kubectl-create",
        "operation": "Update",
        "apiVersion": "v1",
        "time": "2025-02-02T11:03:40Z",
        "fieldsType": "FieldsV1",
        "fieldsV1": {
          "f:metadata": {
            "f:labels": {
              ".": {},
              "f:kubernetes.io/metadata.name": {}
            }
          }
        }
      },
      {
        "manager": "kube-controller-manager",
        "operation": "Update",
        "apiVersion": "v1",
        "time": "2025-04-27T05:30:13Z",
        "fieldsType": "FieldsV1",
        "fieldsV1": {
          "f:status": {
            "f:conditions": {
              ".": {},
              "k:{\"type\":\"NamespaceContentRemaining\"}": {
                ".": {},
                "f:lastTransitionTime": {},
                "f:message": {},
                "f:reason": {},
                "f:status": {},
                "f:type": {}
              },
              "k:{\"type\":\"NamespaceDeletionContentFailure\"}": {
                ".": {},
                "f:lastTransitionTime": {},
                "f:message": {},
                "f:reason": {},
                "f:status": {},
                "f:type": {}
              },
              "k:{\"type\":\"NamespaceDeletionDiscoveryFailure\"}": {
                ".": {},
                "f:lastTransitionTime": {},
                "f:message": {},
                "f:reason": {},
                "f:status": {},
                "f:type": {}
              },
              "k:{\"type\":\"NamespaceDeletionGroupVersionParsingFailure\"}": {
                ".": {},
                "f:lastTransitionTime": {},
                "f:message": {},
                "f:reason": {},
                "f:status": {},
                "f:type": {}
              },
              "k:{\"type\":\"NamespaceFinalizersRemaining\"}": {
                ".": {},
                "f:lastTransitionTime": {},
                "f:message": {},
                "f:reason": {},
                "f:status": {},
                "f:type": {}
              }
            }
          }
        },
        "subresource": "status"
      }
    ]
  },
  "spec": {},
  "status": {
    "phase": "Terminating",
    "conditions": [
      {
        "type": "NamespaceDeletionDiscoveryFailure",
        "status": "False",
        "lastTransitionTime": "2025-04-27T05:30:13Z",
        "reason": "ResourcesDiscovered",
        "message": "All resources successfully discovered"
      },
      {
        "type": "NamespaceDeletionGroupVersionParsingFailure",
        "status": "False",
        "lastTransitionTime": "2025-04-27T05:30:13Z",
        "reason": "ParsedGroupVersions",
        "message": "All legacy kube types successfully parsed"
      },
      {
        "type": "NamespaceDeletionContentFailure",
        "status": "False",
        "lastTransitionTime": "2025-04-27T05:30:13Z",
        "reason": "ContentDeleted",
        "message": "All content successfully deleted, may be waiting on finalization"
      },
      {
        "type": "NamespaceContentRemaining",
        "status": "False",
        "lastTransitionTime": "2025-04-27T05:30:13Z",
        "reason": "ContentRemoved",
        "message": "All content successfully removed"
      },
      {
        "type": "NamespaceFinalizersRemaining",
        "status": "False",
        "lastTransitionTime": "2025-04-27T05:30:13Z",
        "reason": "ContentHasNoFinalizers",
        "message": "All content-preserving finalizers finished"
      }
    ]
  }
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

curl -k -H "Content-Type: application/json" -X PUT http://127.0.0.1:8001/api/v1/namespaces/openunison/finalize -d '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"openunison"}}'

{

"kind": "Namespace",

"apiVersion": "v1",

"metadata": {

"name": "openunison",

"uid": "2525f5b4-b396-4868-ac5c-fa19987a628d",

"resourceVersion": "4324710",

"creationTimestamp": "2025-02-02T11:03:40Z",

"deletionTimestamp": "2025-04-27T05:30:08Z",

"deletionGracePeriodSeconds": 0,

"labels": {

"kubernetes.io/metadata.name": "openunison"

"managedFields": [

{

"manager": "kubectl-create",

"operation": "Update",

"apiVersion": "v1",

"time": "2025-02-02T11:03:40Z",

"fieldsType": "FieldsV1",

"fieldsV1": {

"f:metadata": {

"f:labels": {

".": {},

"f:kubernetes.io/metadata.name": {}

}

{

"manager": "kube-controller-manager",

"operation": "Update",

"apiVersion": "v1",

"time": "2025-04-27T05:30:13Z",

"fieldsType": "FieldsV1",

"fieldsV1": {

"f:status": {

"f:conditions": {

".": {},

"k:{\"type\":\"NamespaceContentRemaining\"}": {

".": {},

"f:lastTransitionTime": {},

"f:message": {},

"f:reason": {},

"f:status": {},

"f:type": {}

"k:{\"type\":\"NamespaceDeletionContentFailure\"}": {

".": {},

"f:lastTransitionTime": {},

"f:message": {},

"f:reason": {},

"f:status": {},

"f:type": {}

"k:{\"type\":\"NamespaceDeletionDiscoveryFailure\"}": {

".": {},

"f:lastTransitionTime": {},

"f:message": {},

"f:reason": {},

"f:status": {},

"f:type": {}

"k:{\"type\":\"NamespaceDeletionGroupVersionParsingFailure\"}": {

".": {},

"f:lastTransitionTime": {},

"f:message": {},

"f:reason": {},

"f:status": {},

"f:type": {}

"k:{\"type\":\"NamespaceFinalizersRemaining\"}": {

".": {},

"f:lastTransitionTime": {},

"f:message": {},

"f:reason": {},

"f:status": {},

"f:type": {}

}

"subresource": "status"

}

]

"spec": {},

"status": {

"phase": "Terminating",

"conditions": [

{

"type": "NamespaceDeletionDiscoveryFailure",

"status": "False",