需求：

K8S中的服务不可用的时候（假定k8s的某个服务出现完全不可用，理论上k8s本身会避免这个问题的出现，这里姑且认为客户需求是合理的），需要为通过CIS已经发布的VS 提供备份访问，及该vs将自动把业务导向其它静态vm提供的服务。

Solution：

需借助BIGIP AS3来实现该需求。即CIS通过部署AS3的方式来部署服务，通过在AS3里直接配置静态的低优先级组member来实现备份，一个例子如下：

[root@k8s-master f5-k8s]# cat f5-vs-as3.yaml 
kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-as3
  labels:
    f5type: virtual-server
    as3: "true"
data:
  template: |
    {
    "class": "AS3",
    "action": "deploy",
    "persist": true,
    "declaration": {
        "class": "ADC",
        "schemaVersion": "3.10.0",
        "id": "123abc",
        "label": "k8s",
        "remark": "HTTPS with predictive-node pool and connection limit",
        "k8sas3": {
        "class": "Tenant",
        "nginxservice": {
            "class": "Application",
            "template": "https",
            "serviceMain": {
            "class": "Service_HTTPS",
            "virtualAddresses": [
                "192.0.2.11"
            ],
            "pool": "web_pool",
            "serverTLS": "webtls"
            },
            "web_pool": {
            "class": "Pool",
            "loadBalancingMode": "predictive-node",
            "monitors": [
                "http"
            ],
            "members": [
            {
                "servicePort": 80,
                "serverAddresses": [],
                "priorityGroup": 5,
                "connectionLimit": 10
            },
            {
                "servicePort": 88,
                "serverAddresses": [
                  "2.54.40.3",
                  "6.65.22.2"
                ],
                "priorityGroup": 0,
                "connectionLimit": 20
            }
             ]
            },
            "webtls": {
            "class": "TLS_Server",
            "certificates": [{
                "certificate": "webcert"
            }]
            },
            "webcert": {
            "class": "Certificate",
            "remark": "in practice we recommend using a passphrase",
            "certificate": "-----BEGIN CERTIFICATE-----\nMIICnDCCAgWgAwIBAgIJAJ5n2b0OCEjwMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMRQwEgYDVQQKDAtmNV9OZXR3b3JrczEbMBkGA1UEAwwSc2FtcGxlLmV4YW1wbGUubmV0MB4XDTE3MTEyNjE5NTAyNFoXDTE4MDIyNTE5NTAyNFowZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxFDASBgNVBAoMC2Y1X05ldHdvcmtzMRswGQYDVQQDDBJzYW1wbGUuZXhhbXBsZS5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALEsuXmSXVQpYjrZPW+WiTBjn491mwZYT7Q92V1HlSBtM6WdWlK1aZN5sovfKtOX7Yrm8xa+e4o/zJ2QYLyyv5O+t2EGN/4qUEjEAPY9mwJdfzRQy6Hyzm84J0QkTuUJ/EjNuPji3D0QJRALUTzu1UqqDCEtiN9OGyXEkh7uvb7BAgMBAAGjUDBOMB0GA1UdDgQWBBSVHPNrGWrjWyZvckQxFYWO59FRFjAfBgNVHSMEGDAWgBSVHPNrGWrjWyZvckQxFYWO59FRFjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GBAJeJ9SEckEwPhkXOm+IuqfbUS/RcziifBCTmVyE+Fa/j9pKSYTgiEBNdbJeBEa+gPMlQtbV7Y2dy8TKx/8axVBHiXC5geDML7caxOrAyHYBpnx690xJTh5OIORBBM/a/NvaR+P3CoVebr/NPRh9oRNxnntnqvqD7SW0U3ZPe3tJc\n-----END CERTIFICATE-----",
            "privateKey": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,D8FFCE6B255601587CB54EC29B737D31\n\nkv4Fc3Jn0Ujkj0yRjt+gQQfBLSNF2aRLUENXnlr7Xpzqu0Ahr3jS1bAAnd8IWnsR\nyILqVmKsYF2DoHh0tWiEAQ7/y/fe5DTFhK7N4Wml6kp2yVMkP6KC4ssyYPw27kjK\nDBwBZ5O8Ioej08A5sgsLCmglbmtSPHJUn14pQnMTmLOpEtOsu6S+2ibPgSNpdg0b\nCAJNG/KHe+Vkx59qNDyDeKb7FZOlsX30+y67zUq9GQqJEDuysPJ2BUNP0IJXAjst\nFIt1qNoZew+5KDYs7u/lPxcMGTirUhgI84Jy4WcDvSOsP/tKlxj04TbIE3epmSKy\n+TihHkwY7ngIGtcm3Sfqk5jz2RXoj1/Ac3SW8kVTYaOUogBhn7zAq4Wju6Et4hQG\nRGapsJp1aCeZ/a4RCDTxspcKoMaRa97/URQb0hBRGx3DGUhzpmX9zl7JI2Xa5D3R\nmdBXtjLKYJTdIMdd27prBEKhMUpae2rz5Mw4J907wZeBq/wu+zp8LAnecfTe2nGY\nE32x1U7gSEdYOGqnwxsOexb1jKgCa67Nw9TmcMPV8zmH7R9qdvgxAbAtwBl1F9OS\nfcGaC7epf1AjJLtaX7krWmzgASHl28Ynh9lmGMdv+5QYMZvKG0LOg/n3m8uJ6sKy\nIzzvaJswwn0j5P5+czyoV5CvvdCfKnNb+3jUEN8I0PPwjBGKr4B1ojwhogTM248V\nHR69D6TxFVMfGpyJhCPkbGEGbpEpcffpgKuC/mEtMqyDQXJNaV5HO6HgAJ9F1P6v\n5ehHHTMRvzCCFiwndHdlMXUjqSNjww6me6dr6LiAPbejdzhL2vWx1YqebOcwQx3G\n-----END RSA PRIVATE KEY-----",
            "passphrase": {
                "ciphertext": "ZjVmNQ==",
                "protected": "eyJhbGciOiJkaXIiLCJlbmMiOiJub25lIn0"
            }
            }
        }
        }
    }
    }

上述配置中的pool members部分是重点，静态添加低优先级组member，高优先级组由CIS自动化发现

            "members": [
            {
                "servicePort": 80,
                "serverAddresses": [],
                "priorityGroup": 5,
                "connectionLimit": 10
            },
            {
                "servicePort": 88,
                "serverAddresses": [
                  "2.54.40.3",
                  "6.65.22.2"
                ],
                "priorityGroup": 0,
                "connectionLimit": 20
            }
             ]

最终在F5上产生的配置效果如下：

root@(v13-common)(cfg-sync Not All Devices Synced)(Active)(/k8sas3/nginxservice)(tmos)# list ltm pool web_pool 
ltm pool web_pool {
    load-balancing-mode predictive-node
    members {
        /k8sas3/10.244.0.182:http {
            address 10.244.0.182
            connection-limit 10
            priority-group 5
            session monitor-enabled
            state up
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/10.244.1.129:http {
            address 10.244.1.129
            connection-limit 10
            priority-group 5
            session monitor-enabled
            state up
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/2.54.40.3:kerberos {
            address 2.54.40.3
            connection-limit 20
            session monitor-enabled
            state down
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/6.65.22.2:kerberos {
            address 6.65.22.2
            connection-limit 20
            session monitor-enabled
            state down
            metadata {
                source {
                    value declaration
                }
            }
        }
    }
    min-active-members 1
    monitor min 1 of { /Common/http }
    partition k8sas3
}

测试：

如果删除相关svc：
[root@k8s-master f5-k8s]# kubectl delete -f nginx-deploy-svc.yaml
CIS将只留下静态部分的pool member

root@(v13-common)(cfg-sync Not All Devices Synced)(Active)(/k8sas3/nginxservice)(tmos)# list ltm pool web_pool
ltm pool web_pool {
    load-balancing-mode predictive-node
    members {
        /k8sas3/2.54.40.3:kerberos {
            address 2.54.40.3
            connection-limit 20
            session monitor-enabled
            state down
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/6.65.22.2:kerberos {
            address 6.65.22.2
            connection-limit 20
            session monitor-enabled
            state down
            metadata {
                source {
                    value declaration
                }
            }
        }
    }
    min-active-members 1
    monitor min 1 of { /Common/http }
    partition k8sas3
}

恢复相关svc：

[root@k8s-master f5-k8s][root@k8s-master f5-k8s]# kubectl create -f nginx-deploy-svc.yaml

pod再次被自动添加

root@(v13-common)(cfg-sync Not All Devices Synced)(Active)(/k8sas3/nginxservice)(tmos)# list ltm pool web_pool
ltm pool web_pool {
    load-balancing-mode predictive-node
    members {
        /k8sas3/10.244.0.182:http {
            address 10.244.0.182
            connection-limit 10
            priority-group 5
            session monitor-enabled
            state up
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/10.244.1.129:http {
            address 10.244.1.129
            connection-limit 10
            priority-group 5
            session monitor-enabled
            state up
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/2.54.40.3:kerberos {
            address 2.54.40.3
            connection-limit 20
            session monitor-enabled
            state down
            metadata {
                source {
                    value declaration
                }
            }
        }
        /k8sas3/6.65.22.2:kerberos {
            address 6.65.22.2
            connection-limit 20
            session monitor-enabled
            state down
            metadata {
                source {
                    value declaration
                }
            }
        }
    }
    min-active-members 1
    monitor min 1 of { /Common/http }
    partition k8sas3
}

此方法需要用户采用AS3来进行配置，用户应充分理解AS3本身的特性和用法，以及了解AS3在与CIS配合时的一些限制