CKS: report drift between a cluster's VM map, its VMs and its Kubernetes nodes

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java

@@ -23,6 +23,7 @@ public class KubernetesClusterEventTypes {
     public static final String EVENT_KUBERNETES_CLUSTER_STOP = "KUBERNETES.CLUSTER.STOP";
     public static final String EVENT_KUBERNETES_CLUSTER_SCALE = "KUBERNETES.CLUSTER.SCALE";
     public static final String EVENT_KUBERNETES_CLUSTER_UPGRADE = "KUBERNETES.CLUSTER.UPGRADE";
+    public static final String EVENT_KUBERNETES_CLUSTER_VM_MAP_CHECK = "KUBERNETES.CLUSTER.VM.MAP.CHECK";
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD";
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE";
 }

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -304,6 +304,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
     private static final String DEFAULT_NETRIS_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering for Netris CloudStack Kubernetes Service on VPC";
 
     protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
+    private KubernetesClusterVmMapChecker vmMapChecker;
 
     protected final static List<String> CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER.name(), CONTROL.name(), ETCD.name());
 
@@ -2563,6 +2564,20 @@ public void reallyRun() {
         }
     }
 
+    /**
+     * Reports the clusters whose VM map does not match their VMs or their Kubernetes nodes. Changes nothing.
+     */
+    protected void checkKubernetesClusterVmMaps() {
+        try {
+            if (vmMapChecker == null) {
+                vmMapChecker = ComponentContext.inject(new KubernetesClusterVmMapChecker(this));
+            }
+            vmMapChecker.checkClustersIfDue();
+        } catch (Exception e) {
+            logger.warn("Caught exception while checking the VM maps of Kubernetes clusters", e);
+        }
+    }
+
     /* Kubernetes cluster scanner checks if the Kubernetes cluster is in desired state. If it detects Kubernetes cluster
        is not in desired state, it will trigger an event and marks the Kubernetes cluster to be 'Alert' state. For e.g a
        Kubernetes cluster in 'Running' state should mean all the cluster of node VM's in the custer should be running and
@@ -2680,6 +2695,8 @@ public void reallyRun() {
             } catch (Exception e) {
                 logger.warn("Caught exception while running Kubernetes cluster state scanner", e);
             }
+            // starts the check on its own thread, as it needs SSH to each cluster
+            checkKubernetesClusterVmMaps();
             firstRun = false;
         }
     }
@@ -2803,6 +2820,7 @@ public String getConfigComponentName() {
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {
             KubernetesServiceEnabled,
+            KubernetesClusterVmMapChecker.KubernetesClusterVmMapCheckEnabled,
             KubernetesClusterNetworkOffering,
             KubernetesClusterStartTimeout,
             KubernetesClusterScaleTimeout,