From e8ac826b7f2bb512c7ca7002e12832834a367f01 Mon Sep 17 00:00:00 2001 From: Marian Fischer Date: Tue, 10 Nov 2026 13:49:44 +0100 Subject: [PATCH] CKS: open the cluster public IP on VPC tiers when the VPC has the Firewall service Since #12706 a VPC offering can provide the Firewall service for public IPs. The VPC virtual router then drops ingress traffic to a public IP that has no firewall rule. CKS only created network ACL items, port forwarding and the API load balancer rule on VPC tiers, so the management server could not reach the nodes (SSH 2222+, API 6443), the control node could not reach its own API through the public IP, and cluster creation failed after the start timeout. When the VPC virtual router provides the Firewall service of the cluster's VPC (the check that sets vpc_firewall_enabled on the VR), CKS now creates the same ingress firewall rules as on isolated networks on create (including etcd nodes), scale and node add/remove. In a VPC the firewall rules are created after the load balancing and port forwarding rules: applying firewall rules on a VPC public IP does not associate the IP with the VR, and if they come first the IP is never associated. Firewall rules on VPC public IPs have no network_id, so per-node rules are looked up by public IP and ports. On delete the rules go with the release of the cluster public IP. Isolated networks, VPCs without the Firewall service and the NSX/Netris paths are unchanged. --- .../KubernetesClusterActionWorker.java | 51 +++++++++ .../KubernetesClusterAddWorker.java | 26 +++-- .../KubernetesClusterRemoveWorker.java | 15 ++- ...esClusterResourceModifierActionWorker.java | 11 +- .../KubernetesClusterScaleWorker.java | 12 ++ .../KubernetesClusterStartWorker.java | 8 ++ .../KubernetesClusterActionWorkerTest.java | 106 +++++++++++++++++- ...usterResourceModifierActionWorkerTest.java | 65 +++++++++++ 8 files changed, 270 insertions(+), 24 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..110be072389b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -51,6 +51,7 @@ import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.firewall.FirewallService; import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -59,6 +60,7 @@ import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic; import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine; @@ -107,10 +109,12 @@ import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.NetworkService; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.vpc.VpcService; +import com.cloud.network.vpc.dao.VpcServiceMapDao; import com.cloud.projects.ProjectService; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Storage; @@ -214,12 +218,16 @@ public class KubernetesClusterActionWorker { @Inject public VpcService vpcService; @Inject + protected VpcServiceMapDao vpcServiceMapDao; + @Inject public PortForwardingRulesDao portForwardingRulesDao; @Inject protected RulesService rulesService; @Inject protected FirewallService firewallService; @Inject + protected FirewallRulesDao firewallRulesDao; + @Inject private NicDao nicDao; @Inject protected AffinityGroupDao affinityGroupDao; @@ -1113,6 +1121,49 @@ protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { return k8sConfig; } + /** + * When the VPC virtual router provides the Firewall service of a VPC, it drops ingress traffic to + * public IPs that has no firewall rule (in addition to the network ACL of each tier). The ports + * used by CKS must then be opened on the cluster public IP, as for isolated networks. This is the + * same check that sets vpc_firewall_enabled for the VPC virtual router. + * + * @param network the network of the Kubernetes cluster + * @return true if the network is a VPC tier and the VPC virtual router provides the Firewall service + */ + protected boolean isVpcFirewallEnabled(Network network) { + if (network == null || network.getVpcId() == null) { + return false; + } + return vpcServiceMapDao.canProviderSupportServiceInVpc(network.getVpcId(), Network.Service.Firewall, Network.Provider.VPCVirtualRouter); + } + + /** + * @param network the network of the Kubernetes cluster + * @return true if ingress firewall rules are needed on the cluster public IP: always for isolated + * networks, and for VPC tiers when the VPC uses the Firewall service + */ + protected boolean isPublicIpFirewallRequired(Network network) { + return network.getVpcId() == null || isVpcFirewallEnabled(network); + } + + /** + * Finds the ingress firewall rule for the given public port range of the cluster. Firewall rules on a + * public IP of a VPC are stored with the VPC and without a network, so they are looked up by the IP. + * + * @return the rule, or null if none exists + */ + protected FirewallRuleVO findIngressFirewallRuleForPorts(Network network, Long publicIpId, int startPort, int endPort) { + if (network.getVpcId() == null) { + return firewallRulesDao.findByNetworkIdAndPorts(network.getId(), startPort, endPort); + } + if (publicIpId == null) { + return null; + } + List rules = firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(publicIpId, startPort, endPort, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall); + return CollectionUtils.isEmpty(rules) ? null : rules.get(0); + } + public Map addFirewallRulesForNodes(IpAddress publicIp, int size) throws ManagementServerException { Map vmIdPortMap = new HashMap<>(); CallContext.register(CallContext.current(), null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java index 8b694adf1cca..0c2b88aa0fa1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -31,7 +31,6 @@ import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.service.ServiceOfferingVO; @@ -49,7 +48,6 @@ import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Level; -import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; @@ -60,8 +58,6 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker { - @Inject - private FirewallRulesDao firewallRulesDao; private long addNodeTimeoutTime; List finalNodeIds = new ArrayList<>(); @@ -229,9 +225,13 @@ private Pair validateAndSetupNode(Network network, IpAddress p provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); } provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT); + if (isVpcFirewallEnabled(network)) { + // In a VPC the firewall rule follows the port forwarding rule, see setupKubernetesClusterVpcTierRules + provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); + } boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort); if (!isCompatible) { - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, publicIp, nodeId, sshStartPort); return new Pair<>(false, nodeIndex); } @@ -249,13 +249,13 @@ private Pair validateAndSetupNode(Network network, IpAddress p InsufficientCapacityException | IllegalAccessException e) { logger.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName())); // remove added Firewall and PF rules - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, publicIp, nodeId, sshStartPort); return new Pair<>( false, nodeIndex); } catch (Exception e) { String errMsg = String.format("Unexpected exception while trying to add the external node %s to the Kubernetes cluster %s: %s", nodeId, kubernetesCluster.getName(), e.getMessage()); logger.error(errMsg, e); - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, publicIp, nodeId, sshStartPort); throw new CloudRuntimeException(e); } return new Pair<>(true, ++nodeIndex); @@ -310,12 +310,14 @@ private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File p } } - private void revertNetworkRules(Network network, long vmId, int port) { + private void revertNetworkRules(Network network, IpAddress publicIp, long vmId, int port) { logger.debug(String.format("Reverting network rules for VM ID %s on network %s", vmId, network.getName())); - FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port); - if (Objects.isNull(network.getVpcId())) { - logger.debug(String.format("Removing firewall rule %s", ruleVO.getId())); - firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + if (isPublicIpFirewallRequired(network)) { + FirewallRuleVO ruleVO = findIngressFirewallRuleForPorts(network, publicIp.getId(), port, port); + if (ruleVO != null) { + logger.debug(String.format("Removing firewall rule %s", ruleVO.getId())); + firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } } List pfRules = portForwardingRulesDao.listByVm(vmId); for (PortForwardingRuleVO pfRule : pfRules) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java index 07d062e23a19..c6443e2afbf7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -26,7 +26,6 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.service.ServiceOfferingVO; @@ -37,7 +36,6 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.context.CallContext; -import javax.inject.Inject; import java.io.File; import java.util.ArrayList; import java.util.List; @@ -47,9 +45,6 @@ public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker { - @Inject - private FirewallRulesDao firewallRulesDao; - private long removeNodeTimeoutTime; public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { @@ -128,9 +123,13 @@ protected boolean removeNodePortForwardingRules(Long nodeId, Network network, Us for (PortForwardingRuleVO pfRule : pfRules) { try { result &= rulesService.revokePortForwardingRule(pfRule.getId(), true); - if (Objects.isNull(network.getVpcId())) { - FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); - result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + if (isPublicIpFirewallRequired(network)) { + FirewallRuleVO ruleVO = findIngressFirewallRuleForPorts(network, pfRule.getSourceIpAddressId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); + if (ruleVO != null) { + result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } else { + logger.warn("Firewall rule for port {} of node {} not found, it may have been removed manually", pfRule.getSourcePortStart(), vm.getName()); + } } } catch (Exception e) { String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 72dd9816858d..3d833409150d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -82,7 +82,6 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.lb.LoadBalancingRulesService; @@ -130,8 +129,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected HostDao hostDao; @Inject - protected FirewallRulesDao firewallRulesDao; - @Inject protected NetworkACLService networkACLService; @Inject protected NetworkACLItemDao networkACLItemDao; @@ -832,6 +829,14 @@ protected void setupKubernetesClusterVpcTierRules(IpAddress publicIp, Network ne } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } + + // The VPC firewall drops traffic to the public IP unless a firewall rule allows it, open the + // API and SSH ports as for isolated networks. Create them after the load balancing and port + // forwarding rules: applying firewall rules on a VPC public IP does not associate the IP with + // the VPC virtual router, the first load balancing or port forwarding rule does. + if (isVpcFirewallEnabled(network)) { + createFirewallRules(publicIp, clusterVMIds, true); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 003a852f3dbe..1a6aa655420c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -173,6 +173,14 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t if (publicIp == null) { throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); } + boolean vpcFirewallEnabled = isVpcFirewallEnabled(network); + if (vpcFirewallEnabled) { + // Remove the existing SSH firewall rule, it is recreated below for the new node count + FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId()); + if (firewallRule == null) { + logger.warn("SSH firewall rule not found for Kubernetes cluster: {}. It may have been manually deleted or modified.", kubernetesCluster.getName()); + } + } try { removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1); } catch (ResourceUnavailableException e) { @@ -184,6 +192,10 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } + if (vpcFirewallEnabled) { + // After the port forwarding rules, as when the cluster is created + addFirewallRulesForNodes(publicIp, clusterVMIds.size()); + } } /** diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a15086c60d6c..df613d392288 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -691,6 +691,14 @@ protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network net throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), etcdStartPort, DEFAULT_SSH_PORT); + if (isVpcFirewallEnabled(network)) { + // In a VPC the firewall rule follows the port forwarding rule, see setupKubernetesClusterVpcTierRules + try { + provisionFirewallRules(publicIp, owner, etcdStartPort, etcdStartPort); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + } } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java index a25ec55cc04a..c35462db331c 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java @@ -44,8 +44,13 @@ import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.network.IpAddress; import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.vpc.dao.VpcServiceMapDao; +import com.cloud.utils.net.NetUtils; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterActionWorkerTest { @@ -74,9 +79,17 @@ public class KubernetesClusterActionWorkerTest { @Mock AffinityGroupDao affinityGroupDao; + @Mock + VpcServiceMapDao vpcServiceMapDao; + + @Mock + FirewallRulesDao firewallRulesDao; + KubernetesClusterActionWorker actionWorker = null; final static Long DEFAULT_ID = 1L; + final static Long PUBLIC_IP_ID = 7L; + final static int NODE_SSH_PORT = 2224; @Before public void setUp() throws Exception { @@ -86,10 +99,12 @@ public void setUp() throws Exception { kubernetesClusterManager.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao; kubernetesClusterManager.kubernetesClusterAffinityGroupMapDao = kubernetesClusterAffinityGroupMapDao; KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); - Mockito.when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID); + Mockito.lenient().when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID); actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager); actionWorker.ipAddressDao = ipAddressDao; actionWorker.affinityGroupDao = affinityGroupDao; + actionWorker.vpcServiceMapDao = vpcServiceMapDao; + actionWorker.firewallRulesDao = firewallRulesDao; } @Test @@ -147,6 +162,95 @@ public void testGetVpcTierKubernetesPublicIpValid() { Assert.assertNotNull(result); } + private Network mockIsolatedNetwork() { + Network network = Mockito.mock(Network.class); + Mockito.when(network.getVpcId()).thenReturn(null); + return network; + } + + private Network mockVpcTier(boolean vpcFirewall) { + Network network = mockNetworkForGetVpcTierKubernetesPublicIpTest(); + Mockito.when(vpcServiceMapDao.canProviderSupportServiceInVpc(DEFAULT_ID, Network.Service.Firewall, Network.Provider.VPCVirtualRouter)) + .thenReturn(vpcFirewall); + return network; + } + + @Test + public void testIsVpcFirewallEnabledIsolatedNetwork() { + Assert.assertFalse(actionWorker.isVpcFirewallEnabled(mockIsolatedNetwork())); + Mockito.verifyNoInteractions(vpcServiceMapDao); + } + + @Test + public void testIsVpcFirewallEnabledNullNetwork() { + Assert.assertFalse(actionWorker.isVpcFirewallEnabled(null)); + Mockito.verifyNoInteractions(vpcServiceMapDao); + } + + @Test + public void testIsVpcFirewallEnabledVpcWithoutVirtualRouterFirewall() { + Assert.assertFalse(actionWorker.isVpcFirewallEnabled(mockVpcTier(false))); + } + + @Test + public void testIsVpcFirewallEnabledVpcWithVirtualRouterFirewall() { + Assert.assertTrue(actionWorker.isVpcFirewallEnabled(mockVpcTier(true))); + } + + @Test + public void testIsPublicIpFirewallRequiredIsolatedNetwork() { + Assert.assertTrue(actionWorker.isPublicIpFirewallRequired(mockIsolatedNetwork())); + } + + @Test + public void testIsPublicIpFirewallRequiredVpcTierWithoutFirewall() { + Assert.assertFalse(actionWorker.isPublicIpFirewallRequired(mockVpcTier(false))); + } + + @Test + public void testIsPublicIpFirewallRequiredVpcTierWithFirewall() { + Assert.assertTrue(actionWorker.isPublicIpFirewallRequired(mockVpcTier(true))); + } + + @Test + public void testFindIngressFirewallRuleForPortsIsolatedNetworkLooksUpByNetwork() { + Network network = mockIsolatedNetwork(); + Mockito.when(network.getId()).thenReturn(DEFAULT_ID); + FirewallRuleVO rule = Mockito.mock(FirewallRuleVO.class); + Mockito.when(firewallRulesDao.findByNetworkIdAndPorts(DEFAULT_ID, NODE_SSH_PORT, NODE_SSH_PORT)).thenReturn(rule); + + Assert.assertEquals(rule, actionWorker.findIngressFirewallRuleForPorts(network, PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT)); + Mockito.verify(firewallRulesDao, Mockito.never()).listByIpPurposePortsProtocolAndNotRevoked( + Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testFindIngressFirewallRuleForPortsVpcTierLooksUpByPublicIp() { + FirewallRuleVO rule = Mockito.mock(FirewallRuleVO.class); + Mockito.when(firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall)).thenReturn(List.of(rule)); + + Assert.assertEquals(rule, actionWorker.findIngressFirewallRuleForPorts(mockNetworkForGetVpcTierKubernetesPublicIpTest(), + PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT)); + Mockito.verify(firewallRulesDao, Mockito.never()).findByNetworkIdAndPorts(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt()); + } + + @Test + public void testFindIngressFirewallRuleForPortsVpcTierNoRule() { + Mockito.when(firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall)).thenReturn(new ArrayList<>()); + + Assert.assertNull(actionWorker.findIngressFirewallRuleForPorts(mockNetworkForGetVpcTierKubernetesPublicIpTest(), + PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT)); + } + + @Test + public void testFindIngressFirewallRuleForPortsVpcTierWithoutPublicIp() { + Assert.assertNull(actionWorker.findIngressFirewallRuleForPorts(mockNetworkForGetVpcTierKubernetesPublicIpTest(), + null, NODE_SSH_PORT, NODE_SSH_PORT)); + Mockito.verifyNoInteractions(firewallRulesDao); + } + @Test public void testGetAffinityGroupIdsForNodeTypeReturnsIds() { Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL")) diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java index c220a3468afb..d2918ec7a8ef 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java @@ -23,14 +23,22 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import java.util.HashMap; +import java.util.List; + @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterResourceModifierActionWorkerTest { @Mock @@ -51,6 +59,18 @@ public class KubernetesClusterResourceModifierActionWorkerTest { @Mock private KubernetesCluster kubernetesClusterMock; + @Mock + private NetworkOfferingDao networkOfferingDaoMock; + + @Mock + private NetworkOfferingVO networkOfferingMock; + + @Mock + private Network networkMock; + + @Mock + private IpAddress publicIpMock; + private KubernetesClusterResourceModifierActionWorker kubernetesClusterResourceModifierActionWorker; @Before @@ -135,4 +155,49 @@ public void getKubernetesClusterNodeNamePrefixTestNormalizedPrefixShouldNotStart Mockito.when(kubernetesClusterMock.getName()).thenReturn(originalPrefix); Assert.assertEquals(expectedPrefix, kubernetesClusterResourceModifierActionWorker.getKubernetesClusterNodeNamePrefix()); } + + private KubernetesClusterResourceModifierActionWorker mockWorkerForVpcTierRules(boolean vpcFirewall) throws Exception { + KubernetesClusterResourceModifierActionWorker worker = Mockito.spy(kubernetesClusterResourceModifierActionWorker); + worker.networkOfferingDao = networkOfferingDaoMock; + Mockito.when(networkMock.getNetworkOfferingId()).thenReturn(1L); + Mockito.when(networkOfferingDaoMock.findById(1L)).thenReturn(networkOfferingMock); + Mockito.when(networkOfferingMock.isConserveMode()).thenReturn(true); + Mockito.doNothing().when(worker).createVpcTierAclRules(networkMock); + Mockito.doReturn(vpcFirewall).when(worker).isVpcFirewallEnabled(networkMock); + Mockito.doNothing().when(worker).provisionLoadBalancerRule(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyInt()); + Mockito.doReturn(new HashMap<>()).when(worker).getVmPortMap(); + Mockito.doNothing().when(worker).provisionSshPortForwardingRules(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyMap()); + return worker; + } + + @Test + public void setupKubernetesClusterVpcTierRulesTestOpensPublicIpFirewallWhenVpcFirewallIsEnabled() throws Exception { + List clusterVmIds = List.of(10L, 11L); + KubernetesClusterResourceModifierActionWorker worker = mockWorkerForVpcTierRules(true); + Mockito.doReturn(new HashMap<>()).when(worker).createFirewallRules(publicIpMock, clusterVmIds, true); + + worker.setupKubernetesClusterVpcTierRules(publicIpMock, networkMock, clusterVmIds); + + // Firewall rules on a VPC public IP must follow the LB/PF rules, which associate the IP with the VR + InOrder inOrder = Mockito.inOrder(worker); + inOrder.verify(worker).createVpcTierAclRules(networkMock); + inOrder.verify(worker).provisionLoadBalancerRule(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyInt()); + inOrder.verify(worker).provisionSshPortForwardingRules(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyMap()); + inOrder.verify(worker).createFirewallRules(publicIpMock, clusterVmIds, true); + } + + @Test + public void setupKubernetesClusterVpcTierRulesTestKeepsAclOnlyWhenVpcHasNoFirewall() throws Exception { + List clusterVmIds = List.of(10L, 11L); + KubernetesClusterResourceModifierActionWorker worker = mockWorkerForVpcTierRules(false); + + worker.setupKubernetesClusterVpcTierRules(publicIpMock, networkMock, clusterVmIds); + + Mockito.verify(worker).createVpcTierAclRules(networkMock); + Mockito.verify(worker, Mockito.never()).createFirewallRules(Mockito.any(), Mockito.anyList(), Mockito.anyBoolean()); + } }