diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..110be072389b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -51,6 +51,7 @@ import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.firewall.FirewallService; import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -59,6 +60,7 @@ import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic; import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine; @@ -107,10 +109,12 @@ import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.NetworkService; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.vpc.VpcService; +import com.cloud.network.vpc.dao.VpcServiceMapDao; import com.cloud.projects.ProjectService; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Storage; @@ -214,12 +218,16 @@ public class KubernetesClusterActionWorker { @Inject public VpcService vpcService; @Inject + protected VpcServiceMapDao vpcServiceMapDao; + @Inject public PortForwardingRulesDao portForwardingRulesDao; @Inject protected RulesService rulesService; @Inject protected FirewallService firewallService; @Inject + protected FirewallRulesDao firewallRulesDao; + @Inject private NicDao nicDao; @Inject protected AffinityGroupDao affinityGroupDao; @@ -1113,6 +1121,49 @@ protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { return k8sConfig; } + /** + * When the VPC virtual router provides the Firewall service of a VPC, it drops ingress traffic to + * public IPs that has no firewall rule (in addition to the network ACL of each tier). The ports + * used by CKS must then be opened on the cluster public IP, as for isolated networks. This is the + * same check that sets vpc_firewall_enabled for the VPC virtual router. + * + * @param network the network of the Kubernetes cluster + * @return true if the network is a VPC tier and the VPC virtual router provides the Firewall service + */ + protected boolean isVpcFirewallEnabled(Network network) { + if (network == null || network.getVpcId() == null) { + return false; + } + return vpcServiceMapDao.canProviderSupportServiceInVpc(network.getVpcId(), Network.Service.Firewall, Network.Provider.VPCVirtualRouter); + } + + /** + * @param network the network of the Kubernetes cluster + * @return true if ingress firewall rules are needed on the cluster public IP: always for isolated + * networks, and for VPC tiers when the VPC uses the Firewall service + */ + protected boolean isPublicIpFirewallRequired(Network network) { + return network.getVpcId() == null || isVpcFirewallEnabled(network); + } + + /** + * Finds the ingress firewall rule for the given public port range of the cluster. Firewall rules on a + * public IP of a VPC are stored with the VPC and without a network, so they are looked up by the IP. + * + * @return the rule, or null if none exists + */ + protected FirewallRuleVO findIngressFirewallRuleForPorts(Network network, Long publicIpId, int startPort, int endPort) { + if (network.getVpcId() == null) { + return firewallRulesDao.findByNetworkIdAndPorts(network.getId(), startPort, endPort); + } + if (publicIpId == null) { + return null; + } + List rules = firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(publicIpId, startPort, endPort, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall); + return CollectionUtils.isEmpty(rules) ? null : rules.get(0); + } + public Map addFirewallRulesForNodes(IpAddress publicIp, int size) throws ManagementServerException { Map vmIdPortMap = new HashMap<>(); CallContext.register(CallContext.current(), null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java index 8b694adf1cca..0c2b88aa0fa1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -31,7 +31,6 @@ import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.service.ServiceOfferingVO; @@ -49,7 +48,6 @@ import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Level; -import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; @@ -60,8 +58,6 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker { - @Inject - private FirewallRulesDao firewallRulesDao; private long addNodeTimeoutTime; List finalNodeIds = new ArrayList<>(); @@ -229,9 +225,13 @@ private Pair validateAndSetupNode(Network network, IpAddress p provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); } provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT); + if (isVpcFirewallEnabled(network)) { + // In a VPC the firewall rule follows the port forwarding rule, see setupKubernetesClusterVpcTierRules + provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); + } boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort); if (!isCompatible) { - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, publicIp, nodeId, sshStartPort); return new Pair<>(false, nodeIndex); } @@ -249,13 +249,13 @@ private Pair validateAndSetupNode(Network network, IpAddress p InsufficientCapacityException | IllegalAccessException e) { logger.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName())); // remove added Firewall and PF rules - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, publicIp, nodeId, sshStartPort); return new Pair<>( false, nodeIndex); } catch (Exception e) { String errMsg = String.format("Unexpected exception while trying to add the external node %s to the Kubernetes cluster %s: %s", nodeId, kubernetesCluster.getName(), e.getMessage()); logger.error(errMsg, e); - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, publicIp, nodeId, sshStartPort); throw new CloudRuntimeException(e); } return new Pair<>(true, ++nodeIndex); @@ -310,12 +310,14 @@ private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File p } } - private void revertNetworkRules(Network network, long vmId, int port) { + private void revertNetworkRules(Network network, IpAddress publicIp, long vmId, int port) { logger.debug(String.format("Reverting network rules for VM ID %s on network %s", vmId, network.getName())); - FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port); - if (Objects.isNull(network.getVpcId())) { - logger.debug(String.format("Removing firewall rule %s", ruleVO.getId())); - firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + if (isPublicIpFirewallRequired(network)) { + FirewallRuleVO ruleVO = findIngressFirewallRuleForPorts(network, publicIp.getId(), port, port); + if (ruleVO != null) { + logger.debug(String.format("Removing firewall rule %s", ruleVO.getId())); + firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } } List pfRules = portForwardingRulesDao.listByVm(vmId); for (PortForwardingRuleVO pfRule : pfRules) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java index 07d062e23a19..c6443e2afbf7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -26,7 +26,6 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.service.ServiceOfferingVO; @@ -37,7 +36,6 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.context.CallContext; -import javax.inject.Inject; import java.io.File; import java.util.ArrayList; import java.util.List; @@ -47,9 +45,6 @@ public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker { - @Inject - private FirewallRulesDao firewallRulesDao; - private long removeNodeTimeoutTime; public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { @@ -128,9 +123,13 @@ protected boolean removeNodePortForwardingRules(Long nodeId, Network network, Us for (PortForwardingRuleVO pfRule : pfRules) { try { result &= rulesService.revokePortForwardingRule(pfRule.getId(), true); - if (Objects.isNull(network.getVpcId())) { - FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); - result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + if (isPublicIpFirewallRequired(network)) { + FirewallRuleVO ruleVO = findIngressFirewallRuleForPorts(network, pfRule.getSourceIpAddressId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); + if (ruleVO != null) { + result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } else { + logger.warn("Firewall rule for port {} of node {} not found, it may have been removed manually", pfRule.getSourcePortStart(), vm.getName()); + } } } catch (Exception e) { String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 72dd9816858d..3d833409150d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -82,7 +82,6 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.lb.LoadBalancingRulesService; @@ -130,8 +129,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected HostDao hostDao; @Inject - protected FirewallRulesDao firewallRulesDao; - @Inject protected NetworkACLService networkACLService; @Inject protected NetworkACLItemDao networkACLItemDao; @@ -832,6 +829,14 @@ protected void setupKubernetesClusterVpcTierRules(IpAddress publicIp, Network ne } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } + + // The VPC firewall drops traffic to the public IP unless a firewall rule allows it, open the + // API and SSH ports as for isolated networks. Create them after the load balancing and port + // forwarding rules: applying firewall rules on a VPC public IP does not associate the IP with + // the VPC virtual router, the first load balancing or port forwarding rule does. + if (isVpcFirewallEnabled(network)) { + createFirewallRules(publicIp, clusterVMIds, true); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 003a852f3dbe..1a6aa655420c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -173,6 +173,14 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t if (publicIp == null) { throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); } + boolean vpcFirewallEnabled = isVpcFirewallEnabled(network); + if (vpcFirewallEnabled) { + // Remove the existing SSH firewall rule, it is recreated below for the new node count + FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId()); + if (firewallRule == null) { + logger.warn("SSH firewall rule not found for Kubernetes cluster: {}. It may have been manually deleted or modified.", kubernetesCluster.getName()); + } + } try { removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1); } catch (ResourceUnavailableException e) { @@ -184,6 +192,10 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } + if (vpcFirewallEnabled) { + // After the port forwarding rules, as when the cluster is created + addFirewallRulesForNodes(publicIp, clusterVMIds.size()); + } } /** diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a15086c60d6c..df613d392288 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -691,6 +691,14 @@ protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network net throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), etcdStartPort, DEFAULT_SSH_PORT); + if (isVpcFirewallEnabled(network)) { + // In a VPC the firewall rule follows the port forwarding rule, see setupKubernetesClusterVpcTierRules + try { + provisionFirewallRules(publicIp, owner, etcdStartPort, etcdStartPort); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + } } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java index a25ec55cc04a..c35462db331c 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java @@ -44,8 +44,13 @@ import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.network.IpAddress; import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.vpc.dao.VpcServiceMapDao; +import com.cloud.utils.net.NetUtils; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterActionWorkerTest { @@ -74,9 +79,17 @@ public class KubernetesClusterActionWorkerTest { @Mock AffinityGroupDao affinityGroupDao; + @Mock + VpcServiceMapDao vpcServiceMapDao; + + @Mock + FirewallRulesDao firewallRulesDao; + KubernetesClusterActionWorker actionWorker = null; final static Long DEFAULT_ID = 1L; + final static Long PUBLIC_IP_ID = 7L; + final static int NODE_SSH_PORT = 2224; @Before public void setUp() throws Exception { @@ -86,10 +99,12 @@ public void setUp() throws Exception { kubernetesClusterManager.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao; kubernetesClusterManager.kubernetesClusterAffinityGroupMapDao = kubernetesClusterAffinityGroupMapDao; KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); - Mockito.when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID); + Mockito.lenient().when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID); actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager); actionWorker.ipAddressDao = ipAddressDao; actionWorker.affinityGroupDao = affinityGroupDao; + actionWorker.vpcServiceMapDao = vpcServiceMapDao; + actionWorker.firewallRulesDao = firewallRulesDao; } @Test @@ -147,6 +162,95 @@ public void testGetVpcTierKubernetesPublicIpValid() { Assert.assertNotNull(result); } + private Network mockIsolatedNetwork() { + Network network = Mockito.mock(Network.class); + Mockito.when(network.getVpcId()).thenReturn(null); + return network; + } + + private Network mockVpcTier(boolean vpcFirewall) { + Network network = mockNetworkForGetVpcTierKubernetesPublicIpTest(); + Mockito.when(vpcServiceMapDao.canProviderSupportServiceInVpc(DEFAULT_ID, Network.Service.Firewall, Network.Provider.VPCVirtualRouter)) + .thenReturn(vpcFirewall); + return network; + } + + @Test + public void testIsVpcFirewallEnabledIsolatedNetwork() { + Assert.assertFalse(actionWorker.isVpcFirewallEnabled(mockIsolatedNetwork())); + Mockito.verifyNoInteractions(vpcServiceMapDao); + } + + @Test + public void testIsVpcFirewallEnabledNullNetwork() { + Assert.assertFalse(actionWorker.isVpcFirewallEnabled(null)); + Mockito.verifyNoInteractions(vpcServiceMapDao); + } + + @Test + public void testIsVpcFirewallEnabledVpcWithoutVirtualRouterFirewall() { + Assert.assertFalse(actionWorker.isVpcFirewallEnabled(mockVpcTier(false))); + } + + @Test + public void testIsVpcFirewallEnabledVpcWithVirtualRouterFirewall() { + Assert.assertTrue(actionWorker.isVpcFirewallEnabled(mockVpcTier(true))); + } + + @Test + public void testIsPublicIpFirewallRequiredIsolatedNetwork() { + Assert.assertTrue(actionWorker.isPublicIpFirewallRequired(mockIsolatedNetwork())); + } + + @Test + public void testIsPublicIpFirewallRequiredVpcTierWithoutFirewall() { + Assert.assertFalse(actionWorker.isPublicIpFirewallRequired(mockVpcTier(false))); + } + + @Test + public void testIsPublicIpFirewallRequiredVpcTierWithFirewall() { + Assert.assertTrue(actionWorker.isPublicIpFirewallRequired(mockVpcTier(true))); + } + + @Test + public void testFindIngressFirewallRuleForPortsIsolatedNetworkLooksUpByNetwork() { + Network network = mockIsolatedNetwork(); + Mockito.when(network.getId()).thenReturn(DEFAULT_ID); + FirewallRuleVO rule = Mockito.mock(FirewallRuleVO.class); + Mockito.when(firewallRulesDao.findByNetworkIdAndPorts(DEFAULT_ID, NODE_SSH_PORT, NODE_SSH_PORT)).thenReturn(rule); + + Assert.assertEquals(rule, actionWorker.findIngressFirewallRuleForPorts(network, PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT)); + Mockito.verify(firewallRulesDao, Mockito.never()).listByIpPurposePortsProtocolAndNotRevoked( + Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testFindIngressFirewallRuleForPortsVpcTierLooksUpByPublicIp() { + FirewallRuleVO rule = Mockito.mock(FirewallRuleVO.class); + Mockito.when(firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall)).thenReturn(List.of(rule)); + + Assert.assertEquals(rule, actionWorker.findIngressFirewallRuleForPorts(mockNetworkForGetVpcTierKubernetesPublicIpTest(), + PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT)); + Mockito.verify(firewallRulesDao, Mockito.never()).findByNetworkIdAndPorts(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt()); + } + + @Test + public void testFindIngressFirewallRuleForPortsVpcTierNoRule() { + Mockito.when(firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall)).thenReturn(new ArrayList<>()); + + Assert.assertNull(actionWorker.findIngressFirewallRuleForPorts(mockNetworkForGetVpcTierKubernetesPublicIpTest(), + PUBLIC_IP_ID, NODE_SSH_PORT, NODE_SSH_PORT)); + } + + @Test + public void testFindIngressFirewallRuleForPortsVpcTierWithoutPublicIp() { + Assert.assertNull(actionWorker.findIngressFirewallRuleForPorts(mockNetworkForGetVpcTierKubernetesPublicIpTest(), + null, NODE_SSH_PORT, NODE_SSH_PORT)); + Mockito.verifyNoInteractions(firewallRulesDao); + } + @Test public void testGetAffinityGroupIdsForNodeTypeReturnsIds() { Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL")) diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java index c220a3468afb..d2918ec7a8ef 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java @@ -23,14 +23,22 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import java.util.HashMap; +import java.util.List; + @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterResourceModifierActionWorkerTest { @Mock @@ -51,6 +59,18 @@ public class KubernetesClusterResourceModifierActionWorkerTest { @Mock private KubernetesCluster kubernetesClusterMock; + @Mock + private NetworkOfferingDao networkOfferingDaoMock; + + @Mock + private NetworkOfferingVO networkOfferingMock; + + @Mock + private Network networkMock; + + @Mock + private IpAddress publicIpMock; + private KubernetesClusterResourceModifierActionWorker kubernetesClusterResourceModifierActionWorker; @Before @@ -135,4 +155,49 @@ public void getKubernetesClusterNodeNamePrefixTestNormalizedPrefixShouldNotStart Mockito.when(kubernetesClusterMock.getName()).thenReturn(originalPrefix); Assert.assertEquals(expectedPrefix, kubernetesClusterResourceModifierActionWorker.getKubernetesClusterNodeNamePrefix()); } + + private KubernetesClusterResourceModifierActionWorker mockWorkerForVpcTierRules(boolean vpcFirewall) throws Exception { + KubernetesClusterResourceModifierActionWorker worker = Mockito.spy(kubernetesClusterResourceModifierActionWorker); + worker.networkOfferingDao = networkOfferingDaoMock; + Mockito.when(networkMock.getNetworkOfferingId()).thenReturn(1L); + Mockito.when(networkOfferingDaoMock.findById(1L)).thenReturn(networkOfferingMock); + Mockito.when(networkOfferingMock.isConserveMode()).thenReturn(true); + Mockito.doNothing().when(worker).createVpcTierAclRules(networkMock); + Mockito.doReturn(vpcFirewall).when(worker).isVpcFirewallEnabled(networkMock); + Mockito.doNothing().when(worker).provisionLoadBalancerRule(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyInt()); + Mockito.doReturn(new HashMap<>()).when(worker).getVmPortMap(); + Mockito.doNothing().when(worker).provisionSshPortForwardingRules(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyMap()); + return worker; + } + + @Test + public void setupKubernetesClusterVpcTierRulesTestOpensPublicIpFirewallWhenVpcFirewallIsEnabled() throws Exception { + List clusterVmIds = List.of(10L, 11L); + KubernetesClusterResourceModifierActionWorker worker = mockWorkerForVpcTierRules(true); + Mockito.doReturn(new HashMap<>()).when(worker).createFirewallRules(publicIpMock, clusterVmIds, true); + + worker.setupKubernetesClusterVpcTierRules(publicIpMock, networkMock, clusterVmIds); + + // Firewall rules on a VPC public IP must follow the LB/PF rules, which associate the IP with the VR + InOrder inOrder = Mockito.inOrder(worker); + inOrder.verify(worker).createVpcTierAclRules(networkMock); + inOrder.verify(worker).provisionLoadBalancerRule(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyInt()); + inOrder.verify(worker).provisionSshPortForwardingRules(Mockito.eq(publicIpMock), Mockito.eq(networkMock), Mockito.any(), + Mockito.anyList(), Mockito.anyMap()); + inOrder.verify(worker).createFirewallRules(publicIpMock, clusterVmIds, true); + } + + @Test + public void setupKubernetesClusterVpcTierRulesTestKeepsAclOnlyWhenVpcHasNoFirewall() throws Exception { + List clusterVmIds = List.of(10L, 11L); + KubernetesClusterResourceModifierActionWorker worker = mockWorkerForVpcTierRules(false); + + worker.setupKubernetesClusterVpcTierRules(publicIpMock, networkMock, clusterVmIds); + + Mockito.verify(worker).createVpcTierAclRules(networkMock); + Mockito.verify(worker, Mockito.never()).createFirewallRules(Mockito.any(), Mockito.anyList(), Mockito.anyBoolean()); + } }